1 /*
2 * Copyright (C) 2013-2014, 2016-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11 #include "mali_soft_job.h"
12 #include "mali_osk.h"
13 #include "mali_timeline.h"
14 #include "mali_session.h"
15 #include "mali_kernel_common.h"
16 #include "mali_uk_types.h"
17 #include "mali_scheduler.h"
18 #include "mali_executor.h"
19
mali_soft_job_system_lock(struct mali_soft_job_system * system)20 MALI_STATIC_INLINE void mali_soft_job_system_lock(struct mali_soft_job_system *system)
21 {
22 MALI_DEBUG_ASSERT_POINTER(system);
23 _mali_osk_spinlock_irq_lock(system->lock);
24 MALI_DEBUG_PRINT(5, ("Mali Soft Job: soft system %p lock taken\n", system));
25 MALI_DEBUG_ASSERT(0 == system->lock_owner);
26 MALI_DEBUG_CODE(system->lock_owner = _mali_osk_get_tid());
27 }
28
mali_soft_job_system_unlock(struct mali_soft_job_system * system)29 MALI_STATIC_INLINE void mali_soft_job_system_unlock(struct mali_soft_job_system *system)
30 {
31 MALI_DEBUG_ASSERT_POINTER(system);
32 MALI_DEBUG_PRINT(5, ("Mali Soft Job: releasing soft system %p lock\n", system));
33 MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner);
34 MALI_DEBUG_CODE(system->lock_owner = 0);
35 _mali_osk_spinlock_irq_unlock(system->lock);
36 }
37
38 #if defined(DEBUG)
mali_soft_job_system_assert_locked(struct mali_soft_job_system * system)39 MALI_STATIC_INLINE void mali_soft_job_system_assert_locked(struct mali_soft_job_system *system)
40 {
41 MALI_DEBUG_ASSERT_POINTER(system);
42 MALI_DEBUG_ASSERT(_mali_osk_get_tid() == system->lock_owner);
43 }
44 #define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system) mali_soft_job_system_assert_locked(system)
45 #else
46 #define MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system)
47 #endif /* defined(DEBUG) */
48
mali_soft_job_system_create(struct mali_session_data * session)49 struct mali_soft_job_system *mali_soft_job_system_create(struct mali_session_data *session)
50 {
51 struct mali_soft_job_system *system;
52
53 MALI_DEBUG_ASSERT_POINTER(session);
54
55 system = (struct mali_soft_job_system *) _mali_osk_calloc(1, sizeof(struct mali_soft_job_system));
56 if (NULL == system) {
57 return NULL;
58 }
59
60 system->session = session;
61
62 system->lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER);
63 if (NULL == system->lock) {
64 mali_soft_job_system_destroy(system);
65 return NULL;
66 }
67 system->lock_owner = 0;
68 system->last_job_id = 0;
69
70 _MALI_OSK_INIT_LIST_HEAD(&(system->jobs_used));
71
72 return system;
73 }
74
mali_soft_job_system_destroy(struct mali_soft_job_system * system)75 void mali_soft_job_system_destroy(struct mali_soft_job_system *system)
76 {
77 MALI_DEBUG_ASSERT_POINTER(system);
78
79 /* All jobs should be free at this point. */
80 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&(system->jobs_used)));
81
82 if (NULL != system) {
83 if (NULL != system->lock) {
84 _mali_osk_spinlock_irq_term(system->lock);
85 }
86 _mali_osk_free(system);
87 }
88 }
89
mali_soft_job_system_free_job(struct mali_soft_job_system * system,struct mali_soft_job * job)90 static void mali_soft_job_system_free_job(struct mali_soft_job_system *system, struct mali_soft_job *job)
91 {
92 MALI_DEBUG_ASSERT_POINTER(job);
93 MALI_DEBUG_ASSERT_POINTER(system);
94
95 mali_soft_job_system_lock(job->system);
96
97 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id);
98 MALI_DEBUG_ASSERT(system == job->system);
99
100 _mali_osk_list_del(&(job->system_list));
101
102 mali_soft_job_system_unlock(job->system);
103
104 _mali_osk_free(job);
105 }
106
mali_soft_job_system_lookup_job(struct mali_soft_job_system * system,u32 job_id)107 MALI_STATIC_INLINE struct mali_soft_job *mali_soft_job_system_lookup_job(struct mali_soft_job_system *system, u32 job_id)
108 {
109 struct mali_soft_job *job, *tmp;
110
111 MALI_DEBUG_ASSERT_POINTER(system);
112 MALI_ASSERT_SOFT_JOB_SYSTEM_LOCKED(system);
113
114 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) {
115 if (job->id == job_id)
116 return job;
117 }
118
119 return NULL;
120 }
121
mali_soft_job_destroy(struct mali_soft_job * job)122 void mali_soft_job_destroy(struct mali_soft_job *job)
123 {
124 MALI_DEBUG_ASSERT_POINTER(job);
125 MALI_DEBUG_ASSERT_POINTER(job->system);
126
127 MALI_DEBUG_PRINT(4, ("Mali Soft Job: destroying soft job %u (0x%08X)\n", job->id, job));
128
129 if (NULL != job) {
130 if (0 < _mali_osk_atomic_dec_return(&job->refcount)) return;
131
132 _mali_osk_atomic_term(&job->refcount);
133
134 if (NULL != job->activated_notification) {
135 _mali_osk_notification_delete(job->activated_notification);
136 job->activated_notification = NULL;
137 }
138
139 mali_soft_job_system_free_job(job->system, job);
140 }
141 }
142
mali_soft_job_create(struct mali_soft_job_system * system,mali_soft_job_type type,u64 user_job)143 struct mali_soft_job *mali_soft_job_create(struct mali_soft_job_system *system, mali_soft_job_type type, u64 user_job)
144 {
145 struct mali_soft_job *job;
146 _mali_osk_notification_t *notification = NULL;
147
148 MALI_DEBUG_ASSERT_POINTER(system);
149 MALI_DEBUG_ASSERT((MALI_SOFT_JOB_TYPE_USER_SIGNALED == type) ||
150 (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == type));
151
152 notification = _mali_osk_notification_create(_MALI_NOTIFICATION_SOFT_ACTIVATED, sizeof(_mali_uk_soft_job_activated_s));
153 if (unlikely(NULL == notification)) {
154 MALI_PRINT_ERROR(("Mali Soft Job: failed to allocate notification"));
155 return NULL;
156 }
157
158 job = _mali_osk_malloc(sizeof(struct mali_soft_job));
159 if (unlikely(NULL == job)) {
160 MALI_DEBUG_PRINT(2, ("Mali Soft Job: system alloc job failed. \n"));
161 return NULL;
162 }
163
164 mali_soft_job_system_lock(system);
165
166 job->system = system;
167 job->id = system->last_job_id++;
168 job->state = MALI_SOFT_JOB_STATE_ALLOCATED;
169
170 _mali_osk_list_add(&(job->system_list), &(system->jobs_used));
171
172 job->type = type;
173 job->user_job = user_job;
174 job->activated = MALI_FALSE;
175
176 job->activated_notification = notification;
177
178 _mali_osk_atomic_init(&job->refcount, 1);
179
180 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state);
181 MALI_DEBUG_ASSERT(system == job->system);
182 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_INVALID_ID != job->id);
183
184 mali_soft_job_system_unlock(system);
185
186 return job;
187 }
188
mali_soft_job_start(struct mali_soft_job * job,struct mali_timeline_fence * fence)189 mali_timeline_point mali_soft_job_start(struct mali_soft_job *job, struct mali_timeline_fence *fence)
190 {
191 mali_timeline_point point;
192 struct mali_soft_job_system *system;
193
194 MALI_DEBUG_ASSERT_POINTER(job);
195 MALI_DEBUG_ASSERT_POINTER(fence);
196
197 MALI_DEBUG_ASSERT_POINTER(job->system);
198 system = job->system;
199
200 MALI_DEBUG_ASSERT_POINTER(system->session);
201 MALI_DEBUG_ASSERT_POINTER(system->session->timeline_system);
202
203 mali_soft_job_system_lock(system);
204
205 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_ALLOCATED == job->state);
206 job->state = MALI_SOFT_JOB_STATE_STARTED;
207
208 mali_soft_job_system_unlock(system);
209
210 MALI_DEBUG_PRINT(4, ("Mali Soft Job: starting soft job %u (0x%08X)\n", job->id, job));
211
212 mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_SOFT, fence, job);
213 point = mali_timeline_system_add_tracker(system->session->timeline_system, &job->tracker, MALI_TIMELINE_SOFT);
214
215 return point;
216 }
217
mali_soft_job_is_activated(void * data)218 static mali_bool mali_soft_job_is_activated(void *data)
219 {
220 struct mali_soft_job *job;
221
222 job = (struct mali_soft_job *) data;
223 MALI_DEBUG_ASSERT_POINTER(job);
224
225 return job->activated;
226 }
227
mali_soft_job_system_signal_job(struct mali_soft_job_system * system,u32 job_id)228 _mali_osk_errcode_t mali_soft_job_system_signal_job(struct mali_soft_job_system *system, u32 job_id)
229 {
230 struct mali_soft_job *job;
231 struct mali_timeline_system *timeline_system;
232 mali_scheduler_mask schedule_mask;
233
234 MALI_DEBUG_ASSERT_POINTER(system);
235
236 mali_soft_job_system_lock(system);
237
238 job = mali_soft_job_system_lookup_job(system, job_id);
239
240 if ((NULL == job) || (MALI_SOFT_JOB_TYPE_USER_SIGNALED != job->type)
241 || !(MALI_SOFT_JOB_STATE_STARTED == job->state || MALI_SOFT_JOB_STATE_TIMED_OUT == job->state)) {
242 mali_soft_job_system_unlock(system);
243 MALI_PRINT_ERROR(("Mali Soft Job: invalid soft job id %u", job_id));
244 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
245 }
246
247 if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) {
248 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
249 mali_soft_job_system_unlock(system);
250
251 MALI_DEBUG_ASSERT(MALI_TRUE == job->activated);
252 MALI_DEBUG_PRINT(4, ("Mali Soft Job: soft job %u (0x%08X) was timed out\n", job->id, job));
253 mali_soft_job_destroy(job);
254
255 return _MALI_OSK_ERR_TIMEOUT;
256 }
257
258 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state);
259
260 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
261 mali_soft_job_system_unlock(system);
262
263 /* Since the job now is in signaled state, timeouts from the timeline system will be
264 * ignored, and it is not possible to signal this job again. */
265
266 timeline_system = system->session->timeline_system;
267 MALI_DEBUG_ASSERT_POINTER(timeline_system);
268
269 /* Wait until activated. */
270 _mali_osk_wait_queue_wait_event(timeline_system->wait_queue, mali_soft_job_is_activated, (void *) job);
271
272 MALI_DEBUG_PRINT(4, ("Mali Soft Job: signaling soft job %u (0x%08X)\n", job->id, job));
273
274 schedule_mask = mali_timeline_tracker_release(&job->tracker);
275 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
276
277 mali_soft_job_destroy(job);
278
279 return _MALI_OSK_ERR_OK;
280 }
281
mali_soft_job_send_activated_notification(struct mali_soft_job * job)282 static void mali_soft_job_send_activated_notification(struct mali_soft_job *job)
283 {
284 if (NULL != job->activated_notification) {
285 _mali_uk_soft_job_activated_s *res = job->activated_notification->result_buffer;
286 res->user_job = job->user_job;
287 mali_session_send_notification(job->system->session, job->activated_notification);
288 }
289 job->activated_notification = NULL;
290 }
291
mali_soft_job_system_activate_job(struct mali_soft_job * job)292 mali_scheduler_mask mali_soft_job_system_activate_job(struct mali_soft_job *job)
293 {
294 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
295
296 MALI_DEBUG_ASSERT_POINTER(job);
297 MALI_DEBUG_ASSERT_POINTER(job->system);
298 MALI_DEBUG_ASSERT_POINTER(job->system->session);
299
300 MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline activation for soft job %u (0x%08X).\n", job->id, job));
301
302 mali_soft_job_system_lock(job->system);
303
304 if (unlikely(job->system->session->is_aborting)) {
305 MALI_DEBUG_PRINT(3, ("Mali Soft Job: Soft job %u (0x%08X) activated while session is aborting.\n", job->id, job));
306
307 mali_soft_job_system_unlock(job->system);
308
309 /* Since we are in shutdown, we can ignore the scheduling bitmask. */
310 mali_timeline_tracker_release(&job->tracker);
311 mali_soft_job_destroy(job);
312 return schedule_mask;
313 }
314
315 /* Send activated notification. */
316 mali_soft_job_send_activated_notification(job);
317
318 /* Wake up sleeping signaler. */
319 job->activated = MALI_TRUE;
320
321 /* If job type is self signaled, release tracker, move soft job to free list, and scheduler at once */
322 if (MALI_SOFT_JOB_TYPE_SELF_SIGNALED == job->type) {
323 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state);
324
325 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
326 mali_soft_job_system_unlock(job->system);
327
328 schedule_mask |= mali_timeline_tracker_release(&job->tracker);
329
330 mali_soft_job_destroy(job);
331 } else {
332 _mali_osk_wait_queue_wake_up(job->tracker.system->wait_queue);
333
334 mali_soft_job_system_unlock(job->system);
335 }
336
337 return schedule_mask;
338 }
339
mali_soft_job_system_timeout_job(struct mali_soft_job * job)340 mali_scheduler_mask mali_soft_job_system_timeout_job(struct mali_soft_job *job)
341 {
342 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
343
344 MALI_DEBUG_ASSERT_POINTER(job);
345 MALI_DEBUG_ASSERT_POINTER(job->system);
346 MALI_DEBUG_ASSERT_POINTER(job->system->session);
347 MALI_DEBUG_ASSERT(MALI_TRUE == job->activated);
348
349 MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeline timeout for soft job %u (0x%08X).\n", job->id, job));
350
351 mali_soft_job_system_lock(job->system);
352
353 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state ||
354 MALI_SOFT_JOB_STATE_SIGNALED == job->state);
355
356 if (unlikely(job->system->session->is_aborting)) {
357 /* The session is aborting. This job will be released and destroyed by @ref
358 * mali_soft_job_system_abort(). */
359 mali_soft_job_system_unlock(job->system);
360
361 return MALI_SCHEDULER_MASK_EMPTY;
362 }
363
364 if (MALI_SOFT_JOB_STATE_STARTED != job->state) {
365 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state);
366
367 /* The job is about to be signaled, ignore timeout. */
368 MALI_DEBUG_PRINT(4, ("Mali Soft Job: Timeout on soft job %u (0x%08X) in signaled state.\n", job->id, job));
369 mali_soft_job_system_unlock(job->system);
370 return schedule_mask;
371 }
372
373 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state);
374
375 job->state = MALI_SOFT_JOB_STATE_TIMED_OUT;
376 _mali_osk_atomic_inc(&job->refcount);
377
378 mali_soft_job_system_unlock(job->system);
379
380 schedule_mask = mali_timeline_tracker_release(&job->tracker);
381
382 mali_soft_job_destroy(job);
383
384 return schedule_mask;
385 }
386
mali_soft_job_system_abort(struct mali_soft_job_system * system)387 void mali_soft_job_system_abort(struct mali_soft_job_system *system)
388 {
389 struct mali_soft_job *job, *tmp;
390 _MALI_OSK_LIST_HEAD_STATIC_INIT(jobs);
391
392 MALI_DEBUG_ASSERT_POINTER(system);
393 MALI_DEBUG_ASSERT_POINTER(system->session);
394 MALI_DEBUG_ASSERT(system->session->is_aborting);
395
396 MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting soft job system for session 0x%08X.\n", system->session));
397
398 mali_soft_job_system_lock(system);
399
400 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &system->jobs_used, struct mali_soft_job, system_list) {
401 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_STARTED == job->state ||
402 MALI_SOFT_JOB_STATE_TIMED_OUT == job->state);
403
404 if (MALI_SOFT_JOB_STATE_STARTED == job->state) {
405 /* If the job has been activated, we have to release the tracker and destroy
406 * the job. If not, the tracker will be released and the job destroyed when
407 * it is activated. */
408 if (MALI_TRUE == job->activated) {
409 MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting unsignaled soft job %u (0x%08X).\n", job->id, job));
410
411 job->state = MALI_SOFT_JOB_STATE_SIGNALED;
412 _mali_osk_list_move(&job->system_list, &jobs);
413 }
414 } else if (MALI_SOFT_JOB_STATE_TIMED_OUT == job->state) {
415 MALI_DEBUG_PRINT(3, ("Mali Soft Job: Aborting timed out soft job %u (0x%08X).\n", job->id, job));
416
417 /* We need to destroy this soft job. */
418 _mali_osk_list_move(&job->system_list, &jobs);
419 }
420 }
421
422 mali_soft_job_system_unlock(system);
423
424 /* Release and destroy jobs. */
425 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &jobs, struct mali_soft_job, system_list) {
426 MALI_DEBUG_ASSERT(MALI_SOFT_JOB_STATE_SIGNALED == job->state ||
427 MALI_SOFT_JOB_STATE_TIMED_OUT == job->state);
428
429 if (MALI_SOFT_JOB_STATE_SIGNALED == job->state) {
430 mali_timeline_tracker_release(&job->tracker);
431 }
432
433 /* Move job back to used list before destroying. */
434 _mali_osk_list_move(&job->system_list, &system->jobs_used);
435
436 mali_soft_job_destroy(job);
437 }
438 }
439