1 /*
2 * Copyright (C) 2011-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11 #include "mali_gp_job.h"
12 #include "mali_osk.h"
13 #include "mali_osk_list.h"
14 #include "mali_uk_types.h"
15 #include "mali_memory_virtual.h"
16 #include "mali_memory_defer_bind.h"
17
18 static u32 gp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */
19 static u32 gp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */
20 static void _mali_gp_del_varying_allocations(struct mali_gp_job *job);
21
22
_mali_gp_add_varying_allocations(struct mali_session_data * session,struct mali_gp_job * job,u32 * alloc,u32 num)23 static int _mali_gp_add_varying_allocations(struct mali_session_data *session,
24 struct mali_gp_job *job,
25 u32 *alloc,
26 u32 num)
27 {
28 int i = 0;
29 struct mali_gp_allocation_node *alloc_node;
30 mali_mem_allocation *mali_alloc = NULL;
31 struct mali_vma_node *mali_vma_node = NULL;
32
33 for (i = 0 ; i < num ; i++) {
34 MALI_DEBUG_ASSERT(alloc[i]);
35 alloc_node = _mali_osk_calloc(1, sizeof(struct mali_gp_allocation_node));
36 if (alloc_node) {
37 INIT_LIST_HEAD(&alloc_node->node);
38 /* find mali allocation structure by vaddress*/
39 mali_vma_node = mali_vma_offset_search(&session->allocation_mgr, alloc[i], 0);
40
41 if (likely(mali_vma_node)) {
42 mali_alloc = container_of(mali_vma_node, struct mali_mem_allocation, mali_vma_node);
43 MALI_DEBUG_ASSERT(alloc[i] == mali_vma_node->vm_node.start);
44 } else {
45 MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,can't find allocation %d by address =0x%x, num=%d\n", i, alloc[i], num));
46 _mali_osk_free(alloc_node);
47 goto fail;
48 }
49 alloc_node->alloc = mali_alloc;
50 /* add to gp job varying alloc list*/
51 list_move(&alloc_node->node, &job->varying_alloc);
52 } else
53 goto fail;
54 }
55
56 return 0;
57 fail:
58 MALI_DEBUG_PRINT(1, ("ERROE!_mali_gp_add_varying_allocations,failed to alloc memory!\n"));
59 _mali_gp_del_varying_allocations(job);
60 return -1;
61 }
62
63
_mali_gp_del_varying_allocations(struct mali_gp_job * job)64 static void _mali_gp_del_varying_allocations(struct mali_gp_job *job)
65 {
66 struct mali_gp_allocation_node *alloc_node, *tmp_node;
67
68 list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) {
69 list_del(&alloc_node->node);
70 kfree(alloc_node);
71 }
72 INIT_LIST_HEAD(&job->varying_alloc);
73 }
74
mali_gp_job_create(struct mali_session_data * session,_mali_uk_gp_start_job_s * uargs,u32 id,struct mali_timeline_tracker * pp_tracker)75 struct mali_gp_job *mali_gp_job_create(struct mali_session_data *session, _mali_uk_gp_start_job_s *uargs, u32 id, struct mali_timeline_tracker *pp_tracker)
76 {
77 struct mali_gp_job *job;
78 u32 perf_counter_flag;
79 u32 __user *memory_list = NULL;
80 struct mali_gp_allocation_node *alloc_node, *tmp_node;
81 _mali_uk_gp_start_job_s copy_of_uargs;
82
83 job = _mali_osk_calloc(1, sizeof(struct mali_gp_job));
84 if (NULL != job) {
85 job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_FINISHED, sizeof(_mali_uk_gp_job_finished_s));
86 if (NULL == job->finished_notification) {
87 goto fail3;
88 }
89
90 job->oom_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s));
91 if (NULL == job->oom_notification) {
92 goto fail2;
93 }
94
95 if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) {
96 goto fail1;
97 }
98
99 perf_counter_flag = mali_gp_job_get_perf_counter_flag(job);
100
101 /* case when no counters came from user space
102 * so pass the debugfs / DS-5 provided global ones to the job object */
103 if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
104 (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
105 mali_gp_job_set_perf_counter_src0(job, mali_gp_job_get_gp_counter_src0());
106 mali_gp_job_set_perf_counter_src1(job, mali_gp_job_get_gp_counter_src1());
107 }
108
109 _mali_osk_list_init(&job->list);
110 job->session = session;
111 job->id = id;
112 job->heap_current_addr = job->uargs.frame_registers[4];
113 job->perf_counter_value0 = 0;
114 job->perf_counter_value1 = 0;
115 job->pid = _mali_osk_get_pid();
116 job->tid = _mali_osk_get_tid();
117
118
119 INIT_LIST_HEAD(&job->varying_alloc);
120 INIT_LIST_HEAD(&job->vary_todo);
121 job->dmem = NULL;
122
123 if (job->uargs.deferred_mem_num > session->allocation_mgr.mali_allocation_num) {
124 MALI_PRINT_ERROR(("Mali GP job: The number of varying buffer to defer bind is invalid !\n"));
125 goto fail1;
126 }
127
128 /* add varying allocation list*/
129 if (job->uargs.deferred_mem_num > 0) {
130 /* copy varying list from user space*/
131 job->varying_list = _mali_osk_calloc(1, sizeof(u32) * job->uargs.deferred_mem_num);
132 if (!job->varying_list) {
133 MALI_PRINT_ERROR(("Mali GP job: allocate varying_list failed varying_alloc_num = %d !\n", job->uargs.deferred_mem_num));
134 goto fail1;
135 }
136
137 if (0 != _mali_osk_copy_from_user(©_of_uargs, uargs, sizeof(_mali_uk_gp_start_job_s))) {
138 goto fail1;
139 }
140 memory_list = (u32 __user *)(uintptr_t)copy_of_uargs.deferred_mem_list;
141
142 if (0 != _mali_osk_copy_from_user(job->varying_list, memory_list, sizeof(u32) * job->uargs.deferred_mem_num)) {
143 MALI_PRINT_ERROR(("Mali GP job: Failed to copy varying list from user space!\n"));
144 goto fail;
145 }
146
147 if (unlikely(_mali_gp_add_varying_allocations(session, job, job->varying_list,
148 job->uargs.deferred_mem_num))) {
149 MALI_PRINT_ERROR(("Mali GP job: _mali_gp_add_varying_allocations failed!\n"));
150 goto fail;
151 }
152
153 /* do preparetion for each allocation */
154 list_for_each_entry_safe(alloc_node, tmp_node, &job->varying_alloc, node) {
155 if (unlikely(_MALI_OSK_ERR_OK != mali_mem_defer_bind_allocation_prepare(alloc_node->alloc, &job->vary_todo, &job->required_varying_memsize))) {
156 MALI_PRINT_ERROR(("Mali GP job: mali_mem_defer_bind_allocation_prepare failed!\n"));
157 goto fail;
158 }
159 }
160
161 _mali_gp_del_varying_allocations(job);
162
163 /* bind varying here, to avoid memory latency issue. */
164 {
165 struct mali_defer_mem_block dmem_block;
166
167 INIT_LIST_HEAD(&dmem_block.free_pages);
168 atomic_set(&dmem_block.num_free_pages, 0);
169
170 if (mali_mem_prepare_mem_for_job(job, &dmem_block)) {
171 MALI_PRINT_ERROR(("Mali GP job: mali_mem_prepare_mem_for_job failed!\n"));
172 goto fail;
173 }
174 if (_MALI_OSK_ERR_OK != mali_mem_defer_bind(job, &dmem_block)) {
175 MALI_PRINT_ERROR(("gp job create, mali_mem_defer_bind failed! GP %x fail!", job));
176 goto fail;
177 }
178 }
179
180 if (job->uargs.varying_memsize > MALI_UK_BIG_VARYING_SIZE) {
181 job->big_job = 1;
182 }
183 }
184 job->pp_tracker = pp_tracker;
185 if (NULL != job->pp_tracker) {
186 /* Take a reference on PP job's tracker that will be released when the GP
187 job is done. */
188 mali_timeline_system_tracker_get(session->timeline_system, pp_tracker);
189 }
190
191 mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_GP, NULL, job);
192 mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
193
194 return job;
195 } else {
196 MALI_PRINT_ERROR(("Mali GP job: _mali_osk_calloc failed!\n"));
197 return NULL;
198 }
199
200
201 fail:
202 _mali_osk_free(job->varying_list);
203 /* Handle allocate fail here, free all varying node */
204 {
205 struct mali_backend_bind_list *bkn, *bkn_tmp;
206 list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) {
207 list_del(&bkn->node);
208 _mali_osk_free(bkn);
209 }
210 }
211 fail1:
212 _mali_osk_notification_delete(job->oom_notification);
213 fail2:
214 _mali_osk_notification_delete(job->finished_notification);
215 fail3:
216 _mali_osk_free(job);
217 return NULL;
218 }
219
mali_gp_job_delete(struct mali_gp_job * job)220 void mali_gp_job_delete(struct mali_gp_job *job)
221 {
222 struct mali_backend_bind_list *bkn, *bkn_tmp;
223 MALI_DEBUG_ASSERT_POINTER(job);
224 MALI_DEBUG_ASSERT(NULL == job->pp_tracker);
225 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list));
226 _mali_osk_free(job->varying_list);
227
228 /* Handle allocate fail here, free all varying node */
229 list_for_each_entry_safe(bkn, bkn_tmp , &job->vary_todo, node) {
230 list_del(&bkn->node);
231 _mali_osk_free(bkn);
232 }
233
234 mali_mem_defer_dmem_free(job);
235
236 /* de-allocate the pre-allocated oom notifications */
237 if (NULL != job->oom_notification) {
238 _mali_osk_notification_delete(job->oom_notification);
239 job->oom_notification = NULL;
240 }
241 if (NULL != job->finished_notification) {
242 _mali_osk_notification_delete(job->finished_notification);
243 job->finished_notification = NULL;
244 }
245
246 _mali_osk_free(job);
247 }
248
mali_gp_job_list_add(struct mali_gp_job * job,_mali_osk_list_t * list)249 void mali_gp_job_list_add(struct mali_gp_job *job, _mali_osk_list_t *list)
250 {
251 struct mali_gp_job *iter;
252 struct mali_gp_job *tmp;
253
254 MALI_DEBUG_ASSERT_POINTER(job);
255 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
256
257 /* Find position in list/queue where job should be added. */
258 _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list,
259 struct mali_gp_job, list) {
260
261 /* A span is used to handle job ID wrapping. */
262 bool job_is_after = (mali_gp_job_get_id(job) -
263 mali_gp_job_get_id(iter)) <
264 MALI_SCHEDULER_JOB_ID_SPAN;
265
266 if (job_is_after) {
267 break;
268 }
269 }
270
271 _mali_osk_list_add(&job->list, &iter->list);
272 }
273
mali_gp_job_get_gp_counter_src0(void)274 u32 mali_gp_job_get_gp_counter_src0(void)
275 {
276 return gp_counter_src0;
277 }
278
mali_gp_job_set_gp_counter_src0(u32 counter)279 void mali_gp_job_set_gp_counter_src0(u32 counter)
280 {
281 gp_counter_src0 = counter;
282 }
283
mali_gp_job_get_gp_counter_src1(void)284 u32 mali_gp_job_get_gp_counter_src1(void)
285 {
286 return gp_counter_src1;
287 }
288
mali_gp_job_set_gp_counter_src1(u32 counter)289 void mali_gp_job_set_gp_counter_src1(u32 counter)
290 {
291 gp_counter_src1 = counter;
292 }
293
mali_gp_job_signal_pp_tracker(struct mali_gp_job * job,mali_bool success)294 mali_scheduler_mask mali_gp_job_signal_pp_tracker(struct mali_gp_job *job, mali_bool success)
295 {
296 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
297
298 MALI_DEBUG_ASSERT_POINTER(job);
299
300 if (NULL != job->pp_tracker) {
301 schedule_mask |= mali_timeline_system_tracker_put(job->session->timeline_system, job->pp_tracker, MALI_FALSE == success);
302 job->pp_tracker = NULL;
303 }
304
305 return schedule_mask;
306 }
307