1 /*
2 * Copyright (C) 2011-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11 #include "mali_pp.h"
12 #include "mali_pp_job.h"
13 #include "mali_osk.h"
14 #include "mali_osk_list.h"
15 #include "mali_kernel_common.h"
16 #include "mali_uk_types.h"
17 #include "mali_executor.h"
18 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
19 #include "linux/mali_memory_dma_buf.h"
20 #endif
21 #include "mali_memory_swap_alloc.h"
22 #include "mali_scheduler.h"
23
24 static u32 pp_counter_src0 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 0, MALI_HW_CORE_NO_COUNTER for disabled */
25 static u32 pp_counter_src1 = MALI_HW_CORE_NO_COUNTER; /**< Performance counter 1, MALI_HW_CORE_NO_COUNTER for disabled */
26 static _mali_osk_atomic_t pp_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */
27 static u32 pp_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER };
28 static u32 pp_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS] = { MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER, MALI_HW_CORE_NO_COUNTER };
29
mali_pp_job_initialize(void)30 void mali_pp_job_initialize(void)
31 {
32 _mali_osk_atomic_init(&pp_counter_per_sub_job_count, 0);
33 }
34
mali_pp_job_terminate(void)35 void mali_pp_job_terminate(void)
36 {
37 _mali_osk_atomic_term(&pp_counter_per_sub_job_count);
38 }
39
mali_pp_job_create(struct mali_session_data * session,_mali_uk_pp_start_job_s __user * uargs,u32 id)40 struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session,
41 _mali_uk_pp_start_job_s __user *uargs, u32 id)
42 {
43 struct mali_pp_job *job;
44 u32 perf_counter_flag;
45
46 job = _mali_osk_calloc(1, sizeof(struct mali_pp_job));
47 if (NULL != job) {
48
49 _mali_osk_list_init(&job->list);
50 _mali_osk_list_init(&job->session_fb_lookup_list);
51 _mali_osk_atomic_inc(&session->number_of_pp_jobs);
52
53 if (0 != _mali_osk_copy_from_user(&job->uargs, uargs, sizeof(_mali_uk_pp_start_job_s))) {
54 goto fail;
55 }
56
57 if (job->uargs.num_cores > _MALI_PP_MAX_SUB_JOBS) {
58 MALI_PRINT_ERROR(("Mali PP job: Too many sub jobs specified in job object\n"));
59 goto fail;
60 }
61
62 if (!mali_pp_job_use_no_notification(job)) {
63 job->finished_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_FINISHED, sizeof(_mali_uk_pp_job_finished_s));
64 if (NULL == job->finished_notification) goto fail;
65 }
66
67 perf_counter_flag = mali_pp_job_get_perf_counter_flag(job);
68
69 /* case when no counters came from user space
70 * so pass the debugfs / DS-5 provided global ones to the job object */
71 if (!((perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC0_ENABLE) ||
72 (perf_counter_flag & _MALI_PERFORMANCE_COUNTER_FLAG_SRC1_ENABLE))) {
73 u32 sub_job_count = _mali_osk_atomic_read(&pp_counter_per_sub_job_count);
74
75 /* These counters apply for all virtual jobs, and where no per sub job counter is specified */
76 job->uargs.perf_counter_src0 = pp_counter_src0;
77 job->uargs.perf_counter_src1 = pp_counter_src1;
78
79 /* We only copy the per sub job array if it is enabled with at least one counter */
80 if (0 < sub_job_count) {
81 job->perf_counter_per_sub_job_count = sub_job_count;
82 _mali_osk_memcpy(job->perf_counter_per_sub_job_src0, pp_counter_per_sub_job_src0, sizeof(pp_counter_per_sub_job_src0));
83 _mali_osk_memcpy(job->perf_counter_per_sub_job_src1, pp_counter_per_sub_job_src1, sizeof(pp_counter_per_sub_job_src1));
84 }
85 }
86
87 job->session = session;
88 job->id = id;
89
90 job->sub_jobs_num = job->uargs.num_cores ? job->uargs.num_cores : 1;
91 job->pid = _mali_osk_get_pid();
92 job->tid = _mali_osk_get_tid();
93
94 _mali_osk_atomic_init(&job->sub_jobs_completed, 0);
95 _mali_osk_atomic_init(&job->sub_job_errors, 0);
96 job->swap_status = MALI_NO_SWAP_IN;
97 job->user_notification = MALI_FALSE;
98 job->num_pp_cores_in_virtual = 0;
99
100 if (job->uargs.num_memory_cookies > session->allocation_mgr.mali_allocation_num) {
101 MALI_PRINT_ERROR(("Mali PP job: The number of memory cookies is invalid !\n"));
102 goto fail;
103 }
104
105 if (job->uargs.num_memory_cookies > 0) {
106 u32 size;
107 u32 __user *memory_cookies = (u32 __user *)(uintptr_t)job->uargs.memory_cookies;
108
109 size = sizeof(*memory_cookies) * (job->uargs.num_memory_cookies);
110
111 job->memory_cookies = _mali_osk_malloc(size);
112 if (NULL == job->memory_cookies) {
113 MALI_PRINT_ERROR(("Mali PP job: Failed to allocate %d bytes of memory cookies!\n", size));
114 goto fail;
115 }
116
117 if (0 != _mali_osk_copy_from_user(job->memory_cookies, memory_cookies, size)) {
118 MALI_PRINT_ERROR(("Mali PP job: Failed to copy %d bytes of memory cookies from user!\n", size));
119 goto fail;
120 }
121 }
122
123 if (_MALI_OSK_ERR_OK != mali_pp_job_check(job)) {
124 /* Not a valid job. */
125 goto fail;
126 }
127
128 mali_timeline_tracker_init(&job->tracker, MALI_TIMELINE_TRACKER_PP, NULL, job);
129 mali_timeline_fence_copy_uk_fence(&(job->tracker.fence), &(job->uargs.fence));
130
131 mali_mem_swap_in_pages(job);
132
133 return job;
134 }
135
136 fail:
137 if (NULL != job) {
138 mali_pp_job_delete(job);
139 }
140
141 return NULL;
142 }
143
mali_pp_job_delete(struct mali_pp_job * job)144 void mali_pp_job_delete(struct mali_pp_job *job)
145 {
146 struct mali_session_data *session;
147
148 MALI_DEBUG_ASSERT_POINTER(job);
149 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list));
150 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->session_fb_lookup_list));
151
152 session = mali_pp_job_get_session(job);
153 MALI_DEBUG_ASSERT_POINTER(session);
154
155 if (NULL != job->memory_cookies) {
156 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
157 /* Unmap buffers attached to job */
158 mali_dma_buf_unmap_job(job);
159 #endif
160 if (MALI_NO_SWAP_IN != job->swap_status) {
161 mali_mem_swap_out_pages(job);
162 }
163
164 _mali_osk_free(job->memory_cookies);
165 }
166
167 if (job->user_notification) {
168 mali_scheduler_return_pp_job_to_user(job,
169 job->num_pp_cores_in_virtual);
170 }
171
172 if (NULL != job->finished_notification) {
173 _mali_osk_notification_delete(job->finished_notification);
174 }
175
176 _mali_osk_atomic_term(&job->sub_jobs_completed);
177 _mali_osk_atomic_term(&job->sub_job_errors);
178 _mali_osk_atomic_dec(&session->number_of_pp_jobs);
179 _mali_osk_free(job);
180
181 _mali_osk_wait_queue_wake_up(session->wait_queue);
182 }
183
mali_pp_job_list_add(struct mali_pp_job * job,_mali_osk_list_t * list)184 void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list)
185 {
186 struct mali_pp_job *iter;
187 struct mali_pp_job *tmp;
188
189 MALI_DEBUG_ASSERT_POINTER(job);
190 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
191
192 /* Find position in list/queue where job should be added. */
193 _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, list,
194 struct mali_pp_job, list) {
195 /* job should be started after iter if iter is in progress. */
196 if (0 < iter->sub_jobs_started) {
197 break;
198 }
199
200 /*
201 * job should be started after iter if it has a higher
202 * job id. A span is used to handle job id wrapping.
203 */
204 if ((mali_pp_job_get_id(job) -
205 mali_pp_job_get_id(iter)) <
206 MALI_SCHEDULER_JOB_ID_SPAN) {
207 break;
208 }
209 }
210
211 _mali_osk_list_add(&job->list, &iter->list);
212 }
213
214
mali_pp_job_get_perf_counter_src0(struct mali_pp_job * job,u32 sub_job)215 u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job)
216 {
217 /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */
218 if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) {
219 return job->uargs.perf_counter_src0;
220 }
221
222 /* Use per sub job counter if enabled... */
223 if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src0[sub_job]) {
224 return job->perf_counter_per_sub_job_src0[sub_job];
225 }
226
227 /* ...else default to global job counter */
228 return job->uargs.perf_counter_src0;
229 }
230
mali_pp_job_get_perf_counter_src1(struct mali_pp_job * job,u32 sub_job)231 u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job)
232 {
233 /* Virtual jobs always use the global job counter (or if there are per sub job counters at all) */
234 if (mali_pp_job_is_virtual(job) || 0 == job->perf_counter_per_sub_job_count) {
235 /* Virtual jobs always use the global job counter */
236 return job->uargs.perf_counter_src1;
237 }
238
239 /* Use per sub job counter if enabled... */
240 if (MALI_HW_CORE_NO_COUNTER != job->perf_counter_per_sub_job_src1[sub_job]) {
241 return job->perf_counter_per_sub_job_src1[sub_job];
242 }
243
244 /* ...else default to global job counter */
245 return job->uargs.perf_counter_src1;
246 }
247
mali_pp_job_set_pp_counter_global_src0(u32 counter)248 void mali_pp_job_set_pp_counter_global_src0(u32 counter)
249 {
250 pp_counter_src0 = counter;
251 }
252
mali_pp_job_set_pp_counter_global_src1(u32 counter)253 void mali_pp_job_set_pp_counter_global_src1(u32 counter)
254 {
255 pp_counter_src1 = counter;
256 }
257
mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job,u32 counter)258 void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter)
259 {
260 MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
261
262 if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src0[sub_job]) {
263 /* increment count since existing counter was disabled */
264 _mali_osk_atomic_inc(&pp_counter_per_sub_job_count);
265 }
266
267 if (MALI_HW_CORE_NO_COUNTER == counter) {
268 /* decrement count since new counter is disabled */
269 _mali_osk_atomic_dec(&pp_counter_per_sub_job_count);
270 }
271
272 /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */
273
274 pp_counter_per_sub_job_src0[sub_job] = counter;
275 }
276
mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job,u32 counter)277 void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter)
278 {
279 MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
280
281 if (MALI_HW_CORE_NO_COUNTER == pp_counter_per_sub_job_src1[sub_job]) {
282 /* increment count since existing counter was disabled */
283 _mali_osk_atomic_inc(&pp_counter_per_sub_job_count);
284 }
285
286 if (MALI_HW_CORE_NO_COUNTER == counter) {
287 /* decrement count since new counter is disabled */
288 _mali_osk_atomic_dec(&pp_counter_per_sub_job_count);
289 }
290
291 /* PS: A change from MALI_HW_CORE_NO_COUNTER to MALI_HW_CORE_NO_COUNTER will inc and dec, result will be 0 change */
292
293 pp_counter_per_sub_job_src1[sub_job] = counter;
294 }
295
mali_pp_job_get_pp_counter_global_src0(void)296 u32 mali_pp_job_get_pp_counter_global_src0(void)
297 {
298 return pp_counter_src0;
299 }
300
mali_pp_job_get_pp_counter_global_src1(void)301 u32 mali_pp_job_get_pp_counter_global_src1(void)
302 {
303 return pp_counter_src1;
304 }
305
mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job)306 u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job)
307 {
308 MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
309 return pp_counter_per_sub_job_src0[sub_job];
310 }
311
mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job)312 u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job)
313 {
314 MALI_DEBUG_ASSERT(sub_job < _MALI_PP_MAX_SUB_JOBS);
315 return pp_counter_per_sub_job_src1[sub_job];
316 }
317