1 /*
2 * Copyright (C) 2010-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11 #include "mali_kernel_common.h"
12 #include "mali_session.h"
13 #include "mali_osk.h"
14 #include "mali_osk_mali.h"
15 #include "mali_ukk.h"
16 #include "mali_kernel_core.h"
17 #include "mali_memory.h"
18 #include "mali_mem_validation.h"
19 #include "mali_mmu.h"
20 #include "mali_mmu_page_directory.h"
21 #include "mali_dlbu.h"
22 #include "mali_broadcast.h"
23 #include "mali_gp.h"
24 #include "mali_pp.h"
25 #include "mali_executor.h"
26 #include "mali_pp_job.h"
27 #include "mali_group.h"
28 #include "mali_pm.h"
29 #include "mali_pmu.h"
30 #include "mali_scheduler.h"
31 #include "mali_kernel_utilization.h"
32 #include "mali_l2_cache.h"
33 #include "mali_timeline.h"
34 #include "mali_soft_job.h"
35 #include "mali_pm_domain.h"
36 #if defined(CONFIG_MALI400_PROFILING)
37 #include "mali_osk_profiling.h"
38 #endif
39 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
40 #include "mali_profiling_internal.h"
41 #endif
42 #include "mali_control_timer.h"
43 #include "mali_dvfs_policy.h"
44 #include <linux/sched.h>
45 #include <linux/atomic.h>
46 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
47 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
48 #include <linux/dma-fence.h>
49 #else
50 #include <linux/fence.h>
51 #endif
52 #endif
53
54 #define MALI_SHARED_MEMORY_DEFAULT_SIZE 0xffffffff
55
56 /* Mali GPU memory. Real values come from module parameter or from device specific data */
57 unsigned int mali_dedicated_mem_start = 0;
58 unsigned int mali_dedicated_mem_size = 0;
59
60 /* Default shared memory size is set to 4G. */
61 unsigned int mali_shared_mem_size = MALI_SHARED_MEMORY_DEFAULT_SIZE;
62
63 /* Frame buffer memory to be accessible by Mali GPU */
64 int mali_fb_start = 0;
65 int mali_fb_size = 0;
66
67 /* Mali max job runtime */
68 extern int mali_max_job_runtime;
69
70 /** Start profiling from module load? */
71 int mali_boot_profiling = 0;
72
73 /** Limits for the number of PP cores behind each L2 cache. */
74 int mali_max_pp_cores_group_1 = 0xFF;
75 int mali_max_pp_cores_group_2 = 0xFF;
76
77 int mali_inited_pp_cores_group_1 = 0;
78 int mali_inited_pp_cores_group_2 = 0;
79
80 static _mali_product_id_t global_product_id = _MALI_PRODUCT_ID_UNKNOWN;
81 static uintptr_t global_gpu_base_address = 0;
82 static u32 global_gpu_major_version = 0;
83 static u32 global_gpu_minor_version = 0;
84
85 mali_bool mali_gpu_class_is_mali450 = MALI_FALSE;
86 mali_bool mali_gpu_class_is_mali470 = MALI_FALSE;
87
mali_set_global_gpu_base_address(void)88 static _mali_osk_errcode_t mali_set_global_gpu_base_address(void)
89 {
90 _mali_osk_errcode_t err = _MALI_OSK_ERR_OK;
91
92 global_gpu_base_address = _mali_osk_resource_base_address();
93 if (0 == global_gpu_base_address) {
94 err = _MALI_OSK_ERR_ITEM_NOT_FOUND;
95 }
96
97 return err;
98 }
99
mali_get_bcast_id(_mali_osk_resource_t * resource_pp)100 static u32 mali_get_bcast_id(_mali_osk_resource_t *resource_pp)
101 {
102 switch (resource_pp->base - global_gpu_base_address) {
103 case 0x08000:
104 case 0x20000: /* fall-through for aliased mapping */
105 return 0x01;
106 case 0x0A000:
107 case 0x22000: /* fall-through for aliased mapping */
108 return 0x02;
109 case 0x0C000:
110 case 0x24000: /* fall-through for aliased mapping */
111 return 0x04;
112 case 0x0E000:
113 case 0x26000: /* fall-through for aliased mapping */
114 return 0x08;
115 case 0x28000:
116 return 0x10;
117 case 0x2A000:
118 return 0x20;
119 case 0x2C000:
120 return 0x40;
121 case 0x2E000:
122 return 0x80;
123 default:
124 return 0;
125 }
126 }
127
mali_parse_product_info(void)128 static _mali_osk_errcode_t mali_parse_product_info(void)
129 {
130 _mali_osk_resource_t first_pp_resource;
131
132 /* Find the first PP core resource (again) */
133 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PP0, &first_pp_resource)) {
134 /* Create a dummy PP object for this core so that we can read the version register */
135 struct mali_group *group = mali_group_create(NULL, NULL, NULL, MALI_DOMAIN_INDEX_PP0);
136 if (NULL != group) {
137 struct mali_pp_core *pp_core = mali_pp_create(&first_pp_resource, group, MALI_FALSE, mali_get_bcast_id(&first_pp_resource));
138 if (NULL != pp_core) {
139 u32 pp_version;
140
141 pp_version = mali_pp_core_get_version(pp_core);
142
143 mali_group_delete(group);
144
145 global_gpu_major_version = (pp_version >> 8) & 0xFF;
146 global_gpu_minor_version = pp_version & 0xFF;
147
148 switch (pp_version >> 16) {
149 case MALI200_PP_PRODUCT_ID:
150 global_product_id = _MALI_PRODUCT_ID_MALI200;
151 MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-200 r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
152 MALI_PRINT_ERROR(("Mali-200 is not supported by this driver.\n"));
153 _mali_osk_abort();
154 break;
155 case MALI300_PP_PRODUCT_ID:
156 global_product_id = _MALI_PRODUCT_ID_MALI300;
157 MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-300 r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
158 break;
159 case MALI400_PP_PRODUCT_ID:
160 global_product_id = _MALI_PRODUCT_ID_MALI400;
161 MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-400 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
162 break;
163 case MALI450_PP_PRODUCT_ID:
164 global_product_id = _MALI_PRODUCT_ID_MALI450;
165 MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-450 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
166 break;
167 case MALI470_PP_PRODUCT_ID:
168 global_product_id = _MALI_PRODUCT_ID_MALI470;
169 MALI_DEBUG_PRINT(2, ("Found Mali GPU Mali-470 MP r%up%u\n", global_gpu_major_version, global_gpu_minor_version));
170 break;
171 default:
172 MALI_DEBUG_PRINT(2, ("Found unknown Mali GPU (r%up%u)\n", global_gpu_major_version, global_gpu_minor_version));
173 return _MALI_OSK_ERR_FAULT;
174 }
175
176 return _MALI_OSK_ERR_OK;
177 } else {
178 MALI_PRINT_ERROR(("Failed to create initial PP object\n"));
179 }
180 } else {
181 MALI_PRINT_ERROR(("Failed to create initial group object\n"));
182 }
183 } else {
184 MALI_PRINT_ERROR(("First PP core not specified in config file\n"));
185 }
186
187 return _MALI_OSK_ERR_FAULT;
188 }
189
mali_delete_groups(void)190 static void mali_delete_groups(void)
191 {
192 struct mali_group *group;
193
194 group = mali_group_get_glob_group(0);
195 while (NULL != group) {
196 mali_group_delete(group);
197 group = mali_group_get_glob_group(0);
198 }
199
200 MALI_DEBUG_ASSERT(0 == mali_group_get_glob_num_groups());
201 }
202
mali_delete_l2_cache_cores(void)203 static void mali_delete_l2_cache_cores(void)
204 {
205 struct mali_l2_cache_core *l2;
206
207 l2 = mali_l2_cache_core_get_glob_l2_core(0);
208 while (NULL != l2) {
209 mali_l2_cache_delete(l2);
210 l2 = mali_l2_cache_core_get_glob_l2_core(0);
211 }
212
213 MALI_DEBUG_ASSERT(0 == mali_l2_cache_core_get_glob_num_l2_cores());
214 }
215
mali_create_l2_cache_core(_mali_osk_resource_t * resource,u32 domain_index)216 static struct mali_l2_cache_core *mali_create_l2_cache_core(_mali_osk_resource_t *resource, u32 domain_index)
217 {
218 struct mali_l2_cache_core *l2_cache = NULL;
219
220 if (NULL != resource) {
221
222 MALI_DEBUG_PRINT(3, ("Found L2 cache %s\n", resource->description));
223
224 l2_cache = mali_l2_cache_create(resource, domain_index);
225 if (NULL == l2_cache) {
226 MALI_PRINT_ERROR(("Failed to create L2 cache object\n"));
227 return NULL;
228 }
229 }
230 MALI_DEBUG_PRINT(3, ("Created L2 cache core object\n"));
231
232 return l2_cache;
233 }
234
mali_parse_config_l2_cache(void)235 static _mali_osk_errcode_t mali_parse_config_l2_cache(void)
236 {
237 struct mali_l2_cache_core *l2_cache = NULL;
238
239 if (mali_is_mali400()) {
240 _mali_osk_resource_t l2_resource;
241 if (_MALI_OSK_ERR_OK != _mali_osk_resource_find(MALI400_OFFSET_L2_CACHE0, &l2_resource)) {
242 MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache in config file\n"));
243 return _MALI_OSK_ERR_FAULT;
244 }
245
246 l2_cache = mali_create_l2_cache_core(&l2_resource, MALI_DOMAIN_INDEX_L20);
247 if (NULL == l2_cache) {
248 return _MALI_OSK_ERR_FAULT;
249 }
250 } else if (mali_is_mali450()) {
251 /*
252 * L2 for GP at 0x10000
253 * L2 for PP0-3 at 0x01000
254 * L2 for PP4-7 at 0x11000 (optional)
255 */
256
257 _mali_osk_resource_t l2_gp_resource;
258 _mali_osk_resource_t l2_pp_grp0_resource;
259 _mali_osk_resource_t l2_pp_grp1_resource;
260
261 /* Make cluster for GP's L2 */
262 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE0, &l2_gp_resource)) {
263 MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for GP\n"));
264 l2_cache = mali_create_l2_cache_core(&l2_gp_resource, MALI_DOMAIN_INDEX_L20);
265 if (NULL == l2_cache) {
266 return _MALI_OSK_ERR_FAULT;
267 }
268 } else {
269 MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for GP in config file\n"));
270 return _MALI_OSK_ERR_FAULT;
271 }
272
273 /* Find corresponding l2 domain */
274 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE1, &l2_pp_grp0_resource)) {
275 MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 0\n"));
276 l2_cache = mali_create_l2_cache_core(&l2_pp_grp0_resource, MALI_DOMAIN_INDEX_L21);
277 if (NULL == l2_cache) {
278 return _MALI_OSK_ERR_FAULT;
279 }
280 } else {
281 MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for PP group 0 in config file\n"));
282 return _MALI_OSK_ERR_FAULT;
283 }
284
285 /* Second PP core group is optional, don't fail if we don't find it */
286 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI450_OFFSET_L2_CACHE2, &l2_pp_grp1_resource)) {
287 MALI_DEBUG_PRINT(3, ("Creating Mali-450 L2 cache core for PP group 1\n"));
288 l2_cache = mali_create_l2_cache_core(&l2_pp_grp1_resource, MALI_DOMAIN_INDEX_L22);
289 if (NULL == l2_cache) {
290 return _MALI_OSK_ERR_FAULT;
291 }
292 }
293 } else if (mali_is_mali470()) {
294 _mali_osk_resource_t l2c1_resource;
295
296 /* Make cluster for L2C1 */
297 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI470_OFFSET_L2_CACHE1, &l2c1_resource)) {
298 MALI_DEBUG_PRINT(3, ("Creating Mali-470 L2 cache 1\n"));
299 l2_cache = mali_create_l2_cache_core(&l2c1_resource, MALI_DOMAIN_INDEX_L21);
300 if (NULL == l2_cache) {
301 return _MALI_OSK_ERR_FAULT;
302 }
303 } else {
304 MALI_DEBUG_PRINT(3, ("Did not find required Mali L2 cache for L2C1\n"));
305 return _MALI_OSK_ERR_FAULT;
306 }
307 }
308
309 return _MALI_OSK_ERR_OK;
310 }
311
mali_create_group(struct mali_l2_cache_core * cache,_mali_osk_resource_t * resource_mmu,_mali_osk_resource_t * resource_gp,_mali_osk_resource_t * resource_pp,u32 domain_index)312 static struct mali_group *mali_create_group(struct mali_l2_cache_core *cache,
313 _mali_osk_resource_t *resource_mmu,
314 _mali_osk_resource_t *resource_gp,
315 _mali_osk_resource_t *resource_pp,
316 u32 domain_index)
317 {
318 struct mali_mmu_core *mmu;
319 struct mali_group *group;
320
321 MALI_DEBUG_PRINT(3, ("Starting new group for MMU %s\n", resource_mmu->description));
322
323 /* Create the group object */
324 group = mali_group_create(cache, NULL, NULL, domain_index);
325 if (NULL == group) {
326 MALI_PRINT_ERROR(("Failed to create group object for MMU %s\n", resource_mmu->description));
327 return NULL;
328 }
329
330 /* Create the MMU object inside group */
331 mmu = mali_mmu_create(resource_mmu, group, MALI_FALSE);
332 if (NULL == mmu) {
333 MALI_PRINT_ERROR(("Failed to create MMU object\n"));
334 mali_group_delete(group);
335 return NULL;
336 }
337
338 if (NULL != resource_gp) {
339 /* Create the GP core object inside this group */
340 struct mali_gp_core *gp_core = mali_gp_create(resource_gp, group);
341 if (NULL == gp_core) {
342 /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */
343 MALI_PRINT_ERROR(("Failed to create GP object\n"));
344 mali_group_delete(group);
345 return NULL;
346 }
347 }
348
349 if (NULL != resource_pp) {
350 struct mali_pp_core *pp_core;
351
352 /* Create the PP core object inside this group */
353 pp_core = mali_pp_create(resource_pp, group, MALI_FALSE, mali_get_bcast_id(resource_pp));
354 if (NULL == pp_core) {
355 /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */
356 MALI_PRINT_ERROR(("Failed to create PP object\n"));
357 mali_group_delete(group);
358 return NULL;
359 }
360 }
361
362 return group;
363 }
364
mali_create_virtual_group(_mali_osk_resource_t * resource_mmu_pp_bcast,_mali_osk_resource_t * resource_pp_bcast,_mali_osk_resource_t * resource_dlbu,_mali_osk_resource_t * resource_bcast)365 static _mali_osk_errcode_t mali_create_virtual_group(_mali_osk_resource_t *resource_mmu_pp_bcast,
366 _mali_osk_resource_t *resource_pp_bcast,
367 _mali_osk_resource_t *resource_dlbu,
368 _mali_osk_resource_t *resource_bcast)
369 {
370 struct mali_mmu_core *mmu_pp_bcast_core;
371 struct mali_pp_core *pp_bcast_core;
372 struct mali_dlbu_core *dlbu_core;
373 struct mali_bcast_unit *bcast_core;
374 struct mali_group *group;
375
376 MALI_DEBUG_PRINT(2, ("Starting new virtual group for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description));
377
378 /* Create the DLBU core object */
379 dlbu_core = mali_dlbu_create(resource_dlbu);
380 if (NULL == dlbu_core) {
381 MALI_PRINT_ERROR(("Failed to create DLBU object \n"));
382 return _MALI_OSK_ERR_FAULT;
383 }
384
385 /* Create the Broadcast unit core */
386 bcast_core = mali_bcast_unit_create(resource_bcast);
387 if (NULL == bcast_core) {
388 MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n"));
389 mali_dlbu_delete(dlbu_core);
390 return _MALI_OSK_ERR_FAULT;
391 }
392
393 /* Create the group object */
394 #if defined(DEBUG)
395 /* Get a physical PP group to temporarily add to broadcast unit. IRQ
396 * verification needs a physical group in the broadcast unit to test
397 * the broadcast unit interrupt line. */
398 {
399 struct mali_group *phys_group = NULL;
400 int i;
401 for (i = 0; i < mali_group_get_glob_num_groups(); i++) {
402 phys_group = mali_group_get_glob_group(i);
403 if (NULL != mali_group_get_pp_core(phys_group)) break;
404 }
405 MALI_DEBUG_ASSERT(NULL != mali_group_get_pp_core(phys_group));
406
407 /* Add the group temporarily to the broadcast, and update the
408 * broadcast HW. Since the HW is not updated when removing the
409 * group the IRQ check will work when the virtual PP is created
410 * later.
411 *
412 * When the virtual group gets populated, the actually used
413 * groups will be added to the broadcast unit and the HW will
414 * be updated.
415 */
416 mali_bcast_add_group(bcast_core, phys_group);
417 mali_bcast_reset(bcast_core);
418 mali_bcast_remove_group(bcast_core, phys_group);
419 }
420 #endif /* DEBUG */
421 group = mali_group_create(NULL, dlbu_core, bcast_core, MALI_DOMAIN_INDEX_DUMMY);
422 if (NULL == group) {
423 MALI_PRINT_ERROR(("Failed to create group object for MMU PP broadcast core %s\n", resource_mmu_pp_bcast->description));
424 mali_bcast_unit_delete(bcast_core);
425 mali_dlbu_delete(dlbu_core);
426 return _MALI_OSK_ERR_FAULT;
427 }
428
429 /* Create the MMU object inside group */
430 mmu_pp_bcast_core = mali_mmu_create(resource_mmu_pp_bcast, group, MALI_TRUE);
431 if (NULL == mmu_pp_bcast_core) {
432 MALI_PRINT_ERROR(("Failed to create MMU PP broadcast object\n"));
433 mali_group_delete(group);
434 return _MALI_OSK_ERR_FAULT;
435 }
436
437 /* Create the PP core object inside this group */
438 pp_bcast_core = mali_pp_create(resource_pp_bcast, group, MALI_TRUE, 0);
439 if (NULL == pp_bcast_core) {
440 /* No need to clean up now, as we will clean up everything linked in from the cluster when we fail this function */
441 MALI_PRINT_ERROR(("Failed to create PP object\n"));
442 mali_group_delete(group);
443 return _MALI_OSK_ERR_FAULT;
444 }
445
446 return _MALI_OSK_ERR_OK;
447 }
448
mali_parse_config_groups(void)449 static _mali_osk_errcode_t mali_parse_config_groups(void)
450 {
451 struct mali_group *group;
452 int cluster_id_gp = 0;
453 int cluster_id_pp_grp0 = 0;
454 int cluster_id_pp_grp1 = 0;
455 int i;
456
457 _mali_osk_resource_t resource_gp;
458 _mali_osk_resource_t resource_gp_mmu;
459 _mali_osk_resource_t resource_pp[8];
460 _mali_osk_resource_t resource_pp_mmu[8];
461 _mali_osk_resource_t resource_pp_mmu_bcast;
462 _mali_osk_resource_t resource_pp_bcast;
463 _mali_osk_resource_t resource_dlbu;
464 _mali_osk_resource_t resource_bcast;
465 _mali_osk_errcode_t resource_gp_found;
466 _mali_osk_errcode_t resource_gp_mmu_found;
467 _mali_osk_errcode_t resource_pp_found[8];
468 _mali_osk_errcode_t resource_pp_mmu_found[8];
469 _mali_osk_errcode_t resource_pp_mmu_bcast_found;
470 _mali_osk_errcode_t resource_pp_bcast_found;
471 _mali_osk_errcode_t resource_dlbu_found;
472 _mali_osk_errcode_t resource_bcast_found;
473
474 if (!(mali_is_mali400() || mali_is_mali450() || mali_is_mali470())) {
475 /* No known HW core */
476 return _MALI_OSK_ERR_FAULT;
477 }
478
479 if (MALI_MAX_JOB_RUNTIME_DEFAULT == mali_max_job_runtime) {
480 /* Group settings are not overridden by module parameters, so use device settings */
481 _mali_osk_device_data data = { 0, };
482
483 if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
484 /* Use device specific settings (if defined) */
485 if (0 != data.max_job_runtime) {
486 mali_max_job_runtime = data.max_job_runtime;
487 }
488 }
489 }
490
491 if (mali_is_mali450()) {
492 /* Mali-450 have separate L2s for GP, and PP core group(s) */
493 cluster_id_pp_grp0 = 1;
494 cluster_id_pp_grp1 = 2;
495 }
496
497 resource_gp_found = _mali_osk_resource_find(MALI_OFFSET_GP, &resource_gp);
498 resource_gp_mmu_found = _mali_osk_resource_find(MALI_OFFSET_GP_MMU, &resource_gp_mmu);
499 resource_pp_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0, &(resource_pp[0]));
500 resource_pp_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1, &(resource_pp[1]));
501 resource_pp_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2, &(resource_pp[2]));
502 resource_pp_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3, &(resource_pp[3]));
503 resource_pp_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4, &(resource_pp[4]));
504 resource_pp_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5, &(resource_pp[5]));
505 resource_pp_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6, &(resource_pp[6]));
506 resource_pp_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7, &(resource_pp[7]));
507 resource_pp_mmu_found[0] = _mali_osk_resource_find(MALI_OFFSET_PP0_MMU, &(resource_pp_mmu[0]));
508 resource_pp_mmu_found[1] = _mali_osk_resource_find(MALI_OFFSET_PP1_MMU, &(resource_pp_mmu[1]));
509 resource_pp_mmu_found[2] = _mali_osk_resource_find(MALI_OFFSET_PP2_MMU, &(resource_pp_mmu[2]));
510 resource_pp_mmu_found[3] = _mali_osk_resource_find(MALI_OFFSET_PP3_MMU, &(resource_pp_mmu[3]));
511 resource_pp_mmu_found[4] = _mali_osk_resource_find(MALI_OFFSET_PP4_MMU, &(resource_pp_mmu[4]));
512 resource_pp_mmu_found[5] = _mali_osk_resource_find(MALI_OFFSET_PP5_MMU, &(resource_pp_mmu[5]));
513 resource_pp_mmu_found[6] = _mali_osk_resource_find(MALI_OFFSET_PP6_MMU, &(resource_pp_mmu[6]));
514 resource_pp_mmu_found[7] = _mali_osk_resource_find(MALI_OFFSET_PP7_MMU, &(resource_pp_mmu[7]));
515
516
517 if (mali_is_mali450() || mali_is_mali470()) {
518 resource_bcast_found = _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast);
519 resource_dlbu_found = _mali_osk_resource_find(MALI_OFFSET_DLBU, &resource_dlbu);
520 resource_pp_mmu_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST_MMU, &resource_pp_mmu_bcast);
521 resource_pp_bcast_found = _mali_osk_resource_find(MALI_OFFSET_PP_BCAST, &resource_pp_bcast);
522
523 if (_MALI_OSK_ERR_OK != resource_bcast_found ||
524 _MALI_OSK_ERR_OK != resource_dlbu_found ||
525 _MALI_OSK_ERR_OK != resource_pp_mmu_bcast_found ||
526 _MALI_OSK_ERR_OK != resource_pp_bcast_found) {
527 /* Missing mandatory core(s) for Mali-450 or Mali-470 */
528 MALI_DEBUG_PRINT(2, ("Missing mandatory resources, Mali-450 needs DLBU, Broadcast unit, virtual PP core and virtual MMU\n"));
529 return _MALI_OSK_ERR_FAULT;
530 }
531 }
532
533 if (_MALI_OSK_ERR_OK != resource_gp_found ||
534 _MALI_OSK_ERR_OK != resource_gp_mmu_found ||
535 _MALI_OSK_ERR_OK != resource_pp_found[0] ||
536 _MALI_OSK_ERR_OK != resource_pp_mmu_found[0]) {
537 /* Missing mandatory core(s) */
538 MALI_DEBUG_PRINT(2, ("Missing mandatory resource, need at least one GP and one PP, both with a separate MMU\n"));
539 return _MALI_OSK_ERR_FAULT;
540 }
541
542 MALI_DEBUG_ASSERT(1 <= mali_l2_cache_core_get_glob_num_l2_cores());
543 group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_gp), &resource_gp_mmu, &resource_gp, NULL, MALI_DOMAIN_INDEX_GP);
544 if (NULL == group) {
545 return _MALI_OSK_ERR_FAULT;
546 }
547
548 /* Create group for first (and mandatory) PP core */
549 MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= (cluster_id_pp_grp0 + 1)); /* >= 1 on Mali-300 and Mali-400, >= 2 on Mali-450 */
550 group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[0], NULL, &resource_pp[0], MALI_DOMAIN_INDEX_PP0);
551 if (NULL == group) {
552 return _MALI_OSK_ERR_FAULT;
553 }
554
555 mali_inited_pp_cores_group_1++;
556
557 /* Create groups for rest of the cores in the first PP core group */
558 for (i = 1; i < 4; i++) { /* First half of the PP cores belong to first core group */
559 if (mali_inited_pp_cores_group_1 < mali_max_pp_cores_group_1) {
560 if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) {
561 group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp0), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i);
562 if (NULL == group) {
563 return _MALI_OSK_ERR_FAULT;
564 }
565
566 mali_inited_pp_cores_group_1++;
567 }
568 }
569 }
570
571 /* Create groups for cores in the second PP core group */
572 for (i = 4; i < 8; i++) { /* Second half of the PP cores belong to second core group */
573 if (mali_inited_pp_cores_group_2 < mali_max_pp_cores_group_2) {
574 if (_MALI_OSK_ERR_OK == resource_pp_found[i] && _MALI_OSK_ERR_OK == resource_pp_mmu_found[i]) {
575 MALI_DEBUG_ASSERT(mali_l2_cache_core_get_glob_num_l2_cores() >= 2); /* Only Mali-450 have a second core group */
576 group = mali_create_group(mali_l2_cache_core_get_glob_l2_core(cluster_id_pp_grp1), &resource_pp_mmu[i], NULL, &resource_pp[i], MALI_DOMAIN_INDEX_PP0 + i);
577 if (NULL == group) {
578 return _MALI_OSK_ERR_FAULT;
579 }
580
581 mali_inited_pp_cores_group_2++;
582 }
583 }
584 }
585
586 if (mali_is_mali450() || mali_is_mali470()) {
587 _mali_osk_errcode_t err = mali_create_virtual_group(&resource_pp_mmu_bcast, &resource_pp_bcast, &resource_dlbu, &resource_bcast);
588 if (_MALI_OSK_ERR_OK != err) {
589 return err;
590 }
591 }
592
593 mali_max_pp_cores_group_1 = mali_inited_pp_cores_group_1;
594 mali_max_pp_cores_group_2 = mali_inited_pp_cores_group_2;
595 MALI_DEBUG_PRINT(2, ("%d+%d PP cores initialized\n", mali_inited_pp_cores_group_1, mali_inited_pp_cores_group_2));
596
597 return _MALI_OSK_ERR_OK;
598 }
599
mali_check_shared_interrupts(void)600 static _mali_osk_errcode_t mali_check_shared_interrupts(void)
601 {
602 #if !defined(CONFIG_MALI_SHARED_INTERRUPTS)
603 if (MALI_TRUE == _mali_osk_shared_interrupts()) {
604 MALI_PRINT_ERROR(("Shared interrupts detected, but driver support is not enabled\n"));
605 return _MALI_OSK_ERR_FAULT;
606 }
607 #endif /* !defined(CONFIG_MALI_SHARED_INTERRUPTS) */
608
609 /* It is OK to compile support for shared interrupts even if Mali is not using it. */
610 return _MALI_OSK_ERR_OK;
611 }
612
mali_parse_config_pmu(void)613 static _mali_osk_errcode_t mali_parse_config_pmu(void)
614 {
615 _mali_osk_resource_t resource_pmu;
616
617 MALI_DEBUG_ASSERT(0 != global_gpu_base_address);
618
619 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_PMU, &resource_pmu)) {
620 struct mali_pmu_core *pmu;
621
622 pmu = mali_pmu_create(&resource_pmu);
623 if (NULL == pmu) {
624 MALI_PRINT_ERROR(("Failed to create PMU\n"));
625 return _MALI_OSK_ERR_FAULT;
626 }
627 }
628
629 /* It's ok if the PMU doesn't exist */
630 return _MALI_OSK_ERR_OK;
631 }
632
mali_parse_config_memory(void)633 static _mali_osk_errcode_t mali_parse_config_memory(void)
634 {
635 _mali_osk_device_data data = { 0, };
636 _mali_osk_errcode_t ret;
637
638 /* The priority of setting the value of mali_shared_mem_size,
639 * mali_dedicated_mem_start and mali_dedicated_mem_size:
640 * 1. module parameter;
641 * 2. platform data;
642 * 3. default value;
643 **/
644 if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
645 /* Memory settings are not overridden by module parameters, so use device settings */
646 if (0 == mali_dedicated_mem_start && 0 == mali_dedicated_mem_size) {
647 /* Use device specific settings (if defined) */
648 mali_dedicated_mem_start = data.dedicated_mem_start;
649 mali_dedicated_mem_size = data.dedicated_mem_size;
650 }
651
652 if (MALI_SHARED_MEMORY_DEFAULT_SIZE == mali_shared_mem_size &&
653 0 != data.shared_mem_size) {
654 mali_shared_mem_size = data.shared_mem_size;
655 }
656 }
657
658 if (0 < mali_dedicated_mem_size && 0 != mali_dedicated_mem_start) {
659 MALI_DEBUG_PRINT(2, ("Mali memory settings (dedicated: 0x%08X@0x%08X)\n",
660 mali_dedicated_mem_size, mali_dedicated_mem_start));
661
662 /* Dedicated memory */
663 ret = mali_memory_core_resource_dedicated_memory(mali_dedicated_mem_start, mali_dedicated_mem_size);
664 if (_MALI_OSK_ERR_OK != ret) {
665 MALI_PRINT_ERROR(("Failed to register dedicated memory\n"));
666 mali_memory_terminate();
667 return ret;
668 }
669 }
670
671 if (0 < mali_shared_mem_size) {
672 MALI_DEBUG_PRINT(2, ("Mali memory settings (shared: 0x%08X)\n", mali_shared_mem_size));
673
674 /* Shared OS memory */
675 ret = mali_memory_core_resource_os_memory(mali_shared_mem_size);
676 if (_MALI_OSK_ERR_OK != ret) {
677 MALI_PRINT_ERROR(("Failed to register shared OS memory\n"));
678 mali_memory_terminate();
679 return ret;
680 }
681 }
682
683 if (0 == mali_fb_start && 0 == mali_fb_size) {
684 /* Frame buffer settings are not overridden by module parameters, so use device settings */
685 _mali_osk_device_data data = { 0, };
686
687 if (_MALI_OSK_ERR_OK == _mali_osk_device_data_get(&data)) {
688 /* Use device specific settings (if defined) */
689 mali_fb_start = data.fb_start;
690 mali_fb_size = data.fb_size;
691 }
692
693 MALI_DEBUG_PRINT(2, ("Using device defined frame buffer settings (0x%08X@0x%08X)\n",
694 mali_fb_size, mali_fb_start));
695 } else {
696 MALI_DEBUG_PRINT(2, ("Using module defined frame buffer settings (0x%08X@0x%08X)\n",
697 mali_fb_size, mali_fb_start));
698 }
699
700 if (0 != mali_fb_size) {
701 /* Register frame buffer */
702 ret = mali_mem_validation_add_range(mali_fb_start, mali_fb_size);
703 if (_MALI_OSK_ERR_OK != ret) {
704 MALI_PRINT_ERROR(("Failed to register frame buffer memory region\n"));
705 mali_memory_terminate();
706 return ret;
707 }
708 }
709
710 return _MALI_OSK_ERR_OK;
711 }
712
mali_detect_gpu_class(void)713 static void mali_detect_gpu_class(void)
714 {
715 if (_mali_osk_identify_gpu_resource() == 0x450)
716 mali_gpu_class_is_mali450 = MALI_TRUE;
717
718 if (_mali_osk_identify_gpu_resource() == 0x470)
719 mali_gpu_class_is_mali470 = MALI_TRUE;
720 }
721
mali_init_hw_reset(void)722 static _mali_osk_errcode_t mali_init_hw_reset(void)
723 {
724 #if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470))
725 _mali_osk_resource_t resource_bcast;
726
727 /* Ensure broadcast unit is in a good state before we start creating
728 * groups and cores.
729 */
730 if (_MALI_OSK_ERR_OK == _mali_osk_resource_find(MALI_OFFSET_BCAST, &resource_bcast)) {
731 struct mali_bcast_unit *bcast_core;
732
733 bcast_core = mali_bcast_unit_create(&resource_bcast);
734 if (NULL == bcast_core) {
735 MALI_PRINT_ERROR(("Failed to create Broadcast unit object!\n"));
736 return _MALI_OSK_ERR_FAULT;
737 }
738 mali_bcast_unit_delete(bcast_core);
739 }
740 #endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */
741
742 return _MALI_OSK_ERR_OK;
743 }
744
mali_initialize_subsystems(void)745 _mali_osk_errcode_t mali_initialize_subsystems(void)
746 {
747 _mali_osk_errcode_t err;
748
749 #ifdef CONFIG_MALI_DT
750 err = _mali_osk_resource_initialize();
751 if (_MALI_OSK_ERR_OK != err) {
752 mali_terminate_subsystems();
753 return err;
754 }
755 #endif
756
757 mali_pp_job_initialize();
758
759 err = mali_timeline_initialize();
760 if (_MALI_OSK_ERR_OK != err) {
761 mali_terminate_subsystems();
762 return err;
763 }
764
765 err = mali_session_initialize();
766 if (_MALI_OSK_ERR_OK != err) {
767 mali_terminate_subsystems();
768 return err;
769 }
770
771 /*Try to init gpu secure mode */
772 _mali_osk_gpu_secure_mode_init();
773
774 #if defined(CONFIG_MALI400_PROFILING)
775 err = _mali_osk_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
776 if (_MALI_OSK_ERR_OK != err) {
777 /* No biggie if we weren't able to initialize the profiling */
778 MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
779 }
780 #endif
781
782 err = mali_memory_initialize();
783 if (_MALI_OSK_ERR_OK != err) {
784 mali_terminate_subsystems();
785 return err;
786 }
787
788 err = mali_executor_initialize();
789 if (_MALI_OSK_ERR_OK != err) {
790 mali_terminate_subsystems();
791 return err;
792 }
793
794 err = mali_scheduler_initialize();
795 if (_MALI_OSK_ERR_OK != err) {
796 mali_terminate_subsystems();
797 return err;
798 }
799
800 /* Configure memory early, needed by mali_mmu_initialize. */
801 err = mali_parse_config_memory();
802 if (_MALI_OSK_ERR_OK != err) {
803 mali_terminate_subsystems();
804 return err;
805 }
806
807 err = mali_set_global_gpu_base_address();
808 if (_MALI_OSK_ERR_OK != err) {
809 mali_terminate_subsystems();
810 return err;
811 }
812
813 /* Detect GPU class (uses L2 cache count) */
814 mali_detect_gpu_class();
815
816 err = mali_check_shared_interrupts();
817 if (_MALI_OSK_ERR_OK != err) {
818 mali_terminate_subsystems();
819 return err;
820 }
821
822 /* Initialize the MALI PMU (will not touch HW!) */
823 err = mali_parse_config_pmu();
824 if (_MALI_OSK_ERR_OK != err) {
825 mali_terminate_subsystems();
826 return err;
827 }
828
829 /* Initialize the power management module */
830 err = mali_pm_initialize();
831 if (_MALI_OSK_ERR_OK != err) {
832 mali_terminate_subsystems();
833 return err;
834 }
835
836 /* Make sure the entire GPU stays on for the rest of this function */
837 mali_pm_init_begin();
838
839 /* Ensure HW is in a good state before starting to access cores. */
840 err = mali_init_hw_reset();
841 if (_MALI_OSK_ERR_OK != err) {
842 mali_terminate_subsystems();
843 return err;
844 }
845
846 /* Detect which Mali GPU we are dealing with */
847 err = mali_parse_product_info();
848 if (_MALI_OSK_ERR_OK != err) {
849 mali_pm_init_end();
850 mali_terminate_subsystems();
851 return err;
852 }
853
854 /* The global_product_id is now populated with the correct Mali GPU */
855
856 /* Start configuring the actual Mali hardware. */
857
858 err = mali_mmu_initialize();
859 if (_MALI_OSK_ERR_OK != err) {
860 mali_pm_init_end();
861 mali_terminate_subsystems();
862 return err;
863 }
864
865 if (mali_is_mali450() || mali_is_mali470()) {
866 err = mali_dlbu_initialize();
867 if (_MALI_OSK_ERR_OK != err) {
868 mali_pm_init_end();
869 mali_terminate_subsystems();
870 return err;
871 }
872 }
873
874 err = mali_parse_config_l2_cache();
875 if (_MALI_OSK_ERR_OK != err) {
876 mali_pm_init_end();
877 mali_terminate_subsystems();
878 return err;
879 }
880
881 err = mali_parse_config_groups();
882 if (_MALI_OSK_ERR_OK != err) {
883 mali_pm_init_end();
884 mali_terminate_subsystems();
885 return err;
886 }
887
888 /* Move groups into executor */
889 mali_executor_populate();
890
891 /* Need call after all group has assigned a domain */
892 mali_pm_power_cost_setup();
893
894 /* Initialize the GPU timer */
895 err = mali_control_timer_init();
896 if (_MALI_OSK_ERR_OK != err) {
897 mali_pm_init_end();
898 mali_terminate_subsystems();
899 return err;
900 }
901
902 /* Initialize the GPU utilization tracking */
903 err = mali_utilization_init();
904 if (_MALI_OSK_ERR_OK != err) {
905 mali_pm_init_end();
906 mali_terminate_subsystems();
907 return err;
908 }
909
910 #if defined(CONFIG_MALI_DVFS)
911 err = mali_dvfs_policy_init();
912 if (_MALI_OSK_ERR_OK != err) {
913 mali_pm_init_end();
914 mali_terminate_subsystems();
915 return err;
916 }
917 #endif
918
919 /* Allowing the system to be turned off */
920 mali_pm_init_end();
921
922 return _MALI_OSK_ERR_OK; /* all ok */
923 }
924
mali_terminate_subsystems(void)925 void mali_terminate_subsystems(void)
926 {
927 struct mali_pmu_core *pmu = mali_pmu_get_global_pmu_core();
928
929 MALI_DEBUG_PRINT(2, ("terminate_subsystems() called\n"));
930
931 mali_utilization_term();
932 mali_control_timer_term();
933
934 mali_executor_depopulate();
935 mali_delete_groups(); /* Delete groups not added to executor */
936 mali_executor_terminate();
937
938 mali_scheduler_terminate();
939 mali_pp_job_terminate();
940 mali_delete_l2_cache_cores();
941 mali_mmu_terminate();
942
943 if (mali_is_mali450() || mali_is_mali470()) {
944 mali_dlbu_terminate();
945 }
946
947 mali_pm_terminate();
948
949 if (NULL != pmu) {
950 mali_pmu_delete(pmu);
951 }
952
953 #if defined(CONFIG_MALI400_PROFILING)
954 _mali_osk_profiling_term();
955 #endif
956
957 _mali_osk_gpu_secure_mode_deinit();
958
959 mali_memory_terminate();
960
961 mali_session_terminate();
962
963 mali_timeline_terminate();
964
965 global_gpu_base_address = 0;
966 }
967
mali_kernel_core_get_product_id(void)968 _mali_product_id_t mali_kernel_core_get_product_id(void)
969 {
970 return global_product_id;
971 }
972
mali_kernel_core_get_gpu_major_version(void)973 u32 mali_kernel_core_get_gpu_major_version(void)
974 {
975 return global_gpu_major_version;
976 }
977
mali_kernel_core_get_gpu_minor_version(void)978 u32 mali_kernel_core_get_gpu_minor_version(void)
979 {
980 return global_gpu_minor_version;
981 }
982
_mali_ukk_get_api_version(_mali_uk_get_api_version_s * args)983 _mali_osk_errcode_t _mali_ukk_get_api_version(_mali_uk_get_api_version_s *args)
984 {
985 MALI_DEBUG_ASSERT_POINTER(args);
986 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
987
988 /* check compatability */
989 if (args->version == _MALI_UK_API_VERSION) {
990 args->compatible = 1;
991 } else {
992 args->compatible = 0;
993 }
994
995 args->version = _MALI_UK_API_VERSION; /* report our version */
996
997 /* success regardless of being compatible or not */
998 MALI_SUCCESS;
999 }
1000
_mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s * args)1001 _mali_osk_errcode_t _mali_ukk_get_api_version_v2(_mali_uk_get_api_version_v2_s *args)
1002 {
1003 MALI_DEBUG_ASSERT_POINTER(args);
1004 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1005
1006 /* check compatability */
1007 if (args->version == _MALI_UK_API_VERSION) {
1008 args->compatible = 1;
1009 } else {
1010 args->compatible = 0;
1011 }
1012
1013 args->version = _MALI_UK_API_VERSION; /* report our version */
1014
1015 /* success regardless of being compatible or not */
1016 return _MALI_OSK_ERR_OK;
1017 }
1018
_mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s * args)1019 _mali_osk_errcode_t _mali_ukk_wait_for_notification(_mali_uk_wait_for_notification_s *args)
1020 {
1021 _mali_osk_errcode_t err;
1022 _mali_osk_notification_t *notification;
1023 _mali_osk_notification_queue_t *queue;
1024 struct mali_session_data *session;
1025
1026 /* check input */
1027 MALI_DEBUG_ASSERT_POINTER(args);
1028 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1029
1030 session = (struct mali_session_data *)(uintptr_t)args->ctx;
1031 queue = session->ioctl_queue;
1032
1033 /* if the queue does not exist we're currently shutting down */
1034 if (NULL == queue) {
1035 MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n"));
1036 args->type = _MALI_NOTIFICATION_CORE_SHUTDOWN_IN_PROGRESS;
1037 return _MALI_OSK_ERR_OK;
1038 }
1039
1040 /* receive a notification, might sleep */
1041 err = _mali_osk_notification_queue_receive(queue, ¬ification);
1042 if (_MALI_OSK_ERR_OK != err) {
1043 MALI_ERROR(err); /* errcode returned, pass on to caller */
1044 }
1045
1046 /* copy the buffer to the user */
1047 args->type = (_mali_uk_notification_type)notification->notification_type;
1048 _mali_osk_memcpy(&args->data, notification->result_buffer, notification->result_buffer_size);
1049
1050 /* finished with the notification */
1051 _mali_osk_notification_delete(notification);
1052
1053 return _MALI_OSK_ERR_OK; /* all ok */
1054 }
1055
_mali_ukk_post_notification(_mali_uk_post_notification_s * args)1056 _mali_osk_errcode_t _mali_ukk_post_notification(_mali_uk_post_notification_s *args)
1057 {
1058 _mali_osk_notification_t *notification;
1059 _mali_osk_notification_queue_t *queue;
1060 struct mali_session_data *session;
1061
1062 /* check input */
1063 MALI_DEBUG_ASSERT_POINTER(args);
1064 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1065
1066 session = (struct mali_session_data *)(uintptr_t)args->ctx;
1067 queue = session->ioctl_queue;
1068
1069 /* if the queue does not exist we're currently shutting down */
1070 if (NULL == queue) {
1071 MALI_DEBUG_PRINT(1, ("No notification queue registered with the session. Asking userspace to stop querying\n"));
1072 return _MALI_OSK_ERR_OK;
1073 }
1074
1075 notification = _mali_osk_notification_create(args->type, 0);
1076 if (NULL == notification) {
1077 MALI_PRINT_ERROR(("Failed to create notification object\n"));
1078 return _MALI_OSK_ERR_NOMEM;
1079 }
1080
1081 _mali_osk_notification_queue_send(queue, notification);
1082
1083 return _MALI_OSK_ERR_OK; /* all ok */
1084 }
1085
_mali_ukk_pending_submit(_mali_uk_pending_submit_s * args)1086 _mali_osk_errcode_t _mali_ukk_pending_submit(_mali_uk_pending_submit_s *args)
1087 {
1088 wait_queue_head_t *queue;
1089
1090 /* check input */
1091 MALI_DEBUG_ASSERT_POINTER(args);
1092 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1093
1094 queue = mali_session_get_wait_queue();
1095
1096 /* check pending big job number, might sleep if larger than MAX allowed number */
1097 if (wait_event_interruptible(*queue, MALI_MAX_PENDING_BIG_JOB > mali_scheduler_job_gp_big_job_count())) {
1098 return _MALI_OSK_ERR_RESTARTSYSCALL;
1099 }
1100
1101 return _MALI_OSK_ERR_OK; /* all ok */
1102 }
1103
1104
_mali_ukk_request_high_priority(_mali_uk_request_high_priority_s * args)1105 _mali_osk_errcode_t _mali_ukk_request_high_priority(_mali_uk_request_high_priority_s *args)
1106 {
1107 struct mali_session_data *session;
1108
1109 MALI_DEBUG_ASSERT_POINTER(args);
1110 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1111
1112 session = (struct mali_session_data *)(uintptr_t)args->ctx;
1113
1114 if (!session->use_high_priority_job_queue) {
1115 session->use_high_priority_job_queue = MALI_TRUE;
1116 MALI_DEBUG_PRINT(2, ("Session 0x%08X with pid %d was granted higher priority.\n", session, _mali_osk_get_pid()));
1117 }
1118
1119 return _MALI_OSK_ERR_OK;
1120 }
1121
_mali_ukk_open(void ** context)1122 _mali_osk_errcode_t _mali_ukk_open(void **context)
1123 {
1124 u32 i;
1125 struct mali_session_data *session;
1126
1127 /* allocated struct to track this session */
1128 session = (struct mali_session_data *)_mali_osk_calloc(1, sizeof(struct mali_session_data));
1129 MALI_CHECK_NON_NULL(session, _MALI_OSK_ERR_NOMEM);
1130
1131 MALI_DEBUG_PRINT(3, ("Session starting\n"));
1132
1133 /* create a response queue for this session */
1134 session->ioctl_queue = _mali_osk_notification_queue_init();
1135 if (NULL == session->ioctl_queue) {
1136 goto err;
1137 }
1138
1139 /*create a wait queue for this session */
1140 session->wait_queue = _mali_osk_wait_queue_init();
1141 if (NULL == session->wait_queue) {
1142 goto err_wait_queue;
1143 }
1144
1145 session->page_directory = mali_mmu_pagedir_alloc();
1146 if (NULL == session->page_directory) {
1147 goto err_mmu;
1148 }
1149
1150 if (_MALI_OSK_ERR_OK != mali_mmu_pagedir_map(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE)) {
1151 MALI_PRINT_ERROR(("Failed to map DLBU page into session\n"));
1152 goto err_mmu;
1153 }
1154
1155 if (0 != mali_dlbu_phys_addr) {
1156 mali_mmu_pagedir_update(session->page_directory, MALI_DLBU_VIRT_ADDR, mali_dlbu_phys_addr,
1157 _MALI_OSK_MALI_PAGE_SIZE, MALI_MMU_FLAGS_DEFAULT);
1158 }
1159
1160 if (_MALI_OSK_ERR_OK != mali_memory_session_begin(session)) {
1161 goto err_session;
1162 }
1163
1164 /* Create soft system. */
1165 session->soft_job_system = mali_soft_job_system_create(session);
1166 if (NULL == session->soft_job_system) {
1167 goto err_soft;
1168 }
1169
1170 /* Initialize the dma fence context.*/
1171 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
1172 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
1173 session->fence_context = dma_fence_context_alloc(1);
1174 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
1175 session->fence_context = fence_context_alloc(1);
1176 _mali_osk_atomic_init(&session->fence_seqno, 0);
1177 #else
1178 MALI_PRINT_ERROR(("The kernel version not support dma fence!\n"));
1179 goto err_time_line;
1180 #endif
1181 #endif
1182
1183 /* Create timeline system. */
1184 session->timeline_system = mali_timeline_system_create(session);
1185 if (NULL == session->timeline_system) {
1186 goto err_time_line;
1187 }
1188
1189 #if defined(CONFIG_MALI_DVFS)
1190 _mali_osk_atomic_init(&session->number_of_window_jobs, 0);
1191 #endif
1192
1193 _mali_osk_atomic_init(&session->number_of_pp_jobs, 0);
1194
1195 session->use_high_priority_job_queue = MALI_FALSE;
1196
1197 /* Initialize list of PP jobs on this session. */
1198 _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_list);
1199
1200 /* Initialize the pp_job_fb_lookup_list array used to quickly lookup jobs from a given frame builder */
1201 for (i = 0; i < MALI_PP_JOB_FB_LOOKUP_LIST_SIZE; ++i) {
1202 _MALI_OSK_INIT_LIST_HEAD(&session->pp_job_fb_lookup_list[i]);
1203 }
1204
1205 session->pid = _mali_osk_get_pid();
1206 session->comm = _mali_osk_get_comm();
1207 session->max_mali_mem_allocated_size = 0;
1208 for (i = 0; i < MALI_MEM_TYPE_MAX; i ++) {
1209 atomic_set(&session->mali_mem_array[i], 0);
1210 }
1211 atomic_set(&session->mali_mem_allocated_pages, 0);
1212 *context = (void *)session;
1213
1214 /* Add session to the list of all sessions. */
1215 mali_session_add(session);
1216
1217 MALI_DEBUG_PRINT(3, ("Session started\n"));
1218 return _MALI_OSK_ERR_OK;
1219
1220 err_time_line:
1221 mali_soft_job_system_destroy(session->soft_job_system);
1222 err_soft:
1223 mali_memory_session_end(session);
1224 err_session:
1225 mali_mmu_pagedir_free(session->page_directory);
1226 err_mmu:
1227 _mali_osk_wait_queue_term(session->wait_queue);
1228 err_wait_queue:
1229 _mali_osk_notification_queue_term(session->ioctl_queue);
1230 err:
1231 _mali_osk_free(session);
1232 MALI_ERROR(_MALI_OSK_ERR_NOMEM);
1233
1234 }
1235
1236 #if defined(DEBUG)
1237 /* parameter used for debug */
1238 extern u32 num_pm_runtime_resume;
1239 extern u32 num_pm_updates;
1240 extern u32 num_pm_updates_up;
1241 extern u32 num_pm_updates_down;
1242 #endif
1243
_mali_ukk_close(void ** context)1244 _mali_osk_errcode_t _mali_ukk_close(void **context)
1245 {
1246 struct mali_session_data *session;
1247 MALI_CHECK_NON_NULL(context, _MALI_OSK_ERR_INVALID_ARGS);
1248 session = (struct mali_session_data *)*context;
1249
1250 MALI_DEBUG_PRINT(3, ("Session ending\n"));
1251
1252 MALI_DEBUG_ASSERT_POINTER(session->soft_job_system);
1253 MALI_DEBUG_ASSERT_POINTER(session->timeline_system);
1254
1255 /* Remove session from list of all sessions. */
1256 mali_session_remove(session);
1257
1258 /* This flag is used to prevent queueing of jobs due to activation. */
1259 session->is_aborting = MALI_TRUE;
1260
1261 /* Stop the soft job timer. */
1262 mali_timeline_system_stop_timer(session->timeline_system);
1263
1264 /* Abort queued jobs */
1265 mali_scheduler_abort_session(session);
1266
1267 /* Abort executing jobs */
1268 mali_executor_abort_session(session);
1269
1270 /* Abort the soft job system. */
1271 mali_soft_job_system_abort(session->soft_job_system);
1272
1273 /* Force execution of all pending bottom half processing for GP and PP. */
1274 _mali_osk_wq_flush();
1275
1276 /* The session PP list should now be empty. */
1277 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&session->pp_job_list));
1278
1279 /* At this point the GP and PP scheduler no longer has any jobs queued or running from this
1280 * session, and all soft jobs in the soft job system has been destroyed. */
1281
1282 /* Any trackers left in the timeline system are directly or indirectly waiting on external
1283 * sync fences. Cancel all sync fence waiters to trigger activation of all remaining
1284 * trackers. This call will sleep until all timelines are empty. */
1285 mali_timeline_system_abort(session->timeline_system);
1286
1287 /* Flush pending work.
1288 * Needed to make sure all bottom half processing related to this
1289 * session has been completed, before we free internal data structures.
1290 */
1291 _mali_osk_wq_flush();
1292
1293 /* Destroy timeline system. */
1294 mali_timeline_system_destroy(session->timeline_system);
1295 session->timeline_system = NULL;
1296
1297 /* Destroy soft system. */
1298 mali_soft_job_system_destroy(session->soft_job_system);
1299 session->soft_job_system = NULL;
1300
1301 /*Wait for the session job lists become empty.*/
1302 _mali_osk_wait_queue_wait_event(session->wait_queue, mali_session_pp_job_is_empty, (void *) session);
1303
1304 /* Free remaining memory allocated to this session */
1305 mali_memory_session_end(session);
1306
1307 #if defined(CONFIG_MALI_DVFS)
1308 _mali_osk_atomic_term(&session->number_of_window_jobs);
1309 #endif
1310
1311 #if defined(CONFIG_MALI400_PROFILING)
1312 _mali_osk_profiling_stop_sampling(session->pid);
1313 #endif
1314
1315 /* Free session data structures */
1316 mali_mmu_pagedir_unmap(session->page_directory, MALI_DLBU_VIRT_ADDR, _MALI_OSK_MALI_PAGE_SIZE);
1317 mali_mmu_pagedir_free(session->page_directory);
1318 _mali_osk_wait_queue_term(session->wait_queue);
1319 _mali_osk_notification_queue_term(session->ioctl_queue);
1320 _mali_osk_free(session);
1321
1322 *context = NULL;
1323
1324 MALI_DEBUG_PRINT(3, ("Session has ended\n"));
1325
1326 #if defined(DEBUG)
1327 MALI_DEBUG_PRINT(3, ("Stats: # runtime resumes: %u\n", num_pm_runtime_resume));
1328 MALI_DEBUG_PRINT(3, (" # PM updates: .... %u (up %u, down %u)\n", num_pm_updates, num_pm_updates_up, num_pm_updates_down));
1329
1330 num_pm_runtime_resume = 0;
1331 num_pm_updates = 0;
1332 num_pm_updates_up = 0;
1333 num_pm_updates_down = 0;
1334 #endif
1335
1336 return _MALI_OSK_ERR_OK;;
1337 }
1338
1339 #if MALI_STATE_TRACKING
_mali_kernel_core_dump_state(char * buf,u32 size)1340 u32 _mali_kernel_core_dump_state(char *buf, u32 size)
1341 {
1342 int n = 0; /* Number of bytes written to buf */
1343
1344 n += mali_scheduler_dump_state(buf + n, size - n);
1345 n += mali_executor_dump_state(buf + n, size - n);
1346
1347 return n;
1348 }
1349 #endif
1350