1 /**
2 * Copyright (C) 2010-2017 ARM Limited. All rights reserved.
3 *
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
6 *
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
9 */
10
11
12 /**
13 * @file mali_kernel_linux.c
14 * Implementation of the Linux device driver entrypoints
15 */
16 #include "../platform/rk/custom_log.h"
17 #include "../platform/rk/rk_ext.h"
18
19 #include <linux/module.h> /* kernel module definitions */
20 #include <linux/fs.h> /* file system operations */
21 #include <linux/cdev.h> /* character device definitions */
22 #include <linux/mm.h> /* memory manager definitions */
23 #include <linux/mali/mali_utgard_ioctl.h>
24 #include <linux/version.h>
25 #include <linux/device.h>
26 #include "mali_kernel_license.h"
27 #include <linux/platform_device.h>
28 #include <linux/miscdevice.h>
29 #include <linux/bug.h>
30 #include <linux/of.h>
31 #include <linux/clk.h>
32 #include <linux/regulator/consumer.h>
33 #include <linux/mali/mali_utgard.h>
34 #include <linux/pm_runtime.h>
35 #include <soc/rockchip/rockchip_opp_select.h>
36
37 #include "mali_kernel_common.h"
38 #include "mali_session.h"
39 #include "mali_kernel_core.h"
40 #include "mali_osk.h"
41 #include "mali_kernel_linux.h"
42 #include "mali_ukk.h"
43 #include "mali_ukk_wrappers.h"
44 #include "mali_kernel_sysfs.h"
45 #include "mali_pm.h"
46 #include "mali_kernel_license.h"
47 #include "mali_memory.h"
48 #include "mali_memory_dma_buf.h"
49 #include "mali_memory_manager.h"
50 #include "mali_memory_swap_alloc.h"
51 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
52 #include "mali_profiling_internal.h"
53 #endif
54 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
55 #include "mali_osk_profiling.h"
56 #include "mali_dvfs_policy.h"
57
58 static int is_first_resume = 1;
59 /*Store the clk and vol for boot/insmod and mali_resume*/
60 static struct mali_gpu_clk_item mali_gpu_clk[2];
61 #endif
62
63 /* Streamline support for the Mali driver */
64 #if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING)
65 /* Ask Linux to create the tracepoints */
66 #define CREATE_TRACE_POINTS
67 #include "mali_linux_trace.h"
68
69 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event);
70 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter);
71 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters);
72 #endif /* CONFIG_TRACEPOINTS */
73
74 #ifdef CONFIG_MALI_DEVFREQ
75 #include "mali_devfreq.h"
76 #include "mali_osk_mali.h"
77
78 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)
79 #include <linux/pm_opp.h>
80 #else
81 /* In 3.13 the OPP include header file, types, and functions were all
82 * renamed. Use the old filename for the include, and define the new names to
83 * the old, when an old kernel is detected.
84 */
85 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
86 #include <linux/pm_opp.h>
87 #else
88 #include <linux/opp.h>
89 #endif /* Linux >= 3.13*/
90 #define dev_pm_opp_of_add_table of_init_opp_table
91 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)
92 #define dev_pm_opp_of_remove_table of_free_opp_table
93 #endif /* Linux >= 3.19 */
94 #endif /* Linux >= 4.4.0 */
95 #endif
96
97 /* from the __malidrv_build_info.c file that is generated during build */
98 extern const char *__malidrv_build_info(void);
99
100 /* Module parameter to control log level */
101 int mali_debug_level = 2;
102 module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
103 MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output");
104
105 extern int mali_max_job_runtime;
106 module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
107 MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what");
108
109 extern int mali_l2_max_reads;
110 module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH);
111 MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache");
112
113 extern unsigned int mali_dedicated_mem_start;
114 module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH);
115 MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory.");
116
117 extern unsigned int mali_dedicated_mem_size;
118 module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
119 MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory.");
120
121 extern unsigned int mali_shared_mem_size;
122 module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
123 MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory.");
124
125 #if defined(CONFIG_MALI400_PROFILING)
126 extern int mali_boot_profiling;
127 module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH);
128 MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization");
129 #endif
130
131 extern int mali_max_pp_cores_group_1;
132 module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH);
133 MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group.");
134
135 extern int mali_max_pp_cores_group_2;
136 module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH);
137 MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only).");
138
139 extern unsigned int mali_mem_swap_out_threshold_value;
140 module_param(mali_mem_swap_out_threshold_value, uint, S_IRUSR | S_IRGRP | S_IROTH);
141 MODULE_PARM_DESC(mali_mem_swap_out_threshold_value, "Threshold value used to limit how much swappable memory cached in Mali driver.");
142
143 #if defined(CONFIG_MALI_DVFS)
144 /** the max fps the same as display vsync default 60, can set by module insert parameter */
145 extern int mali_max_system_fps;
146 module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
147 MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC.");
148
149 /** a lower limit on their desired FPS default 58, can set by module insert parameter*/
150 extern int mali_desired_fps;
151 module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
152 MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps");
153 #endif
154
155 #if MALI_ENABLE_CPU_CYCLES
156 #include <linux/cpumask.h>
157 #include <linux/timer.h>
158 #include <asm/smp.h>
159 static struct timer_list mali_init_cpu_clock_timers[8];
160 static u32 mali_cpu_clock_last_value[8] = {0,};
161 #endif
162
163 /* Export symbols from common code: mali_user_settings.c */
164 #include "mali_user_settings_db.h"
165 EXPORT_SYMBOL(mali_set_user_setting);
166 EXPORT_SYMBOL(mali_get_user_setting);
167
168 static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */
169
170 /* This driver only supports one Mali device, and this variable stores this single platform device */
171 struct platform_device *mali_platform_device = NULL;
172
173 /* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */
174 static struct miscdevice mali_miscdevice = { 0, };
175
176 static int mali_miscdevice_register(struct platform_device *pdev);
177 static void mali_miscdevice_unregister(void);
178
179 static int mali_open(struct inode *inode, struct file *filp);
180 static int mali_release(struct inode *inode, struct file *filp);
181 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
182
183 static int mali_probe(struct platform_device *pdev);
184 static int mali_remove(struct platform_device *pdev);
185
186 static int mali_driver_suspend_scheduler(struct device *dev);
187 static int mali_driver_resume_scheduler(struct device *dev);
188
189 #ifdef CONFIG_PM_RUNTIME
190 static int mali_driver_runtime_suspend(struct device *dev);
191 static int mali_driver_runtime_resume(struct device *dev);
192 static int mali_driver_runtime_idle(struct device *dev);
193 #endif
194
195 #if defined(MALI_FAKE_PLATFORM_DEVICE)
196 #if defined(CONFIG_MALI_DT)
197 extern int mali_platform_device_init(struct platform_device *device);
198 extern int mali_platform_device_deinit(struct platform_device *device);
199 #else
200 extern int mali_platform_device_register(void);
201 extern int mali_platform_device_unregister(void);
202 #endif
203 #endif
204
205 extern int rk_platform_init_opp_table(struct device *dev);
206
207 /* Linux power management operations provided by the Mali device driver */
208 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
209 struct pm_ext_ops mali_dev_ext_pm_ops = {
210 .base =
211 {
212 .suspend = mali_driver_suspend_scheduler,
213 .resume = mali_driver_resume_scheduler,
214 .freeze = mali_driver_suspend_scheduler,
215 .thaw = mali_driver_resume_scheduler,
216 },
217 };
218 #else
219 static const struct dev_pm_ops mali_dev_pm_ops = {
220 #ifdef CONFIG_PM_RUNTIME
221 .runtime_suspend = mali_driver_runtime_suspend,
222 .runtime_resume = mali_driver_runtime_resume,
223 .runtime_idle = mali_driver_runtime_idle,
224 #endif
225 .suspend = mali_driver_suspend_scheduler,
226 .resume = mali_driver_resume_scheduler,
227 .freeze = mali_driver_suspend_scheduler,
228 .thaw = mali_driver_resume_scheduler,
229 .poweroff = mali_driver_suspend_scheduler,
230 };
231 #endif
232
233 #ifdef CONFIG_MALI_DT
234 static struct of_device_id base_dt_ids[] = {
235 {.compatible = "arm,mali-300"},
236 /*-------------------------------------------------------*/
237 /* rk_ext : to use dts_for_mali_ko_befor_r5p0-01rel0. */
238 // {.compatible = "arm,mali-400"},
239 {.compatible = "arm,mali400"},
240 /*-------------------------------------------------------*/
241 {.compatible = "arm,mali-450"},
242 {.compatible = "arm,mali-470"},
243 {},
244 };
245
246 MODULE_DEVICE_TABLE(of, base_dt_ids);
247 #endif
248
249 /* The Mali device driver struct */
250 static struct platform_driver mali_platform_driver = {
251 .probe = mali_probe,
252 .remove = mali_remove,
253 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
254 .pm = &mali_dev_ext_pm_ops,
255 #endif
256 .driver =
257 {
258 .name = MALI_GPU_NAME_UTGARD,
259 .owner = THIS_MODULE,
260 .bus = &platform_bus_type,
261 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
262 .pm = &mali_dev_pm_ops,
263 #endif
264 #ifdef CONFIG_MALI_DT
265 .of_match_table = of_match_ptr(base_dt_ids),
266 #endif
267 },
268 };
269
270 /* Linux misc device operations (/dev/mali) */
271 struct file_operations mali_fops = {
272 .owner = THIS_MODULE,
273 .open = mali_open,
274 .release = mali_release,
275 .unlocked_ioctl = mali_ioctl,
276 .compat_ioctl = mali_ioctl,
277 .mmap = mali_mmap
278 };
279
280 #if MALI_ENABLE_CPU_CYCLES
mali_init_cpu_time_counters(int reset,int enable_divide_by_64)281 void mali_init_cpu_time_counters(int reset, int enable_divide_by_64)
282 {
283 /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */
284 u32 write_value;
285
286 /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */
287 /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */
288 asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f));
289
290
291 /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */
292 write_value = 1 << 0; /* Bit 0 set. Enable counters */
293 if (reset) {
294 write_value |= 1 << 1; /* Reset event counters */
295 write_value |= 1 << 2; /* Reset cycle counter */
296 }
297 if (enable_divide_by_64) {
298 write_value |= 1 << 3; /* Enable the Clock divider by 64 */
299 }
300 write_value |= 1 << 4; /* Export enable. Not needed */
301 asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value));
302
303 /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */
304 asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f));
305
306
307 /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */
308 /* User mode access to the Performance Monitors enabled. */
309 /* Lets User space read cpu clock cycles */
310 asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1));
311 }
312
313 /** A timer function that configures the cycle clock counter on current CPU.
314 * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this
315 * function to trigger on all Cpus during module load.
316 */
mali_init_cpu_clock_timer_func(unsigned long data)317 static void mali_init_cpu_clock_timer_func(unsigned long data)
318 {
319 int reset_counters, enable_divide_clock_counter_by_64;
320 int current_cpu = raw_smp_processor_id();
321 unsigned int sample0;
322 unsigned int sample1;
323
324 MALI_IGNORE(data);
325
326 reset_counters = 1;
327 enable_divide_clock_counter_by_64 = 0;
328 mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64);
329
330 sample0 = mali_get_cpu_cyclecount();
331 sample1 = mali_get_cpu_cyclecount();
332
333 MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1));
334 }
335
336 /** A timer functions for storing current time on all cpus.
337 * Used for checking if the clocks have similar values or if they are drifting.
338 */
mali_print_cpu_clock_timer_func(unsigned long data)339 static void mali_print_cpu_clock_timer_func(unsigned long data)
340 {
341 int current_cpu = raw_smp_processor_id();
342 unsigned int sample0;
343
344 MALI_IGNORE(data);
345 sample0 = mali_get_cpu_cyclecount();
346 if (current_cpu < 8) {
347 mali_cpu_clock_last_value[current_cpu] = sample0;
348 }
349 }
350
351 /** Init the performance registers on all CPUs to count clock cycles.
352 * For init \a print_only should be 0.
353 * If \a print_only is 1, it will intead print the current clock value of all CPUs.
354 */
mali_init_cpu_time_counters_on_all_cpus(int print_only)355 void mali_init_cpu_time_counters_on_all_cpus(int print_only)
356 {
357 int i = 0;
358 int cpu_number;
359 int jiffies_trigger;
360 int jiffies_wait;
361
362 jiffies_wait = 2;
363 jiffies_trigger = jiffies + jiffies_wait;
364
365 for (i = 0 ; i < 8 ; i++) {
366 init_timer(&mali_init_cpu_clock_timers[i]);
367 if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func;
368 else mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func;
369 mali_init_cpu_clock_timers[i].expires = jiffies_trigger ;
370 }
371 cpu_number = cpumask_first(cpu_online_mask);
372 for (i = 0 ; i < 8 ; i++) {
373 int next_cpu;
374 add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number);
375 next_cpu = cpumask_next(cpu_number, cpu_online_mask);
376 if (next_cpu >= nr_cpu_ids) break;
377 cpu_number = next_cpu;
378 }
379
380 while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait);
381
382 for (i = 0 ; i < 8 ; i++) {
383 del_timer_sync(&mali_init_cpu_clock_timers[i]);
384 }
385
386 if (print_only) {
387 if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) {
388 /* Diff can be printed if we want to check if the clocks are in sync
389 int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/
390 MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1]));
391 } else {
392 MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3]));
393 }
394 }
395 }
396 #endif
397
mali_module_init(void)398 int mali_module_init(void)
399 {
400 int err = 0;
401
402 MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION));
403 MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__));
404 MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING));
405
406 I("svn_rev_string_from_arm of this mali_ko is '%s', rk_ko_ver is '%d', built at '%s', on '%s'.",
407 SVN_REV_STRING,
408 RK_KO_VER,
409 __TIME__,
410 __DATE__);
411
412 #if MALI_ENABLE_CPU_CYCLES
413 mali_init_cpu_time_counters_on_all_cpus(0);
414 MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n"));
415 /* Printing the current cpu counters */
416 mali_init_cpu_time_counters_on_all_cpus(1);
417 #endif
418
419 /* Initialize module wide settings */
420 #ifdef MALI_FAKE_PLATFORM_DEVICE
421 #ifndef CONFIG_MALI_DT
422 MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n"));
423 err = mali_platform_device_register();
424 if (0 != err) {
425 return err;
426 }
427 #endif
428 #endif
429
430 MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n"));
431
432 err = platform_driver_register(&mali_platform_driver);
433
434 if (0 != err) {
435 MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err));
436 #ifdef MALI_FAKE_PLATFORM_DEVICE
437 #ifndef CONFIG_MALI_DT
438 mali_platform_device_unregister();
439 #endif
440 #endif
441 mali_platform_device = NULL;
442 return err;
443 }
444
445 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
446 err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
447 if (0 != err) {
448 /* No biggie if we wheren't able to initialize the profiling */
449 MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
450 }
451 #endif
452
453 /* Tracing the current frequency and voltage from boot/insmod*/
454 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
455 /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/
456 mali_get_current_gpu_clk_item(&mali_gpu_clk[0]);
457 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
458 MALI_PROFILING_EVENT_CHANNEL_GPU |
459 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
460 mali_gpu_clk[0].clock,
461 mali_gpu_clk[0].vol / 1000,
462 0, 0, 0);
463 #endif
464
465 MALI_PRINT(("Mali device driver loaded\n"));
466
467 return 0; /* Success */
468 }
469
mali_module_exit(void)470 void mali_module_exit(void)
471 {
472 MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION));
473
474 MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n"));
475
476 platform_driver_unregister(&mali_platform_driver);
477
478 #if defined(MALI_FAKE_PLATFORM_DEVICE)
479 #ifndef CONFIG_MALI_DT
480 MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n"));
481 mali_platform_device_unregister();
482 #endif
483 #endif
484
485 /* Tracing the current frequency and voltage from rmmod*/
486 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
487 MALI_PROFILING_EVENT_CHANNEL_GPU |
488 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
489 0,
490 0,
491 0, 0, 0);
492
493 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
494 _mali_internal_profiling_term();
495 #endif
496
497 MALI_PRINT(("Mali device driver unloaded\n"));
498 }
499
500 #ifdef CONFIG_MALI_DEVFREQ
mali_device_alloc(void)501 struct mali_device *mali_device_alloc(void)
502 {
503 return kzalloc(sizeof(struct mali_device), GFP_KERNEL);
504 }
505
mali_device_free(struct mali_device * mdev)506 void mali_device_free(struct mali_device *mdev)
507 {
508 kfree(mdev);
509 }
510 #endif
511
mali_probe(struct platform_device * pdev)512 static int mali_probe(struct platform_device *pdev)
513 {
514 int err;
515 #ifdef CONFIG_MALI_DEVFREQ
516 struct mali_device *mdev;
517 const char *name = "mali";
518 #endif
519
520 MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name));
521
522 if (NULL != mali_platform_device) {
523 /* Already connected to a device, return error */
524 MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device."));
525 return -EEXIST;
526 }
527
528 mali_platform_device = pdev;
529
530 dev_info(&pdev->dev, "mali_platform_device->num_resources = %d\n",
531 mali_platform_device->num_resources);
532
533 {
534 int i = 0;
535
536 for(i = 0; i < mali_platform_device->num_resources; i++)
537 dev_info(&pdev->dev,
538 "resource[%d].start = 0x%pa\n",
539 i,
540 &mali_platform_device->resource[i].start);
541 }
542
543 #ifdef CONFIG_MALI_DT
544 /* If we use DT to initialize our DDK, we have to prepare somethings. */
545 err = mali_platform_device_init(mali_platform_device);
546 if (0 != err) {
547 MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device."));
548 mali_platform_device = NULL;
549 return -EFAULT;
550 }
551 #endif
552
553 #ifdef CONFIG_MALI_DEVFREQ
554 mdev = mali_device_alloc();
555 if (!mdev) {
556 MALI_PRINT_ERROR(("Can't allocate mali device private data\n"));
557 return -ENOMEM;
558 }
559
560 mdev->dev = &pdev->dev;
561 dev_set_drvdata(mdev->dev, mdev);
562
563 /*Initilization clock and regulator*/
564 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
565 && defined(CONFIG_REGULATOR)
566 mdev->regulator = regulator_get_optional(mdev->dev, "mali");
567 if (IS_ERR_OR_NULL(mdev->regulator)) {
568 MALI_DEBUG_PRINT(2, ("Continuing without Mali regulator control\n"));
569 mdev->regulator = NULL;
570 /* Allow probe to continue without regulator */
571 }
572 if (mdev->regulator) {
573 mdev->opp_table = dev_pm_opp_set_regulators(mdev->dev, &name, 1);
574 if (IS_ERR(mdev->opp_table)) {
575 mdev->opp_table = NULL;
576 MALI_DEBUG_PRINT(2, ("Continuing without opp regulator\n"));
577 }
578 }
579 #endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
580
581 mdev->num_clks = devm_clk_bulk_get_all(mdev->dev, &mdev->clks);
582 if (mdev->num_clks < 1) {
583 MALI_DEBUG_PRINT(2, ("Continuing without Mali clock control\n"));
584 mdev->num_clks = 0;
585 mdev->clock = NULL;
586 } else {
587 /* Get "clk_mali" in the device tree for gpu dvfs */
588 mdev->clock = clk_get(mdev->dev, "clk_mali");
589 if (IS_ERR_OR_NULL(mdev->clock)) {
590 MALI_DEBUG_PRINT(2, ("Continuing without Mali dvfs clock\n"));
591 /* Allow probe to continue without clock. */
592 mdev->clock = NULL;
593 }
594 }
595 err = clk_bulk_prepare_enable(mdev->num_clks, mdev->clks);
596 if (err) {
597 MALI_PRINT_ERROR(("Failed to prepare clock (%d)\n", err));
598 goto clock_prepare_failed;
599 }
600
601 err = rk_platform_init_opp_table(mdev->dev);
602 if (err)
603 MALI_DEBUG_PRINT(3, ("Failed to init_opp_table\n"));
604
605 /* initilize pm metrics related */
606 if (mali_pm_metrics_init(mdev) < 0) {
607 MALI_DEBUG_PRINT(2, ("mali pm metrics init failed\n"));
608 goto pm_metrics_init_failed;
609 }
610
611 if (mali_devfreq_init(mdev) < 0) {
612 MALI_DEBUG_PRINT(2, ("mali devfreq init failed\n"));
613 goto devfreq_init_failed;
614 }
615 clk_bulk_disable(mdev->num_clks, mdev->clks);
616 #endif
617
618
619 if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) {
620 /* Initialize the Mali GPU HW specified by pdev */
621 if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) {
622 /* Register a misc device (so we are accessible from user space) */
623 err = mali_miscdevice_register(pdev);
624 if (0 == err) {
625 /* Setup sysfs entries */
626 err = mali_sysfs_register(mali_dev_name);
627
628 if (0 == err) {
629 MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name));
630
631 return 0;
632 } else {
633 MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries"));
634 }
635 mali_miscdevice_unregister();
636 } else {
637 MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device."));
638 }
639 mali_terminate_subsystems();
640 } else {
641 MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver."));
642 }
643 _mali_osk_wq_term();
644 }
645
646 #ifdef CONFIG_MALI_DEVFREQ
647 mali_devfreq_term(mdev);
648 devfreq_init_failed:
649 mali_pm_metrics_term(mdev);
650 pm_metrics_init_failed:
651 clk_bulk_disable_unprepare(mdev->num_clks, mdev->clks);
652 clock_prepare_failed:
653 clk_bulk_put(mdev->num_clks, mdev->clks);
654 clk_put(mdev->clock);
655 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \
656 && defined(CONFIG_PM_OPP)
657 dev_pm_opp_of_remove_table(mdev->dev);
658 #endif
659
660 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
661 && defined(CONFIG_REGULATOR)
662 regulator_put(mdev->regulator);
663 dev_pm_opp_put_regulators(mdev->opp_table);
664 #endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
665 mali_device_free(mdev);
666 #endif
667
668 #ifdef CONFIG_MALI_DT
669 mali_platform_device_deinit(mali_platform_device);
670 #endif
671 mali_platform_device = NULL;
672 return -EFAULT;
673 }
674
mali_remove(struct platform_device * pdev)675 static int mali_remove(struct platform_device *pdev)
676 {
677 #ifdef CONFIG_MALI_DEVFREQ
678 struct mali_device *mdev = dev_get_drvdata(&pdev->dev);
679 #endif
680
681 MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name));
682 mali_sysfs_unregister();
683 mali_miscdevice_unregister();
684 mali_terminate_subsystems();
685 _mali_osk_wq_term();
686
687 #ifdef CONFIG_MALI_DEVFREQ
688 mali_devfreq_term(mdev);
689
690 mali_pm_metrics_term(mdev);
691
692 if (mdev->clock) {
693 clk_put(mdev->clock);
694 mdev->clock = NULL;
695 }
696 clk_bulk_unprepare(mdev->num_clks, mdev->clks);
697 clk_bulk_put(mdev->num_clks, mdev->clks);
698
699 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) && defined(CONFIG_OF) \
700 && defined(CONFIG_PM_OPP)
701 dev_pm_opp_of_remove_table(mdev->dev);
702 #endif
703
704 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \
705 && defined(CONFIG_REGULATOR)
706 regulator_put(mdev->regulator);
707 dev_pm_opp_put_regulators(mdev->opp_table);
708 #endif /* LINUX_VERSION_CODE >= 3, 12, 0 */
709 mali_device_free(mdev);
710 #endif
711
712 #ifdef CONFIG_MALI_DT
713 mali_platform_device_deinit(mali_platform_device);
714 #endif
715 mali_platform_device = NULL;
716 return 0;
717 }
718
mali_miscdevice_register(struct platform_device * pdev)719 static int mali_miscdevice_register(struct platform_device *pdev)
720 {
721 int err;
722
723 mali_miscdevice.minor = MISC_DYNAMIC_MINOR;
724 mali_miscdevice.name = mali_dev_name;
725 mali_miscdevice.fops = &mali_fops;
726 mali_miscdevice.parent = get_device(&pdev->dev);
727
728 err = misc_register(&mali_miscdevice);
729 if (0 != err) {
730 MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err));
731 }
732
733 return err;
734 }
735
mali_miscdevice_unregister(void)736 static void mali_miscdevice_unregister(void)
737 {
738 misc_deregister(&mali_miscdevice);
739 }
740
mali_driver_suspend_scheduler(struct device * dev)741 static int mali_driver_suspend_scheduler(struct device *dev)
742 {
743 #ifdef CONFIG_MALI_DEVFREQ
744 struct mali_device *mdev = dev_get_drvdata(dev);
745 if (!mdev)
746 return -ENODEV;
747 #endif
748
749 pm_runtime_force_suspend(dev);
750 #if defined(CONFIG_MALI_DEVFREQ) && \
751 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
752 devfreq_suspend_device(mdev->devfreq);
753 #endif
754
755 mali_pm_os_suspend(MALI_TRUE);
756 /* Tracing the frequency and voltage after mali is suspended */
757 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
758 MALI_PROFILING_EVENT_CHANNEL_GPU |
759 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
760 0,
761 0,
762 0, 0, 0);
763 return 0;
764 }
765
mali_driver_resume_scheduler(struct device * dev)766 static int mali_driver_resume_scheduler(struct device *dev)
767 {
768 #ifdef CONFIG_MALI_DEVFREQ
769 struct mali_device *mdev = dev_get_drvdata(dev);
770 if (!mdev)
771 return -ENODEV;
772 #endif
773
774 /* Tracing the frequency and voltage after mali is resumed */
775 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
776 /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
777 if (is_first_resume == 1) {
778 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
779 is_first_resume = 0;
780 }
781 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
782 MALI_PROFILING_EVENT_CHANNEL_GPU |
783 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
784 mali_gpu_clk[1].clock,
785 mali_gpu_clk[1].vol / 1000,
786 0, 0, 0);
787 #endif
788 mali_pm_os_resume();
789
790 #if defined(CONFIG_MALI_DEVFREQ) && \
791 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
792 devfreq_resume_device(mdev->devfreq);
793 #endif
794 pm_runtime_force_resume(dev);
795
796 return 0;
797 }
798
799 #ifdef CONFIG_PM_RUNTIME
mali_driver_runtime_suspend(struct device * dev)800 static int mali_driver_runtime_suspend(struct device *dev)
801 {
802 #ifdef CONFIG_MALI_DEVFREQ
803 struct mali_device *mdev = dev_get_drvdata(dev);
804 if (!mdev)
805 return -ENODEV;
806 #endif
807
808 if (MALI_TRUE == mali_pm_runtime_suspend()) {
809 /* Tracing the frequency and voltage after mali is suspended */
810 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
811 MALI_PROFILING_EVENT_CHANNEL_GPU |
812 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
813 0,
814 0,
815 0, 0, 0);
816
817 #if defined(CONFIG_MALI_DEVFREQ) && \
818 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
819 MALI_DEBUG_PRINT(4, ("devfreq_suspend_device: stop devfreq monitor\n"));
820 devfreq_suspend_device(mdev->devfreq);
821 #endif
822
823 return 0;
824 } else {
825 return -EBUSY;
826 }
827 }
828
mali_driver_runtime_resume(struct device * dev)829 static int mali_driver_runtime_resume(struct device *dev)
830 {
831 #ifdef CONFIG_MALI_DEVFREQ
832 struct mali_device *mdev = dev_get_drvdata(dev);
833 if (!mdev)
834 return -ENODEV;
835 #endif
836
837 /* Tracing the frequency and voltage after mali is resumed */
838 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
839 /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
840 if (is_first_resume == 1) {
841 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
842 is_first_resume = 0;
843 }
844 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
845 MALI_PROFILING_EVENT_CHANNEL_GPU |
846 MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
847 mali_gpu_clk[1].clock,
848 mali_gpu_clk[1].vol / 1000,
849 0, 0, 0);
850 #endif
851
852 mali_pm_runtime_resume();
853
854 #if defined(CONFIG_MALI_DEVFREQ) && \
855 (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0))
856 MALI_DEBUG_PRINT(4, ("devfreq_resume_device: start devfreq monitor\n"));
857 devfreq_resume_device(mdev->devfreq);
858 #endif
859 return 0;
860 }
861
mali_driver_runtime_idle(struct device * dev)862 static int mali_driver_runtime_idle(struct device *dev)
863 {
864 /* Nothing to do */
865 return 0;
866 }
867 #endif
868
mali_open(struct inode * inode,struct file * filp)869 static int mali_open(struct inode *inode, struct file *filp)
870 {
871 struct mali_session_data *session_data;
872 _mali_osk_errcode_t err;
873
874 /* input validation */
875 if (mali_miscdevice.minor != iminor(inode)) {
876 MALI_PRINT_ERROR(("mali_open() Minor does not match\n"));
877 return -ENODEV;
878 }
879
880 /* allocated struct to track this session */
881 err = _mali_ukk_open((void **)&session_data);
882 if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
883
884 /* initialize file pointer */
885 filp->f_pos = 0;
886
887 /* link in our session data */
888 filp->private_data = (void *)session_data;
889
890 filp->f_mapping = mali_mem_swap_get_global_swap_file()->f_mapping;
891
892 return 0;
893 }
894
mali_release(struct inode * inode,struct file * filp)895 static int mali_release(struct inode *inode, struct file *filp)
896 {
897 _mali_osk_errcode_t err;
898
899 /* input validation */
900 if (mali_miscdevice.minor != iminor(inode)) {
901 MALI_PRINT_ERROR(("mali_release() Minor does not match\n"));
902 return -ENODEV;
903 }
904
905 err = _mali_ukk_close((void **)&filp->private_data);
906 if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
907
908 return 0;
909 }
910
map_errcode(_mali_osk_errcode_t err)911 int map_errcode(_mali_osk_errcode_t err)
912 {
913 switch (err) {
914 case _MALI_OSK_ERR_OK :
915 return 0;
916 case _MALI_OSK_ERR_FAULT:
917 return -EFAULT;
918 case _MALI_OSK_ERR_INVALID_FUNC:
919 return -ENOTTY;
920 case _MALI_OSK_ERR_INVALID_ARGS:
921 return -EINVAL;
922 case _MALI_OSK_ERR_NOMEM:
923 return -ENOMEM;
924 case _MALI_OSK_ERR_TIMEOUT:
925 return -ETIMEDOUT;
926 case _MALI_OSK_ERR_RESTARTSYSCALL:
927 return -ERESTARTSYS;
928 case _MALI_OSK_ERR_ITEM_NOT_FOUND:
929 return -ENOENT;
930 default:
931 return -EFAULT;
932 }
933 }
934
mali_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)935 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
936 {
937 int err;
938 struct mali_session_data *session_data;
939
940 MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg));
941
942 session_data = (struct mali_session_data *)filp->private_data;
943 if (NULL == session_data) {
944 MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n"));
945 return -ENOTTY;
946 }
947
948 if (NULL == (void *)arg) {
949 MALI_DEBUG_PRINT(7, ("arg was NULL\n"));
950 return -ENOTTY;
951 }
952
953 switch (cmd) {
954 case MALI_IOC_WAIT_FOR_NOTIFICATION:
955 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64)));
956 err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg);
957 break;
958
959 case MALI_IOC_GET_API_VERSION_V2:
960 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64)));
961 err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg);
962 break;
963
964 case MALI_IOC_GET_API_VERSION:
965 err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg);
966 break;
967
968 case MALI_IOC_POST_NOTIFICATION:
969 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64)));
970 err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg);
971 break;
972
973 /* rk_ext : 从对 r5p0-01rel0 集成开始, 不再使用. */
974 #if 0
975 case MALI_IOC_GET_MALI_VERSION_IN_RK30:
976 err = get_mali_version_in_rk30_wrapper(session_data, (_mali_uk_get_mali_version_in_rk30_s __user *)arg);
977 break;
978 #else
979 case MALI_IOC_GET_RK_KO_VERSION:
980 err = get_rk_ko_version_wrapper(session_data, (_mali_rk_ko_version_s __user *)arg);
981 break;
982 #endif
983
984 case MALI_IOC_GET_USER_SETTINGS:
985 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64)));
986 err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg);
987 break;
988
989 case MALI_IOC_REQUEST_HIGH_PRIORITY:
990 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64)));
991 err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg);
992 break;
993
994 case MALI_IOC_PENDING_SUBMIT:
995 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pending_submit_s), sizeof(u64)));
996 err = pending_submit_wrapper(session_data, (_mali_uk_pending_submit_s __user *)arg);
997 break;
998
999 #if defined(CONFIG_MALI400_PROFILING)
1000 case MALI_IOC_PROFILING_ADD_EVENT:
1001 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64)));
1002 err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg);
1003 break;
1004
1005 case MALI_IOC_PROFILING_REPORT_SW_COUNTERS:
1006 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64)));
1007 err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg);
1008 break;
1009
1010 case MALI_IOC_PROFILING_STREAM_FD_GET:
1011 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_stream_fd_get_s), sizeof(u64)));
1012 err = profiling_get_stream_fd_wrapper(session_data, (_mali_uk_profiling_stream_fd_get_s __user *)arg);
1013 break;
1014
1015 case MALI_IOC_PROILING_CONTROL_SET:
1016 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_control_set_s), sizeof(u64)));
1017 err = profiling_control_set_wrapper(session_data, (_mali_uk_profiling_control_set_s __user *)arg);
1018 break;
1019 #else
1020
1021 case MALI_IOC_PROFILING_ADD_EVENT: /* FALL-THROUGH */
1022 case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */
1023 MALI_DEBUG_PRINT(2, ("Profiling not supported\n"));
1024 err = -ENOTTY;
1025 break;
1026 #endif
1027
1028 case MALI_IOC_PROFILING_MEMORY_USAGE_GET:
1029 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64)));
1030 err = mem_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg);
1031 break;
1032
1033 case MALI_IOC_MEM_ALLOC:
1034 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_alloc_mem_s), sizeof(u64)));
1035 err = mem_alloc_wrapper(session_data, (_mali_uk_alloc_mem_s __user *)arg);
1036 break;
1037
1038 case MALI_IOC_MEM_FREE:
1039 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_free_mem_s), sizeof(u64)));
1040 err = mem_free_wrapper(session_data, (_mali_uk_free_mem_s __user *)arg);
1041 break;
1042
1043 case MALI_IOC_MEM_BIND:
1044 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_bind_mem_s), sizeof(u64)));
1045 err = mem_bind_wrapper(session_data, (_mali_uk_bind_mem_s __user *)arg);
1046 break;
1047
1048 case MALI_IOC_MEM_UNBIND:
1049 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unbind_mem_s), sizeof(u64)));
1050 err = mem_unbind_wrapper(session_data, (_mali_uk_unbind_mem_s __user *)arg);
1051 break;
1052
1053 case MALI_IOC_MEM_COW:
1054 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_mem_s), sizeof(u64)));
1055 err = mem_cow_wrapper(session_data, (_mali_uk_cow_mem_s __user *)arg);
1056 break;
1057
1058 case MALI_IOC_MEM_COW_MODIFY_RANGE:
1059 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_cow_modify_range_s), sizeof(u64)));
1060 err = mem_cow_modify_range_wrapper(session_data, (_mali_uk_cow_modify_range_s __user *)arg);
1061 break;
1062
1063 case MALI_IOC_MEM_RESIZE:
1064 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_resize_s), sizeof(u64)));
1065 err = mem_resize_mem_wrapper(session_data, (_mali_uk_mem_resize_s __user *)arg);
1066 break;
1067
1068 case MALI_IOC_MEM_WRITE_SAFE:
1069 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64)));
1070 err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg);
1071 break;
1072
1073 case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE:
1074 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64)));
1075 err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg);
1076 break;
1077
1078 case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE:
1079 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64)));
1080 err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg);
1081 break;
1082
1083 case MALI_IOC_MEM_DMA_BUF_GET_SIZE:
1084 #ifdef CONFIG_DMA_SHARED_BUFFER
1085 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64)));
1086 err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg);
1087 #else
1088 MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n"));
1089 err = -ENOTTY;
1090 #endif
1091 break;
1092
1093 case MALI_IOC_PP_START_JOB:
1094 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64)));
1095 err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg);
1096 break;
1097
1098 case MALI_IOC_PP_AND_GP_START_JOB:
1099 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64)));
1100 err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg);
1101 break;
1102
1103 case MALI_IOC_PP_NUMBER_OF_CORES_GET:
1104 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64)));
1105 err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg);
1106 break;
1107
1108 case MALI_IOC_PP_CORE_VERSION_GET:
1109 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64)));
1110 err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg);
1111 break;
1112
1113 case MALI_IOC_PP_DISABLE_WB:
1114 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64)));
1115 err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg);
1116 break;
1117
1118 case MALI_IOC_GP2_START_JOB:
1119 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64)));
1120 err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg);
1121 break;
1122
1123 case MALI_IOC_GP2_NUMBER_OF_CORES_GET:
1124 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64)));
1125 err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg);
1126 break;
1127
1128 case MALI_IOC_GP2_CORE_VERSION_GET:
1129 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64)));
1130 err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg);
1131 break;
1132
1133 case MALI_IOC_GP2_SUSPEND_RESPONSE:
1134 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64)));
1135 err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg);
1136 break;
1137
1138 case MALI_IOC_VSYNC_EVENT_REPORT:
1139 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64)));
1140 err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg);
1141 break;
1142
1143 case MALI_IOC_TIMELINE_GET_LATEST_POINT:
1144 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64)));
1145 err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg);
1146 break;
1147 case MALI_IOC_TIMELINE_WAIT:
1148 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64)));
1149 err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg);
1150 break;
1151 case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE:
1152 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64)));
1153 err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg);
1154 break;
1155 case MALI_IOC_SOFT_JOB_START:
1156 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64)));
1157 err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg);
1158 break;
1159 case MALI_IOC_SOFT_JOB_SIGNAL:
1160 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64)));
1161 err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg);
1162 break;
1163
1164 default:
1165 MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg));
1166 err = -ENOTTY;
1167 };
1168
1169 return err;
1170 }
1171
1172 late_initcall_sync(mali_module_init);
1173 module_exit(mali_module_exit);
1174
1175 MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE);
1176 MODULE_AUTHOR("ARM Ltd.");
1177 MODULE_VERSION(SVN_REV_STRING);
1178