1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) Rockchip Electronics Co.Ltd
4 * Author: Felix Zeng <felix.zeng@rock-chips.com>
5 */
6
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/fs.h>
10 #include <linux/interrupt.h>
11 #include <linux/irq.h>
12 #include <linux/irqdomain.h>
13 #include <linux/iopoll.h>
14 #include <linux/module.h>
15 #include <linux/of.h>
16 #include <linux/of_device.h>
17 #include <linux/of_irq.h>
18 #include <linux/of_platform.h>
19 #include <linux/of_reserved_mem.h>
20 #include <linux/platform_device.h>
21 #include <linux/printk.h>
22 #include <linux/slab.h>
23 #include <linux/spinlock.h>
24 #include <linux/time.h>
25 #include <linux/uaccess.h>
26 #include <linux/ktime.h>
27 #include <linux/delay.h>
28 #include <linux/wait.h>
29 #include <linux/sched.h>
30 #include <linux/clk.h>
31 #include <linux/clk-provider.h>
32 #include <linux/pm_domain.h>
33 #include <linux/pm_runtime.h>
34 #include <linux/devfreq_cooling.h>
35 #include <linux/regmap.h>
36 #include <linux/of_address.h>
37
38 #ifndef FPGA_PLATFORM
39 #include <soc/rockchip/rockchip_iommu.h>
40 #include <soc/rockchip/rockchip_opp_select.h>
41 #include <soc/rockchip/rockchip_system_monitor.h>
42 #include <soc/rockchip/rockchip_ipa.h>
43 #ifdef CONFIG_PM_DEVFREQ
44 #include <../drivers/devfreq/governor.h>
45 #endif
46 #endif
47
48 #include "rknpu_ioctl.h"
49 #include "rknpu_reset.h"
50 #include "rknpu_fence.h"
51 #include "rknpu_drv.h"
52 #include "rknpu_gem.h"
53
54 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
55 #include <drm/drm_device.h>
56 #include <drm/drm_ioctl.h>
57 #include <drm/drm_file.h>
58 #include <drm/drm_drv.h>
59 #include "rknpu_gem.h"
60 #endif
61
62 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
63 #include <linux/rk-dma-heap.h>
64 #include "rknpu_mem.h"
65 #endif
66
67 #define POWER_DOWN_FREQ 200000000
68 #define NPU_MMU_DISABLED_POLL_PERIOD_US 1000
69 #define NPU_MMU_DISABLED_POLL_TIMEOUT_US 20000
70
71 static int bypass_irq_handler;
72 module_param(bypass_irq_handler, int, 0644);
73 MODULE_PARM_DESC(bypass_irq_handler,
74 "bypass RKNPU irq handler if set it to 1, disabled by default");
75
76 static int bypass_soft_reset;
77 module_param(bypass_soft_reset, int, 0644);
78 MODULE_PARM_DESC(bypass_soft_reset,
79 "bypass RKNPU soft reset if set it to 1, disabled by default");
80
81 struct rknpu_irqs_data {
82 const char *name;
83 irqreturn_t (*irq_hdl)(int irq, void *ctx);
84 };
85
86 static const struct rknpu_irqs_data rknpu_irqs[] = {
87 { "npu_irq", rknpu_core0_irq_handler }
88 };
89
90 static const struct rknpu_irqs_data rk3588_npu_irqs[] = {
91 { "npu0_irq", rknpu_core0_irq_handler },
92 { "npu1_irq", rknpu_core1_irq_handler },
93 { "npu2_irq", rknpu_core2_irq_handler }
94 };
95
96 static const struct rknpu_reset_data rknpu_resets[] = { { "srst_a",
97 "srst_h" } };
98
99 static const struct rknpu_reset_data rk3588_npu_resets[] = {
100 { "srst_a0", "srst_h0" },
101 { "srst_a1", "srst_h1" },
102 { "srst_a2", "srst_h2" }
103 };
104
105 static const struct rknpu_config rk356x_rknpu_config = {
106 .bw_priority_addr = 0xfe180008,
107 .bw_priority_length = 0x10,
108 .dma_mask = DMA_BIT_MASK(32),
109 .pc_data_amount_scale = 1,
110 .pc_task_number_bits = 12,
111 .pc_task_number_mask = 0xfff,
112 .pc_task_status_offset = 0x3c,
113 .pc_dma_ctrl = 0,
114 .bw_enable = 1,
115 .irqs = rknpu_irqs,
116 .resets = rknpu_resets,
117 .num_irqs = ARRAY_SIZE(rknpu_irqs),
118 .num_resets = ARRAY_SIZE(rknpu_resets),
119 .nbuf_phyaddr = 0,
120 .nbuf_size = 0
121 };
122
123 static const struct rknpu_config rk3588_rknpu_config = {
124 .bw_priority_addr = 0x0,
125 .bw_priority_length = 0x0,
126 .dma_mask = DMA_BIT_MASK(40),
127 .pc_data_amount_scale = 2,
128 .pc_task_number_bits = 12,
129 .pc_task_number_mask = 0xfff,
130 .pc_task_status_offset = 0x3c,
131 .pc_dma_ctrl = 0,
132 .bw_enable = 0,
133 .irqs = rk3588_npu_irqs,
134 .resets = rk3588_npu_resets,
135 .num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
136 .num_resets = ARRAY_SIZE(rk3588_npu_resets),
137 .nbuf_phyaddr = 0,
138 .nbuf_size = 0
139 };
140
141 static const struct rknpu_config rv1106_rknpu_config = {
142 .bw_priority_addr = 0x0,
143 .bw_priority_length = 0x0,
144 .dma_mask = DMA_BIT_MASK(32),
145 .pc_data_amount_scale = 2,
146 .pc_task_number_bits = 16,
147 .pc_task_number_mask = 0xffff,
148 .pc_task_status_offset = 0x3c,
149 .pc_dma_ctrl = 0,
150 .bw_enable = 1,
151 .irqs = rknpu_irqs,
152 .resets = rknpu_resets,
153 .num_irqs = ARRAY_SIZE(rknpu_irqs),
154 .num_resets = ARRAY_SIZE(rknpu_resets),
155 .nbuf_phyaddr = 0,
156 .nbuf_size = 0
157 };
158
159 static const struct rknpu_config rk3562_rknpu_config = {
160 .bw_priority_addr = 0x0,
161 .bw_priority_length = 0x0,
162 .dma_mask = DMA_BIT_MASK(40),
163 .pc_data_amount_scale = 2,
164 .pc_task_number_bits = 16,
165 .pc_task_number_mask = 0xffff,
166 .pc_task_status_offset = 0x48,
167 .pc_dma_ctrl = 1,
168 .bw_enable = 1,
169 .irqs = rknpu_irqs,
170 .resets = rknpu_resets,
171 .num_irqs = ARRAY_SIZE(rknpu_irqs),
172 .num_resets = ARRAY_SIZE(rknpu_resets),
173 .nbuf_phyaddr = 0xfe400000,
174 .nbuf_size = 256 * 1024
175 };
176
177 /* driver probe and init */
178 static const struct of_device_id rknpu_of_match[] = {
179 {
180 .compatible = "rockchip,rknpu",
181 .data = &rk356x_rknpu_config,
182 },
183 {
184 .compatible = "rockchip,rk3568-rknpu",
185 .data = &rk356x_rknpu_config,
186 },
187 {
188 .compatible = "rockchip,rk3588-rknpu",
189 .data = &rk3588_rknpu_config,
190 },
191 {
192 .compatible = "rockchip,rv1106-rknpu",
193 .data = &rv1106_rknpu_config,
194 },
195 {
196 .compatible = "rockchip,rk3562-rknpu",
197 .data = &rk3562_rknpu_config,
198 },
199 {},
200 };
201
rknpu_get_drv_version(uint32_t * version)202 static int rknpu_get_drv_version(uint32_t *version)
203 {
204 *version = RKNPU_GET_DRV_VERSION_CODE(DRIVER_MAJOR, DRIVER_MINOR,
205 DRIVER_PATCHLEVEL);
206 return 0;
207 }
208
209 static int rknpu_power_on(struct rknpu_device *rknpu_dev);
210 static int rknpu_power_off(struct rknpu_device *rknpu_dev);
211
rknpu_power_off_delay_work(struct work_struct * power_off_work)212 static void rknpu_power_off_delay_work(struct work_struct *power_off_work)
213 {
214 struct rknpu_device *rknpu_dev =
215 container_of(to_delayed_work(power_off_work),
216 struct rknpu_device, power_off_work);
217 mutex_lock(&rknpu_dev->power_lock);
218 if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0)
219 rknpu_power_off(rknpu_dev);
220 mutex_unlock(&rknpu_dev->power_lock);
221 }
222
rknpu_power_get(struct rknpu_device * rknpu_dev)223 int rknpu_power_get(struct rknpu_device *rknpu_dev)
224 {
225 int ret = 0;
226
227 mutex_lock(&rknpu_dev->power_lock);
228 if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
229 ret = rknpu_power_on(rknpu_dev);
230 mutex_unlock(&rknpu_dev->power_lock);
231
232 return ret;
233 }
234
rknpu_power_put(struct rknpu_device * rknpu_dev)235 int rknpu_power_put(struct rknpu_device *rknpu_dev)
236 {
237 int ret = 0;
238
239 mutex_lock(&rknpu_dev->power_lock);
240 if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0)
241 ret = rknpu_power_off(rknpu_dev);
242 mutex_unlock(&rknpu_dev->power_lock);
243
244 return ret;
245 }
246
rknpu_power_put_delay(struct rknpu_device * rknpu_dev)247 static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
248 {
249 if (rknpu_dev->power_put_delay == 0)
250 return rknpu_power_put(rknpu_dev);
251
252 mutex_lock(&rknpu_dev->power_lock);
253 if (atomic_read(&rknpu_dev->power_refcount) == 1)
254 queue_delayed_work(
255 rknpu_dev->power_off_wq, &rknpu_dev->power_off_work,
256 msecs_to_jiffies(rknpu_dev->power_put_delay));
257 else
258 atomic_dec_if_positive(&rknpu_dev->power_refcount);
259 mutex_unlock(&rknpu_dev->power_lock);
260
261 return 0;
262 }
263
rknpu_action(struct rknpu_device * rknpu_dev,struct rknpu_action * args)264 static int rknpu_action(struct rknpu_device *rknpu_dev,
265 struct rknpu_action *args)
266 {
267 int ret = -EINVAL;
268
269 switch (args->flags) {
270 case RKNPU_GET_HW_VERSION:
271 ret = rknpu_get_hw_version(rknpu_dev, &args->value);
272 break;
273 case RKNPU_GET_DRV_VERSION:
274 ret = rknpu_get_drv_version(&args->value);
275 break;
276 case RKNPU_GET_FREQ:
277 #ifndef FPGA_PLATFORM
278 args->value = clk_get_rate(rknpu_dev->clks[0].clk);
279 #endif
280 ret = 0;
281 break;
282 case RKNPU_SET_FREQ:
283 break;
284 case RKNPU_GET_VOLT:
285 #ifndef FPGA_PLATFORM
286 args->value = regulator_get_voltage(rknpu_dev->vdd);
287 #endif
288 ret = 0;
289 break;
290 case RKNPU_SET_VOLT:
291 break;
292 case RKNPU_ACT_RESET:
293 ret = rknpu_soft_reset(rknpu_dev);
294 break;
295 case RKNPU_GET_BW_PRIORITY:
296 ret = rknpu_get_bw_priority(rknpu_dev, &args->value, NULL,
297 NULL);
298 break;
299 case RKNPU_SET_BW_PRIORITY:
300 ret = rknpu_set_bw_priority(rknpu_dev, args->value, 0, 0);
301 break;
302 case RKNPU_GET_BW_EXPECT:
303 ret = rknpu_get_bw_priority(rknpu_dev, NULL, &args->value,
304 NULL);
305 break;
306 case RKNPU_SET_BW_EXPECT:
307 ret = rknpu_set_bw_priority(rknpu_dev, 0, args->value, 0);
308 break;
309 case RKNPU_GET_BW_TW:
310 ret = rknpu_get_bw_priority(rknpu_dev, NULL, NULL,
311 &args->value);
312 break;
313 case RKNPU_SET_BW_TW:
314 ret = rknpu_set_bw_priority(rknpu_dev, 0, 0, args->value);
315 break;
316 case RKNPU_ACT_CLR_TOTAL_RW_AMOUNT:
317 ret = rknpu_clear_rw_amount(rknpu_dev);
318 break;
319 case RKNPU_GET_DT_WR_AMOUNT:
320 ret = rknpu_get_rw_amount(rknpu_dev, &args->value, NULL, NULL);
321 break;
322 case RKNPU_GET_DT_RD_AMOUNT:
323 ret = rknpu_get_rw_amount(rknpu_dev, NULL, &args->value, NULL);
324 break;
325 case RKNPU_GET_WT_RD_AMOUNT:
326 ret = rknpu_get_rw_amount(rknpu_dev, NULL, NULL, &args->value);
327 break;
328 case RKNPU_GET_TOTAL_RW_AMOUNT:
329 ret = rknpu_get_total_rw_amount(rknpu_dev, &args->value);
330 break;
331 case RKNPU_GET_IOMMU_EN:
332 args->value = rknpu_dev->iommu_en;
333 ret = 0;
334 break;
335 case RKNPU_SET_PROC_NICE:
336 set_user_nice(current, *(int32_t *)&args->value);
337 ret = 0;
338 break;
339 case RKNPU_GET_TOTAL_SRAM_SIZE:
340 if (rknpu_dev->sram_mm)
341 args->value = rknpu_dev->sram_mm->total_chunks *
342 rknpu_dev->sram_mm->chunk_size;
343 else
344 args->value = 0;
345 ret = 0;
346 break;
347 case RKNPU_GET_FREE_SRAM_SIZE:
348 if (rknpu_dev->sram_mm)
349 args->value = rknpu_dev->sram_mm->free_chunks *
350 rknpu_dev->sram_mm->chunk_size;
351 else
352 args->value = 0;
353 ret = 0;
354 break;
355 default:
356 ret = -EINVAL;
357 break;
358 }
359
360 return ret;
361 }
362
363 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
rknpu_open(struct inode * inode,struct file * file)364 static int rknpu_open(struct inode *inode, struct file *file)
365 {
366 struct rknpu_device *rknpu_dev =
367 container_of(file->private_data, struct rknpu_device, miscdev);
368 struct rknpu_session *session = NULL;
369
370 session = kzalloc(sizeof(*session), GFP_KERNEL);
371 if (!session) {
372 LOG_ERROR("rknpu session alloc failed\n");
373 return -ENOMEM;
374 }
375
376 session->rknpu_dev = rknpu_dev;
377 INIT_LIST_HEAD(&session->list);
378
379 file->private_data = (void *)session;
380
381 return nonseekable_open(inode, file);
382 }
383
rknpu_release(struct inode * inode,struct file * file)384 static int rknpu_release(struct inode *inode, struct file *file)
385 {
386 struct rknpu_mem_object *entry;
387 struct rknpu_session *session = file->private_data;
388 struct rknpu_device *rknpu_dev = session->rknpu_dev;
389 LIST_HEAD(local_list);
390
391 spin_lock(&rknpu_dev->lock);
392 list_replace_init(&session->list, &local_list);
393 file->private_data = NULL;
394 spin_unlock(&rknpu_dev->lock);
395
396 while (!list_empty(&local_list)) {
397 entry = list_first_entry(&local_list, struct rknpu_mem_object,
398 head);
399
400 LOG_DEBUG(
401 "Fd close free rknpu_obj: %#llx, rknpu_obj->dma_addr: %#llx\n",
402 (__u64)(uintptr_t)entry, (__u64)entry->dma_addr);
403
404 vunmap(entry->kv_addr);
405 entry->kv_addr = NULL;
406
407 if (!entry->owner)
408 dma_buf_put(entry->dmabuf);
409
410 list_del(&entry->head);
411 kfree(entry);
412 }
413
414 kfree(session);
415
416 return 0;
417 }
418
rknpu_action_ioctl(struct rknpu_device * rknpu_dev,unsigned long data)419 static int rknpu_action_ioctl(struct rknpu_device *rknpu_dev,
420 unsigned long data)
421 {
422 struct rknpu_action args;
423 int ret = -EINVAL;
424
425 if (unlikely(copy_from_user(&args, (struct rknpu_action *)data,
426 sizeof(struct rknpu_action)))) {
427 LOG_ERROR("%s: copy_from_user failed\n", __func__);
428 ret = -EFAULT;
429 return ret;
430 }
431
432 ret = rknpu_action(rknpu_dev, &args);
433
434 if (unlikely(copy_to_user((struct rknpu_action *)data, &args,
435 sizeof(struct rknpu_action)))) {
436 LOG_ERROR("%s: copy_to_user failed\n", __func__);
437 ret = -EFAULT;
438 return ret;
439 }
440
441 return ret;
442 }
443
rknpu_ioctl(struct file * file,uint32_t cmd,unsigned long arg)444 static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
445 {
446 long ret = -EINVAL;
447 struct rknpu_device *rknpu_dev = NULL;
448
449 if (!file->private_data)
450 return -EINVAL;
451
452 rknpu_dev = ((struct rknpu_session *)file->private_data)->rknpu_dev;
453
454 rknpu_power_get(rknpu_dev);
455
456 switch (cmd) {
457 case IOCTL_RKNPU_ACTION:
458 ret = rknpu_action_ioctl(rknpu_dev, arg);
459 break;
460 case IOCTL_RKNPU_SUBMIT:
461 ret = rknpu_submit_ioctl(rknpu_dev, arg);
462 break;
463 case IOCTL_RKNPU_MEM_CREATE:
464 ret = rknpu_mem_create_ioctl(rknpu_dev, arg, file);
465 break;
466 case RKNPU_MEM_MAP:
467 break;
468 case IOCTL_RKNPU_MEM_DESTROY:
469 ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg, file);
470 break;
471 case IOCTL_RKNPU_MEM_SYNC:
472 ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
473 break;
474 default:
475 break;
476 }
477
478 rknpu_power_put_delay(rknpu_dev);
479
480 return ret;
481 }
482 const struct file_operations rknpu_fops = {
483 .owner = THIS_MODULE,
484 .open = rknpu_open,
485 .release = rknpu_release,
486 .unlocked_ioctl = rknpu_ioctl,
487 #ifdef CONFIG_COMPAT
488 .compat_ioctl = rknpu_ioctl,
489 #endif
490 };
491 #endif
492
493 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
494 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
495 static const struct vm_operations_struct rknpu_gem_vm_ops = {
496 .fault = rknpu_gem_fault,
497 .open = drm_gem_vm_open,
498 .close = drm_gem_vm_close,
499 };
500 #endif
501
rknpu_action_ioctl(struct drm_device * dev,void * data,struct drm_file * file_priv)502 static int rknpu_action_ioctl(struct drm_device *dev, void *data,
503 struct drm_file *file_priv)
504 {
505 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
506
507 return rknpu_action(rknpu_dev, (struct rknpu_action *)data);
508 }
509
510 #define RKNPU_IOCTL(func) \
511 static int __##func(struct drm_device *dev, void *data, \
512 struct drm_file *file_priv) \
513 { \
514 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); \
515 int ret = -EINVAL; \
516 rknpu_power_get(rknpu_dev); \
517 ret = func(dev, data, file_priv); \
518 rknpu_power_put_delay(rknpu_dev); \
519 return ret; \
520 }
521
522 RKNPU_IOCTL(rknpu_action_ioctl);
523 RKNPU_IOCTL(rknpu_submit_ioctl);
524 RKNPU_IOCTL(rknpu_gem_create_ioctl);
525 RKNPU_IOCTL(rknpu_gem_map_ioctl);
526 RKNPU_IOCTL(rknpu_gem_destroy_ioctl);
527 RKNPU_IOCTL(rknpu_gem_sync_ioctl);
528
529 static const struct drm_ioctl_desc rknpu_ioctls[] = {
530 DRM_IOCTL_DEF_DRV(RKNPU_ACTION, __rknpu_action_ioctl, DRM_RENDER_ALLOW),
531 DRM_IOCTL_DEF_DRV(RKNPU_SUBMIT, __rknpu_submit_ioctl, DRM_RENDER_ALLOW),
532 DRM_IOCTL_DEF_DRV(RKNPU_MEM_CREATE, __rknpu_gem_create_ioctl,
533 DRM_RENDER_ALLOW),
534 DRM_IOCTL_DEF_DRV(RKNPU_MEM_MAP, __rknpu_gem_map_ioctl,
535 DRM_RENDER_ALLOW),
536 DRM_IOCTL_DEF_DRV(RKNPU_MEM_DESTROY, __rknpu_gem_destroy_ioctl,
537 DRM_RENDER_ALLOW),
538 DRM_IOCTL_DEF_DRV(RKNPU_MEM_SYNC, __rknpu_gem_sync_ioctl,
539 DRM_RENDER_ALLOW),
540 };
541
542 static const struct file_operations rknpu_drm_driver_fops = {
543 .owner = THIS_MODULE,
544 .open = drm_open,
545 .mmap = rknpu_gem_mmap,
546 .poll = drm_poll,
547 .read = drm_read,
548 .unlocked_ioctl = drm_ioctl,
549 #ifdef CONFIG_COMPAT
550 .compat_ioctl = drm_compat_ioctl,
551 #endif
552 .release = drm_release,
553 .llseek = noop_llseek,
554 };
555
556 static struct drm_driver rknpu_drm_driver = {
557 #if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
558 .driver_features = DRIVER_GEM | DRIVER_RENDER,
559 #else
560 .driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
561 #endif
562 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
563 .gem_free_object_unlocked = rknpu_gem_free_object,
564 .gem_vm_ops = &rknpu_gem_vm_ops,
565 .dumb_destroy = drm_gem_dumb_destroy,
566 .gem_prime_export = drm_gem_prime_export,
567 .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
568 .gem_prime_vmap = rknpu_gem_prime_vmap,
569 .gem_prime_vunmap = rknpu_gem_prime_vunmap,
570 #endif
571 .dumb_create = rknpu_gem_dumb_create,
572 #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
573 .dumb_map_offset = rknpu_gem_dumb_map_offset,
574 #else
575 .dumb_map_offset = drm_gem_dumb_map_offset,
576 #endif
577 .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
578 .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
579 #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
580 .gem_prime_import = rknpu_gem_prime_import,
581 #else
582 .gem_prime_import = drm_gem_prime_import,
583 #endif
584 .gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
585 .gem_prime_mmap = rknpu_gem_prime_mmap,
586 .ioctls = rknpu_ioctls,
587 .num_ioctls = ARRAY_SIZE(rknpu_ioctls),
588 .fops = &rknpu_drm_driver_fops,
589 .name = DRIVER_NAME,
590 .desc = DRIVER_DESC,
591 .date = DRIVER_DATE,
592 .major = DRIVER_MAJOR,
593 .minor = DRIVER_MINOR,
594 .patchlevel = DRIVER_PATCHLEVEL,
595 };
596
597 #endif
598
hrtimer_handler(struct hrtimer * timer)599 static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer)
600 {
601 struct rknpu_device *rknpu_dev =
602 container_of(timer, struct rknpu_device, timer);
603 struct rknpu_subcore_data *subcore_data = NULL;
604 struct rknpu_job *job = NULL;
605 ktime_t now = ktime_get();
606 unsigned long flags;
607 int i;
608
609 for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
610 subcore_data = &rknpu_dev->subcore_datas[i];
611
612 spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
613
614 job = subcore_data->job;
615 if (job) {
616 subcore_data->timer.busy_time +=
617 ktime_us_delta(now, job->hw_recoder_time);
618 job->hw_recoder_time = ktime_get();
619 }
620
621 subcore_data->timer.busy_time_record =
622 subcore_data->timer.busy_time;
623 subcore_data->timer.busy_time = 0;
624 spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
625 }
626
627 hrtimer_forward_now(timer, rknpu_dev->kt);
628 return HRTIMER_RESTART;
629 }
630
rknpu_init_timer(struct rknpu_device * rknpu_dev)631 static void rknpu_init_timer(struct rknpu_device *rknpu_dev)
632 {
633 rknpu_dev->kt = ktime_set(0, RKNPU_LOAD_INTERVAL);
634 hrtimer_init(&rknpu_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
635 rknpu_dev->timer.function = hrtimer_handler;
636 hrtimer_start(&rknpu_dev->timer, rknpu_dev->kt, HRTIMER_MODE_REL);
637 }
638
rknpu_cancel_timer(struct rknpu_device * rknpu_dev)639 static void rknpu_cancel_timer(struct rknpu_device *rknpu_dev)
640 {
641 hrtimer_cancel(&rknpu_dev->timer);
642 }
643
rknpu_is_iommu_enable(struct device * dev)644 static bool rknpu_is_iommu_enable(struct device *dev)
645 {
646 struct device_node *iommu = NULL;
647
648 iommu = of_parse_phandle(dev->of_node, "iommus", 0);
649 if (!iommu) {
650 LOG_DEV_INFO(
651 dev,
652 "rknpu iommu device-tree entry not found!, using non-iommu mode\n");
653 return false;
654 }
655
656 if (!of_device_is_available(iommu)) {
657 LOG_DEV_INFO(dev,
658 "rknpu iommu is disabled, using non-iommu mode\n");
659 of_node_put(iommu);
660 return false;
661 }
662 of_node_put(iommu);
663
664 LOG_DEV_INFO(dev, "rknpu iommu is enabled, using iommu mode\n");
665
666 return true;
667 }
668
669 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
rknpu_drm_probe(struct rknpu_device * rknpu_dev)670 static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
671 {
672 struct device *dev = rknpu_dev->dev;
673 struct drm_device *drm_dev = NULL;
674 int ret = -EINVAL;
675
676 drm_dev = drm_dev_alloc(&rknpu_drm_driver, dev);
677 if (IS_ERR(drm_dev))
678 return PTR_ERR(drm_dev);
679
680 /* register the DRM device */
681 ret = drm_dev_register(drm_dev, 0);
682 if (ret < 0)
683 goto err_free_drm;
684
685 drm_dev->dev_private = rknpu_dev;
686 rknpu_dev->drm_dev = drm_dev;
687
688 return 0;
689
690 err_free_drm:
691 #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
692 drm_dev_put(drm_dev);
693 #else
694 drm_dev_unref(drm_dev);
695 #endif
696
697 return ret;
698 }
699
rknpu_drm_remove(struct rknpu_device * rknpu_dev)700 static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
701 {
702 struct drm_device *drm_dev = rknpu_dev->drm_dev;
703
704 drm_dev_unregister(drm_dev);
705
706 #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
707 drm_dev_put(drm_dev);
708 #else
709 drm_dev_unref(drm_dev);
710 #endif
711 }
712 #endif
713
rknpu_power_on(struct rknpu_device * rknpu_dev)714 static int rknpu_power_on(struct rknpu_device *rknpu_dev)
715 {
716 struct device *dev = rknpu_dev->dev;
717 int ret = -EINVAL;
718
719 #ifndef FPGA_PLATFORM
720 if (rknpu_dev->vdd) {
721 ret = regulator_enable(rknpu_dev->vdd);
722 if (ret) {
723 LOG_DEV_ERROR(
724 dev,
725 "failed to enable vdd reg for rknpu, ret: %d\n",
726 ret);
727 return ret;
728 }
729 }
730
731 if (rknpu_dev->mem) {
732 ret = regulator_enable(rknpu_dev->mem);
733 if (ret) {
734 LOG_DEV_ERROR(
735 dev,
736 "failed to enable mem reg for rknpu, ret: %d\n",
737 ret);
738 return ret;
739 }
740 }
741 #endif
742
743 ret = clk_bulk_prepare_enable(rknpu_dev->num_clks, rknpu_dev->clks);
744 if (ret) {
745 LOG_DEV_ERROR(dev, "failed to enable clk for rknpu, ret: %d\n",
746 ret);
747 return ret;
748 }
749
750 #ifndef FPGA_PLATFORM
751 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
752 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
753 rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
754 #endif
755 #endif
756
757 if (rknpu_dev->multiple_domains) {
758 if (rknpu_dev->genpd_dev_npu0) {
759 #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
760 ret = pm_runtime_resume_and_get(
761 rknpu_dev->genpd_dev_npu0);
762 #else
763 ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu0);
764 #endif
765 if (ret < 0) {
766 LOG_DEV_ERROR(
767 dev,
768 "failed to get pm runtime for npu0, ret: %d\n",
769 ret);
770 goto out;
771 }
772 }
773 if (rknpu_dev->genpd_dev_npu1) {
774 #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
775 ret = pm_runtime_resume_and_get(
776 rknpu_dev->genpd_dev_npu1);
777 #else
778 ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu1);
779 #endif
780 if (ret < 0) {
781 LOG_DEV_ERROR(
782 dev,
783 "failed to get pm runtime for npu1, ret: %d\n",
784 ret);
785 goto out;
786 }
787 }
788 if (rknpu_dev->genpd_dev_npu2) {
789 #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
790 ret = pm_runtime_resume_and_get(
791 rknpu_dev->genpd_dev_npu2);
792 #else
793 ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu2);
794 #endif
795 if (ret < 0) {
796 LOG_DEV_ERROR(
797 dev,
798 "failed to get pm runtime for npu2, ret: %d\n",
799 ret);
800 goto out;
801 }
802 }
803 }
804 ret = pm_runtime_get_sync(dev);
805 if (ret < 0) {
806 LOG_DEV_ERROR(dev,
807 "failed to get pm runtime for rknpu, ret: %d\n",
808 ret);
809 }
810
811 out:
812 #ifndef FPGA_PLATFORM
813 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
814 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
815 rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
816 #endif
817 #endif
818
819 return ret;
820 }
821
rknpu_power_off(struct rknpu_device * rknpu_dev)822 static int rknpu_power_off(struct rknpu_device *rknpu_dev)
823 {
824 struct device *dev = rknpu_dev->dev;
825
826 #ifndef FPGA_PLATFORM
827 int ret;
828 bool val;
829
830 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
831 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
832 rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
833 #endif
834 #endif
835
836 pm_runtime_put_sync(dev);
837
838 if (rknpu_dev->multiple_domains) {
839 #ifndef FPGA_PLATFORM
840 /*
841 * Because IOMMU's runtime suspend callback is asynchronous,
842 * So it may be executed after the NPU is turned off after PD/CLK/VD,
843 * and the runtime suspend callback has a register access.
844 * If the PD/VD/CLK is closed, the register access will crash.
845 * As a workaround, it's safe to close pd stuff until iommu disabled.
846 * If pm runtime framework can handle this issue in the future, remove
847 * this.
848 */
849 ret = readx_poll_timeout(rockchip_iommu_is_enabled, dev, val,
850 !val, NPU_MMU_DISABLED_POLL_PERIOD_US,
851 NPU_MMU_DISABLED_POLL_TIMEOUT_US);
852 if (ret) {
853 LOG_DEV_ERROR(dev, "iommu still enabled\n");
854 pm_runtime_get_sync(dev);
855 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
856 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
857 rockchip_monitor_volt_adjust_unlock(
858 rknpu_dev->mdev_info);
859 #endif
860 return ret;
861 }
862 #else
863 if (rknpu_dev->iommu_en)
864 msleep(20);
865 #endif
866 if (rknpu_dev->genpd_dev_npu2)
867 pm_runtime_put_sync(rknpu_dev->genpd_dev_npu2);
868 if (rknpu_dev->genpd_dev_npu1)
869 pm_runtime_put_sync(rknpu_dev->genpd_dev_npu1);
870 if (rknpu_dev->genpd_dev_npu0)
871 pm_runtime_put_sync(rknpu_dev->genpd_dev_npu0);
872 }
873
874 #ifndef FPGA_PLATFORM
875 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
876 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
877 rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
878 #endif
879 #endif
880
881 clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
882
883 #ifndef FPGA_PLATFORM
884 if (rknpu_dev->vdd)
885 regulator_disable(rknpu_dev->vdd);
886
887 if (rknpu_dev->mem)
888 regulator_disable(rknpu_dev->mem);
889 #endif
890
891 return 0;
892 }
893
894 #ifndef FPGA_PLATFORM
895 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
896 static struct monitor_dev_profile npu_mdevp = {
897 .type = MONITOR_TYPE_DEV,
898 .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
899 .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
900 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
901 .update_volt = rockchip_monitor_check_rate_volt,
902 #endif
903 };
904
905 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
npu_opp_helper(struct dev_pm_set_opp_data * data)906 static int npu_opp_helper(struct dev_pm_set_opp_data *data)
907 {
908 struct device *dev = data->dev;
909 struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
910 struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
911 struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
912 struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
913 struct regulator *vdd_reg = data->regulators[0];
914 struct regulator *mem_reg = data->regulators[1];
915 struct clk *clk = data->clk;
916 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
917 struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
918 unsigned long old_freq = data->old_opp.rate;
919 unsigned long new_freq = data->new_opp.rate;
920 bool is_set_rm = true;
921 bool is_set_clk = true;
922 u32 target_rm = UINT_MAX;
923 int ret = 0;
924
925 if (!pm_runtime_active(dev)) {
926 is_set_rm = false;
927 if (opp_info->scmi_clk)
928 is_set_clk = false;
929 }
930
931 ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
932 if (ret < 0) {
933 LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
934 return ret;
935 }
936 rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
937 &target_rm);
938
939 /* Change frequency */
940 LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
941 new_freq);
942 /* Scaling up? Scale voltage before frequency */
943 if (new_freq >= old_freq) {
944 rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
945 new_freq, true, is_set_clk);
946 ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
947 INT_MAX);
948 if (ret) {
949 LOG_DEV_ERROR(dev,
950 "failed to set volt %lu uV for mem reg\n",
951 new_supply_mem->u_volt);
952 goto restore_voltage;
953 }
954 ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
955 INT_MAX);
956 if (ret) {
957 LOG_DEV_ERROR(dev,
958 "failed to set volt %lu uV for vdd reg\n",
959 new_supply_vdd->u_volt);
960 goto restore_voltage;
961 }
962 rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
963 if (is_set_clk && clk_set_rate(clk, new_freq)) {
964 ret = -EINVAL;
965 LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
966 goto restore_rm;
967 }
968 /* Scaling down? Scale voltage after frequency */
969 } else {
970 rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
971 new_freq, false, is_set_clk);
972 rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
973 if (is_set_clk && clk_set_rate(clk, new_freq)) {
974 ret = -EINVAL;
975 LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
976 goto restore_rm;
977 }
978 ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
979 INT_MAX);
980 if (ret) {
981 LOG_DEV_ERROR(dev,
982 "failed to set volt %lu uV for vdd reg\n",
983 new_supply_vdd->u_volt);
984 goto restore_freq;
985 }
986 ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
987 INT_MAX);
988 if (ret) {
989 LOG_DEV_ERROR(dev,
990 "failed to set volt %lu uV for mem reg\n",
991 new_supply_mem->u_volt);
992 goto restore_freq;
993 }
994 }
995
996 clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
997
998 return 0;
999
1000 restore_freq:
1001 if (is_set_clk && clk_set_rate(clk, old_freq))
1002 LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
1003 old_freq);
1004 restore_rm:
1005 rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
1006 &target_rm);
1007 rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
1008 is_set_rm);
1009 restore_voltage:
1010 regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
1011 regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
1012 clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
1013
1014 return ret;
1015 }
1016
npu_devfreq_target(struct device * dev,unsigned long * freq,u32 flags)1017 static int npu_devfreq_target(struct device *dev, unsigned long *freq,
1018 u32 flags)
1019 {
1020 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1021 struct dev_pm_opp *opp;
1022 unsigned long opp_volt;
1023 int ret = 0;
1024
1025 if (!npu_mdevp.is_checked)
1026 return -EINVAL;
1027
1028 opp = devfreq_recommended_opp(dev, freq, flags);
1029 if (IS_ERR(opp))
1030 return PTR_ERR(opp);
1031 opp_volt = dev_pm_opp_get_voltage(opp);
1032 dev_pm_opp_put(opp);
1033
1034 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1035 rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
1036 #endif
1037 ret = dev_pm_opp_set_rate(dev, *freq);
1038 if (!ret) {
1039 rknpu_dev->current_freq = *freq;
1040 if (rknpu_dev->devfreq)
1041 rknpu_dev->devfreq->last_status.current_frequency =
1042 *freq;
1043 rknpu_dev->current_volt = opp_volt;
1044 LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
1045 rknpu_dev->current_freq, rknpu_dev->current_volt);
1046 }
1047 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
1048 rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
1049 #endif
1050
1051 return ret;
1052 }
1053
1054 #else
1055
npu_devfreq_target(struct device * dev,unsigned long * target_freq,u32 flags)1056 static int npu_devfreq_target(struct device *dev, unsigned long *target_freq,
1057 u32 flags)
1058 {
1059 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1060 struct dev_pm_opp *opp = NULL;
1061 unsigned long freq = *target_freq;
1062 unsigned long old_freq = rknpu_dev->current_freq;
1063 unsigned long volt, old_volt = rknpu_dev->current_volt;
1064 int ret = -EINVAL;
1065
1066 #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1067 rcu_read_lock();
1068 #endif
1069
1070 opp = devfreq_recommended_opp(dev, &freq, flags);
1071 if (IS_ERR(opp)) {
1072 #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1073 rcu_read_unlock();
1074 #endif
1075 LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
1076 return PTR_ERR(opp);
1077 }
1078 volt = dev_pm_opp_get_voltage(opp);
1079 #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1080 rcu_read_unlock();
1081 #endif
1082
1083 /*
1084 * Only update if there is a change of frequency
1085 */
1086 if (old_freq == freq) {
1087 *target_freq = freq;
1088 if (old_volt == volt)
1089 return 0;
1090 ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1091 if (ret) {
1092 LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1093 return ret;
1094 }
1095 rknpu_dev->current_volt = volt;
1096 return 0;
1097 }
1098
1099 if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
1100 ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1101 if (ret) {
1102 LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
1103 volt);
1104 return ret;
1105 }
1106 }
1107 LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
1108 freq, volt);
1109 ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1110 if (ret) {
1111 LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1112 return ret;
1113 }
1114 *target_freq = freq;
1115 rknpu_dev->current_freq = freq;
1116
1117 if (rknpu_dev->devfreq)
1118 rknpu_dev->devfreq->last_status.current_frequency = freq;
1119
1120 if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
1121 ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1122 if (ret) {
1123 LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
1124 volt);
1125 return ret;
1126 }
1127 }
1128 rknpu_dev->current_volt = volt;
1129
1130 LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
1131 rknpu_dev->current_freq, rknpu_dev->current_volt);
1132
1133 return ret;
1134 }
1135 #endif
1136
npu_devfreq_get_dev_status(struct device * dev,struct devfreq_dev_status * stat)1137 static int npu_devfreq_get_dev_status(struct device *dev,
1138 struct devfreq_dev_status *stat)
1139 {
1140 return 0;
1141 }
1142
npu_devfreq_get_cur_freq(struct device * dev,unsigned long * freq)1143 static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
1144 {
1145 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1146
1147 *freq = rknpu_dev->current_freq;
1148
1149 return 0;
1150 }
1151
1152 static struct devfreq_dev_profile npu_devfreq_profile = {
1153 .polling_ms = 50,
1154 .target = npu_devfreq_target,
1155 .get_dev_status = npu_devfreq_get_dev_status,
1156 .get_cur_freq = npu_devfreq_get_cur_freq,
1157 };
1158 #endif
1159
1160 #ifdef CONFIG_PM_DEVFREQ
devfreq_rknpu_ondemand_func(struct devfreq * df,unsigned long * freq)1161 static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
1162 {
1163 struct rknpu_device *rknpu_dev = df->data;
1164
1165 if (rknpu_dev)
1166 *freq = rknpu_dev->ondemand_freq;
1167 else
1168 *freq = df->previous_freq;
1169
1170 return 0;
1171 }
1172
devfreq_rknpu_ondemand_handler(struct devfreq * devfreq,unsigned int event,void * data)1173 static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
1174 unsigned int event, void *data)
1175 {
1176 return 0;
1177 }
1178
1179 static struct devfreq_governor devfreq_rknpu_ondemand = {
1180 .name = "rknpu_ondemand",
1181 .get_target_freq = devfreq_rknpu_ondemand_func,
1182 .event_handler = devfreq_rknpu_ondemand_handler,
1183 };
1184 #endif
1185
1186 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
npu_get_static_power(struct devfreq * devfreq,unsigned long voltage)1187 static unsigned long npu_get_static_power(struct devfreq *devfreq,
1188 unsigned long voltage)
1189 {
1190 struct device *dev = devfreq->dev.parent;
1191 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
1192
1193 if (!rknpu_dev->model_data)
1194 return 0;
1195
1196 return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
1197 }
1198
1199 static struct devfreq_cooling_power npu_cooling_power = {
1200 .get_static_power = &npu_get_static_power,
1201 };
1202
1203 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
rk3588_npu_get_soc_info(struct device * dev,struct device_node * np,int * bin,int * process)1204 static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
1205 int *bin, int *process)
1206 {
1207 int ret = 0;
1208 u8 value = 0;
1209
1210 if (!bin)
1211 return 0;
1212
1213 if (of_property_match_string(np, "nvmem-cell-names",
1214 "specification_serial_number") >= 0) {
1215 ret = rockchip_nvmem_cell_read_u8(
1216 np, "specification_serial_number", &value);
1217 if (ret) {
1218 LOG_DEV_ERROR(
1219 dev,
1220 "Failed to get specification_serial_number\n");
1221 return ret;
1222 }
1223 /* RK3588M */
1224 if (value == 0xd)
1225 *bin = 1;
1226 /* RK3588J */
1227 else if (value == 0xa)
1228 *bin = 2;
1229 }
1230 if (*bin < 0)
1231 *bin = 0;
1232 LOG_DEV_INFO(dev, "bin=%d\n", *bin);
1233
1234 return ret;
1235 }
1236
rk3588_npu_set_soc_info(struct device * dev,struct device_node * np,int bin,int process,int volt_sel)1237 static int rk3588_npu_set_soc_info(struct device *dev, struct device_node *np,
1238 int bin, int process, int volt_sel)
1239 {
1240 struct opp_table *opp_table;
1241 u32 supported_hw[2];
1242
1243 if (volt_sel < 0)
1244 return 0;
1245 if (bin < 0)
1246 bin = 0;
1247
1248 if (!of_property_read_bool(np, "rockchip,supported-hw"))
1249 return 0;
1250
1251 /* SoC Version */
1252 supported_hw[0] = BIT(bin);
1253 /* Speed Grade */
1254 supported_hw[1] = BIT(volt_sel);
1255 opp_table = dev_pm_opp_set_supported_hw(dev, supported_hw, 2);
1256 if (IS_ERR(opp_table)) {
1257 LOG_DEV_ERROR(dev, "failed to set supported opp\n");
1258 return PTR_ERR(opp_table);
1259 }
1260
1261 return 0;
1262 }
1263
rk3588_npu_set_read_margin(struct device * dev,struct rockchip_opp_info * opp_info,u32 rm)1264 static int rk3588_npu_set_read_margin(struct device *dev,
1265 struct rockchip_opp_info *opp_info,
1266 u32 rm)
1267 {
1268 u32 offset = 0, val = 0;
1269 int i, ret = 0;
1270
1271 if (!opp_info->grf || !opp_info->volt_rm_tbl)
1272 return 0;
1273
1274 if (rm == opp_info->current_rm || rm == UINT_MAX)
1275 return 0;
1276
1277 LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
1278
1279 for (i = 0; i < 3; i++) {
1280 ret = regmap_read(opp_info->grf, offset, &val);
1281 if (ret < 0) {
1282 LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
1283 offset);
1284 return ret;
1285 }
1286 val &= ~0x1c;
1287 regmap_write(opp_info->grf, offset, val | (rm << 2));
1288 offset += 4;
1289 }
1290 opp_info->current_rm = rm;
1291
1292 return 0;
1293 }
1294
1295 static const struct rockchip_opp_data rk3588_npu_opp_data = {
1296 .get_soc_info = rk3588_npu_get_soc_info,
1297 .set_soc_info = rk3588_npu_set_soc_info,
1298 .set_read_margin = rk3588_npu_set_read_margin,
1299 };
1300
1301 static const struct of_device_id rockchip_npu_of_match[] = {
1302 {
1303 .compatible = "rockchip,rk3588",
1304 .data = (void *)&rk3588_npu_opp_data,
1305 },
1306 {},
1307 };
1308
rknpu_devfreq_init(struct rknpu_device * rknpu_dev)1309 static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1310 {
1311 struct device *dev = rknpu_dev->dev;
1312 struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1313 struct dev_pm_opp *opp;
1314 struct opp_table *reg_table = NULL;
1315 struct opp_table *opp_table = NULL;
1316 const char *const reg_names[] = { "rknpu", "mem" };
1317 int ret = -EINVAL;
1318
1319 if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
1320 of_find_property(dev->of_node, "mem-supply", NULL)) {
1321 reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
1322 if (IS_ERR(reg_table))
1323 return PTR_ERR(reg_table);
1324 opp_table =
1325 dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
1326 if (IS_ERR(opp_table)) {
1327 dev_pm_opp_put_regulators(reg_table);
1328 return PTR_ERR(opp_table);
1329 }
1330 } else {
1331 reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
1332 if (IS_ERR(reg_table))
1333 return PTR_ERR(reg_table);
1334 }
1335
1336 rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
1337 ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
1338 "rknpu");
1339 if (ret) {
1340 LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1341 return ret;
1342 }
1343
1344 rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1345
1346 opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
1347 if (IS_ERR(opp)) {
1348 ret = PTR_ERR(opp);
1349 goto err_remove_table;
1350 }
1351 dev_pm_opp_put(opp);
1352 dp->initial_freq = rknpu_dev->current_freq;
1353
1354 #ifdef CONFIG_PM_DEVFREQ
1355 ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1356 if (ret) {
1357 LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1358 goto err_remove_table;
1359 }
1360 #endif
1361
1362 rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1363 (void *)rknpu_dev);
1364 if (IS_ERR(rknpu_dev->devfreq)) {
1365 LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1366 ret = PTR_ERR(rknpu_dev->devfreq);
1367 goto err_remove_governor;
1368 }
1369 devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1370
1371 rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1372 rknpu_dev->devfreq->last_status.total_time = 1;
1373 rknpu_dev->devfreq->last_status.busy_time = 1;
1374
1375 npu_mdevp.data = rknpu_dev->devfreq;
1376 npu_mdevp.opp_info = &rknpu_dev->opp_info;
1377 rknpu_dev->mdev_info =
1378 rockchip_system_monitor_register(dev, &npu_mdevp);
1379 if (IS_ERR(rknpu_dev->mdev_info)) {
1380 LOG_DEV_DEBUG(dev, "without system monitor\n");
1381 rknpu_dev->mdev_info = NULL;
1382 npu_mdevp.is_checked = true;
1383 }
1384 rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1385 rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1386
1387 of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1388 (u32 *)&npu_cooling_power.dyn_power_coeff);
1389 rknpu_dev->model_data =
1390 rockchip_ipa_power_model_init(dev, "npu_leakage");
1391 if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1392 rknpu_dev->model_data = NULL;
1393 LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1394 } else if (rknpu_dev->model_data->dynamic_coefficient) {
1395 npu_cooling_power.dyn_power_coeff =
1396 rknpu_dev->model_data->dynamic_coefficient;
1397 }
1398 if (!npu_cooling_power.dyn_power_coeff) {
1399 LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1400 goto out;
1401 }
1402
1403 rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1404 dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1405 if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1406 LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1407
1408 out:
1409 return 0;
1410
1411 err_remove_governor:
1412 #ifdef CONFIG_PM_DEVFREQ
1413 devfreq_remove_governor(&devfreq_rknpu_ondemand);
1414 #endif
1415 err_remove_table:
1416 dev_pm_opp_of_remove_table(dev);
1417
1418 rknpu_dev->devfreq = NULL;
1419
1420 return ret;
1421 }
1422
1423 #else
1424
npu_devfreq_adjust_current_freq_volt(struct device * dev,struct rknpu_device * rknpu_dev)1425 static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
1426 struct rknpu_device *rknpu_dev)
1427 {
1428 unsigned long volt, old_freq, freq;
1429 struct dev_pm_opp *opp = NULL;
1430 int ret = -EINVAL;
1431
1432 old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1433 freq = old_freq;
1434
1435 #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1436 rcu_read_lock();
1437 #endif
1438
1439 opp = devfreq_recommended_opp(dev, &freq, 0);
1440 volt = dev_pm_opp_get_voltage(opp);
1441
1442 #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
1443 rcu_read_unlock();
1444 #endif
1445
1446 if (freq >= old_freq && rknpu_dev->vdd) {
1447 ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1448 if (ret) {
1449 LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1450 return ret;
1451 }
1452 }
1453 LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
1454 volt);
1455 ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
1456 if (ret) {
1457 LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
1458 return ret;
1459 }
1460 if (freq < old_freq && rknpu_dev->vdd) {
1461 ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
1462 if (ret) {
1463 LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
1464 return ret;
1465 }
1466 }
1467 rknpu_dev->current_freq = freq;
1468 rknpu_dev->current_volt = volt;
1469
1470 return 0;
1471 }
1472
rknpu_devfreq_init(struct rknpu_device * rknpu_dev)1473 static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
1474 {
1475 struct device *dev = rknpu_dev->dev;
1476 struct devfreq_dev_profile *dp = &npu_devfreq_profile;
1477 int ret = -EINVAL;
1478
1479 ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
1480 if (ret) {
1481 LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
1482 return ret;
1483 }
1484
1485 ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
1486 if (ret) {
1487 LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
1488 goto err_remove_table;
1489 }
1490 dp->initial_freq = rknpu_dev->current_freq;
1491
1492 #ifdef CONFIG_PM_DEVFREQ
1493 ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
1494 if (ret) {
1495 LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
1496 goto err_remove_table;
1497 }
1498 #endif
1499
1500 rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
1501 (void *)rknpu_dev);
1502 if (IS_ERR(rknpu_dev->devfreq)) {
1503 LOG_DEV_ERROR(dev, "failed to add devfreq\n");
1504 ret = PTR_ERR(rknpu_dev->devfreq);
1505 goto err_remove_governor;
1506 }
1507 devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
1508
1509 rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
1510 rknpu_dev->devfreq->last_status.total_time = 1;
1511 rknpu_dev->devfreq->last_status.busy_time = 1;
1512
1513 npu_mdevp.data = rknpu_dev->devfreq;
1514 rknpu_dev->mdev_info =
1515 rockchip_system_monitor_register(dev, &npu_mdevp);
1516 if (IS_ERR(rknpu_dev->mdev_info)) {
1517 LOG_DEV_DEBUG(dev, "without system monitor\n");
1518 rknpu_dev->mdev_info = NULL;
1519 }
1520 rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
1521 rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
1522
1523 of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
1524 (u32 *)&npu_cooling_power.dyn_power_coeff);
1525 rknpu_dev->model_data =
1526 rockchip_ipa_power_model_init(dev, "npu_leakage");
1527 if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
1528 rknpu_dev->model_data = NULL;
1529 LOG_DEV_ERROR(dev, "failed to initialize power model\n");
1530 } else if (rknpu_dev->model_data->dynamic_coefficient) {
1531 npu_cooling_power.dyn_power_coeff =
1532 rknpu_dev->model_data->dynamic_coefficient;
1533 }
1534
1535 if (!npu_cooling_power.dyn_power_coeff) {
1536 LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
1537 goto out;
1538 }
1539
1540 rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
1541 dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
1542 if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
1543 LOG_DEV_ERROR(dev, "failed to register cooling device\n");
1544
1545 out:
1546 return 0;
1547
1548 err_remove_governor:
1549 #ifdef CONFIG_PM_DEVFREQ
1550 devfreq_remove_governor(&devfreq_rknpu_ondemand);
1551 #endif
1552 err_remove_table:
1553 dev_pm_opp_of_remove_table(dev);
1554
1555 rknpu_dev->devfreq = NULL;
1556
1557 return ret;
1558 }
1559 #endif
1560 #endif
1561
rknpu_devfreq_remove(struct rknpu_device * rknpu_dev)1562 static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
1563 {
1564 if (rknpu_dev->devfreq) {
1565 devfreq_unregister_opp_notifier(rknpu_dev->dev,
1566 rknpu_dev->devfreq);
1567 dev_pm_opp_of_remove_table(rknpu_dev->dev);
1568 #ifdef CONFIG_PM_DEVFREQ
1569 devfreq_remove_governor(&devfreq_rknpu_ondemand);
1570 #endif
1571 }
1572
1573 return 0;
1574 }
1575
1576 #endif
1577
rknpu_register_irq(struct platform_device * pdev,struct rknpu_device * rknpu_dev)1578 static int rknpu_register_irq(struct platform_device *pdev,
1579 struct rknpu_device *rknpu_dev)
1580 {
1581 const struct rknpu_config *config = rknpu_dev->config;
1582 struct device *dev = &pdev->dev;
1583 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1584 struct resource *res;
1585 #endif
1586 int i, ret, irq;
1587
1588 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1589 res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
1590 config->irqs[0].name);
1591 if (res) {
1592 /* there are irq names in dts */
1593 for (i = 0; i < config->num_irqs; i++) {
1594 irq = platform_get_irq_byname(pdev,
1595 config->irqs[i].name);
1596 if (irq < 0) {
1597 LOG_DEV_ERROR(dev, "no npu %s in dts\n",
1598 config->irqs[i].name);
1599 return irq;
1600 }
1601
1602 ret = devm_request_irq(dev, irq,
1603 config->irqs[i].irq_hdl,
1604 IRQF_SHARED, dev_name(dev),
1605 rknpu_dev);
1606 if (ret < 0) {
1607 LOG_DEV_ERROR(dev, "request %s failed: %d\n",
1608 config->irqs[i].name, ret);
1609 return ret;
1610 }
1611 }
1612 } else {
1613 /* no irq names in dts */
1614 irq = platform_get_irq(pdev, 0);
1615 if (irq < 0) {
1616 LOG_DEV_ERROR(dev, "no npu irq in dts\n");
1617 return irq;
1618 }
1619
1620 ret = devm_request_irq(dev, irq, rknpu_core0_irq_handler,
1621 IRQF_SHARED, dev_name(dev), rknpu_dev);
1622 if (ret < 0) {
1623 LOG_DEV_ERROR(dev, "request irq failed: %d\n", ret);
1624 return ret;
1625 }
1626 }
1627 #else
1628 /* there are irq names in dts */
1629 for (i = 0; i < config->num_irqs; i++) {
1630 irq = platform_get_irq_byname(pdev, config->irqs[i].name);
1631 if (irq < 0) {
1632 LOG_DEV_ERROR(dev, "no npu %s in dts\n",
1633 config->irqs[i].name);
1634 return irq;
1635 }
1636
1637 ret = devm_request_irq(dev, irq, config->irqs[i].irq_hdl,
1638 IRQF_SHARED, dev_name(dev), rknpu_dev);
1639 if (ret < 0) {
1640 LOG_DEV_ERROR(dev, "request %s failed: %d\n",
1641 config->irqs[i].name, ret);
1642 return ret;
1643 }
1644 }
1645 #endif
1646
1647 return 0;
1648 }
1649
rknpu_find_sram_resource(struct rknpu_device * rknpu_dev)1650 static int rknpu_find_sram_resource(struct rknpu_device *rknpu_dev)
1651 {
1652 struct device *dev = rknpu_dev->dev;
1653 struct device_node *sram_node = NULL;
1654 struct resource sram_res;
1655 uint32_t sram_size = 0;
1656 int ret = -EINVAL;
1657
1658 /* get sram device node */
1659 sram_node = of_parse_phandle(dev->of_node, "rockchip,sram", 0);
1660 rknpu_dev->sram_size = 0;
1661 if (!sram_node)
1662 return -EINVAL;
1663
1664 /* get sram start and size */
1665 ret = of_address_to_resource(sram_node, 0, &sram_res);
1666 of_node_put(sram_node);
1667 if (ret)
1668 return ret;
1669
1670 /* check sram start and size is PAGE_SIZE align */
1671 rknpu_dev->sram_start = round_up(sram_res.start, PAGE_SIZE);
1672 rknpu_dev->sram_end = round_down(
1673 sram_res.start + resource_size(&sram_res), PAGE_SIZE);
1674 if (rknpu_dev->sram_end <= rknpu_dev->sram_start) {
1675 LOG_DEV_WARN(
1676 dev,
1677 "invalid sram resource, sram start %pa, sram end %pa\n",
1678 &rknpu_dev->sram_start, &rknpu_dev->sram_end);
1679 return -EINVAL;
1680 }
1681
1682 sram_size = rknpu_dev->sram_end - rknpu_dev->sram_start;
1683
1684 rknpu_dev->sram_base_io =
1685 devm_ioremap(dev, rknpu_dev->sram_start, sram_size);
1686 if (IS_ERR(rknpu_dev->sram_base_io)) {
1687 LOG_DEV_ERROR(dev, "failed to remap sram base io!\n");
1688 rknpu_dev->sram_base_io = NULL;
1689 }
1690
1691 rknpu_dev->sram_size = sram_size;
1692
1693 LOG_DEV_INFO(dev, "sram region: [%pa, %pa), sram size: %#x\n",
1694 &rknpu_dev->sram_start, &rknpu_dev->sram_end,
1695 rknpu_dev->sram_size);
1696
1697 return 0;
1698 }
1699
rknpu_find_nbuf_resource(struct rknpu_device * rknpu_dev)1700 static int rknpu_find_nbuf_resource(struct rknpu_device *rknpu_dev)
1701 {
1702 struct device *dev = rknpu_dev->dev;
1703
1704 if (rknpu_dev->config->nbuf_size == 0)
1705 return -EINVAL;
1706
1707 rknpu_dev->nbuf_start = rknpu_dev->config->nbuf_phyaddr;
1708 rknpu_dev->nbuf_size = rknpu_dev->config->nbuf_size;
1709 rknpu_dev->nbuf_base_io =
1710 devm_ioremap(dev, rknpu_dev->nbuf_start, rknpu_dev->nbuf_size);
1711 if (IS_ERR(rknpu_dev->nbuf_base_io)) {
1712 LOG_DEV_ERROR(dev, "failed to remap nbuf base io!\n");
1713 rknpu_dev->nbuf_base_io = NULL;
1714 }
1715
1716 rknpu_dev->nbuf_end = rknpu_dev->nbuf_start + rknpu_dev->nbuf_size;
1717
1718 LOG_DEV_INFO(dev, "nbuf region: [%pa, %pa), nbuf size: %#x\n",
1719 &rknpu_dev->nbuf_start, &rknpu_dev->nbuf_end,
1720 rknpu_dev->nbuf_size);
1721
1722 return 0;
1723 }
1724
rknpu_probe(struct platform_device * pdev)1725 static int rknpu_probe(struct platform_device *pdev)
1726 {
1727 struct resource *res = NULL;
1728 struct rknpu_device *rknpu_dev = NULL;
1729 struct device *dev = &pdev->dev;
1730 struct device *virt_dev = NULL;
1731 const struct of_device_id *match = NULL;
1732 const struct rknpu_config *config = NULL;
1733 int ret = -EINVAL, i = 0;
1734
1735 if (!pdev->dev.of_node) {
1736 LOG_DEV_ERROR(dev, "rknpu device-tree data is missing!\n");
1737 return -ENODEV;
1738 }
1739
1740 match = of_match_device(rknpu_of_match, dev);
1741 if (!match) {
1742 LOG_DEV_ERROR(dev, "rknpu device-tree entry is missing!\n");
1743 return -ENODEV;
1744 }
1745
1746 rknpu_dev = devm_kzalloc(dev, sizeof(*rknpu_dev), GFP_KERNEL);
1747 if (!rknpu_dev) {
1748 LOG_DEV_ERROR(dev, "failed to allocate rknpu device!\n");
1749 return -ENOMEM;
1750 }
1751
1752 config = of_device_get_match_data(dev);
1753 if (!config)
1754 return -EINVAL;
1755
1756 rknpu_dev->config = config;
1757 rknpu_dev->dev = dev;
1758
1759 rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev);
1760 if (!rknpu_dev->iommu_en) {
1761 /* Initialize reserved memory resources */
1762 ret = of_reserved_mem_device_init(dev);
1763 if (!ret) {
1764 LOG_DEV_INFO(
1765 dev,
1766 "initialize reserved memory for rknpu device!\n");
1767 }
1768 }
1769
1770 rknpu_dev->bypass_irq_handler = bypass_irq_handler;
1771 rknpu_dev->bypass_soft_reset = bypass_soft_reset;
1772
1773 rknpu_reset_get(rknpu_dev);
1774
1775 rknpu_dev->num_clks = devm_clk_bulk_get_all(dev, &rknpu_dev->clks);
1776 if (rknpu_dev->num_clks < 1) {
1777 LOG_DEV_ERROR(dev, "failed to get clk source for rknpu\n");
1778 #ifndef FPGA_PLATFORM
1779 return -ENODEV;
1780 #endif
1781 }
1782
1783 #ifndef FPGA_PLATFORM
1784 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
1785 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1786 if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
1787 rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
1788 #endif
1789
1790 rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
1791 if (IS_ERR(rknpu_dev->vdd)) {
1792 if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
1793 ret = PTR_ERR(rknpu_dev->vdd);
1794 LOG_DEV_ERROR(
1795 dev,
1796 "failed to get vdd regulator for rknpu: %d\n",
1797 ret);
1798 return ret;
1799 }
1800 rknpu_dev->vdd = NULL;
1801 }
1802
1803 rknpu_dev->mem = devm_regulator_get_optional(dev, "mem");
1804 if (IS_ERR(rknpu_dev->mem)) {
1805 if (PTR_ERR(rknpu_dev->mem) != -ENODEV) {
1806 ret = PTR_ERR(rknpu_dev->mem);
1807 LOG_DEV_ERROR(
1808 dev,
1809 "failed to get mem regulator for rknpu: %d\n",
1810 ret);
1811 return ret;
1812 }
1813 rknpu_dev->mem = NULL;
1814 }
1815 #endif
1816
1817 spin_lock_init(&rknpu_dev->lock);
1818 spin_lock_init(&rknpu_dev->irq_lock);
1819 mutex_init(&rknpu_dev->power_lock);
1820 mutex_init(&rknpu_dev->reset_lock);
1821 for (i = 0; i < config->num_irqs; i++) {
1822 INIT_LIST_HEAD(&rknpu_dev->subcore_datas[i].todo_list);
1823 init_waitqueue_head(&rknpu_dev->subcore_datas[i].job_done_wq);
1824 rknpu_dev->subcore_datas[i].task_num = 0;
1825 res = platform_get_resource(pdev, IORESOURCE_MEM, i);
1826 if (!res) {
1827 LOG_DEV_ERROR(
1828 dev,
1829 "failed to get memory resource for rknpu\n");
1830 return -ENXIO;
1831 }
1832
1833 rknpu_dev->base[i] = devm_ioremap_resource(dev, res);
1834 if (PTR_ERR(rknpu_dev->base[i]) == -EBUSY) {
1835 rknpu_dev->base[i] = devm_ioremap(dev, res->start,
1836 resource_size(res));
1837 }
1838
1839 if (IS_ERR(rknpu_dev->base[i])) {
1840 LOG_DEV_ERROR(dev,
1841 "failed to remap register for rknpu\n");
1842 return PTR_ERR(rknpu_dev->base[i]);
1843 }
1844 }
1845
1846 if (config->bw_priority_length > 0) {
1847 rknpu_dev->bw_priority_base =
1848 devm_ioremap(dev, config->bw_priority_addr,
1849 config->bw_priority_length);
1850 if (IS_ERR(rknpu_dev->bw_priority_base)) {
1851 LOG_DEV_ERROR(
1852 rknpu_dev->dev,
1853 "failed to remap bw priority register for rknpu\n");
1854 rknpu_dev->bw_priority_base = NULL;
1855 }
1856 }
1857
1858 if (!rknpu_dev->bypass_irq_handler) {
1859 ret = rknpu_register_irq(pdev, rknpu_dev);
1860 if (ret)
1861 return ret;
1862 } else {
1863 LOG_DEV_WARN(dev, "bypass irq handler!\n");
1864 }
1865
1866 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
1867 ret = rknpu_drm_probe(rknpu_dev);
1868 if (ret) {
1869 LOG_DEV_ERROR(dev, "failed to probe device for rknpu\n");
1870 return ret;
1871 }
1872 #endif
1873 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
1874 rknpu_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
1875 rknpu_dev->miscdev.name = "rknpu";
1876 rknpu_dev->miscdev.fops = &rknpu_fops;
1877
1878 ret = misc_register(&rknpu_dev->miscdev);
1879 if (ret) {
1880 LOG_DEV_ERROR(dev, "cannot register miscdev (%d)\n", ret);
1881 return ret;
1882 }
1883
1884 rknpu_dev->heap = rk_dma_heap_find("rk-dma-heap-cma");
1885 if (!rknpu_dev->heap) {
1886 LOG_DEV_ERROR(dev, "failed to find cma heap\n");
1887 return -ENOMEM;
1888 }
1889 rk_dma_heap_set_dev(dev);
1890 LOG_DEV_INFO(dev, "Initialized %s: v%d.%d.%d for %s\n", DRIVER_DESC,
1891 DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL,
1892 DRIVER_DATE);
1893 #endif
1894
1895 #ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
1896 ret = rknpu_fence_context_alloc(rknpu_dev);
1897 if (ret) {
1898 LOG_DEV_ERROR(dev,
1899 "failed to allocate fence context for rknpu\n");
1900 goto err_remove_drv;
1901 }
1902 #endif
1903
1904 platform_set_drvdata(pdev, rknpu_dev);
1905
1906 pm_runtime_enable(dev);
1907
1908 if (of_count_phandle_with_args(dev->of_node, "power-domains",
1909 "#power-domain-cells") > 1) {
1910 virt_dev = dev_pm_domain_attach_by_name(dev, "npu0");
1911 if (!IS_ERR(virt_dev))
1912 rknpu_dev->genpd_dev_npu0 = virt_dev;
1913 virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
1914 if (!IS_ERR(virt_dev))
1915 rknpu_dev->genpd_dev_npu1 = virt_dev;
1916 virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
1917 if (!IS_ERR(virt_dev))
1918 rknpu_dev->genpd_dev_npu2 = virt_dev;
1919 rknpu_dev->multiple_domains = true;
1920 }
1921
1922 ret = rknpu_power_on(rknpu_dev);
1923 if (ret)
1924 goto err_remove_drv;
1925
1926 #ifndef FPGA_PLATFORM
1927 #if KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
1928 rknpu_devfreq_init(rknpu_dev);
1929 #endif
1930 #endif
1931
1932 // set default power put delay to 3s
1933 rknpu_dev->power_put_delay = 3000;
1934 rknpu_dev->power_off_wq =
1935 create_freezable_workqueue("rknpu_power_off_wq");
1936 if (!rknpu_dev->power_off_wq) {
1937 LOG_DEV_ERROR(dev, "rknpu couldn't create power_off workqueue");
1938 ret = -ENOMEM;
1939 goto err_devfreq_remove;
1940 }
1941 INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
1942 rknpu_power_off_delay_work);
1943
1944 if (IS_ENABLED(CONFIG_NO_GKI) &&
1945 IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
1946 if (!rknpu_find_sram_resource(rknpu_dev)) {
1947 ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
1948 &rknpu_dev->sram_mm);
1949 if (ret != 0)
1950 goto err_remove_wq;
1951 } else {
1952 LOG_DEV_WARN(dev, "could not find sram resource!\n");
1953 }
1954 }
1955
1956 if (IS_ENABLED(CONFIG_NO_GKI) && rknpu_dev->iommu_en &&
1957 rknpu_dev->config->nbuf_size > 0)
1958 rknpu_find_nbuf_resource(rknpu_dev);
1959
1960 rknpu_power_off(rknpu_dev);
1961 atomic_set(&rknpu_dev->power_refcount, 0);
1962 atomic_set(&rknpu_dev->cmdline_power_refcount, 0);
1963
1964 rknpu_debugger_init(rknpu_dev);
1965 rknpu_init_timer(rknpu_dev);
1966
1967 return 0;
1968
1969 err_remove_wq:
1970 destroy_workqueue(rknpu_dev->power_off_wq);
1971
1972 err_devfreq_remove:
1973 #ifndef FPGA_PLATFORM
1974 rknpu_devfreq_remove(rknpu_dev);
1975 #endif
1976
1977 err_remove_drv:
1978 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
1979 rknpu_drm_remove(rknpu_dev);
1980 #endif
1981 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
1982 misc_deregister(&(rknpu_dev->miscdev));
1983 #endif
1984
1985 return ret;
1986 }
1987
rknpu_remove(struct platform_device * pdev)1988 static int rknpu_remove(struct platform_device *pdev)
1989 {
1990 struct rknpu_device *rknpu_dev = platform_get_drvdata(pdev);
1991 int i = 0;
1992
1993 cancel_delayed_work_sync(&rknpu_dev->power_off_work);
1994 destroy_workqueue(rknpu_dev->power_off_wq);
1995
1996 if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm)
1997 rknpu_mm_destroy(rknpu_dev->sram_mm);
1998
1999 rknpu_debugger_remove(rknpu_dev);
2000 rknpu_cancel_timer(rknpu_dev);
2001
2002 for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
2003 WARN_ON(rknpu_dev->subcore_datas[i].job);
2004 WARN_ON(!list_empty(&rknpu_dev->subcore_datas[i].todo_list));
2005 }
2006
2007 #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
2008 rknpu_drm_remove(rknpu_dev);
2009 #endif
2010 #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
2011 misc_deregister(&(rknpu_dev->miscdev));
2012 #endif
2013
2014 #ifndef FPGA_PLATFORM
2015 rknpu_devfreq_remove(rknpu_dev);
2016 #endif
2017
2018 mutex_lock(&rknpu_dev->power_lock);
2019 if (atomic_read(&rknpu_dev->power_refcount) > 0)
2020 rknpu_power_off(rknpu_dev);
2021 mutex_unlock(&rknpu_dev->power_lock);
2022
2023 if (rknpu_dev->multiple_domains) {
2024 if (rknpu_dev->genpd_dev_npu0)
2025 dev_pm_domain_detach(rknpu_dev->genpd_dev_npu0, true);
2026 if (rknpu_dev->genpd_dev_npu1)
2027 dev_pm_domain_detach(rknpu_dev->genpd_dev_npu1, true);
2028 if (rknpu_dev->genpd_dev_npu2)
2029 dev_pm_domain_detach(rknpu_dev->genpd_dev_npu2, true);
2030 }
2031
2032 pm_runtime_disable(&pdev->dev);
2033
2034 return 0;
2035 }
2036
2037 #ifndef FPGA_PLATFORM
2038 #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE && \
2039 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
rknpu_runtime_suspend(struct device * dev)2040 static int rknpu_runtime_suspend(struct device *dev)
2041 {
2042 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
2043 struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
2044
2045 if (opp_info->scmi_clk) {
2046 if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
2047 LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
2048 }
2049 opp_info->current_rm = UINT_MAX;
2050
2051 return 0;
2052 }
2053
rknpu_runtime_resume(struct device * dev)2054 static int rknpu_runtime_resume(struct device *dev)
2055 {
2056 struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
2057 struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
2058 int ret = 0;
2059
2060 if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
2061 return 0;
2062
2063 ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
2064 if (ret) {
2065 LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
2066 return ret;
2067 }
2068
2069 if (opp_info->data && opp_info->data->set_read_margin)
2070 opp_info->data->set_read_margin(dev, opp_info,
2071 opp_info->target_rm);
2072 if (opp_info->scmi_clk) {
2073 if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
2074 LOG_DEV_ERROR(dev, "failed to set power down rate\n");
2075 }
2076
2077 clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
2078
2079 return ret;
2080 }
2081
2082 static const struct dev_pm_ops rknpu_pm_ops = {
2083 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2084 pm_runtime_force_resume)
2085 SET_RUNTIME_PM_OPS(rknpu_runtime_suspend, rknpu_runtime_resume,
2086 NULL)
2087 };
2088 #endif
2089 #endif
2090
2091 static struct platform_driver rknpu_driver = {
2092 .probe = rknpu_probe,
2093 .remove = rknpu_remove,
2094 .driver = {
2095 .owner = THIS_MODULE,
2096 .name = "RKNPU",
2097 #ifndef FPGA_PLATFORM
2098 #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE && \
2099 KERNEL_VERSION(6, 1, 0) > LINUX_VERSION_CODE
2100 .pm = &rknpu_pm_ops,
2101 #endif
2102 #endif
2103 .of_match_table = of_match_ptr(rknpu_of_match),
2104 },
2105 };
2106
rknpu_init(void)2107 static int rknpu_init(void)
2108 {
2109 return platform_driver_register(&rknpu_driver);
2110 }
2111
rknpu_exit(void)2112 static void rknpu_exit(void)
2113 {
2114 platform_driver_unregister(&rknpu_driver);
2115 }
2116
2117 late_initcall(rknpu_init);
2118 module_exit(rknpu_exit);
2119
2120 MODULE_DESCRIPTION("RKNPU driver");
2121 MODULE_AUTHOR("Felix Zeng <felix.zeng@rock-chips.com>");
2122 MODULE_ALIAS("rockchip-rknpu");
2123 MODULE_LICENSE("GPL v2");
2124 MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
2125 DRIVER_PATCHLEVEL));
2126 #if defined(CONFIG_ROCKCHIP_RKNPU_DMA_HEAP) && KERNEL_VERSION(5, 16, 0) < LINUX_VERSION_CODE
2127 MODULE_IMPORT_NS(DMA_BUF);
2128 #endif
2129