1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) Rockchip Electronics Co., Ltd.
4 *
5 * Author: Huang Lee <Putin.li@rock-chips.com>
6 */
7
8 #ifndef __LINUX_RGA_DRV_H_
9 #define __LINUX_RGA_DRV_H_
10
11 #include <linux/clk.h>
12 #include <linux/completion.h>
13 #include <linux/debugfs.h>
14 #include <linux/delay.h>
15 #include <linux/device.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/err.h>
18 #include <linux/fb.h>
19 #include <linux/fdtable.h>
20 #include <linux/fs.h>
21 #include <linux/init.h>
22 #include <linux/interrupt.h>
23 #include <linux/io.h>
24 #include <linux/irq.h>
25 #include <linux/kernel.h>
26 #include <linux/kref.h>
27 #include <linux/miscdevice.h>
28 #include <linux/module.h>
29 #include <linux/mutex.h>
30 #include <linux/of_device.h>
31 #include <linux/platform_device.h>
32 #include <linux/poll.h>
33 #include <linux/regulator/consumer.h>
34 #include <linux/scatterlist.h>
35 #include <linux/sched.h>
36 #include <linux/slab.h>
37 #include <linux/spinlock.h>
38 #include <linux/syscalls.h>
39 #include <linux/time.h>
40 #include <linux/timer.h>
41 #include <linux/uaccess.h>
42 #include <linux/version.h>
43 #include <linux/wait.h>
44 #include <linux/pm_runtime.h>
45 #include <linux/sched/mm.h>
46 #include <linux/string_helpers.h>
47
48 #include <asm/cacheflush.h>
49
50 #include <linux/iommu.h>
51 #include <linux/iova.h>
52 #include <linux/pagemap.h>
53
54 #ifdef CONFIG_DMABUF_CACHE
55 #include <linux/dma-buf-cache.h>
56 #else
57 #include <linux/dma-buf.h>
58 #endif
59
60 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
61 #include <linux/dma-map-ops.h>
62 #endif
63
64 #include <linux/hrtimer.h>
65
66 #include "rga.h"
67
68 #define RGA_CORE_REG_OFFSET 0x10000
69
70 /* load interval: 1000ms */
71 #define RGA_LOAD_INTERVAL_US 1000000
72
73 /* timer interval: 1000ms */
74 #define RGA_TIMER_INTERVAL_NS 1000000000
75
76 #if ((defined(CONFIG_RK_IOMMU) || defined(CONFIG_ROCKCHIP_IOMMU)) \
77 && defined(CONFIG_ION_ROCKCHIP))
78 #define CONFIG_RGA_IOMMU
79 #endif
80
81 /* Driver information */
82 #define DRIVER_DESC "RGA multicore Device Driver"
83 #define DRIVER_NAME "rga_multicore"
84
85 #define STR_HELPER(x) #x
86 #define STR(x) STR_HELPER(x)
87
88 #define DRIVER_MAJOR_VERISON 1
89 #define DRIVER_MINOR_VERSION 2
90 #define DRIVER_REVISION_VERSION 27
91 #define DRIVER_PATCH_VERSION
92
93 #define DRIVER_VERSION (STR(DRIVER_MAJOR_VERISON) "." STR(DRIVER_MINOR_VERSION) \
94 "." STR(DRIVER_REVISION_VERSION) STR(DRIVER_PATCH_VERSION))
95
96 /* time limit */
97 #define RGA_JOB_TIMEOUT_DELAY HZ
98 #define RGA_RESET_TIMEOUT 1000
99
100 #define RGA_MAX_SCHEDULER 3
101 #define RGA_MAX_BUS_CLK 10
102
103 #define RGA_BUFFER_POOL_MAX_SIZE 64
104
105 #ifndef ABS
106 #define ABS(X) (((X) < 0) ? (-(X)) : (X))
107 #endif
108
109 #ifndef CLIP
110 #define CLIP(x, a, b) (((x) < (a)) \
111 ? (a) : (((x) > (b)) ? (b) : (x)))
112 #endif
113
114 extern struct rga_drvdata_t *rga_drvdata;
115
116 enum {
117 RGA3_SCHEDULER_CORE0 = 1 << 0,
118 RGA3_SCHEDULER_CORE1 = 1 << 1,
119 RGA2_SCHEDULER_CORE0 = 1 << 2,
120 RGA_CORE_MASK = 0x7,
121 RGA_NONE_CORE = 0x0,
122 };
123
124 enum {
125 RGA_CMD_SLAVE = 1,
126 RGA_CMD_MASTER = 2,
127 };
128
129 enum iommu_dma_cookie_type {
130 IOMMU_DMA_IOVA_COOKIE,
131 IOMMU_DMA_MSI_COOKIE,
132 };
133
134 enum rga_scheduler_status {
135 RGA_SCHEDULER_IDLE = 0,
136 RGA_SCHEDULER_WORKING,
137 RGA_SCHEDULER_ABORT,
138 };
139
140 enum rga_job_state {
141 RGA_JOB_STATE_PENDING = 0,
142 RGA_JOB_STATE_PREPARE,
143 RGA_JOB_STATE_RUNNING,
144 RGA_JOB_STATE_FINISH,
145 RGA_JOB_STATE_DONE,
146 RGA_JOB_STATE_INTR_ERR,
147 RGA_JOB_STATE_HW_TIMEOUT,
148 RGA_JOB_STATE_ABORT,
149 };
150
151 struct rga_iommu_dma_cookie {
152 enum iommu_dma_cookie_type type;
153
154 /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
155 struct iova_domain iovad;
156 };
157
158 struct rga_iommu_info {
159 struct device *dev;
160 struct device *default_dev; /* for dma-buf_api */
161 struct iommu_domain *domain;
162 struct iommu_group *group;
163 };
164
165 struct rga_dma_buffer {
166 /* DMABUF information */
167 struct dma_buf *dma_buf;
168 struct dma_buf_attachment *attach;
169 struct sg_table *sgt;
170 void *vmap_ptr;
171
172 struct iommu_domain *domain;
173
174 enum dma_data_direction dir;
175
176 dma_addr_t iova;
177 unsigned long size;
178 /*
179 * The offset of the first page of the sgt.
180 * Since alloc iova must be page aligned, the offset of the first page is
181 * identified separately.
182 */
183 size_t offset;
184
185 /* The scheduler of the mapping */
186 struct rga_scheduler_t *scheduler;
187 };
188
189 struct rga_virt_addr {
190 uint64_t addr;
191
192 struct page **pages;
193 int pages_order;
194 int page_count;
195 unsigned long size;
196
197 /* The offset of the first page of the virtual address */
198 size_t offset;
199
200 int result;
201 };
202
203 struct rga_internal_buffer {
204 /* DMA buffer */
205 struct rga_dma_buffer *dma_buffer;
206
207 /* virtual address */
208 struct rga_virt_addr *virt_addr;
209
210 /* physical address */
211 uint64_t phys_addr;
212
213 /* buffer size */
214 unsigned long size;
215
216 struct rga_memory_parm memory_parm;
217
218
219 struct mm_struct *current_mm;
220
221 /* memory type. */
222 uint32_t type;
223
224 uint32_t handle;
225
226 uint32_t mm_flag;
227
228 struct kref refcount;
229 struct rga_session *session;
230 };
231
232 struct rga_scheduler_t;
233
234 struct rga_session {
235 int id;
236
237 pid_t tgid;
238
239 char *pname;
240 };
241
242 struct rga_job_buffer {
243 union {
244 struct {
245 struct rga_external_buffer *ex_y_addr;
246 struct rga_external_buffer *ex_uv_addr;
247 struct rga_external_buffer *ex_v_addr;
248 };
249 struct rga_external_buffer *ex_addr;
250 };
251
252 union {
253 struct {
254 struct rga_internal_buffer *y_addr;
255 struct rga_internal_buffer *uv_addr;
256 struct rga_internal_buffer *v_addr;
257 };
258 struct rga_internal_buffer *addr;
259 };
260
261 uint32_t *page_table;
262 int order;
263 int page_count;
264 };
265
266 struct rga_job {
267 struct list_head head;
268
269 struct rga_scheduler_t *scheduler;
270 struct rga_session *session;
271
272 struct rga_req rga_command_base;
273 uint32_t cmd_reg[32 * 8];
274 struct rga_full_csc full_csc;
275 struct rga_pre_intr_info pre_intr_info;
276
277 struct rga_job_buffer src_buffer;
278 struct rga_job_buffer src1_buffer;
279 struct rga_job_buffer dst_buffer;
280 /* used by rga2 */
281 struct rga_job_buffer els_buffer;
282
283 /* for rga2 virtual_address */
284 struct mm_struct *mm;
285
286 /* job time stamp */
287 ktime_t timestamp;
288 /* The time when the job is actually executed on the hardware */
289 ktime_t hw_running_time;
290 /* The time only for hrtimer to calculate the load */
291 ktime_t hw_recoder_time;
292 unsigned int flags;
293 int request_id;
294 int priority;
295 int core;
296 int ret;
297 pid_t pid;
298 bool use_batch_mode;
299
300 struct kref refcount;
301 unsigned long state;
302 uint32_t intr_status;
303 uint32_t hw_status;
304 uint32_t cmd_status;
305 };
306
307 struct rga_backend_ops {
308 int (*get_version)(struct rga_scheduler_t *scheduler);
309 int (*set_reg)(struct rga_job *job, struct rga_scheduler_t *scheduler);
310 int (*init_reg)(struct rga_job *job);
311 void (*soft_reset)(struct rga_scheduler_t *scheduler);
312 int (*read_back_reg)(struct rga_job *job, struct rga_scheduler_t *scheduler);
313 int (*irq)(struct rga_scheduler_t *scheduler);
314 int (*isr_thread)(struct rga_job *job, struct rga_scheduler_t *scheduler);
315 };
316
317 struct rga_timer {
318 u32 busy_time;
319 u32 busy_time_record;
320 };
321
322 struct rga_scheduler_t {
323 struct device *dev;
324 void __iomem *rga_base;
325 struct rga_iommu_info *iommu_info;
326
327 struct clk *clks[RGA_MAX_BUS_CLK];
328 int num_clks;
329
330 enum rga_scheduler_status status;
331 int pd_refcount;
332
333 struct rga_job *running_job;
334 struct list_head todo_list;
335 spinlock_t irq_lock;
336 wait_queue_head_t job_done_wq;
337 const struct rga_backend_ops *ops;
338 const struct rga_hw_data *data;
339 int job_count;
340 int irq;
341 struct rga_version_t version;
342 int core;
343
344 struct rga_timer timer;
345 };
346
347 struct rga_request {
348 struct rga_req *task_list;
349 int task_count;
350 uint32_t finished_task_count;
351 uint32_t failed_task_count;
352
353 bool use_batch_mode;
354 bool is_running;
355 bool is_done;
356 int ret;
357 uint32_t sync_mode;
358
359 int32_t acquire_fence_fd;
360 int32_t release_fence_fd;
361 struct dma_fence *release_fence;
362 spinlock_t fence_lock;
363
364 wait_queue_head_t finished_wq;
365
366 int flags;
367 uint8_t mpi_config_flags;
368 int id;
369 struct rga_session *session;
370
371 spinlock_t lock;
372 struct kref refcount;
373
374 pid_t pid;
375
376 /*
377 * The mapping of virtual addresses to obtain physical addresses requires
378 * the memory mapping information of the current process.
379 */
380 struct mm_struct *current_mm;
381
382 /* TODO: add some common work */
383 };
384
385 struct rga_pending_request_manager {
386 struct mutex lock;
387
388 /*
389 * @request_idr:
390 *
391 * Mapping of request id to object pointers. Used by the GEM
392 * subsystem. Protected by @lock.
393 */
394 struct idr request_idr;
395
396 int request_count;
397 };
398
399 struct rga_session_manager {
400 struct mutex lock;
401
402 struct idr ctx_id_idr;
403
404 int session_cnt;
405 };
406
407 struct rga_drvdata_t {
408 /* used by rga2's mmu lock */
409 struct mutex lock;
410
411 struct rga_scheduler_t *scheduler[RGA_MAX_SCHEDULER];
412 int num_of_scheduler;
413 /* The scheduler_index used by default for memory mapping. */
414 int map_scheduler_index;
415 struct rga_mmu_base *mmu_base;
416
417 struct delayed_work power_off_work;
418
419 struct rga_mm *mm;
420
421 /* rga_job pending manager, import by RGA_START_CONFIG */
422 struct rga_pending_request_manager *pend_request_manager;
423
424 struct rga_session_manager *session_manager;
425
426 #ifdef CONFIG_ROCKCHIP_RGA_ASYNC
427 struct rga_fence_context *fence_ctx;
428 #endif
429
430 #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER
431 struct rga_debugger *debugger;
432 #endif
433 };
434
435 struct rga_irqs_data_t {
436 const char *name;
437 irqreturn_t (*irq_hdl)(int irq, void *ctx);
438 irqreturn_t (*irq_thread)(int irq, void *ctx);
439 };
440
441 struct rga_match_data_t {
442 const char * const *clks;
443 int num_clks;
444 };
445
rga_read(int offset,struct rga_scheduler_t * scheduler)446 static inline int rga_read(int offset, struct rga_scheduler_t *scheduler)
447 {
448 return readl(scheduler->rga_base + offset);
449 }
450
rga_write(int value,int offset,struct rga_scheduler_t * scheduler)451 static inline void rga_write(int value, int offset, struct rga_scheduler_t *scheduler)
452 {
453 writel(value, scheduler->rga_base + offset);
454 }
455
456 int rga_power_enable(struct rga_scheduler_t *scheduler);
457 int rga_power_disable(struct rga_scheduler_t *scheduler);
458
459 int rga_kernel_commit(struct rga_req *cmd);
460
461 #endif /* __LINUX_RGA_FENCE_H_ */
462