1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2019 Rockchip Electronics Co., Ltd */
3
4 #include <media/videobuf2-dma-contig.h>
5 #include <media/videobuf2-dma-sg.h>
6 #include <linux/of_platform.h>
7 #include <linux/slab.h>
8 #include "dev.h"
9 #include "isp_ispp.h"
10 #include "regs.h"
11
rkisp_write(struct rkisp_device * dev,u32 reg,u32 val,bool is_direct)12 void rkisp_write(struct rkisp_device *dev, u32 reg, u32 val, bool is_direct)
13 {
14 u32 *mem = dev->sw_base_addr + reg;
15 u32 *flag = dev->sw_base_addr + reg + RKISP_ISP_SW_REG_SIZE;
16
17 *mem = val;
18 *flag = SW_REG_CACHE;
19 if (dev->hw_dev->is_single || is_direct) {
20 *flag = SW_REG_CACHE_SYNC;
21 if (dev->isp_ver == ISP_V32 && reg <= 0x200)
22 rv1106_sdmmc_get_lock();
23 writel(val, dev->hw_dev->base_addr + reg);
24 if (dev->isp_ver == ISP_V32 && reg <= 0x200)
25 rv1106_sdmmc_put_lock();
26 }
27 }
28
rkisp_next_write(struct rkisp_device * dev,u32 reg,u32 val,bool is_direct)29 void rkisp_next_write(struct rkisp_device *dev, u32 reg, u32 val, bool is_direct)
30 {
31 u32 offset = RKISP_ISP_SW_MAX_SIZE + reg;
32 u32 *mem = dev->sw_base_addr + offset;
33 u32 *flag = dev->sw_base_addr + offset + RKISP_ISP_SW_REG_SIZE;
34
35 *mem = val;
36 *flag = SW_REG_CACHE;
37 if (dev->hw_dev->is_single || is_direct) {
38 *flag = SW_REG_CACHE_SYNC;
39 writel(val, dev->hw_dev->base_next_addr + reg);
40 }
41 }
42
rkisp_read(struct rkisp_device * dev,u32 reg,bool is_direct)43 u32 rkisp_read(struct rkisp_device *dev, u32 reg, bool is_direct)
44 {
45 u32 val;
46
47 if (dev->hw_dev->is_single || is_direct)
48 val = readl(dev->hw_dev->base_addr + reg);
49 else
50 val = *(u32 *)(dev->sw_base_addr + reg);
51 return val;
52 }
53
rkisp_next_read(struct rkisp_device * dev,u32 reg,bool is_direct)54 u32 rkisp_next_read(struct rkisp_device *dev, u32 reg, bool is_direct)
55 {
56 u32 val;
57
58 if (dev->hw_dev->is_single || is_direct)
59 val = readl(dev->hw_dev->base_next_addr + reg);
60 else
61 val = *(u32 *)(dev->sw_base_addr + RKISP_ISP_SW_MAX_SIZE + reg);
62 return val;
63 }
64
rkisp_set_bits(struct rkisp_device * dev,u32 reg,u32 mask,u32 val,bool is_direct)65 void rkisp_set_bits(struct rkisp_device *dev, u32 reg, u32 mask, u32 val, bool is_direct)
66 {
67 u32 tmp = rkisp_read(dev, reg, is_direct) & ~mask;
68
69 rkisp_write(dev, reg, val | tmp, is_direct);
70 }
71
rkisp_next_set_bits(struct rkisp_device * dev,u32 reg,u32 mask,u32 val,bool is_direct)72 void rkisp_next_set_bits(struct rkisp_device *dev, u32 reg, u32 mask, u32 val, bool is_direct)
73 {
74 u32 tmp = rkisp_next_read(dev, reg, is_direct) & ~mask;
75
76 rkisp_next_write(dev, reg, val | tmp, is_direct);
77 }
78
rkisp_clear_bits(struct rkisp_device * dev,u32 reg,u32 mask,bool is_direct)79 void rkisp_clear_bits(struct rkisp_device *dev, u32 reg, u32 mask, bool is_direct)
80 {
81 u32 tmp = rkisp_read(dev, reg, is_direct);
82
83 rkisp_write(dev, reg, tmp & ~mask, is_direct);
84 }
85
rkisp_next_clear_bits(struct rkisp_device * dev,u32 reg,u32 mask,bool is_direct)86 void rkisp_next_clear_bits(struct rkisp_device *dev, u32 reg, u32 mask, bool is_direct)
87 {
88 u32 tmp = rkisp_next_read(dev, reg, is_direct);
89
90 rkisp_next_write(dev, reg, tmp & ~mask, is_direct);
91 }
92
rkisp_write_reg_cache(struct rkisp_device * dev,u32 reg,u32 val)93 void rkisp_write_reg_cache(struct rkisp_device *dev, u32 reg, u32 val)
94 {
95 u32 *mem = dev->sw_base_addr + reg;
96
97 *mem = val;
98 }
99
rkisp_next_write_reg_cache(struct rkisp_device * dev,u32 reg,u32 val)100 void rkisp_next_write_reg_cache(struct rkisp_device *dev, u32 reg, u32 val)
101 {
102 u32 offset = RKISP_ISP_SW_MAX_SIZE + reg;
103 u32 *mem = dev->sw_base_addr + offset;
104
105 *mem = val;
106 }
107
rkisp_read_reg_cache(struct rkisp_device * dev,u32 reg)108 u32 rkisp_read_reg_cache(struct rkisp_device *dev, u32 reg)
109 {
110 return *(u32 *)(dev->sw_base_addr + reg);
111 }
112
rkisp_next_read_reg_cache(struct rkisp_device * dev,u32 reg)113 u32 rkisp_next_read_reg_cache(struct rkisp_device *dev, u32 reg)
114 {
115 return *(u32 *)(dev->sw_base_addr + RKISP_ISP_SW_MAX_SIZE + reg);
116 }
117
rkisp_set_reg_cache_bits(struct rkisp_device * dev,u32 reg,u32 mask,u32 val)118 void rkisp_set_reg_cache_bits(struct rkisp_device *dev, u32 reg, u32 mask, u32 val)
119 {
120 u32 tmp = rkisp_read_reg_cache(dev, reg) & ~mask;
121
122 rkisp_write_reg_cache(dev, reg, val | tmp);
123 }
124
rkisp_next_set_reg_cache_bits(struct rkisp_device * dev,u32 reg,u32 mask,u32 val)125 void rkisp_next_set_reg_cache_bits(struct rkisp_device *dev, u32 reg, u32 mask, u32 val)
126 {
127 u32 tmp = rkisp_next_read_reg_cache(dev, reg) & ~mask;
128
129 rkisp_next_write_reg_cache(dev, reg, val | tmp);
130 }
131
rkisp_clear_reg_cache_bits(struct rkisp_device * dev,u32 reg,u32 mask)132 void rkisp_clear_reg_cache_bits(struct rkisp_device *dev, u32 reg, u32 mask)
133 {
134 u32 tmp = rkisp_read_reg_cache(dev, reg);
135
136 rkisp_write_reg_cache(dev, reg, tmp & ~mask);
137 }
138
rkisp_next_clear_reg_cache_bits(struct rkisp_device * dev,u32 reg,u32 mask)139 void rkisp_next_clear_reg_cache_bits(struct rkisp_device *dev, u32 reg, u32 mask)
140 {
141 u32 tmp = rkisp_next_read_reg_cache(dev, reg);
142
143 rkisp_next_write_reg_cache(dev, reg, tmp & ~mask);
144 }
145
rkisp_update_regs(struct rkisp_device * dev,u32 start,u32 end)146 void rkisp_update_regs(struct rkisp_device *dev, u32 start, u32 end)
147 {
148 struct rkisp_hw_dev *hw = dev->hw_dev;
149 void __iomem *base = hw->base_addr;
150 u32 i;
151
152 if (end > RKISP_ISP_SW_REG_SIZE - 4) {
153 dev_err(dev->dev, "%s out of range\n", __func__);
154 return;
155 }
156 for (i = start; i <= end; i += 4) {
157 u32 *val = dev->sw_base_addr + i;
158 u32 *flag = dev->sw_base_addr + i + RKISP_ISP_SW_REG_SIZE;
159
160 if (dev->procfs.mode & RKISP_PROCFS_FIL_SW) {
161 if (!((i >= ISP3X_ISP_ACQ_H_OFFS && i <= ISP3X_ISP_ACQ_V_SIZE) ||
162 (i >= ISP3X_ISP_OUT_H_OFFS && i <= ISP3X_ISP_OUT_V_SIZE) ||
163 (i >= ISP3X_DUAL_CROP_BASE && i < ISP3X_DUAL_CROP_FBC_V_SIZE_SHD) ||
164 (i >= ISP3X_MAIN_RESIZE_BASE && i < ISP3X_CSI2RX_VERSION) ||
165 (i >= CIF_ISP_IS_BASE && i < CIF_ISP_IS_V_SIZE_SHD)))
166 continue;
167 }
168
169 if (*flag == SW_REG_CACHE) {
170 if ((i == ISP3X_MAIN_RESIZE_CTRL ||
171 i == ISP32_BP_RESIZE_CTRL ||
172 i == ISP3X_SELF_RESIZE_CTRL) && *val == 0)
173 *val = CIF_RSZ_CTRL_CFG_UPD;
174 writel(*val, base + i);
175 if (hw->is_unite) {
176 val = dev->sw_base_addr + i + RKISP_ISP_SW_MAX_SIZE;
177 if ((i == ISP3X_MAIN_RESIZE_CTRL ||
178 i == ISP32_BP_RESIZE_CTRL ||
179 i == ISP3X_SELF_RESIZE_CTRL) && *val == 0)
180 *val = CIF_RSZ_CTRL_CFG_UPD;
181 writel(*val, hw->base_next_addr + i);
182 }
183 }
184 }
185 }
186
rkisp_alloc_buffer(struct rkisp_device * dev,struct rkisp_dummy_buffer * buf)187 int rkisp_alloc_buffer(struct rkisp_device *dev,
188 struct rkisp_dummy_buffer *buf)
189 {
190 unsigned long attrs = buf->is_need_vaddr ? 0 : DMA_ATTR_NO_KERNEL_MAPPING;
191 const struct vb2_mem_ops *g_ops = dev->hw_dev->mem_ops;
192 struct sg_table *sg_tbl;
193 void *mem_priv;
194 int ret = 0;
195
196 mutex_lock(&dev->buf_lock);
197
198 if (!buf->size) {
199 ret = -EINVAL;
200 goto err;
201 }
202
203 if (dev->hw_dev->is_dma_contig)
204 attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
205 buf->size = PAGE_ALIGN(buf->size);
206 mem_priv = g_ops->alloc(dev->hw_dev->dev, attrs, buf->size,
207 DMA_BIDIRECTIONAL, GFP_KERNEL | GFP_DMA32);
208 if (IS_ERR_OR_NULL(mem_priv)) {
209 ret = -ENOMEM;
210 goto err;
211 }
212
213 buf->mem_priv = mem_priv;
214 if (dev->hw_dev->is_dma_sg_ops) {
215 sg_tbl = (struct sg_table *)g_ops->cookie(mem_priv);
216 buf->dma_addr = sg_dma_address(sg_tbl->sgl);
217 g_ops->prepare(mem_priv);
218 } else {
219 buf->dma_addr = *((dma_addr_t *)g_ops->cookie(mem_priv));
220 }
221 if (buf->is_need_vaddr)
222 buf->vaddr = g_ops->vaddr(mem_priv);
223 if (buf->is_need_dbuf) {
224 buf->dbuf = g_ops->get_dmabuf(mem_priv, O_RDWR);
225 if (buf->is_need_dmafd) {
226 buf->dma_fd = dma_buf_fd(buf->dbuf, O_CLOEXEC);
227 if (buf->dma_fd < 0) {
228 dma_buf_put(buf->dbuf);
229 ret = buf->dma_fd;
230 goto err;
231 }
232 get_dma_buf(buf->dbuf);
233 }
234 }
235 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
236 "%s buf:0x%x~0x%x size:%d\n", __func__,
237 (u32)buf->dma_addr, (u32)buf->dma_addr + buf->size, buf->size);
238 mutex_unlock(&dev->buf_lock);
239 return ret;
240 err:
241 mutex_unlock(&dev->buf_lock);
242 dev_err(dev->dev, "%s failed ret:%d\n", __func__, ret);
243 return ret;
244 }
245
rkisp_free_buffer(struct rkisp_device * dev,struct rkisp_dummy_buffer * buf)246 void rkisp_free_buffer(struct rkisp_device *dev,
247 struct rkisp_dummy_buffer *buf)
248 {
249 const struct vb2_mem_ops *g_ops = dev->hw_dev->mem_ops;
250
251 mutex_lock(&dev->buf_lock);
252 if (buf && buf->mem_priv) {
253 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
254 "%s buf:0x%x~0x%x\n", __func__,
255 (u32)buf->dma_addr, (u32)buf->dma_addr + buf->size);
256 if (buf->dbuf)
257 dma_buf_put(buf->dbuf);
258 g_ops->put(buf->mem_priv);
259 buf->size = 0;
260 buf->dbuf = NULL;
261 buf->vaddr = NULL;
262 buf->mem_priv = NULL;
263 buf->is_need_dbuf = false;
264 buf->is_need_vaddr = false;
265 buf->is_need_dmafd = false;
266 }
267 mutex_unlock(&dev->buf_lock);
268 }
269
rkisp_prepare_buffer(struct rkisp_device * dev,struct rkisp_dummy_buffer * buf)270 void rkisp_prepare_buffer(struct rkisp_device *dev,
271 struct rkisp_dummy_buffer *buf)
272 {
273 const struct vb2_mem_ops *g_ops = dev->hw_dev->mem_ops;
274
275 if (buf && buf->mem_priv)
276 g_ops->prepare(buf->mem_priv);
277 }
278
rkisp_finish_buffer(struct rkisp_device * dev,struct rkisp_dummy_buffer * buf)279 void rkisp_finish_buffer(struct rkisp_device *dev,
280 struct rkisp_dummy_buffer *buf)
281 {
282 const struct vb2_mem_ops *g_ops = dev->hw_dev->mem_ops;
283
284 if (buf && buf->mem_priv)
285 g_ops->finish(buf->mem_priv);
286 }
287
rkisp_attach_hw(struct rkisp_device * isp)288 int rkisp_attach_hw(struct rkisp_device *isp)
289 {
290 struct device_node *np;
291 struct platform_device *pdev;
292 struct rkisp_hw_dev *hw;
293
294 np = of_parse_phandle(isp->dev->of_node, "rockchip,hw", 0);
295 if (!np || !of_device_is_available(np)) {
296 dev_err(isp->dev, "failed to get isp hw node\n");
297 return -ENODEV;
298 }
299
300 pdev = of_find_device_by_node(np);
301 of_node_put(np);
302 if (!pdev) {
303 dev_err(isp->dev, "failed to get isp hw from node\n");
304 return -ENODEV;
305 }
306
307 hw = platform_get_drvdata(pdev);
308 if (!hw) {
309 dev_err(isp->dev, "failed attach isp hw\n");
310 return -EINVAL;
311 }
312
313 isp->dev_id = hw->dev_num;
314 hw->isp[hw->dev_num] = isp;
315 hw->dev_num++;
316 isp->hw_dev = hw;
317 isp->isp_ver = hw->isp_ver;
318 isp->base_addr = hw->base_addr;
319
320 return 0;
321 }
322
rkisp_alloc_page_dummy_buf(struct rkisp_device * dev,u32 size)323 static int rkisp_alloc_page_dummy_buf(struct rkisp_device *dev, u32 size)
324 {
325 struct rkisp_hw_dev *hw = dev->hw_dev;
326 struct rkisp_dummy_buffer *dummy_buf = &hw->dummy_buf;
327 u32 i, n_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
328 struct page *page = NULL, **pages = NULL;
329 struct sg_table *sg = NULL;
330 int ret = -ENOMEM;
331
332 page = alloc_pages(GFP_KERNEL | GFP_DMA32, 0);
333 if (!page)
334 goto err;
335
336 pages = kvmalloc_array(n_pages, sizeof(struct page *), GFP_KERNEL);
337 if (!pages)
338 goto free_page;
339 for (i = 0; i < n_pages; i++)
340 pages[i] = page;
341
342 sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
343 if (!sg)
344 goto free_pages;
345 ret = sg_alloc_table_from_pages(sg, pages, n_pages, 0,
346 n_pages << PAGE_SHIFT, GFP_KERNEL);
347 if (ret)
348 goto free_sg;
349
350 ret = dma_map_sg(hw->dev, sg->sgl, sg->nents, DMA_BIDIRECTIONAL);
351 dummy_buf->dma_addr = sg_dma_address(sg->sgl);
352 dummy_buf->mem_priv = sg;
353 dummy_buf->pages = pages;
354 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
355 "%s buf:0x%x map cnt:%d size:%d\n", __func__,
356 (u32)dummy_buf->dma_addr, ret, size);
357 return 0;
358 free_sg:
359 kfree(sg);
360 free_pages:
361 kvfree(pages);
362 free_page:
363 __free_pages(page, 0);
364 err:
365 return ret;
366 }
367
rkisp_free_page_dummy_buf(struct rkisp_device * dev)368 static void rkisp_free_page_dummy_buf(struct rkisp_device *dev)
369 {
370 struct rkisp_dummy_buffer *dummy_buf = &dev->hw_dev->dummy_buf;
371 struct sg_table *sg = dummy_buf->mem_priv;
372
373 if (!sg)
374 return;
375 dma_unmap_sg(dev->hw_dev->dev, sg->sgl, sg->nents, DMA_BIDIRECTIONAL);
376 sg_free_table(sg);
377 kfree(sg);
378 __free_pages(dummy_buf->pages[0], 0);
379 kvfree(dummy_buf->pages);
380 dummy_buf->mem_priv = NULL;
381 dummy_buf->pages = NULL;
382 }
383
rkisp_alloc_common_dummy_buf(struct rkisp_device * dev)384 int rkisp_alloc_common_dummy_buf(struct rkisp_device *dev)
385 {
386 struct rkisp_hw_dev *hw = dev->hw_dev;
387 struct rkisp_dummy_buffer *dummy_buf = &hw->dummy_buf;
388 struct rkisp_stream *stream;
389 struct rkisp_device *isp;
390 u32 i, j, val, size = 0;
391 int ret = 0;
392
393 if (dummy_buf->mem_priv)
394 goto end;
395
396 if (hw->max_in.w && hw->max_in.h)
397 size = hw->max_in.w * hw->max_in.h * 2;
398 for (i = 0; i < hw->dev_num; i++) {
399 isp = hw->isp[i];
400 if (!isp || (isp && !isp->is_hw_link))
401 continue;
402 for (j = 0; j < RKISP_MAX_STREAM; j++) {
403 stream = &isp->cap_dev.stream[j];
404 if (!stream->linked)
405 continue;
406 val = stream->out_isp_fmt.fmt_type == FMT_FBC ?
407 stream->out_fmt.plane_fmt[1].sizeimage :
408 stream->out_fmt.plane_fmt[0].bytesperline *
409 stream->out_fmt.height;
410 size = max(size, val);
411 }
412 }
413 if (size == 0)
414 goto end;
415
416 if (hw->is_mmu) {
417 ret = rkisp_alloc_page_dummy_buf(dev, size);
418 goto end;
419 }
420
421 dummy_buf->size = size;
422 ret = rkisp_alloc_buffer(dev, dummy_buf);
423 if (!ret)
424 v4l2_dbg(1, rkisp_debug, &dev->v4l2_dev,
425 "%s buf:0x%x size:%d\n", __func__,
426 (u32)dummy_buf->dma_addr, dummy_buf->size);
427 end:
428 if (ret < 0)
429 v4l2_err(&dev->v4l2_dev, "%s failed:%d\n", __func__, ret);
430 return ret;
431 }
432
rkisp_free_common_dummy_buf(struct rkisp_device * dev)433 void rkisp_free_common_dummy_buf(struct rkisp_device *dev)
434 {
435 struct rkisp_hw_dev *hw = dev->hw_dev;
436
437 if (atomic_read(&hw->refcnt) ||
438 atomic_read(&dev->cap_dev.refcnt) > 1)
439 return;
440
441 if (hw->is_mmu)
442 rkisp_free_page_dummy_buf(dev);
443 else
444 rkisp_free_buffer(dev, &hw->dummy_buf);
445 }
446