97 static void cio2_fbpt_exit_dummy(struct cio2_device *cio2)  in cio2_fbpt_exit_dummy()  argument
99 	if (cio2->dummy_lop) {  in cio2_fbpt_exit_dummy()
100 		dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_exit_dummy()
101 				  cio2->dummy_lop, cio2->dummy_lop_bus_addr);  in cio2_fbpt_exit_dummy()
102 		cio2->dummy_lop = NULL;  in cio2_fbpt_exit_dummy()
104 	if (cio2->dummy_page) {  in cio2_fbpt_exit_dummy()
105 		dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_exit_dummy()
106 				  cio2->dummy_page, cio2->dummy_page_bus_addr);  in cio2_fbpt_exit_dummy()
107 		cio2->dummy_page = NULL;  in cio2_fbpt_exit_dummy()
111 static int cio2_fbpt_init_dummy(struct cio2_device *cio2)  in cio2_fbpt_init_dummy()  argument
115 	cio2->dummy_page = dma_alloc_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_init_dummy()
116 					      &cio2->dummy_page_bus_addr,  in cio2_fbpt_init_dummy()
118 	cio2->dummy_lop = dma_alloc_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_fbpt_init_dummy()
119 					     &cio2->dummy_lop_bus_addr,  in cio2_fbpt_init_dummy()
121 	if (!cio2->dummy_page || !cio2->dummy_lop) {  in cio2_fbpt_init_dummy()
122 		cio2_fbpt_exit_dummy(cio2);  in cio2_fbpt_init_dummy()
130 		cio2->dummy_lop[i] = PFN_DOWN(cio2->dummy_page_bus_addr);  in cio2_fbpt_init_dummy()
135 static void cio2_fbpt_entry_enable(struct cio2_device *cio2,  in cio2_fbpt_entry_enable()  argument
155 static void cio2_fbpt_entry_init_dummy(struct cio2_device *cio2,  in cio2_fbpt_entry_init_dummy()  argument
166 		entry[i].lop_page_addr = PFN_DOWN(cio2->dummy_lop_bus_addr);  in cio2_fbpt_entry_init_dummy()
168 	cio2_fbpt_entry_enable(cio2, entry);  in cio2_fbpt_entry_init_dummy()
172 static void cio2_fbpt_entry_init_buf(struct cio2_device *cio2,  in cio2_fbpt_entry_init_buf()  argument
208 	entry->lop_page_addr = PFN_DOWN(cio2->dummy_lop_bus_addr);  in cio2_fbpt_entry_init_buf()
210 	cio2_fbpt_entry_enable(cio2, entry);  in cio2_fbpt_entry_init_buf()
213 static int cio2_fbpt_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_fbpt_init()  argument
215 	struct device *dev = &cio2->pci_dev->dev;  in cio2_fbpt_init()
290 static int cio2_csi2_calc_timing(struct cio2_device *cio2, struct cio2_queue *q,  in cio2_csi2_calc_timing()  argument
293 	struct device *dev = &cio2->pci_dev->dev;  in cio2_csi2_calc_timing()
346 static int cio2_hw_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_hw_init()  argument
355 	void __iomem *const base = cio2->base;  in cio2_hw_init()
367 	r = cio2_csi2_calc_timing(cio2, q, &timing);  in cio2_hw_init()
508 static void cio2_hw_exit(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_hw_exit()  argument
510 	void __iomem *const base = cio2->base;  in cio2_hw_exit()
527 		dev_err(&cio2->pci_dev->dev,  in cio2_hw_exit()
538 static void cio2_buffer_done(struct cio2_device *cio2, unsigned int dma_chan)  in cio2_buffer_done()  argument
540 	struct device *dev = &cio2->pci_dev->dev;  in cio2_buffer_done()
541 	struct cio2_queue *q = cio2->cur_queue;  in cio2_buffer_done()
552 		dev_warn(&cio2->pci_dev->dev,  in cio2_buffer_done()
568 			dev_dbg(&cio2->pci_dev->dev,  in cio2_buffer_done()
581 		cio2_fbpt_entry_init_dummy(cio2, entry);  in cio2_buffer_done()
587 static void cio2_queue_event_sof(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_event_sof()  argument
633 static void cio2_irq_handle_once(struct cio2_device *cio2, u32 int_status)  in cio2_irq_handle_once()  argument
635 	void __iomem *const base = cio2->base;  in cio2_irq_handle_once()
636 	struct device *dev = &cio2->pci_dev->dev;  in cio2_irq_handle_once()
676 				cio2_buffer_done(cio2, d);  in cio2_irq_handle_once()
690 					cio2_queue_event_sof(cio2,  in cio2_irq_handle_once()
691 							     cio2->cur_queue);  in cio2_irq_handle_once()
761 	struct cio2_device *cio2 = cio2_ptr;  in cio2_irq()  local
762 	void __iomem *const base = cio2->base;  in cio2_irq()
763 	struct device *dev = &cio2->pci_dev->dev;  in cio2_irq()
773 		cio2_irq_handle_once(cio2, int_status);  in cio2_irq()
805 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_queue_setup()  local
813 		alloc_devs[i] = &cio2->pci_dev->dev;  in cio2_vb2_queue_setup()
821 		cio2_fbpt_entry_init_dummy(cio2, &q->fbpt[i * CIO2_MAX_LOPS]);  in cio2_vb2_queue_setup()
833 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_init()  local
834 	struct device *dev = &cio2->pci_dev->dev;  in cio2_vb2_buf_init()
878 	b->lop[i][j] = PFN_DOWN(cio2->dummy_page_bus_addr);  in cio2_vb2_buf_init()
889 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_queue()  local
900 	dev_dbg(&cio2->pci_dev->dev, "queue buffer %d\n", vb->index);  in cio2_vb2_buf_queue()
917 	fbpt_rp = (readl(cio2->base + CIO2_REG_CDMARI(CIO2_DMA_CHAN))  in cio2_vb2_buf_queue()
942 			cio2_fbpt_entry_init_buf(cio2, b, entry);  in cio2_vb2_buf_queue()
951 		dev_dbg(&cio2->pci_dev->dev, "entry %i was full!\n", next);  in cio2_vb2_buf_queue()
956 	dev_err(&cio2->pci_dev->dev, "error: all cio2 entries were full!\n");  in cio2_vb2_buf_queue()
964 	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);  in cio2_vb2_buf_cleanup()  local
972 			dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,  in cio2_vb2_buf_cleanup()
980 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_start_streaming()  local
983 	cio2->cur_queue = q;  in cio2_vb2_start_streaming()
986 	r = pm_runtime_get_sync(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
988 		dev_info(&cio2->pci_dev->dev, "failed to set power %d\n", r);  in cio2_vb2_start_streaming()
989 		pm_runtime_put_noidle(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
997 	r = cio2_hw_init(cio2, q);  in cio2_vb2_start_streaming()
1006 	cio2->streaming = true;  in cio2_vb2_start_streaming()
1011 	cio2_hw_exit(cio2, q);  in cio2_vb2_start_streaming()
1015 	dev_dbg(&cio2->pci_dev->dev, "failed to start streaming (%d)\n", r);  in cio2_vb2_start_streaming()
1017 	pm_runtime_put(&cio2->pci_dev->dev);  in cio2_vb2_start_streaming()
1025 	struct cio2_device *cio2 = vb2_get_drv_priv(vq);  in cio2_vb2_stop_streaming()  local
1028 		dev_err(&cio2->pci_dev->dev,  in cio2_vb2_stop_streaming()
1031 	cio2_hw_exit(cio2, q);  in cio2_vb2_stop_streaming()
1032 	synchronize_irq(cio2->pci_dev->irq);  in cio2_vb2_stop_streaming()
1035 	pm_runtime_put(&cio2->pci_dev->dev);  in cio2_vb2_stop_streaming()
1036 	cio2->streaming = false;  in cio2_vb2_stop_streaming()
1055 	struct cio2_device *cio2 = video_drvdata(file);  in cio2_v4l2_querycap()  local
1060 		 "PCI:%s", pci_name(cio2->pci_dev));  in cio2_v4l2_querycap()
1329 	struct cio2_device *cio2 = video_get_drvdata(vd);  in cio2_video_link_validate()  local
1334 		dev_info(&cio2->pci_dev->dev,  in cio2_video_link_validate()
1345 		dev_err(&cio2->pci_dev->dev,  in cio2_video_link_validate()
1391 	struct cio2_device *cio2 = container_of(notifier,  in cio2_notifier_bound()  local
1397 	if (cio2->queue[s_asd->csi2.port].sensor)  in cio2_notifier_bound()
1400 	q = &cio2->queue[s_asd->csi2.port];  in cio2_notifier_bound()
1404 	q->csi_rx_base = cio2->base + CIO2_REG_PIPE_BASE(q->csi2.port);  in cio2_notifier_bound()
1414 	struct cio2_device *cio2 = container_of(notifier,  in cio2_notifier_unbind()  local
1419 	cio2->queue[s_asd->csi2.port].sensor = NULL;  in cio2_notifier_unbind()
1425 	struct cio2_device *cio2 = container_of(notifier, struct cio2_device,  in cio2_notifier_complete()  local
1433 	list_for_each_entry(asd, &cio2->notifier.asd_list, asd_list) {  in cio2_notifier_complete()
1435 		q = &cio2->queue[s_asd->csi2.port];  in cio2_notifier_complete()
1443 			dev_err(&cio2->pci_dev->dev,  in cio2_notifier_complete()
1454 			dev_err(&cio2->pci_dev->dev,  in cio2_notifier_complete()
1461 	return v4l2_device_register_subdev_nodes(&cio2->v4l2_dev);  in cio2_notifier_complete()
1470 static int cio2_parse_firmware(struct cio2_device *cio2)  in cio2_parse_firmware()  argument
1484 			dev_fwnode(&cio2->pci_dev->dev), i, 0,  in cio2_parse_firmware()
1495 				&cio2->notifier, ep, sizeof(*s_asd));  in cio2_parse_firmware()
1518 	cio2->notifier.ops = &cio2_async_ops;  in cio2_parse_firmware()
1519 	ret = v4l2_async_notifier_register(&cio2->v4l2_dev, &cio2->notifier);  in cio2_parse_firmware()
1521 		dev_err(&cio2->pci_dev->dev,  in cio2_parse_firmware()
1536 static int cio2_queue_init(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_init()  argument
1571 	r = cio2_fbpt_init(cio2, q);  in cio2_queue_init()
1583 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1592 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1602 		 CIO2_ENTITY_NAME " %td", q - cio2->queue);  in cio2_queue_init()
1604 	v4l2_set_subdevdata(subdev, cio2);  in cio2_queue_init()
1605 	r = v4l2_device_register_subdev(&cio2->v4l2_dev, subdev);  in cio2_queue_init()
1607 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1620 	vbq->drv_priv = cio2;  in cio2_queue_init()
1624 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1631 		 "%s %td", CIO2_NAME, q - cio2->queue);  in cio2_queue_init()
1635 	vdev->lock = &cio2->lock;  in cio2_queue_init()
1636 	vdev->v4l2_dev = &cio2->v4l2_dev;  in cio2_queue_init()
1639 	video_set_drvdata(vdev, cio2);  in cio2_queue_init()
1642 		dev_err(&cio2->pci_dev->dev,  in cio2_queue_init()
1665 	cio2_fbpt_exit(q, &cio2->pci_dev->dev);  in cio2_queue_init()
1673 static void cio2_queue_exit(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_queue_exit()  argument
1679 	cio2_fbpt_exit(q, &cio2->pci_dev->dev);  in cio2_queue_exit()
1684 static int cio2_queues_init(struct cio2_device *cio2)  in cio2_queues_init()  argument
1689 		r = cio2_queue_init(cio2, &cio2->queue[i]);  in cio2_queues_init()
1698 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_queues_init()
1703 static void cio2_queues_exit(struct cio2_device *cio2)  in cio2_queues_exit()  argument
1708 		cio2_queue_exit(cio2, &cio2->queue[i]);  in cio2_queues_exit()
1716 	struct cio2_device *cio2;  in cio2_pci_probe()  local
1719 	cio2 = devm_kzalloc(&pci_dev->dev, sizeof(*cio2), GFP_KERNEL);  in cio2_pci_probe()
1720 	if (!cio2)  in cio2_pci_probe()
1722 	cio2->pci_dev = pci_dev;  in cio2_pci_probe()
1739 	cio2->base = pcim_iomap_table(pci_dev)[CIO2_PCI_BAR];  in cio2_pci_probe()
1741 	pci_set_drvdata(pci_dev, cio2);  in cio2_pci_probe()
1757 	r = cio2_fbpt_init_dummy(cio2);  in cio2_pci_probe()
1761 	mutex_init(&cio2->lock);  in cio2_pci_probe()
1763 	cio2->media_dev.dev = &cio2->pci_dev->dev;  in cio2_pci_probe()
1764 	strscpy(cio2->media_dev.model, CIO2_DEVICE_NAME,  in cio2_pci_probe()
1765 		sizeof(cio2->media_dev.model));  in cio2_pci_probe()
1766 	snprintf(cio2->media_dev.bus_info, sizeof(cio2->media_dev.bus_info),  in cio2_pci_probe()
1767 		 "PCI:%s", pci_name(cio2->pci_dev));  in cio2_pci_probe()
1768 	cio2->media_dev.hw_revision = 0;  in cio2_pci_probe()
1770 	media_device_init(&cio2->media_dev);  in cio2_pci_probe()
1771 	r = media_device_register(&cio2->media_dev);  in cio2_pci_probe()
1775 	cio2->v4l2_dev.mdev = &cio2->media_dev;  in cio2_pci_probe()
1776 	r = v4l2_device_register(&pci_dev->dev, &cio2->v4l2_dev);  in cio2_pci_probe()
1783 	r = cio2_queues_init(cio2);  in cio2_pci_probe()
1787 	v4l2_async_notifier_init(&cio2->notifier);  in cio2_pci_probe()
1790 	r = cio2_parse_firmware(cio2);  in cio2_pci_probe()
1795 			     IRQF_SHARED, CIO2_NAME, cio2);  in cio2_pci_probe()
1807 	v4l2_async_notifier_unregister(&cio2->notifier);  in cio2_pci_probe()
1808 	v4l2_async_notifier_cleanup(&cio2->notifier);  in cio2_pci_probe()
1809 	cio2_queues_exit(cio2);  in cio2_pci_probe()
1811 	v4l2_device_unregister(&cio2->v4l2_dev);  in cio2_pci_probe()
1813 	media_device_unregister(&cio2->media_dev);  in cio2_pci_probe()
1814 	media_device_cleanup(&cio2->media_dev);  in cio2_pci_probe()
1816 	mutex_destroy(&cio2->lock);  in cio2_pci_probe()
1817 	cio2_fbpt_exit_dummy(cio2);  in cio2_pci_probe()
1824 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_pci_remove()  local
1826 	media_device_unregister(&cio2->media_dev);  in cio2_pci_remove()
1827 	v4l2_async_notifier_unregister(&cio2->notifier);  in cio2_pci_remove()
1828 	v4l2_async_notifier_cleanup(&cio2->notifier);  in cio2_pci_remove()
1829 	cio2_queues_exit(cio2);  in cio2_pci_remove()
1830 	cio2_fbpt_exit_dummy(cio2);  in cio2_pci_remove()
1831 	v4l2_device_unregister(&cio2->v4l2_dev);  in cio2_pci_remove()
1832 	media_device_cleanup(&cio2->media_dev);  in cio2_pci_remove()
1833 	mutex_destroy(&cio2->lock);  in cio2_pci_remove()
1839 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_runtime_suspend()  local
1840 	void __iomem *const base = cio2->base;  in cio2_runtime_suspend()
1857 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_runtime_resume()  local
1858 	void __iomem *const base = cio2->base;  in cio2_runtime_resume()
1921 static void cio2_fbpt_rearrange(struct cio2_device *cio2, struct cio2_queue *q)  in cio2_fbpt_rearrange()  argument
1948 		cio2_fbpt_entry_enable(cio2, q->fbpt + i * CIO2_MAX_LOPS);  in cio2_fbpt_rearrange()
1954 	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);  in cio2_suspend()  local
1955 	struct cio2_queue *q = cio2->cur_queue;  in cio2_suspend()
1958 	if (!cio2->streaming)  in cio2_suspend()
1962 	cio2_hw_exit(cio2, q);  in cio2_suspend()
1971 	cio2_fbpt_rearrange(cio2, q);  in cio2_suspend()
1980 	struct cio2_device *cio2 = dev_get_drvdata(dev);  in cio2_resume()  local
1981 	struct cio2_queue *q = cio2->cur_queue;  in cio2_resume()
1985 	if (!cio2->streaming)  in cio2_resume()
1988 	r = pm_runtime_force_resume(&cio2->pci_dev->dev);  in cio2_resume()
1990 		dev_err(&cio2->pci_dev->dev,  in cio2_resume()
1995 	r = cio2_hw_init(cio2, q);  in cio2_resume()