241 static inline bool imxdma_chan_is_doing_cyclic(struct imxdma_channel *imxdmac)  in imxdma_chan_is_doing_cyclic()  argument
245 	if (!list_empty(&imxdmac->ld_active)) {  in imxdma_chan_is_doing_cyclic()
246 		desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc,  in imxdma_chan_is_doing_cyclic()
267 static int imxdma_hw_chain(struct imxdma_channel *imxdmac)  in imxdma_hw_chain()  argument
269 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_hw_chain()
272 		return imxdmac->hw_chaining;  in imxdma_hw_chain()
282 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_sg_next()  local
283 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_sg_next()
293 				 DMA_DAR(imxdmac->channel));  in imxdma_sg_next()
296 				 DMA_SAR(imxdmac->channel));  in imxdma_sg_next()
298 	imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));  in imxdma_sg_next()
301 		"size 0x%08x\n", __func__, imxdmac->channel,  in imxdma_sg_next()
302 		 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),  in imxdma_sg_next()
303 		 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),  in imxdma_sg_next()
304 		 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));  in imxdma_sg_next()
309 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_enable_hw()  local
310 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_enable_hw()
311 	int channel = imxdmac->channel;  in imxdma_enable_hw()
325 			d->sg && imxdma_hw_chain(imxdmac)) {  in imxdma_enable_hw()
339 static void imxdma_disable_hw(struct imxdma_channel *imxdmac)  in imxdma_disable_hw()  argument
341 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_disable_hw()
342 	int channel = imxdmac->channel;  in imxdma_disable_hw()
347 	if (imxdma_hw_chain(imxdmac))  in imxdma_disable_hw()
348 		del_timer(&imxdmac->watchdog);  in imxdma_disable_hw()
361 	struct imxdma_channel *imxdmac = from_timer(imxdmac, t, watchdog);  in imxdma_watchdog()  local
362 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_watchdog()
363 	int channel = imxdmac->channel;  in imxdma_watchdog()
368 	tasklet_schedule(&imxdmac->dma_tasklet);  in imxdma_watchdog()
370 		imxdmac->channel);  in imxdma_watchdog()
426 static void dma_irq_handle_channel(struct imxdma_channel *imxdmac)  in dma_irq_handle_channel()  argument
428 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in dma_irq_handle_channel()
429 	int chno = imxdmac->channel;  in dma_irq_handle_channel()
434 	if (list_empty(&imxdmac->ld_active)) {  in dma_irq_handle_channel()
439 	desc = list_first_entry(&imxdmac->ld_active,  in dma_irq_handle_channel()
453 			if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
457 				mod_timer(&imxdmac->watchdog,  in dma_irq_handle_channel()
470 			if (imxdma_chan_is_doing_cyclic(imxdmac))  in dma_irq_handle_channel()
472 				tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
477 		if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
478 			del_timer(&imxdmac->watchdog);  in dma_irq_handle_channel()
486 	tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
512 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_xfer_desc()  local
513 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_xfer_desc()
538 		imxdmac->slot_2d = slot;  in imxdma_xfer_desc()
539 		imxdmac->enabled_2d = true;  in imxdma_xfer_desc()
560 		imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
561 		imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
563 			 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
565 		imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));  in imxdma_xfer_desc()
569 			__func__, imxdmac->channel,  in imxdma_xfer_desc()
578 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
579 					 DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
580 			imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,  in imxdma_xfer_desc()
581 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
585 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
587 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
589 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
590 					 DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
591 			imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,  in imxdma_xfer_desc()
592 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
596 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
598 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
601 				__func__, imxdmac->channel);  in imxdma_xfer_desc()
617 	struct imxdma_channel *imxdmac = from_tasklet(imxdmac, t, dma_tasklet);  in imxdma_tasklet()  local
618 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tasklet()
624 	if (list_empty(&imxdmac->ld_active)) {  in imxdma_tasklet()
629 	desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node);  in imxdma_tasklet()
635 	if (imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_tasklet()
641 	if (imxdmac->enabled_2d) {  in imxdma_tasklet()
642 		imxdma->slots_2d[imxdmac->slot_2d].count--;  in imxdma_tasklet()
643 		imxdmac->enabled_2d = false;  in imxdma_tasklet()
646 	list_move_tail(imxdmac->ld_active.next, &imxdmac->ld_free);  in imxdma_tasklet()
648 	if (!list_empty(&imxdmac->ld_queue)) {  in imxdma_tasklet()
649 		next_desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_tasklet()
651 		list_move_tail(imxdmac->ld_queue.next, &imxdmac->ld_active);  in imxdma_tasklet()
654 				 __func__, imxdmac->channel);  in imxdma_tasklet()
664 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_terminate_all()  local
665 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_terminate_all()
668 	imxdma_disable_hw(imxdmac);  in imxdma_terminate_all()
671 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_terminate_all()
672 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_terminate_all()
681 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config_write()  local
682 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_config_write()
686 		imxdmac->per_address = dmaengine_cfg->src_addr;  in imxdma_config_write()
687 		imxdmac->watermark_level = dmaengine_cfg->src_maxburst;  in imxdma_config_write()
688 		imxdmac->word_size = dmaengine_cfg->src_addr_width;  in imxdma_config_write()
690 		imxdmac->per_address = dmaengine_cfg->dst_addr;  in imxdma_config_write()
691 		imxdmac->watermark_level = dmaengine_cfg->dst_maxburst;  in imxdma_config_write()
692 		imxdmac->word_size = dmaengine_cfg->dst_addr_width;  in imxdma_config_write()
695 	switch (imxdmac->word_size) {  in imxdma_config_write()
708 	imxdmac->hw_chaining = 0;  in imxdma_config_write()
710 	imxdmac->ccr_from_device = (mode | IMX_DMA_TYPE_FIFO) |  in imxdma_config_write()
713 	imxdmac->ccr_to_device =  in imxdma_config_write()
716 	imx_dmav1_writel(imxdma, imxdmac->dma_request,  in imxdma_config_write()
717 			 DMA_RSSR(imxdmac->channel));  in imxdma_config_write()
720 	imx_dmav1_writel(imxdma, imxdmac->watermark_level *  in imxdma_config_write()
721 			 imxdmac->word_size, DMA_BLR(imxdmac->channel));  in imxdma_config_write()
729 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config()  local
731 	memcpy(&imxdmac->config, dmaengine_cfg, sizeof(*dmaengine_cfg));  in imxdma_config()
745 	struct imxdma_channel *imxdmac = to_imxdma_chan(tx->chan);  in imxdma_tx_submit()  local
746 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tx_submit()
751 	list_move_tail(imxdmac->ld_free.next, &imxdmac->ld_queue);  in imxdma_tx_submit()
760 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_alloc_chan_resources()  local
764 		imxdmac->dma_request = data->dma_request;  in imxdma_alloc_chan_resources()
766 	while (imxdmac->descs_allocated < IMXDMA_MAX_CHAN_DESCRIPTORS) {  in imxdma_alloc_chan_resources()
779 		list_add_tail(&desc->node, &imxdmac->ld_free);  in imxdma_alloc_chan_resources()
780 		imxdmac->descs_allocated++;  in imxdma_alloc_chan_resources()
783 	if (!imxdmac->descs_allocated)  in imxdma_alloc_chan_resources()
786 	return imxdmac->descs_allocated;  in imxdma_alloc_chan_resources()
791 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_free_chan_resources()  local
792 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_free_chan_resources()
798 	imxdma_disable_hw(imxdmac);  in imxdma_free_chan_resources()
799 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_free_chan_resources()
800 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_free_chan_resources()
804 	list_for_each_entry_safe(desc, _desc, &imxdmac->ld_free, node) {  in imxdma_free_chan_resources()
806 		imxdmac->descs_allocated--;  in imxdma_free_chan_resources()
808 	INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_free_chan_resources()
810 	kfree(imxdmac->sg_list);  in imxdma_free_chan_resources()
811 	imxdmac->sg_list = NULL;  in imxdma_free_chan_resources()
819 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_slave_sg()  local
824 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_slave_sg()
825 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_slave_sg()
828 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_slave_sg()
834 	imxdma_config_write(chan, &imxdmac->config, direction);  in imxdma_prep_slave_sg()
836 	switch (imxdmac->word_size) {  in imxdma_prep_slave_sg()
857 		desc->src = imxdmac->per_address;  in imxdma_prep_slave_sg()
859 		desc->dest = imxdmac->per_address;  in imxdma_prep_slave_sg()
872 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_cyclic()  local
873 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_cyclic()
879 			__func__, imxdmac->channel, buf_len, period_len);  in imxdma_prep_dma_cyclic()
881 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_cyclic()
882 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_cyclic()
885 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_cyclic()
887 	kfree(imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
889 	imxdmac->sg_list = kcalloc(periods + 1,  in imxdma_prep_dma_cyclic()
891 	if (!imxdmac->sg_list)  in imxdma_prep_dma_cyclic()
894 	sg_init_table(imxdmac->sg_list, periods);  in imxdma_prep_dma_cyclic()
897 		sg_assign_page(&imxdmac->sg_list[i], NULL);  in imxdma_prep_dma_cyclic()
898 		imxdmac->sg_list[i].offset = 0;  in imxdma_prep_dma_cyclic()
899 		imxdmac->sg_list[i].dma_address = dma_addr;  in imxdma_prep_dma_cyclic()
900 		sg_dma_len(&imxdmac->sg_list[i]) = period_len;  in imxdma_prep_dma_cyclic()
905 	sg_chain(imxdmac->sg_list, periods + 1, imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
908 	desc->sg = imxdmac->sg_list;  in imxdma_prep_dma_cyclic()
913 		desc->src = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
915 		desc->dest = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
920 	imxdma_config_write(chan, &imxdmac->config, direction);  in imxdma_prep_dma_cyclic()
929 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_memcpy()  local
930 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_memcpy()
934 		__func__, imxdmac->channel, (unsigned long long)src,  in imxdma_prep_dma_memcpy()
937 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_memcpy()
938 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_memcpy()
941 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_memcpy()
960 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_interleaved()  local
961 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_interleaved()
966 		imxdmac->channel, (unsigned long long)xt->src_start,  in imxdma_prep_dma_interleaved()
971 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_interleaved()
972 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_interleaved()
978 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_interleaved()
1002 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_issue_pending()  local
1003 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_issue_pending()
1008 	if (list_empty(&imxdmac->ld_active) &&  in imxdma_issue_pending()
1009 	    !list_empty(&imxdmac->ld_queue)) {  in imxdma_issue_pending()
1010 		desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_issue_pending()
1016 				 __func__, imxdmac->channel);  in imxdma_issue_pending()
1018 			list_move_tail(imxdmac->ld_queue.next,  in imxdma_issue_pending()
1019 				       &imxdmac->ld_active);  in imxdma_issue_pending()
1151 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_probe()  local
1163 			imxdmac->irq = irq + i;  in imxdma_probe()
1164 			timer_setup(&imxdmac->watchdog, imxdma_watchdog, 0);  in imxdma_probe()
1167 		imxdmac->imxdma = imxdma;  in imxdma_probe()
1169 		INIT_LIST_HEAD(&imxdmac->ld_queue);  in imxdma_probe()
1170 		INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_probe()
1171 		INIT_LIST_HEAD(&imxdmac->ld_active);  in imxdma_probe()
1173 		tasklet_setup(&imxdmac->dma_tasklet, imxdma_tasklet);  in imxdma_probe()
1174 		imxdmac->chan.device = &imxdma->dma_device;  in imxdma_probe()
1175 		dma_cookie_init(&imxdmac->chan);  in imxdma_probe()
1176 		imxdmac->channel = i;  in imxdma_probe()
1179 		list_add_tail(&imxdmac->chan.device_node,  in imxdma_probe()
1237 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_free_irq()  local
1240 			disable_irq(imxdmac->irq);  in imxdma_free_irq()
1242 		tasklet_kill(&imxdmac->dma_tasklet);  in imxdma_free_irq()