103 static struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan)  in tdchantotd()  argument
105 	int id = td_chan->chan.chan_id;  in tdchantotd()
106 	return (struct timb_dma *)((u8 *)td_chan -  in tdchantotd()
111 static void __td_enable_chan_irq(struct timb_dma_chan *td_chan)  in __td_enable_chan_irq()  argument
113 	int id = td_chan->chan.chan_id;  in __td_enable_chan_irq()
114 	struct timb_dma *td = tdchantotd(td_chan);  in __td_enable_chan_irq()
120 	dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id,  in __td_enable_chan_irq()
126 static bool __td_dma_done_ack(struct timb_dma_chan *td_chan)  in __td_dma_done_ack()  argument
128 	int id = td_chan->chan.chan_id;  in __td_dma_done_ack()
129 	struct timb_dma *td = (struct timb_dma *)((u8 *)td_chan -  in __td_dma_done_ack()
134 	dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td);  in __td_dma_done_ack()
145 static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,  in td_fill_desc()  argument
149 		dev_err(chan2dev(&td_chan->chan), "Too big sg element\n");  in td_fill_desc()
155 		dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n",  in td_fill_desc()
160 	dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n",  in td_fill_desc()
178 static void __td_start_dma(struct timb_dma_chan *td_chan)  in __td_start_dma()  argument
182 	if (td_chan->ongoing) {  in __td_start_dma()
183 		dev_err(chan2dev(&td_chan->chan),  in __td_start_dma()
188 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,  in __td_start_dma()
191 	dev_dbg(chan2dev(&td_chan->chan),  in __td_start_dma()
193 		td_chan, td_chan->chan.chan_id, td_chan->membase);  in __td_start_dma()
195 	if (td_chan->direction == DMA_DEV_TO_MEM) {  in __td_start_dma()
198 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_DHAR);  in __td_start_dma()
199 		iowrite32(td_desc->txd.phys, td_chan->membase +  in __td_start_dma()
202 		iowrite32(td_chan->bytes_per_line, td_chan->membase +  in __td_start_dma()
205 		iowrite32(TIMBDMA_RX_EN, td_chan->membase + TIMBDMA_OFFS_RX_ER);  in __td_start_dma()
208 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DHAR);  in __td_start_dma()
209 		iowrite32(td_desc->txd.phys, td_chan->membase +  in __td_start_dma()
213 	td_chan->ongoing = true;  in __td_start_dma()
216 		__td_enable_chan_irq(td_chan);  in __td_start_dma()
219 static void __td_finish(struct timb_dma_chan *td_chan)  in __td_finish()  argument
226 	if (list_empty(&td_chan->active_list))  in __td_finish()
229 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,  in __td_finish()
233 	dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n",  in __td_finish()
237 	if (td_chan->direction == DMA_DEV_TO_MEM)  in __td_finish()
238 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_ER);  in __td_finish()
244 	td_chan->ongoing = false;  in __td_finish()
248 	list_move(&td_desc->desc_node, &td_chan->free_list);  in __td_finish()
264 		struct timb_dma_chan *td_chan = td->channels + i;  in __td_ier_mask()  local
265 		if (td_chan->ongoing) {  in __td_ier_mask()
267 				list_entry(td_chan->active_list.next,  in __td_ier_mask()
277 static void __td_start_next(struct timb_dma_chan *td_chan)  in __td_start_next()  argument
281 	BUG_ON(list_empty(&td_chan->queue));  in __td_start_next()
282 	BUG_ON(td_chan->ongoing);  in __td_start_next()
284 	td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc,  in __td_start_next()
287 	dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n",  in __td_start_next()
290 	list_move(&td_desc->desc_node, &td_chan->active_list);  in __td_start_next()
291 	__td_start_dma(td_chan);  in __td_start_next()
298 	struct timb_dma_chan *td_chan = container_of(txd->chan,  in td_tx_submit()  local
302 	spin_lock_bh(&td_chan->lock);  in td_tx_submit()
305 	if (list_empty(&td_chan->active_list)) {  in td_tx_submit()
308 		list_add_tail(&td_desc->desc_node, &td_chan->active_list);  in td_tx_submit()
309 		__td_start_dma(td_chan);  in td_tx_submit()
314 		list_add_tail(&td_desc->desc_node, &td_chan->queue);  in td_tx_submit()
317 	spin_unlock_bh(&td_chan->lock);  in td_tx_submit()
322 static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)  in td_alloc_init_desc()  argument
324 	struct dma_chan *chan = &td_chan->chan;  in td_alloc_init_desc()
332 	td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE;  in td_alloc_init_desc()
370 static void td_desc_put(struct timb_dma_chan *td_chan,  in td_desc_put()  argument
373 	dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc);  in td_desc_put()
375 	spin_lock_bh(&td_chan->lock);  in td_desc_put()
376 	list_add(&td_desc->desc_node, &td_chan->free_list);  in td_desc_put()
377 	spin_unlock_bh(&td_chan->lock);  in td_desc_put()
380 static struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan)  in td_desc_get()  argument
385 	spin_lock_bh(&td_chan->lock);  in td_desc_get()
386 	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->free_list,  in td_desc_get()
393 		dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n",  in td_desc_get()
396 	spin_unlock_bh(&td_chan->lock);  in td_desc_get()
403 	struct timb_dma_chan *td_chan =  in td_alloc_chan_resources()  local
409 	BUG_ON(!list_empty(&td_chan->free_list));  in td_alloc_chan_resources()
410 	for (i = 0; i < td_chan->descs; i++) {  in td_alloc_chan_resources()
411 		struct timb_dma_desc *td_desc = td_alloc_init_desc(td_chan);  in td_alloc_chan_resources()
422 		td_desc_put(td_chan, td_desc);  in td_alloc_chan_resources()
425 	spin_lock_bh(&td_chan->lock);  in td_alloc_chan_resources()
427 	spin_unlock_bh(&td_chan->lock);  in td_alloc_chan_resources()
434 	struct timb_dma_chan *td_chan =  in td_free_chan_resources()  local
442 	BUG_ON(!list_empty(&td_chan->active_list));  in td_free_chan_resources()
443 	BUG_ON(!list_empty(&td_chan->queue));  in td_free_chan_resources()
445 	spin_lock_bh(&td_chan->lock);  in td_free_chan_resources()
446 	list_splice_init(&td_chan->free_list, &list);  in td_free_chan_resources()
447 	spin_unlock_bh(&td_chan->lock);  in td_free_chan_resources()
472 	struct timb_dma_chan *td_chan =  in td_issue_pending()  local
476 	spin_lock_bh(&td_chan->lock);  in td_issue_pending()
478 	if (!list_empty(&td_chan->active_list))  in td_issue_pending()
480 		if (__td_dma_done_ack(td_chan))  in td_issue_pending()
481 			__td_finish(td_chan);  in td_issue_pending()
483 	if (list_empty(&td_chan->active_list) && !list_empty(&td_chan->queue))  in td_issue_pending()
484 		__td_start_next(td_chan);  in td_issue_pending()
486 	spin_unlock_bh(&td_chan->lock);  in td_issue_pending()
494 	struct timb_dma_chan *td_chan =  in td_prep_slave_sg()  local
507 	if (td_chan->direction != direction) {  in td_prep_slave_sg()
513 	td_desc = td_desc_get(td_chan);  in td_prep_slave_sg()
528 		err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg,  in td_prep_slave_sg()
533 			td_desc_put(td_chan, td_desc);  in td_prep_slave_sg()
547 	struct timb_dma_chan *td_chan =  in td_terminate_all()  local
554 	spin_lock_bh(&td_chan->lock);  in td_terminate_all()
555 	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->queue,  in td_terminate_all()
557 		list_move(&td_desc->desc_node, &td_chan->free_list);  in td_terminate_all()
560 	__td_finish(td_chan);  in td_terminate_all()
561 	spin_unlock_bh(&td_chan->lock);  in td_terminate_all()
582 			struct timb_dma_chan *td_chan = td->channels + i;  in td_tasklet()  local
583 			spin_lock(&td_chan->lock);  in td_tasklet()
584 			__td_finish(td_chan);  in td_tasklet()
585 			if (!list_empty(&td_chan->queue))  in td_tasklet()
586 				__td_start_next(td_chan);  in td_tasklet()
587 			spin_unlock(&td_chan->lock);  in td_tasklet()
684 		struct timb_dma_chan *td_chan = &td->channels[i];  in td_probe()  local
695 		td_chan->chan.device = &td->dma;  in td_probe()
696 		dma_cookie_init(&td_chan->chan);  in td_probe()
697 		spin_lock_init(&td_chan->lock);  in td_probe()
698 		INIT_LIST_HEAD(&td_chan->active_list);  in td_probe()
699 		INIT_LIST_HEAD(&td_chan->queue);  in td_probe()
700 		INIT_LIST_HEAD(&td_chan->free_list);  in td_probe()
702 		td_chan->descs = pchan->descriptors;  in td_probe()
703 		td_chan->desc_elems = pchan->descriptor_elements;  in td_probe()
704 		td_chan->bytes_per_line = pchan->bytes_per_line;  in td_probe()
705 		td_chan->direction = pchan->rx ? DMA_DEV_TO_MEM :  in td_probe()
708 		td_chan->membase = td->membase +  in td_probe()
713 			i, td_chan->membase);  in td_probe()
715 		list_add_tail(&td_chan->chan.device_node, &td->dma.channels);  in td_probe()