227 	struct edma_desc		*edesc;  member
779 	struct edma_desc *edesc;  in edma_execute()  local
783 	if (!echan->edesc) {  in edma_execute()
789 		echan->edesc = to_edma_desc(&vdesc->tx);  in edma_execute()
792 	edesc = echan->edesc;  in edma_execute()
795 	left = edesc->pset_nr - edesc->processed;  in edma_execute()
797 	edesc->sg_len = 0;  in edma_execute()
801 		j = i + edesc->processed;  in edma_execute()
802 		edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);  in edma_execute()
803 		edesc->sg_len += edesc->pset[j].len;  in edma_execute()
817 			 edesc->pset[j].param.opt,  in edma_execute()
818 			 edesc->pset[j].param.src,  in edma_execute()
819 			 edesc->pset[j].param.dst,  in edma_execute()
820 			 edesc->pset[j].param.a_b_cnt,  in edma_execute()
821 			 edesc->pset[j].param.ccnt,  in edma_execute()
822 			 edesc->pset[j].param.src_dst_bidx,  in edma_execute()
823 			 edesc->pset[j].param.src_dst_cidx,  in edma_execute()
824 			 edesc->pset[j].param.link_bcntrld);  in edma_execute()
830 	edesc->processed += nslots;  in edma_execute()
837 	if (edesc->processed == edesc->pset_nr) {  in edma_execute()
838 		if (edesc->cyclic)  in edma_execute()
857 	} else if (edesc->processed <= MAX_NR_SG) {  in edma_execute()
863 			echan->ch_num, edesc->processed);  in edma_execute()
881 	if (echan->edesc) {  in edma_terminate_all()
884 		if (!echan->tc && echan->edesc->cyclic)  in edma_terminate_all()
887 		vchan_terminate_vdesc(&echan->edesc->vdesc);  in edma_terminate_all()
888 		echan->edesc = NULL;  in edma_terminate_all()
927 	if (!echan->edesc)  in edma_dma_pause()
1073 	struct edma_desc *edesc;  in edma_prep_slave_sg()  local
1101 	edesc = kzalloc(struct_size(edesc, pset, sg_len), GFP_ATOMIC);  in edma_prep_slave_sg()
1102 	if (!edesc)  in edma_prep_slave_sg()
1105 	edesc->pset_nr = sg_len;  in edma_prep_slave_sg()
1106 	edesc->residue = 0;  in edma_prep_slave_sg()
1107 	edesc->direction = direction;  in edma_prep_slave_sg()
1108 	edesc->echan = echan;  in edma_prep_slave_sg()
1118 				kfree(edesc);  in edma_prep_slave_sg()
1134 		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,  in edma_prep_slave_sg()
1138 			kfree(edesc);  in edma_prep_slave_sg()
1142 		edesc->absync = ret;  in edma_prep_slave_sg()
1143 		edesc->residue += sg_dma_len(sg);  in edma_prep_slave_sg()
1147 			edesc->pset[i].param.opt |= TCINTEN;  in edma_prep_slave_sg()
1155 			edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);  in edma_prep_slave_sg()
1157 	edesc->residue_stat = edesc->residue;  in edma_prep_slave_sg()
1159 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_slave_sg()
1167 	struct edma_desc *edesc;  in edma_prep_dma_memcpy()  local
1218 	edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);  in edma_prep_dma_memcpy()
1219 	if (!edesc)  in edma_prep_dma_memcpy()
1222 	edesc->pset_nr = nslots;  in edma_prep_dma_memcpy()
1223 	edesc->residue = edesc->residue_stat = len;  in edma_prep_dma_memcpy()
1224 	edesc->direction = DMA_MEM_TO_MEM;  in edma_prep_dma_memcpy()
1225 	edesc->echan = echan;  in edma_prep_dma_memcpy()
1227 	ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,  in edma_prep_dma_memcpy()
1230 		kfree(edesc);  in edma_prep_dma_memcpy()
1234 	edesc->absync = ret;  in edma_prep_dma_memcpy()
1236 	edesc->pset[0].param.opt |= ITCCHEN;  in edma_prep_dma_memcpy()
1240 			edesc->pset[0].param.opt |= TCINTEN;  in edma_prep_dma_memcpy()
1243 		edesc->pset[0].param.opt |= TCCHEN;  in edma_prep_dma_memcpy()
1249 				kfree(edesc);  in edma_prep_dma_memcpy()
1259 		ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,  in edma_prep_dma_memcpy()
1262 			kfree(edesc);  in edma_prep_dma_memcpy()
1266 		edesc->pset[1].param.opt |= ITCCHEN;  in edma_prep_dma_memcpy()
1269 			edesc->pset[1].param.opt |= TCINTEN;  in edma_prep_dma_memcpy()
1273 		edesc->polled = true;  in edma_prep_dma_memcpy()
1275 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_dma_memcpy()
1286 	struct edma_desc *edesc;  in edma_prep_dma_interleaved()  local
1325 	edesc = kzalloc(struct_size(edesc, pset, 1), GFP_ATOMIC);  in edma_prep_dma_interleaved()
1326 	if (!edesc)  in edma_prep_dma_interleaved()
1329 	edesc->direction = DMA_MEM_TO_MEM;  in edma_prep_dma_interleaved()
1330 	edesc->echan = echan;  in edma_prep_dma_interleaved()
1331 	edesc->pset_nr = 1;  in edma_prep_dma_interleaved()
1333 	param = &edesc->pset[0].param;  in edma_prep_dma_interleaved()
1348 		edesc->polled = true;  in edma_prep_dma_interleaved()
1350 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_dma_interleaved()
1360 	struct edma_desc *edesc;  in edma_prep_dma_cyclic()  local
1421 	edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);  in edma_prep_dma_cyclic()
1422 	if (!edesc)  in edma_prep_dma_cyclic()
1425 	edesc->cyclic = 1;  in edma_prep_dma_cyclic()
1426 	edesc->pset_nr = nslots;  in edma_prep_dma_cyclic()
1427 	edesc->residue = edesc->residue_stat = buf_len;  in edma_prep_dma_cyclic()
1428 	edesc->direction = direction;  in edma_prep_dma_cyclic()
1429 	edesc->echan = echan;  in edma_prep_dma_cyclic()
1440 				kfree(edesc);  in edma_prep_dma_cyclic()
1448 			memcpy(&edesc->pset[i], &edesc->pset[0],  in edma_prep_dma_cyclic()
1449 			       sizeof(edesc->pset[0]));  in edma_prep_dma_cyclic()
1453 		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,  in edma_prep_dma_cyclic()
1457 			kfree(edesc);  in edma_prep_dma_cyclic()
1480 			edesc->pset[i].param.opt,  in edma_prep_dma_cyclic()
1481 			edesc->pset[i].param.src,  in edma_prep_dma_cyclic()
1482 			edesc->pset[i].param.dst,  in edma_prep_dma_cyclic()
1483 			edesc->pset[i].param.a_b_cnt,  in edma_prep_dma_cyclic()
1484 			edesc->pset[i].param.ccnt,  in edma_prep_dma_cyclic()
1485 			edesc->pset[i].param.src_dst_bidx,  in edma_prep_dma_cyclic()
1486 			edesc->pset[i].param.src_dst_cidx,  in edma_prep_dma_cyclic()
1487 			edesc->pset[i].param.link_bcntrld);  in edma_prep_dma_cyclic()
1489 		edesc->absync = ret;  in edma_prep_dma_cyclic()
1495 			edesc->pset[i].param.opt |= TCINTEN;  in edma_prep_dma_cyclic()
1499 				edesc->pset[i].param.opt |= ITCINTEN;  in edma_prep_dma_cyclic()
1507 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_dma_cyclic()
1513 	struct edma_desc *edesc;  in edma_completion_handler()  local
1516 	edesc = echan->edesc;  in edma_completion_handler()
1517 	if (edesc) {  in edma_completion_handler()
1518 		if (edesc->cyclic) {  in edma_completion_handler()
1519 			vchan_cyclic_callback(&edesc->vdesc);  in edma_completion_handler()
1522 		} else if (edesc->processed == edesc->pset_nr) {  in edma_completion_handler()
1523 			edesc->residue = 0;  in edma_completion_handler()
1525 			vchan_cookie_complete(&edesc->vdesc);  in edma_completion_handler()
1526 			echan->edesc = NULL;  in edma_completion_handler()
1537 			edesc->residue -= edesc->sg_len;  in edma_completion_handler()
1538 			edesc->residue_stat = edesc->residue;  in edma_completion_handler()
1539 			edesc->processed_stat = edesc->processed;  in edma_completion_handler()
1600 	if (!echan->edesc)  in edma_error_handler()
1809 	if (vchan_issue_pending(&echan->vchan) && !echan->edesc)  in edma_issue_pending()
1823 static u32 edma_residue(struct edma_desc *edesc)  in edma_residue()  argument
1825 	bool dst = edesc->direction == DMA_DEV_TO_MEM;  in edma_residue()
1827 	struct edma_chan *echan = edesc->echan;  in edma_residue()
1828 	struct edma_pset *pset = edesc->pset;  in edma_residue()
1850 	if (is_slave_direction(edesc->direction))  in edma_residue()
1878 	if (edesc->cyclic) {  in edma_residue()
1880 		edesc->residue_stat = edesc->residue - done;  in edma_residue()
1881 		return edesc->residue_stat;  in edma_residue()
1894 	pset += edesc->processed_stat;  in edma_residue()
1896 	for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {  in edma_residue()
1903 			return edesc->residue_stat - (pos - pset->addr);  in edma_residue()
1906 		edesc->processed_stat++;  in edma_residue()
1907 		edesc->residue_stat -= pset->len;  in edma_residue()
1909 	return edesc->residue_stat;  in edma_residue()
1932 	if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {  in edma_tx_status()
1933 		txstate->residue = edma_residue(echan->edesc);  in edma_tx_status()
1949 	    echan->edesc && echan->edesc->polled &&  in edma_tx_status()
1950 	    echan->edesc->vdesc.tx.cookie == cookie) {  in edma_tx_status()
1952 		vchan_cookie_complete(&echan->edesc->vdesc);  in edma_tx_status()
1953 		echan->edesc = NULL;  in edma_tx_status()