1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
19  * typedef dma_cookie_t - an opaque DMA cookie
32  * enum dma_status - DMA transaction status
47  * enum dma_transaction_type - DMA transaction types/indexes
74  * enum dma_transfer_direction - dma transfer mode and direction indicator
90  * ----------------------------
92  * The gap(in bytes) between two chunks is called inter-chunk-gap(ICG).
97  *  it is to be repeated and other per-transfer attributes.
104  *  |      Frame-1        |       Frame-2       | ~ |       Frame-'numf'  |
112  * struct data_chunk - Element of scatter-gather list that makes a frame.
134  * struct dma_interleaved_template - Template to convey DMAC the transfer pattern
165  * enum dma_ctrl_flags - DMA flags to augment operation preparation,
167  * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
169  * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
172  * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q
173  * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P
174  * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as
177  * @DMA_PREP_FENCE - tell the driver that subsequent operations depend
209  * enum sum_check_bits - bit position of pq_check_flags
217  * enum pq_check_flags - result of async_{xor,pq}_zero_sum operations
218  * @SUM_CHECK_P_RESULT - 1 if xor zero sum error, 0 otherwise
219  * @SUM_CHECK_Q_RESULT - 1 if reed-solomon zero sum error, 0 otherwise
228  * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
234  * struct dma_chan_percpu - the per-CPU part of struct dma_chan
240  * enum dma_desc_metadata_mode - per descriptor metadata mode types supported
241  * @DESC_METADATA_CLIENT - the metadata buffer is allocated/provided by the
246  * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
252  * - DMA_DEV_TO_MEM:
260  * @DESC_METADATA_ENGINE - the metadata buffer is allocated/managed by the DMA
271  * - DMA_MEM_TO_DEV / DEV_MEM_TO_MEM:
279  * - DMA_DEV_TO_MEM:
302  * struct dma_router - DMA router structure
312  * struct dma_chan - devices supply DMA channels, clients use them
323  * @local: per-cpu pointer to a struct dma_chan_percpu
325  * @table_count: number of appearances in the mem-to-mem allocation table
328  * @private: private data for certain client-channel associations
357  * struct dma_chan_dev - relate sysfs device node to backing channel device
369  * enum dma_slave_buswidth - defines bus width of the DMA slave
385  * struct dma_slave_config - dma slave channel runtime config
405  * in one burst to the device. Typically something like half the
459  * enum dma_residue_granularity - Granularity of the reported transfer residue
469  *  the hardware supports scatter-gather and the segment descriptor has a field
473  *  burst. This is typically only supported if the hardware has a progress
485  * struct dma_slave_caps - expose capabilities of a slave channel only
494  * @min_burst: min burst capability per-transfer
495  * @max_burst: max burst capability per-transfer
496  * @max_sg_burst: max number of SG list entries executed in a single burst
523 	return dev_name(&chan->dev->device);  in dma_chan_name()
529  * typedef dma_filter_fn - callback filter for dma_request_channel
535  * being returned.  Where 'suitable' indicates a non-busy channel that
586  * struct dma_async_tx_descriptor - async transaction descriptor
587  * ---dma generic offload fields---
588  * @cookie: tracking cookie for this transaction, set to -EBUSY if
598  * @desc_metadata_mode: core managed metadata mode to protect mixed use of
603  * ---async_tx api specific fields---
632 	kref_get(&unmap->kref);  in dma_set_unmap()
633 	tx->unmap = unmap;  in dma_set_unmap()
656 	if (!tx->unmap)  in dma_descriptor_unmap()
659 	dmaengine_unmap_put(tx->unmap);  in dma_descriptor_unmap()
660 	tx->unmap = NULL;  in dma_descriptor_unmap()
692 	spin_lock_bh(&txd->lock);  in txd_lock()
696 	spin_unlock_bh(&txd->lock);  in txd_unlock()
700 	txd->next = next;  in txd_chain()
701 	next->parent = txd;  in txd_chain()
705 	txd->parent = NULL;  in txd_clear_parent()
709 	txd->next = NULL;  in txd_clear_next()
713 	return txd->parent;  in txd_parent()
717 	return txd->next;  in txd_next()
722  * struct dma_tx_state - filled in to report the status of
739  * enum dmaengine_alignment - defines alignment of the DMA async tx
753  * struct dma_slave_map - associates slave device and it's slave channel with
766  * struct dma_filter - information for slave device/channel to filter_fn/param
779  * struct dma_device - info on the entity supplying DMA services
788  * @max_pq: maximum number of PQ sources and PQ-continue capability
804  * @min_burst: min burst capability per-transfer
805  * @max_burst: max burst capability per-transfer
806  * @max_sg_burst: max number of SG list entries executed in a single burst
829  *	with per-channel specific ones
959 	if (chan->device->device_config)  in dmaengine_slave_config()
960 		return chan->device->device_config(chan, config);  in dmaengine_slave_config()
962 	return -ENOSYS;  in dmaengine_slave_config()
979 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)  in dmaengine_prep_slave_single()
982 	return chan->device->device_prep_slave_sg(chan, &sg, 1,  in dmaengine_prep_slave_single()
990 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)  in dmaengine_prep_slave_sg()
993 	return chan->device->device_prep_slave_sg(chan, sgl, sg_len,  in dmaengine_prep_slave_sg()
1004 	if (!chan || !chan->device || !chan->device->device_prep_slave_sg)  in dmaengine_prep_rio_sg()
1007 	return chan->device->device_prep_slave_sg(chan, sgl, sg_len,  in dmaengine_prep_rio_sg()
1017 	if (!chan || !chan->device || !chan->device->device_prep_dma_cyclic)  in dmaengine_prep_dma_cyclic()
1020 	return chan->device->device_prep_dma_cyclic(chan, buf_addr, buf_len,  in dmaengine_prep_dma_cyclic()
1028 	if (!chan || !chan->device || !chan->device->device_prep_interleaved_dma)  in dmaengine_prep_interleaved_dma()
1031 	    !test_bit(DMA_REPEAT, chan->device->cap_mask.bits))  in dmaengine_prep_interleaved_dma()
1034 	return chan->device->device_prep_interleaved_dma(chan, xt, flags);  in dmaengine_prep_interleaved_dma()
1041 	if (!chan || !chan->device || !chan->device->device_prep_dma_memset)  in dmaengine_prep_dma_memset()
1044 	return chan->device->device_prep_dma_memset(chan, dest, value,  in dmaengine_prep_dma_memset()
1052 	if (!chan || !chan->device || !chan->device->device_prep_dma_memcpy)  in dmaengine_prep_dma_memcpy()
1055 	return chan->device->device_prep_dma_memcpy(chan, dest, src,  in dmaengine_prep_dma_memcpy()
1065 	return !!(chan->device->desc_metadata_modes & mode);  in dmaengine_is_metadata_mode_supported()
1079 	return -EINVAL;  in dmaengine_desc_attach_metadata()
1090 	return -EINVAL;  in dmaengine_desc_set_metadata_len()
1095  * dmaengine_terminate_all() - Terminate all active DMA transfers
1103 	if (chan->device->device_terminate_all)  in dmaengine_terminate_all()
1104 		return chan->device->device_terminate_all(chan);  in dmaengine_terminate_all()
1106 	return -ENOSYS;  in dmaengine_terminate_all()
1110  * dmaengine_terminate_async() - Terminate all active DMA transfers
1132 	if (chan->device->device_terminate_all)  in dmaengine_terminate_async()
1133 		return chan->device->device_terminate_all(chan);  in dmaengine_terminate_async()
1135 	return -EINVAL;  in dmaengine_terminate_async()
1139  * dmaengine_synchronize() - Synchronize DMA channel termination
1152  * This function must only be called from non-atomic context and must not be
1160 	if (chan->device->device_synchronize)  in dmaengine_synchronize()
1161 		chan->device->device_synchronize(chan);  in dmaengine_synchronize()
1165  * dmaengine_terminate_sync() - Terminate all active DMA transfers
1174  * This function must only be called from non-atomic context and must not be
1193 	if (chan->device->device_pause)  in dmaengine_pause()
1194 		return chan->device->device_pause(chan);  in dmaengine_pause()
1196 	return -ENOSYS;  in dmaengine_pause()
1201 	if (chan->device->device_resume)  in dmaengine_resume()
1202 		return chan->device->device_resume(chan);  in dmaengine_resume()
1204 	return -ENOSYS;  in dmaengine_resume()
1210 	return chan->device->device_tx_status(chan, cookie, state);  in dmaengine_tx_status()
1215 	return desc->tx_submit(desc);  in dmaengine_submit()
1221 	return !(((1 << align) - 1) & (off1 | off2 | len));  in dmaengine_check_align()
1227 	return dmaengine_check_align(dev->copy_align, off1, off2, len);  in is_dma_copy_aligned()
1233 	return dmaengine_check_align(dev->xor_align, off1, off2, len);  in is_dma_xor_aligned()
1239 	return dmaengine_check_align(dev->pq_align, off1, off2, len);  in is_dma_pq_aligned()
1245 	return dmaengine_check_align(dev->fill_align, off1, off2, len);  in is_dma_fill_aligned()
1251 	dma->max_pq = maxpq;  in dma_set_maxpq()
1253 		dma->max_pq |= DMA_HAS_PQ_CONTINUE;  in dma_set_maxpq()
1270 	return (dma->max_pq & DMA_HAS_PQ_CONTINUE) == DMA_HAS_PQ_CONTINUE;  in dma_dev_has_pq_continue()
1275 	return dma->max_pq & ~DMA_HAS_PQ_CONTINUE;  in dma_dev_to_maxpq()
1278 /* dma_maxpq - reduce maxpq in the face of continued operations
1279  * @dma - dma device with PQ capability
1280  * @flags - to check if DMA_PREP_CONTINUE and DMA_PREP_PQ_DISABLE_P are set
1296 		return dma_dev_to_maxpq(dma) - 1;  in dma_maxpq()
1298 		return dma_dev_to_maxpq(dma) - 3;  in dma_maxpq()
1318 	return dmaengine_get_icg(xt->dst_inc, xt->dst_sgl,  in dmaengine_get_dst_icg()
1319 				 chunk->icg, chunk->dst_icg);  in dmaengine_get_dst_icg()
1325 	return dmaengine_get_icg(xt->src_inc, xt->src_sgl,  in dmaengine_get_src_icg()
1326 				 chunk->icg, chunk->src_icg);  in dmaengine_get_src_icg()
1329 /* --- public DMA engine API --- */
1369 	tx->flags |= DMA_CTRL_ACK;  in async_tx_ack()
1374 	tx->flags &= ~DMA_CTRL_ACK;  in async_tx_clear_ack()
1379 	return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;  in async_tx_test_ack()
1386 	set_bit(tx_type, dstp->bits);  in __dma_cap_set()
1393 	clear_bit(tx_type, dstp->bits);  in __dma_cap_clear()
1399 	bitmap_zero(dstp->bits, DMA_TX_TYPE_END);  in __dma_cap_zero()
1406 	return test_bit(tx_type, srcp->bits);  in __dma_has_cap()
1413  * dma_async_issue_pending - flush pending transactions to HW
1421 	chan->device->device_issue_pending(chan);  in dma_async_issue_pending()
1425  * dma_async_is_tx_complete - poll for transaction completion
1433  * the status of multiple cookies without re-checking hardware state.
1441 	status = chan->device->device_tx_status(chan, cookie, &state);  in dma_async_is_tx_complete()
1450  * dma_async_is_complete - test a cookie against chan state
1477 	st->last = last;  in dma_set_tx_state()
1478 	st->used = used;  in dma_set_tx_state()
1479 	st->residue = residue;  in dma_set_tx_state()
1522 	return ERR_PTR(-ENODEV);  in dma_request_chan()
1527 	return ERR_PTR(-ENODEV);  in dma_request_chan_by_mask()
1535 	return -ENXIO;  in dma_get_slave_caps()
1544 	ret = dma_get_slave_caps(tx->chan, &caps);  in dmaengine_desc_set_reuse()
1549 		return -EPERM;  in dmaengine_desc_set_reuse()
1551 	tx->flags |= DMA_CTRL_REUSE;  in dmaengine_desc_set_reuse()
1557 	tx->flags &= ~DMA_CTRL_REUSE;  in dmaengine_desc_clear_reuse()
1562 	return (tx->flags & DMA_CTRL_REUSE) == DMA_CTRL_REUSE;  in dmaengine_desc_test_reuse()
1569 		return -EPERM;  in dmaengine_desc_free()
1571 	return desc->desc_free(desc);  in dmaengine_desc_free()
1574 /* --- DMA device --- */