248 static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)  in at_xdmac_chan_reg_base()  argument
250 	return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);  in at_xdmac_chan_reg_base()
253 #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))  argument
254 #define at_xdmac_write(atxdmac, reg, value) \  argument
255 	writel_relaxed((value), (atxdmac)->regs + (reg))
322 static void at_xdmac_off(struct at_xdmac *atxdmac)  in at_xdmac_off()  argument
324 	at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L);  in at_xdmac_off()
327 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS))  in at_xdmac_off()
330 	at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L);  in at_xdmac_off()
337 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()  local
402 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
406 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
442 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_alloc_desc()  local
445 	desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);  in at_xdmac_alloc_desc()
512 	struct at_xdmac		*atxdmac = of_dma->of_dma_data;  in at_xdmac_xlate()  local
515 	struct device		*dev = atxdmac->dma.dev;  in at_xdmac_xlate()
522 	chan = dma_get_any_slave_channel(&atxdmac->dma);  in at_xdmac_xlate()
1392 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()  local
1439 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1497 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1581 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_handle_error()  local
1600 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_handle_error()
1601 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_handle_error()
1671 	struct at_xdmac		*atxdmac = (struct at_xdmac *)dev_id;  in at_xdmac_interrupt()  local
1678 		imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in at_xdmac_interrupt()
1679 		status = at_xdmac_read(atxdmac, AT_XDMAC_GIS);  in at_xdmac_interrupt()
1682 		dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1690 		for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_interrupt()
1694 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1698 			dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1712 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1756 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()  local
1765 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1777 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()  local
1788 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1799 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()  local
1805 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1806 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1863 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_free_chan_resources()  local
1869 		dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);  in at_xdmac_free_chan_resources()
1878 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_prepare()  local
1881 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_prepare()
1897 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_suspend()  local
1900 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_suspend()
1912 	atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in atmel_xdmac_suspend()
1914 	at_xdmac_off(atxdmac);  in atmel_xdmac_suspend()
1915 	clk_disable_unprepare(atxdmac->clk);  in atmel_xdmac_suspend()
1921 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_resume()  local
1927 	ret = clk_prepare_enable(atxdmac->clk);  in atmel_xdmac_resume()
1932 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in atmel_xdmac_resume()
1933 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1938 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);  in atmel_xdmac_resume()
1939 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_resume()
1949 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
1958 	struct at_xdmac	*atxdmac;  in at_xdmac_probe()  local
1984 	size = sizeof(*atxdmac);  in at_xdmac_probe()
1986 	atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in at_xdmac_probe()
1987 	if (!atxdmac) {  in at_xdmac_probe()
1992 	atxdmac->regs = base;  in at_xdmac_probe()
1993 	atxdmac->irq = irq;  in at_xdmac_probe()
1995 	atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");  in at_xdmac_probe()
1996 	if (IS_ERR(atxdmac->clk)) {  in at_xdmac_probe()
1998 		return PTR_ERR(atxdmac->clk);  in at_xdmac_probe()
2002 	ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac);  in at_xdmac_probe()
2008 	ret = clk_prepare_enable(atxdmac->clk);  in at_xdmac_probe()
2014 	atxdmac->at_xdmac_desc_pool =  in at_xdmac_probe()
2017 	if (!atxdmac->at_xdmac_desc_pool) {  in at_xdmac_probe()
2023 	dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2024 	dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2025 	dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2026 	dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2027 	dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2028 	dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2033 	dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2034 	atxdmac->dma.dev				= &pdev->dev;  in at_xdmac_probe()
2035 	atxdmac->dma.device_alloc_chan_resources	= at_xdmac_alloc_chan_resources;  in at_xdmac_probe()
2036 	atxdmac->dma.device_free_chan_resources		= at_xdmac_free_chan_resources;  in at_xdmac_probe()
2037 	atxdmac->dma.device_tx_status			= at_xdmac_tx_status;  in at_xdmac_probe()
2038 	atxdmac->dma.device_issue_pending		= at_xdmac_issue_pending;  in at_xdmac_probe()
2039 	atxdmac->dma.device_prep_dma_cyclic		= at_xdmac_prep_dma_cyclic;  in at_xdmac_probe()
2040 	atxdmac->dma.device_prep_interleaved_dma	= at_xdmac_prep_interleaved;  in at_xdmac_probe()
2041 	atxdmac->dma.device_prep_dma_memcpy		= at_xdmac_prep_dma_memcpy;  in at_xdmac_probe()
2042 	atxdmac->dma.device_prep_dma_memset		= at_xdmac_prep_dma_memset;  in at_xdmac_probe()
2043 	atxdmac->dma.device_prep_dma_memset_sg		= at_xdmac_prep_dma_memset_sg;  in at_xdmac_probe()
2044 	atxdmac->dma.device_prep_slave_sg		= at_xdmac_prep_slave_sg;  in at_xdmac_probe()
2045 	atxdmac->dma.device_config			= at_xdmac_device_config;  in at_xdmac_probe()
2046 	atxdmac->dma.device_pause			= at_xdmac_device_pause;  in at_xdmac_probe()
2047 	atxdmac->dma.device_resume			= at_xdmac_device_resume;  in at_xdmac_probe()
2048 	atxdmac->dma.device_terminate_all		= at_xdmac_device_terminate_all;  in at_xdmac_probe()
2049 	atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2050 	atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2051 	atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_xdmac_probe()
2052 	atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_xdmac_probe()
2055 	at_xdmac_off(atxdmac);  in at_xdmac_probe()
2058 	INIT_LIST_HEAD(&atxdmac->dma.channels);  in at_xdmac_probe()
2060 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()
2062 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2064 			      &atxdmac->dma.channels);  in at_xdmac_probe()
2066 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2078 	platform_set_drvdata(pdev, atxdmac);  in at_xdmac_probe()
2080 	ret = dma_async_device_register(&atxdmac->dma);  in at_xdmac_probe()
2087 					 at_xdmac_xlate, atxdmac);  in at_xdmac_probe()
2094 		 nr_channels, atxdmac->regs);  in at_xdmac_probe()
2099 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_probe()
2101 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_probe()
2103 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_probe()
2109 	struct at_xdmac	*atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);  in at_xdmac_remove()  local
2112 	at_xdmac_off(atxdmac);  in at_xdmac_remove()
2114 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_remove()
2115 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_remove()
2117 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_remove()
2119 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_remove()
2120 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()