106 	struct at_dma	*atdma = to_at_dma(chan->device);  in atc_alloc_descriptor()  local
109 	desc = dma_pool_zalloc(atdma->dma_desc_pool, gfp_flags, &phys);  in atc_alloc_descriptor()
211 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_dostart()  local
242 	dma_writel(atdma, CHER, atchan->mask);  in atc_dostart()
437 	struct at_dma			*atdma = to_at_dma(atchan->chan_common.device);  in atc_chain_complete()  local
468 		dma_pool_free(atdma->memset_pool, desc->memset_vaddr,  in atc_chain_complete()
587 	struct at_dma		*atdma = (struct at_dma *)dev_id;  in at_dma_interrupt()  local
594 		imr = dma_readl(atdma, EBCIMR);  in at_dma_interrupt()
595 		status = dma_readl(atdma, EBCISR);  in at_dma_interrupt()
601 		dev_vdbg(atdma->dma_common.dev,  in at_dma_interrupt()
605 		for (i = 0; i < atdma->dma_common.chancnt; i++) {  in at_dma_interrupt()
606 			atchan = &atdma->chan[i];  in at_dma_interrupt()
610 					dma_writel(atdma, CHDR,  in at_dma_interrupt()
885 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset()  local
904 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr);  in atc_prep_dma_memset()
934 	dma_pool_free(atdma->memset_pool, vaddr, paddr);  in atc_prep_dma_memset()
945 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset_sg()  local
962 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr);  in atc_prep_dma_memset_sg()
1341 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_pause()  local
1349 	dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));  in atc_pause()
1360 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_resume()  local
1371 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));  in atc_resume()
1382 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_terminate_all()  local
1397 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);  in atc_terminate_all()
1400 	while (dma_readl(atdma, CHSR) & atchan->mask)  in atc_terminate_all()
1498 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_alloc_chan_resources()  local
1525 		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);  in atc_alloc_chan_resources()
1536 			dev_err(atdma->dma_common.dev,  in atc_alloc_chan_resources()
1561 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_free_chan_resources()  local
1574 		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);  in atc_free_chan_resources()
1730 static void at_dma_off(struct at_dma *atdma)  in at_dma_off()  argument
1732 	dma_writel(atdma, EN, 0);  in at_dma_off()
1735 	dma_writel(atdma, EBCIDR, -1L);  in at_dma_off()
1738 	while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)  in at_dma_off()
1745 	struct at_dma		*atdma;  in at_dma_probe()  local
1776 	atdma = kzalloc(size, GFP_KERNEL);  in at_dma_probe()
1777 	if (!atdma)  in at_dma_probe()
1781 	atdma->dma_common.cap_mask = plat_dat->cap_mask;  in at_dma_probe()
1782 	atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;  in at_dma_probe()
1790 	atdma->regs = ioremap(io->start, size);  in at_dma_probe()
1791 	if (!atdma->regs) {  in at_dma_probe()
1796 	atdma->clk = clk_get(&pdev->dev, "dma_clk");  in at_dma_probe()
1797 	if (IS_ERR(atdma->clk)) {  in at_dma_probe()
1798 		err = PTR_ERR(atdma->clk);  in at_dma_probe()
1801 	err = clk_prepare_enable(atdma->clk);  in at_dma_probe()
1806 	at_dma_off(atdma);  in at_dma_probe()
1808 	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);  in at_dma_probe()
1812 	platform_set_drvdata(pdev, atdma);  in at_dma_probe()
1815 	atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",  in at_dma_probe()
1818 	if (!atdma->dma_desc_pool) {  in at_dma_probe()
1825 	atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool",  in at_dma_probe()
1827 	if (!atdma->memset_pool) {  in at_dma_probe()
1834 	while (dma_readl(atdma, EBCISR))  in at_dma_probe()
1838 	INIT_LIST_HEAD(&atdma->dma_common.channels);  in at_dma_probe()
1840 		struct at_dma_chan	*atchan = &atdma->chan[i];  in at_dma_probe()
1844 		atchan->chan_common.device = &atdma->dma_common;  in at_dma_probe()
1847 				&atdma->dma_common.channels);  in at_dma_probe()
1849 		atchan->ch_regs = atdma->regs + ch_regs(i);  in at_dma_probe()
1858 		atc_enable_chan_irq(atdma, i);  in at_dma_probe()
1862 	atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;  in at_dma_probe()
1863 	atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;  in at_dma_probe()
1864 	atdma->dma_common.device_tx_status = atc_tx_status;  in at_dma_probe()
1865 	atdma->dma_common.device_issue_pending = atc_issue_pending;  in at_dma_probe()
1866 	atdma->dma_common.dev = &pdev->dev;  in at_dma_probe()
1869 	if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask))  in at_dma_probe()
1870 		atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved;  in at_dma_probe()
1872 	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))  in at_dma_probe()
1873 		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;  in at_dma_probe()
1875 	if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) {  in at_dma_probe()
1876 		atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset;  in at_dma_probe()
1877 		atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg;  in at_dma_probe()
1878 		atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES;  in at_dma_probe()
1881 	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {  in at_dma_probe()
1882 		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;  in at_dma_probe()
1884 		dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask);  in at_dma_probe()
1885 		atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;  in at_dma_probe()
1886 		atdma->dma_common.device_config = atc_config;  in at_dma_probe()
1887 		atdma->dma_common.device_pause = atc_pause;  in at_dma_probe()
1888 		atdma->dma_common.device_resume = atc_resume;  in at_dma_probe()
1889 		atdma->dma_common.device_terminate_all = atc_terminate_all;  in at_dma_probe()
1890 		atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
1891 		atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
1892 		atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_dma_probe()
1893 		atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_dma_probe()
1896 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_probe()
1899 	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",  in at_dma_probe()
1900 	  dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",  in at_dma_probe()
1901 	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",  in at_dma_probe()
1904 	err = dma_async_device_register(&atdma->dma_common);  in at_dma_probe()
1917 						 at_dma_xlate, atdma);  in at_dma_probe()
1927 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_probe()
1929 	dma_pool_destroy(atdma->memset_pool);  in at_dma_probe()
1931 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_probe()
1933 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_probe()
1935 	clk_disable_unprepare(atdma->clk);  in at_dma_probe()
1937 	clk_put(atdma->clk);  in at_dma_probe()
1939 	iounmap(atdma->regs);  in at_dma_probe()
1940 	atdma->regs = NULL;  in at_dma_probe()
1944 	kfree(atdma);  in at_dma_probe()
1950 	struct at_dma		*atdma = platform_get_drvdata(pdev);  in at_dma_remove()  local
1954 	at_dma_off(atdma);  in at_dma_remove()
1957 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_remove()
1959 	dma_pool_destroy(atdma->memset_pool);  in at_dma_remove()
1960 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_remove()
1961 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_remove()
1963 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_remove()
1968 		atc_disable_chan_irq(atdma, chan->chan_id);  in at_dma_remove()
1974 	clk_disable_unprepare(atdma->clk);  in at_dma_remove()
1975 	clk_put(atdma->clk);  in at_dma_remove()
1977 	iounmap(atdma->regs);  in at_dma_remove()
1978 	atdma->regs = NULL;  in at_dma_remove()
1983 	kfree(atdma);  in at_dma_remove()
1990 	struct at_dma	*atdma = platform_get_drvdata(pdev);  in at_dma_shutdown()  local
1993 	clk_disable_unprepare(atdma->clk);  in at_dma_shutdown()
1998 	struct at_dma *atdma = dev_get_drvdata(dev);  in at_dma_prepare()  local
2001 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_prepare()
2032 	struct at_dma *atdma = dev_get_drvdata(dev);  in at_dma_suspend_noirq()  local
2036 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_suspend_noirq()
2044 	atdma->save_imr = dma_readl(atdma, EBCIMR);  in at_dma_suspend_noirq()
2047 	at_dma_off(atdma);  in at_dma_suspend_noirq()
2048 	clk_disable_unprepare(atdma->clk);  in at_dma_suspend_noirq()
2054 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_resume_cyclic()  local
2063 	dma_writel(atdma, CHER, atchan->mask);  in atc_resume_cyclic()
2073 	struct at_dma *atdma = dev_get_drvdata(dev);  in at_dma_resume_noirq()  local
2077 	clk_prepare_enable(atdma->clk);  in at_dma_resume_noirq()
2078 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_resume_noirq()
2081 	while (dma_readl(atdma, EBCISR))  in at_dma_resume_noirq()
2085 	dma_writel(atdma, EBCIER, atdma->save_imr);  in at_dma_resume_noirq()
2086 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_resume_noirq()