132 static void fsl_lpspi_buf_rx_##type(struct fsl_lpspi_data *fsl_lpspi)	\
134 	unsigned int val = readl(fsl_lpspi->base + IMX7ULP_RDR);	\
136 	if (fsl_lpspi->rx_buf) {					\
137 		*(type *)fsl_lpspi->rx_buf = val;			\
138 		fsl_lpspi->rx_buf += sizeof(type);                      \
143 static void fsl_lpspi_buf_tx_##type(struct fsl_lpspi_data *fsl_lpspi)	\
147 	if (fsl_lpspi->tx_buf) {					\
148 		val = *(type *)fsl_lpspi->tx_buf;			\
149 		fsl_lpspi->tx_buf += sizeof(type);			\
152 	fsl_lpspi->remain -= sizeof(type);				\
153 	writel(val, fsl_lpspi->base + IMX7ULP_TDR);			\
163 static void fsl_lpspi_intctrl(struct fsl_lpspi_data *fsl_lpspi,  in LPSPI_BUF_TX()
166 	writel(enable, fsl_lpspi->base + IMX7ULP_IER);  in LPSPI_BUF_TX()
199 	struct fsl_lpspi_data *fsl_lpspi =  in lpspi_prepare_xfer_hardware()  local
203 	ret = pm_runtime_resume_and_get(fsl_lpspi->dev);  in lpspi_prepare_xfer_hardware()
205 		dev_err(fsl_lpspi->dev, "failed to enable clock\n");  in lpspi_prepare_xfer_hardware()
214 	struct fsl_lpspi_data *fsl_lpspi =  in lpspi_unprepare_xfer_hardware()  local
217 	pm_runtime_mark_last_busy(fsl_lpspi->dev);  in lpspi_unprepare_xfer_hardware()
218 	pm_runtime_put_autosuspend(fsl_lpspi->dev);  in lpspi_unprepare_xfer_hardware()
223 static void fsl_lpspi_write_tx_fifo(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_write_tx_fifo()  argument
228 	txfifo_cnt = readl(fsl_lpspi->base + IMX7ULP_FSR) & 0xff;  in fsl_lpspi_write_tx_fifo()
230 	while (txfifo_cnt < fsl_lpspi->txfifosize) {  in fsl_lpspi_write_tx_fifo()
231 		if (!fsl_lpspi->remain)  in fsl_lpspi_write_tx_fifo()
233 		fsl_lpspi->tx(fsl_lpspi);  in fsl_lpspi_write_tx_fifo()
237 	if (txfifo_cnt < fsl_lpspi->txfifosize) {  in fsl_lpspi_write_tx_fifo()
238 		if (!fsl_lpspi->is_slave) {  in fsl_lpspi_write_tx_fifo()
239 			temp = readl(fsl_lpspi->base + IMX7ULP_TCR);  in fsl_lpspi_write_tx_fifo()
241 			writel(temp, fsl_lpspi->base + IMX7ULP_TCR);  in fsl_lpspi_write_tx_fifo()
244 		fsl_lpspi_intctrl(fsl_lpspi, IER_FCIE);  in fsl_lpspi_write_tx_fifo()
246 		fsl_lpspi_intctrl(fsl_lpspi, IER_TDIE);  in fsl_lpspi_write_tx_fifo()
249 static void fsl_lpspi_read_rx_fifo(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_read_rx_fifo()  argument
251 	while (!(readl(fsl_lpspi->base + IMX7ULP_RSR) & RSR_RXEMPTY))  in fsl_lpspi_read_rx_fifo()
252 		fsl_lpspi->rx(fsl_lpspi);  in fsl_lpspi_read_rx_fifo()
255 static void fsl_lpspi_set_cmd(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_set_cmd()  argument
259 	temp |= fsl_lpspi->config.bpw - 1;  in fsl_lpspi_set_cmd()
260 	temp |= (fsl_lpspi->config.mode & 0x3) << 30;  in fsl_lpspi_set_cmd()
261 	temp |= (fsl_lpspi->config.chip_select & 0x3) << 24;  in fsl_lpspi_set_cmd()
262 	if (!fsl_lpspi->is_slave) {  in fsl_lpspi_set_cmd()
263 		temp |= fsl_lpspi->config.prescale << 27;  in fsl_lpspi_set_cmd()
269 		if (!fsl_lpspi->usedma) {  in fsl_lpspi_set_cmd()
271 			if (fsl_lpspi->is_first_byte)  in fsl_lpspi_set_cmd()
277 	writel(temp, fsl_lpspi->base + IMX7ULP_TCR);  in fsl_lpspi_set_cmd()
279 	dev_dbg(fsl_lpspi->dev, "TCR=0x%x\n", temp);  in fsl_lpspi_set_cmd()
282 static void fsl_lpspi_set_watermark(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_set_watermark()  argument
286 	if (!fsl_lpspi->usedma)  in fsl_lpspi_set_watermark()
287 		temp = fsl_lpspi->watermark >> 1 |  in fsl_lpspi_set_watermark()
288 		       (fsl_lpspi->watermark >> 1) << 16;  in fsl_lpspi_set_watermark()
290 		temp = fsl_lpspi->watermark >> 1;  in fsl_lpspi_set_watermark()
292 	writel(temp, fsl_lpspi->base + IMX7ULP_FCR);  in fsl_lpspi_set_watermark()
294 	dev_dbg(fsl_lpspi->dev, "FCR=0x%x\n", temp);  in fsl_lpspi_set_watermark()
297 static int fsl_lpspi_set_bitrate(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_set_bitrate()  argument
299 	struct lpspi_config config = fsl_lpspi->config;  in fsl_lpspi_set_bitrate()
303 	perclk_rate = clk_get_rate(fsl_lpspi->clk_per);  in fsl_lpspi_set_bitrate()
306 		dev_err(fsl_lpspi->dev,  in fsl_lpspi_set_bitrate()
314 			fsl_lpspi->config.prescale = prescale;  in fsl_lpspi_set_bitrate()
323 					fsl_lpspi->base + IMX7ULP_CCR);  in fsl_lpspi_set_bitrate()
325 	dev_dbg(fsl_lpspi->dev, "perclk=%d, speed=%d, prescale=%d, scldiv=%d\n",  in fsl_lpspi_set_bitrate()
336 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_dma_configure()  local
339 	switch (fsl_lpspi_bytes_per_word(fsl_lpspi->config.bpw)) {  in fsl_lpspi_dma_configure()
354 	tx.dst_addr = fsl_lpspi->base_phys + IMX7ULP_TDR;  in fsl_lpspi_dma_configure()
359 		dev_err(fsl_lpspi->dev, "TX dma configuration failed with %d\n",  in fsl_lpspi_dma_configure()
365 	rx.src_addr = fsl_lpspi->base_phys + IMX7ULP_RDR;  in fsl_lpspi_dma_configure()
370 		dev_err(fsl_lpspi->dev, "RX dma configuration failed with %d\n",  in fsl_lpspi_dma_configure()
378 static int fsl_lpspi_config(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_config()  argument
383 	if (!fsl_lpspi->is_slave) {  in fsl_lpspi_config()
384 		ret = fsl_lpspi_set_bitrate(fsl_lpspi);  in fsl_lpspi_config()
389 	fsl_lpspi_set_watermark(fsl_lpspi);  in fsl_lpspi_config()
391 	if (!fsl_lpspi->is_slave)  in fsl_lpspi_config()
395 	if (fsl_lpspi->config.mode & SPI_CS_HIGH)  in fsl_lpspi_config()
397 	writel(temp, fsl_lpspi->base + IMX7ULP_CFGR1);  in fsl_lpspi_config()
399 	temp = readl(fsl_lpspi->base + IMX7ULP_CR);  in fsl_lpspi_config()
401 	writel(temp, fsl_lpspi->base + IMX7ULP_CR);  in fsl_lpspi_config()
404 	if (fsl_lpspi->usedma)  in fsl_lpspi_config()
406 	writel(temp, fsl_lpspi->base + IMX7ULP_DER);  in fsl_lpspi_config()
415 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_setup_transfer()  local
421 	fsl_lpspi->config.mode = spi->mode;  in fsl_lpspi_setup_transfer()
422 	fsl_lpspi->config.bpw = t->bits_per_word;  in fsl_lpspi_setup_transfer()
423 	fsl_lpspi->config.speed_hz = t->speed_hz;  in fsl_lpspi_setup_transfer()
424 	if (fsl_lpspi->is_only_cs1)  in fsl_lpspi_setup_transfer()
425 		fsl_lpspi->config.chip_select = 1;  in fsl_lpspi_setup_transfer()
427 		fsl_lpspi->config.chip_select = spi->chip_select;  in fsl_lpspi_setup_transfer()
429 	if (!fsl_lpspi->config.speed_hz)  in fsl_lpspi_setup_transfer()
430 		fsl_lpspi->config.speed_hz = spi->max_speed_hz;  in fsl_lpspi_setup_transfer()
431 	if (!fsl_lpspi->config.bpw)  in fsl_lpspi_setup_transfer()
432 		fsl_lpspi->config.bpw = spi->bits_per_word;  in fsl_lpspi_setup_transfer()
435 	if (fsl_lpspi->config.bpw <= 8) {  in fsl_lpspi_setup_transfer()
436 		fsl_lpspi->rx = fsl_lpspi_buf_rx_u8;  in fsl_lpspi_setup_transfer()
437 		fsl_lpspi->tx = fsl_lpspi_buf_tx_u8;  in fsl_lpspi_setup_transfer()
438 	} else if (fsl_lpspi->config.bpw <= 16) {  in fsl_lpspi_setup_transfer()
439 		fsl_lpspi->rx = fsl_lpspi_buf_rx_u16;  in fsl_lpspi_setup_transfer()
440 		fsl_lpspi->tx = fsl_lpspi_buf_tx_u16;  in fsl_lpspi_setup_transfer()
442 		fsl_lpspi->rx = fsl_lpspi_buf_rx_u32;  in fsl_lpspi_setup_transfer()
443 		fsl_lpspi->tx = fsl_lpspi_buf_tx_u32;  in fsl_lpspi_setup_transfer()
446 	if (t->len <= fsl_lpspi->txfifosize)  in fsl_lpspi_setup_transfer()
447 		fsl_lpspi->watermark = t->len;  in fsl_lpspi_setup_transfer()
449 		fsl_lpspi->watermark = fsl_lpspi->txfifosize;  in fsl_lpspi_setup_transfer()
452 		fsl_lpspi->usedma = true;  in fsl_lpspi_setup_transfer()
454 		fsl_lpspi->usedma = false;  in fsl_lpspi_setup_transfer()
456 	return fsl_lpspi_config(fsl_lpspi);  in fsl_lpspi_setup_transfer()
461 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_slave_abort()  local
464 	fsl_lpspi->slave_aborted = true;  in fsl_lpspi_slave_abort()
465 	if (!fsl_lpspi->usedma)  in fsl_lpspi_slave_abort()
466 		complete(&fsl_lpspi->xfer_done);  in fsl_lpspi_slave_abort()
468 		complete(&fsl_lpspi->dma_tx_completion);  in fsl_lpspi_slave_abort()
469 		complete(&fsl_lpspi->dma_rx_completion);  in fsl_lpspi_slave_abort()
477 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_wait_for_completion()  local
480 	if (fsl_lpspi->is_slave) {  in fsl_lpspi_wait_for_completion()
481 		if (wait_for_completion_interruptible(&fsl_lpspi->xfer_done) ||  in fsl_lpspi_wait_for_completion()
482 			fsl_lpspi->slave_aborted) {  in fsl_lpspi_wait_for_completion()
483 			dev_dbg(fsl_lpspi->dev, "interrupted\n");  in fsl_lpspi_wait_for_completion()
487 		if (!wait_for_completion_timeout(&fsl_lpspi->xfer_done, HZ)) {  in fsl_lpspi_wait_for_completion()
488 			dev_dbg(fsl_lpspi->dev, "wait for completion timeout\n");  in fsl_lpspi_wait_for_completion()
496 static int fsl_lpspi_reset(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_reset()  argument
500 	if (!fsl_lpspi->usedma) {  in fsl_lpspi_reset()
502 		fsl_lpspi_intctrl(fsl_lpspi, 0);  in fsl_lpspi_reset()
507 	writel(temp, fsl_lpspi->base + IMX7ULP_SR);  in fsl_lpspi_reset()
511 	writel(temp, fsl_lpspi->base + IMX7ULP_CR);  in fsl_lpspi_reset()
518 	struct fsl_lpspi_data *fsl_lpspi = (struct fsl_lpspi_data *)cookie;  in fsl_lpspi_dma_rx_callback()  local
520 	complete(&fsl_lpspi->dma_rx_completion);  in fsl_lpspi_dma_rx_callback()
525 	struct fsl_lpspi_data *fsl_lpspi = (struct fsl_lpspi_data *)cookie;  in fsl_lpspi_dma_tx_callback()  local
527 	complete(&fsl_lpspi->dma_tx_completion);  in fsl_lpspi_dma_tx_callback()
530 static int fsl_lpspi_calculate_timeout(struct fsl_lpspi_data *fsl_lpspi,  in fsl_lpspi_calculate_timeout()  argument
536 	timeout = (8 + 4) * size / fsl_lpspi->config.speed_hz;  in fsl_lpspi_calculate_timeout()
546 				struct fsl_lpspi_data *fsl_lpspi,  in fsl_lpspi_dma_transfer()  argument
566 	desc_rx->callback_param = (void *)fsl_lpspi;  in fsl_lpspi_dma_transfer()
568 	reinit_completion(&fsl_lpspi->dma_rx_completion);  in fsl_lpspi_dma_transfer()
580 	desc_tx->callback_param = (void *)fsl_lpspi;  in fsl_lpspi_dma_transfer()
582 	reinit_completion(&fsl_lpspi->dma_tx_completion);  in fsl_lpspi_dma_transfer()
585 	fsl_lpspi->slave_aborted = false;  in fsl_lpspi_dma_transfer()
587 	if (!fsl_lpspi->is_slave) {  in fsl_lpspi_dma_transfer()
588 		transfer_timeout = fsl_lpspi_calculate_timeout(fsl_lpspi,  in fsl_lpspi_dma_transfer()
592 		timeout = wait_for_completion_timeout(&fsl_lpspi->dma_tx_completion,  in fsl_lpspi_dma_transfer()
595 			dev_err(fsl_lpspi->dev, "I/O Error in DMA TX\n");  in fsl_lpspi_dma_transfer()
598 			fsl_lpspi_reset(fsl_lpspi);  in fsl_lpspi_dma_transfer()
602 		timeout = wait_for_completion_timeout(&fsl_lpspi->dma_rx_completion,  in fsl_lpspi_dma_transfer()
605 			dev_err(fsl_lpspi->dev, "I/O Error in DMA RX\n");  in fsl_lpspi_dma_transfer()
608 			fsl_lpspi_reset(fsl_lpspi);  in fsl_lpspi_dma_transfer()
612 		if (wait_for_completion_interruptible(&fsl_lpspi->dma_tx_completion) ||  in fsl_lpspi_dma_transfer()
613 			fsl_lpspi->slave_aborted) {  in fsl_lpspi_dma_transfer()
614 			dev_dbg(fsl_lpspi->dev,  in fsl_lpspi_dma_transfer()
618 			fsl_lpspi_reset(fsl_lpspi);  in fsl_lpspi_dma_transfer()
622 		if (wait_for_completion_interruptible(&fsl_lpspi->dma_rx_completion) ||  in fsl_lpspi_dma_transfer()
623 			fsl_lpspi->slave_aborted) {  in fsl_lpspi_dma_transfer()
624 			dev_dbg(fsl_lpspi->dev,  in fsl_lpspi_dma_transfer()
628 			fsl_lpspi_reset(fsl_lpspi);  in fsl_lpspi_dma_transfer()
633 	fsl_lpspi_reset(fsl_lpspi);  in fsl_lpspi_dma_transfer()
652 			      struct fsl_lpspi_data *fsl_lpspi,  in fsl_lpspi_dma_init()  argument
675 	init_completion(&fsl_lpspi->dma_rx_completion);  in fsl_lpspi_dma_init()
676 	init_completion(&fsl_lpspi->dma_tx_completion);  in fsl_lpspi_dma_init()
689 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_pio_transfer()  local
693 	fsl_lpspi->tx_buf = t->tx_buf;  in fsl_lpspi_pio_transfer()
694 	fsl_lpspi->rx_buf = t->rx_buf;  in fsl_lpspi_pio_transfer()
695 	fsl_lpspi->remain = t->len;  in fsl_lpspi_pio_transfer()
697 	reinit_completion(&fsl_lpspi->xfer_done);  in fsl_lpspi_pio_transfer()
698 	fsl_lpspi->slave_aborted = false;  in fsl_lpspi_pio_transfer()
700 	fsl_lpspi_write_tx_fifo(fsl_lpspi);  in fsl_lpspi_pio_transfer()
706 	fsl_lpspi_reset(fsl_lpspi);  in fsl_lpspi_pio_transfer()
715 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_transfer_one()  local
719 	fsl_lpspi->is_first_byte = true;  in fsl_lpspi_transfer_one()
724 	fsl_lpspi_set_cmd(fsl_lpspi);  in fsl_lpspi_transfer_one()
725 	fsl_lpspi->is_first_byte = false;  in fsl_lpspi_transfer_one()
727 	if (fsl_lpspi->usedma)  in fsl_lpspi_transfer_one()
728 		ret = fsl_lpspi_dma_transfer(controller, fsl_lpspi, t);  in fsl_lpspi_transfer_one()
740 	struct fsl_lpspi_data *fsl_lpspi = dev_id;  in fsl_lpspi_isr()  local
742 	temp_IER = readl(fsl_lpspi->base + IMX7ULP_IER);  in fsl_lpspi_isr()
743 	fsl_lpspi_intctrl(fsl_lpspi, 0);  in fsl_lpspi_isr()
744 	temp_SR = readl(fsl_lpspi->base + IMX7ULP_SR);  in fsl_lpspi_isr()
746 	fsl_lpspi_read_rx_fifo(fsl_lpspi);  in fsl_lpspi_isr()
749 		fsl_lpspi_write_tx_fifo(fsl_lpspi);  in fsl_lpspi_isr()
754 	    readl(fsl_lpspi->base + IMX7ULP_FSR) & FSR_TXCOUNT) {  in fsl_lpspi_isr()
755 		writel(SR_FCF, fsl_lpspi->base + IMX7ULP_SR);  in fsl_lpspi_isr()
756 		fsl_lpspi_intctrl(fsl_lpspi, IER_FCIE);  in fsl_lpspi_isr()
761 		writel(SR_FCF, fsl_lpspi->base + IMX7ULP_SR);  in fsl_lpspi_isr()
762 		complete(&fsl_lpspi->xfer_done);  in fsl_lpspi_isr()
773 	struct fsl_lpspi_data *fsl_lpspi;  in fsl_lpspi_runtime_resume()  local
776 	fsl_lpspi = spi_controller_get_devdata(controller);  in fsl_lpspi_runtime_resume()
778 	ret = clk_prepare_enable(fsl_lpspi->clk_per);  in fsl_lpspi_runtime_resume()
782 	ret = clk_prepare_enable(fsl_lpspi->clk_ipg);  in fsl_lpspi_runtime_resume()
784 		clk_disable_unprepare(fsl_lpspi->clk_per);  in fsl_lpspi_runtime_resume()
794 	struct fsl_lpspi_data *fsl_lpspi;  in fsl_lpspi_runtime_suspend()  local
796 	fsl_lpspi = spi_controller_get_devdata(controller);  in fsl_lpspi_runtime_suspend()
798 	clk_disable_unprepare(fsl_lpspi->clk_per);  in fsl_lpspi_runtime_suspend()
799 	clk_disable_unprepare(fsl_lpspi->clk_ipg);  in fsl_lpspi_runtime_suspend()
805 static int fsl_lpspi_init_rpm(struct fsl_lpspi_data *fsl_lpspi)  in fsl_lpspi_init_rpm()  argument
807 	struct device *dev = fsl_lpspi->dev;  in fsl_lpspi_init_rpm()
818 	struct fsl_lpspi_data *fsl_lpspi;  in fsl_lpspi_probe()  local
838 	fsl_lpspi = spi_controller_get_devdata(controller);  in fsl_lpspi_probe()
839 	fsl_lpspi->dev = &pdev->dev;  in fsl_lpspi_probe()
840 	fsl_lpspi->is_slave = is_slave;  in fsl_lpspi_probe()
841 	fsl_lpspi->is_only_cs1 = of_property_read_bool((&pdev->dev)->of_node,  in fsl_lpspi_probe()
853 	if (!fsl_lpspi->is_slave)  in fsl_lpspi_probe()
856 	init_completion(&fsl_lpspi->xfer_done);  in fsl_lpspi_probe()
859 	fsl_lpspi->base = devm_ioremap_resource(&pdev->dev, res);  in fsl_lpspi_probe()
860 	if (IS_ERR(fsl_lpspi->base)) {  in fsl_lpspi_probe()
861 		ret = PTR_ERR(fsl_lpspi->base);  in fsl_lpspi_probe()
864 	fsl_lpspi->base_phys = res->start;  in fsl_lpspi_probe()
873 			       dev_name(&pdev->dev), fsl_lpspi);  in fsl_lpspi_probe()
879 	fsl_lpspi->clk_per = devm_clk_get(&pdev->dev, "per");  in fsl_lpspi_probe()
880 	if (IS_ERR(fsl_lpspi->clk_per)) {  in fsl_lpspi_probe()
881 		ret = PTR_ERR(fsl_lpspi->clk_per);  in fsl_lpspi_probe()
885 	fsl_lpspi->clk_ipg = devm_clk_get(&pdev->dev, "ipg");  in fsl_lpspi_probe()
886 	if (IS_ERR(fsl_lpspi->clk_ipg)) {  in fsl_lpspi_probe()
887 		ret = PTR_ERR(fsl_lpspi->clk_ipg);  in fsl_lpspi_probe()
892 	ret = fsl_lpspi_init_rpm(fsl_lpspi);  in fsl_lpspi_probe()
896 	ret = pm_runtime_get_sync(fsl_lpspi->dev);  in fsl_lpspi_probe()
898 		dev_err(fsl_lpspi->dev, "failed to enable clock\n");  in fsl_lpspi_probe()
902 	temp = readl(fsl_lpspi->base + IMX7ULP_PARAM);  in fsl_lpspi_probe()
903 	fsl_lpspi->txfifosize = 1 << (temp & 0x0f);  in fsl_lpspi_probe()
904 	fsl_lpspi->rxfifosize = 1 << ((temp >> 8) & 0x0f);  in fsl_lpspi_probe()
906 	ret = fsl_lpspi_dma_init(&pdev->dev, fsl_lpspi, controller);  in fsl_lpspi_probe()
919 	pm_runtime_mark_last_busy(fsl_lpspi->dev);  in fsl_lpspi_probe()
920 	pm_runtime_put_autosuspend(fsl_lpspi->dev);  in fsl_lpspi_probe()
925 	pm_runtime_dont_use_autosuspend(fsl_lpspi->dev);  in fsl_lpspi_probe()
926 	pm_runtime_put_sync(fsl_lpspi->dev);  in fsl_lpspi_probe()
927 	pm_runtime_disable(fsl_lpspi->dev);  in fsl_lpspi_probe()
937 	struct fsl_lpspi_data *fsl_lpspi =  in fsl_lpspi_remove()  local
940 	pm_runtime_disable(fsl_lpspi->dev);  in fsl_lpspi_remove()