121 		  struct ioatdma_chan *ioat_chan, int idx);
405 	struct ioatdma_chan *ioat_chan;  in ioat_dma_setup_interrupts()  local
434 		ioat_chan = ioat_chan_by_index(ioat_dma, i);  in ioat_dma_setup_interrupts()
437 				       "ioat-msix", ioat_chan);  in ioat_dma_setup_interrupts()
441 				ioat_chan = ioat_chan_by_index(ioat_dma, j);  in ioat_dma_setup_interrupts()
442 				devm_free_irq(dev, msix->vector, ioat_chan);  in ioat_dma_setup_interrupts()
567 	struct ioatdma_chan *ioat_chan;  in ioat_enumerate_channels()  local
588 		ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);  in ioat_enumerate_channels()
589 		if (!ioat_chan)  in ioat_enumerate_channels()
592 		ioat_init_channel(ioat_dma, ioat_chan, i);  in ioat_enumerate_channels()
593 		ioat_chan->xfercap_log = xfercap_log;  in ioat_enumerate_channels()
594 		spin_lock_init(&ioat_chan->prep_lock);  in ioat_enumerate_channels()
595 		if (ioat_reset_hw(ioat_chan)) {  in ioat_enumerate_channels()
609 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);  in ioat_free_chan_resources()  local
610 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;  in ioat_free_chan_resources()
612 	const int total_descs = 1 << ioat_chan->alloc_order;  in ioat_free_chan_resources()
619 	if (!ioat_chan->ring)  in ioat_free_chan_resources()
622 	ioat_stop(ioat_chan);  in ioat_free_chan_resources()
624 	if (!test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) {  in ioat_free_chan_resources()
625 		ioat_reset_hw(ioat_chan);  in ioat_free_chan_resources()
630 			       ioat_chan->reg_base +  in ioat_free_chan_resources()
634 	spin_lock_bh(&ioat_chan->cleanup_lock);  in ioat_free_chan_resources()
635 	spin_lock_bh(&ioat_chan->prep_lock);  in ioat_free_chan_resources()
636 	descs = ioat_ring_space(ioat_chan);  in ioat_free_chan_resources()
637 	dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);  in ioat_free_chan_resources()
639 		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);  in ioat_free_chan_resources()
644 		dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",  in ioat_free_chan_resources()
648 		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i);  in ioat_free_chan_resources()
649 		dump_desc_dbg(ioat_chan, desc);  in ioat_free_chan_resources()
653 	for (i = 0; i < ioat_chan->desc_chunks; i++) {  in ioat_free_chan_resources()
654 		dma_free_coherent(to_dev(ioat_chan), IOAT_CHUNK_SIZE,  in ioat_free_chan_resources()
655 				  ioat_chan->descs[i].virt,  in ioat_free_chan_resources()
656 				  ioat_chan->descs[i].hw);  in ioat_free_chan_resources()
657 		ioat_chan->descs[i].virt = NULL;  in ioat_free_chan_resources()
658 		ioat_chan->descs[i].hw = 0;  in ioat_free_chan_resources()
660 	ioat_chan->desc_chunks = 0;  in ioat_free_chan_resources()
662 	kfree(ioat_chan->ring);  in ioat_free_chan_resources()
663 	ioat_chan->ring = NULL;  in ioat_free_chan_resources()
664 	ioat_chan->alloc_order = 0;  in ioat_free_chan_resources()
665 	dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion,  in ioat_free_chan_resources()
666 		      ioat_chan->completion_dma);  in ioat_free_chan_resources()
667 	spin_unlock_bh(&ioat_chan->prep_lock);  in ioat_free_chan_resources()
668 	spin_unlock_bh(&ioat_chan->cleanup_lock);  in ioat_free_chan_resources()
670 	ioat_chan->last_completion = 0;  in ioat_free_chan_resources()
671 	ioat_chan->completion_dma = 0;  in ioat_free_chan_resources()
672 	ioat_chan->dmacount = 0;  in ioat_free_chan_resources()
680 	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);  in ioat_alloc_chan_resources()  local
688 	if (ioat_chan->ring)  in ioat_alloc_chan_resources()
689 		return 1 << ioat_chan->alloc_order;  in ioat_alloc_chan_resources()
692 	writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);  in ioat_alloc_chan_resources()
696 	ioat_chan->completion =  in ioat_alloc_chan_resources()
697 		dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,  in ioat_alloc_chan_resources()
698 				GFP_NOWAIT, &ioat_chan->completion_dma);  in ioat_alloc_chan_resources()
699 	if (!ioat_chan->completion)  in ioat_alloc_chan_resources()
702 	writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,  in ioat_alloc_chan_resources()
703 	       ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);  in ioat_alloc_chan_resources()
704 	writel(((u64)ioat_chan->completion_dma) >> 32,  in ioat_alloc_chan_resources()
705 	       ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);  in ioat_alloc_chan_resources()
712 	spin_lock_bh(&ioat_chan->cleanup_lock);  in ioat_alloc_chan_resources()
713 	spin_lock_bh(&ioat_chan->prep_lock);  in ioat_alloc_chan_resources()
714 	ioat_chan->ring = ring;  in ioat_alloc_chan_resources()
715 	ioat_chan->head = 0;  in ioat_alloc_chan_resources()
716 	ioat_chan->issued = 0;  in ioat_alloc_chan_resources()
717 	ioat_chan->tail = 0;  in ioat_alloc_chan_resources()
718 	ioat_chan->alloc_order = order;  in ioat_alloc_chan_resources()
719 	set_bit(IOAT_RUN, &ioat_chan->state);  in ioat_alloc_chan_resources()
720 	spin_unlock_bh(&ioat_chan->prep_lock);  in ioat_alloc_chan_resources()
721 	spin_unlock_bh(&ioat_chan->cleanup_lock);  in ioat_alloc_chan_resources()
724 	if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) {  in ioat_alloc_chan_resources()
730 		writel(lat_val, ioat_chan->reg_base +  in ioat_alloc_chan_resources()
736 		writel(lat_val, ioat_chan->reg_base +  in ioat_alloc_chan_resources()
741 		       ioat_chan->reg_base +  in ioat_alloc_chan_resources()
745 	ioat_start_null_desc(ioat_chan);  in ioat_alloc_chan_resources()
750 		status = ioat_chansts(ioat_chan);  in ioat_alloc_chan_resources()
754 		return 1 << ioat_chan->alloc_order;  in ioat_alloc_chan_resources()
756 	chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);  in ioat_alloc_chan_resources()
758 	dev_WARN(to_dev(ioat_chan),  in ioat_alloc_chan_resources()
767 		  struct ioatdma_chan *ioat_chan, int idx)  in ioat_init_channel()  argument
771 	ioat_chan->ioat_dma = ioat_dma;  in ioat_init_channel()
772 	ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));  in ioat_init_channel()
773 	spin_lock_init(&ioat_chan->cleanup_lock);  in ioat_init_channel()
774 	ioat_chan->dma_chan.device = dma;  in ioat_init_channel()
775 	dma_cookie_init(&ioat_chan->dma_chan);  in ioat_init_channel()
776 	list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);  in ioat_init_channel()
777 	ioat_dma->idx[idx] = ioat_chan;  in ioat_init_channel()
778 	timer_setup(&ioat_chan->timer, ioat_timer_event, 0);  in ioat_init_channel()
779 	tasklet_setup(&ioat_chan->cleanup_task, ioat_cleanup_event);  in ioat_init_channel()
1065 	struct ioatdma_chan *ioat_chan;  in ioat_intr_quirk()  local
1076 			ioat_chan = to_ioat_chan(c);  in ioat_intr_quirk()
1077 			errmask = readl(ioat_chan->reg_base +  in ioat_intr_quirk()
1081 			writel(errmask, ioat_chan->reg_base +  in ioat_intr_quirk()
1093 	struct ioatdma_chan *ioat_chan;  in ioat3_dma_probe()  local
1179 		ioat_chan = to_ioat_chan(c);  in ioat3_dma_probe()
1181 		       ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);  in ioat3_dma_probe()
1214 	struct ioatdma_chan *ioat_chan;  in ioat_shutdown()  local
1221 		ioat_chan = ioat_dma->idx[i];  in ioat_shutdown()
1222 		if (!ioat_chan)  in ioat_shutdown()
1225 		spin_lock_bh(&ioat_chan->prep_lock);  in ioat_shutdown()
1226 		set_bit(IOAT_CHAN_DOWN, &ioat_chan->state);  in ioat_shutdown()
1227 		spin_unlock_bh(&ioat_chan->prep_lock);  in ioat_shutdown()
1234 		del_timer_sync(&ioat_chan->timer);  in ioat_shutdown()
1237 		ioat_reset_hw(ioat_chan);  in ioat_shutdown()
1245 	struct ioatdma_chan *ioat_chan;  in ioat_resume()  local
1250 		ioat_chan = ioat_dma->idx[i];  in ioat_resume()
1251 		if (!ioat_chan)  in ioat_resume()
1254 		spin_lock_bh(&ioat_chan->prep_lock);  in ioat_resume()
1255 		clear_bit(IOAT_CHAN_DOWN, &ioat_chan->state);  in ioat_resume()
1256 		spin_unlock_bh(&ioat_chan->prep_lock);  in ioat_resume()
1258 		chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);  in ioat_resume()
1259 		writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);  in ioat_resume()