1899 	struct ppc440spe_adma_desc_slot *sw_desc;  in ppc440spe_adma_tx_submit()  local
1906 	sw_desc = tx_to_ppc440spe_adma_slot(tx);  in ppc440spe_adma_tx_submit()
1908 	group_start = sw_desc->group_head;  in ppc440spe_adma_tx_submit()
1917 		list_splice_init(&sw_desc->group_list, &chan->chain);  in ppc440spe_adma_tx_submit()
1924 		list_splice_init(&sw_desc->group_list,  in ppc440spe_adma_tx_submit()
1938 		sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);  in ppc440spe_adma_tx_submit()
1950 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_adma_prep_dma_interrupt()  local
1961 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_interrupt()
1963 	if (sw_desc) {  in ppc440spe_adma_prep_dma_interrupt()
1964 		group_start = sw_desc->group_head;  in ppc440spe_adma_prep_dma_interrupt()
1967 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_interrupt()
1971 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_interrupt()
1982 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_adma_prep_dma_memcpy()  local
1999 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_memcpy()
2001 	if (sw_desc) {  in ppc440spe_adma_prep_dma_memcpy()
2002 		group_start = sw_desc->group_head;  in ppc440spe_adma_prep_dma_memcpy()
2007 		sw_desc->unmap_len = len;  in ppc440spe_adma_prep_dma_memcpy()
2008 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_memcpy()
2012 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_memcpy()
2024 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_adma_prep_dma_xor()  local
2042 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_xor()
2044 	if (sw_desc) {  in ppc440spe_adma_prep_dma_xor()
2045 		group_start = sw_desc->group_head;  in ppc440spe_adma_prep_dma_xor()
2052 		sw_desc->unmap_len = len;  in ppc440spe_adma_prep_dma_xor()
2053 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_xor()
2057 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_xor()
2090 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;  in ppc440spe_dma01_prep_mult()  local
2100 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma01_prep_mult()
2101 	if (sw_desc) {  in ppc440spe_dma01_prep_mult()
2106 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_dma01_prep_mult()
2107 		set_bits(op, &sw_desc->flags);  in ppc440spe_dma01_prep_mult()
2108 		sw_desc->src_cnt = src_cnt;  in ppc440spe_dma01_prep_mult()
2109 		sw_desc->dst_cnt = dst_cnt;  in ppc440spe_dma01_prep_mult()
2113 		iter = list_first_entry(&sw_desc->group_list,  in ppc440spe_dma01_prep_mult()
2158 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma01_prep_mult()
2163 	return sw_desc;  in ppc440spe_dma01_prep_mult()
2176 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;  in ppc440spe_dma01_prep_sum_product()  local
2186 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma01_prep_sum_product()
2187 	if (sw_desc) {  in ppc440spe_dma01_prep_sum_product()
2192 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_dma01_prep_sum_product()
2193 		set_bits(op, &sw_desc->flags);  in ppc440spe_dma01_prep_sum_product()
2194 		sw_desc->src_cnt = src_cnt;  in ppc440spe_dma01_prep_sum_product()
2195 		sw_desc->dst_cnt = 1;  in ppc440spe_dma01_prep_sum_product()
2197 		iter = list_first_entry(&sw_desc->group_list,  in ppc440spe_dma01_prep_sum_product()
2267 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma01_prep_sum_product()
2272 	return sw_desc;  in ppc440spe_dma01_prep_sum_product()
2281 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;  in ppc440spe_dma01_prep_pq()  local
2381 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma01_prep_pq()
2382 	if (sw_desc) {  in ppc440spe_dma01_prep_pq()
2383 		ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,  in ppc440spe_dma01_prep_pq()
2389 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);  in ppc440spe_dma01_prep_pq()
2391 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],  in ppc440spe_dma01_prep_pq()
2402 			ppc440spe_adma_pq_set_src_mult(sw_desc,  in ppc440spe_dma01_prep_pq()
2407 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma01_prep_pq()
2408 		list_for_each_entry(iter, &sw_desc->group_list,  in ppc440spe_dma01_prep_pq()
2417 	return sw_desc;  in ppc440spe_dma01_prep_pq()
2426 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;  in ppc440spe_dma2_prep_pq()  local
2444 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);  in ppc440spe_dma2_prep_pq()
2445 	if (sw_desc) {  in ppc440spe_dma2_prep_pq()
2447 		sw_desc->async_tx.flags = flags;  in ppc440spe_dma2_prep_pq()
2448 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_dma2_prep_pq()
2460 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_dma2_prep_pq()
2466 						 &sw_desc->group_list))) {  in ppc440spe_dma2_prep_pq()
2480 		sw_desc->dst_cnt = dst_cnt;  in ppc440spe_dma2_prep_pq()
2482 			set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);  in ppc440spe_dma2_prep_pq()
2484 			set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);  in ppc440spe_dma2_prep_pq()
2487 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);  in ppc440spe_dma2_prep_pq()
2493 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],  in ppc440spe_dma2_prep_pq()
2497 			ppc440spe_adma_pq_set_src_mult(sw_desc,  in ppc440spe_dma2_prep_pq()
2503 	return sw_desc;  in ppc440spe_dma2_prep_pq()
2515 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;  in ppc440spe_adma_prep_dma_pq()  local
2533 		sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2535 		return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pq()
2539 		sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2541 		return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pq()
2566 		sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2572 		sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,  in ppc440spe_adma_prep_dma_pq()
2578 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pq()
2591 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;  in ppc440spe_adma_prep_dma_pqzero_sum()  local
2622 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,  in ppc440spe_adma_prep_dma_pqzero_sum()
2624 	if (sw_desc) {  in ppc440spe_adma_prep_dma_pqzero_sum()
2625 		ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);  in ppc440spe_adma_prep_dma_pqzero_sum()
2628 		sw_desc->async_tx.flags = flags;  in ppc440spe_adma_prep_dma_pqzero_sum()
2629 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_adma_prep_dma_pqzero_sum()
2639 			iter = sw_desc->group_head;  in ppc440spe_adma_prep_dma_pqzero_sum()
2662 			iter = list_first_entry(&sw_desc->group_list,  in ppc440spe_adma_prep_dma_pqzero_sum()
2692 		ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);  in ppc440spe_adma_prep_dma_pqzero_sum()
2696 		list_for_each_entry_reverse(iter, &sw_desc->group_list,  in ppc440spe_adma_prep_dma_pqzero_sum()
2735 		list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,  in ppc440spe_adma_prep_dma_pqzero_sum()
2757 	return sw_desc ? &sw_desc->async_tx : NULL;  in ppc440spe_adma_prep_dma_pqzero_sum()
2785 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_set_dest()  argument
2790 	BUG_ON(index >= sw_desc->dst_cnt);  in ppc440spe_adma_set_dest()
2792 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_set_dest()
2800 		ppc440spe_desc_set_dest_addr(sw_desc->group_head,  in ppc440spe_adma_set_dest()
2804 		sw_desc = ppc440spe_get_group_entry(sw_desc, index);  in ppc440spe_adma_set_dest()
2805 		ppc440spe_desc_set_dest_addr(sw_desc,  in ppc440spe_adma_set_dest()
2832 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pq_set_dest()  argument
2841 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pq_set_dest()
2862 		if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2864 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))  in ppc440spe_adma_pq_set_dest()
2866 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))  in ppc440spe_adma_pq_set_dest()
2869 			iter = ppc440spe_get_group_entry(sw_desc, index);  in ppc440spe_adma_pq_set_dest()
2873 					&sw_desc->group_list, chain_node)  in ppc440spe_adma_pq_set_dest()
2879 					&sw_desc->group_list, chain_node) {  in ppc440spe_adma_pq_set_dest()
2893 						&sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2895 							sw_desc, index++);  in ppc440spe_adma_pq_set_dest()
2901 						&sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2903 							sw_desc, index++);  in ppc440spe_adma_pq_set_dest()
2916 			ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
2920 			qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
2926 			iter = ppc440spe_get_group_entry(sw_desc, index++);  in ppc440spe_adma_pq_set_dest()
2932 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_dest()
2938 			if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_dest()
2942 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_dest()
2947 					    &sw_desc->group_list,  in ppc440spe_adma_pq_set_dest()
2957 					    &sw_desc->group_list,  in ppc440spe_adma_pq_set_dest()
2980 		ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
2985 		qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?  in ppc440spe_adma_pq_set_dest()
2990 		iter = ppc440spe_get_group_entry(sw_desc, 0);  in ppc440spe_adma_pq_set_dest()
2991 		for (i = 0; i < sw_desc->descs_per_op; i++) {  in ppc440spe_adma_pq_set_dest()
3002 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_dest()
3003 				sw_desc->descs_per_op);  in ppc440spe_adma_pq_set_dest()
3004 			for (i = 0; i < sw_desc->descs_per_op; i++) {  in ppc440spe_adma_pq_set_dest()
3022 		struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pqzero_sum_set_dest()  argument
3030 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pqzero_sum_set_dest()
3037 	list_for_each_entry_reverse(end, &sw_desc->group_list,  in ppc440spe_adma_pqzero_sum_set_dest()
3044 	iter = ppc440spe_get_group_entry(sw_desc, idx);  in ppc440spe_adma_pqzero_sum_set_dest()
3048 		list_for_each_entry_from(iter, &sw_desc->group_list,  in ppc440spe_adma_pqzero_sum_set_dest()
3060 		list_for_each_entry_from(iter, &sw_desc->group_list,  in ppc440spe_adma_pqzero_sum_set_dest()
3098 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pq_set_src()  argument
3105 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pq_set_src()
3112 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_src()
3115 				&sw_desc->flags) ?  2 : 3;  in ppc440spe_adma_pq_set_src()
3123 						&sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3127 				    &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3131 				    &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3135 				    &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3141 				iter = ppc440spe_get_group_entry(sw_desc, 0);  in ppc440spe_adma_pq_set_src()
3153 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src()
3154 				    index - iskip + sw_desc->dst_cnt);  in ppc440spe_adma_pq_set_src()
3162 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3164 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))  in ppc440spe_adma_pq_set_src()
3168 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src()
3176 			    test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&  in ppc440spe_adma_pq_set_src()
3177 			    sw_desc->dst_cnt == 2) {  in ppc440spe_adma_pq_set_src()
3181 				iter = ppc440spe_get_group_entry(sw_desc, 1);  in ppc440spe_adma_pq_set_src()
3190 		iter = sw_desc->group_head;  in ppc440spe_adma_pq_set_src()
3196 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src()
3197 				sw_desc->descs_per_op);  in ppc440spe_adma_pq_set_src()
3208 		struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_memcpy_xor_set_src()  argument
3213 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_memcpy_xor_set_src()
3214 	sw_desc = sw_desc->group_head;  in ppc440spe_adma_memcpy_xor_set_src()
3216 	if (likely(sw_desc))  in ppc440spe_adma_memcpy_xor_set_src()
3217 		ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);  in ppc440spe_adma_memcpy_xor_set_src()
3450 		struct ppc440spe_adma_desc_slot *sw_desc,  in ppc440spe_adma_pq_set_src_mult()  argument
3457 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);  in ppc440spe_adma_pq_set_src_mult()
3462 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {  in ppc440spe_adma_pq_set_src_mult()
3464 					&sw_desc->flags) ? 2 : 3;  in ppc440spe_adma_pq_set_src_mult()
3468 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src_mult()
3469 					sw_desc->dst_cnt - 1);  in ppc440spe_adma_pq_set_src_mult()
3470 				if (sw_desc->dst_cnt == 2)  in ppc440spe_adma_pq_set_src_mult()
3472 							sw_desc, 0);  in ppc440spe_adma_pq_set_src_mult()
3478 				iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src_mult()
3480 							sw_desc->dst_cnt);  in ppc440spe_adma_pq_set_src_mult()
3492 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))  in ppc440spe_adma_pq_set_src_mult()
3494 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))  in ppc440spe_adma_pq_set_src_mult()
3497 			iter = ppc440spe_get_group_entry(sw_desc, index + znum);  in ppc440spe_adma_pq_set_src_mult()
3518 		iter = sw_desc->group_head;  in ppc440spe_adma_pq_set_src_mult()
3519 		if (sw_desc->dst_cnt == 2) {  in ppc440spe_adma_pq_set_src_mult()
3524 			iter = ppc440spe_get_group_entry(sw_desc,  in ppc440spe_adma_pq_set_src_mult()
3525 			       sw_desc->descs_per_op);  in ppc440spe_adma_pq_set_src_mult()
3658 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;  in ppc440spe_chan_start_null_xor()  local
3667 	sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);  in ppc440spe_chan_start_null_xor()
3668 	if (sw_desc) {  in ppc440spe_chan_start_null_xor()
3669 		group_start = sw_desc->group_head;  in ppc440spe_chan_start_null_xor()
3670 		list_splice_init(&sw_desc->group_list, &chan->chain);  in ppc440spe_chan_start_null_xor()
3671 		async_tx_ack(&sw_desc->async_tx);  in ppc440spe_chan_start_null_xor()
3674 		cookie = dma_cookie_assign(&sw_desc->async_tx);  in ppc440spe_chan_start_null_xor()
3685 		ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);  in ppc440spe_chan_start_null_xor()
3704 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;  in ppc440spe_test_raid6()  local
3718 	sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);  in ppc440spe_test_raid6()
3719 	if (sw_desc) {  in ppc440spe_test_raid6()
3721 		ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);  in ppc440spe_test_raid6()
3722 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {  in ppc440spe_test_raid6()
3739 	ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);  in ppc440spe_test_raid6()
3740 	ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);  in ppc440spe_test_raid6()
3743 	ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);  in ppc440spe_test_raid6()
3745 	async_tx_ack(&sw_desc->async_tx);  in ppc440spe_test_raid6()
3746 	sw_desc->async_tx.callback = ppc440spe_test_callback;  in ppc440spe_test_raid6()
3747 	sw_desc->async_tx.callback_param = NULL;  in ppc440spe_test_raid6()
3751 	ppc440spe_adma_tx_submit(&sw_desc->async_tx);  in ppc440spe_test_raid6()