71 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,  in pio_copy()  argument
74 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in pio_copy()
125 		if (pbuf->end <= dend) {  in pio_copy()
126 			while (dest < pbuf->end) {  in pio_copy()
132 			dest -= pbuf->sc->size;  in pio_copy()
133 			dend -= pbuf->sc->size;  in pio_copy()
164 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in pio_copy()
224 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,  in read_low_bytes()  argument
227 	pbuf->carry.val64 = 0;  in read_low_bytes()
228 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
229 	pbuf->carry_bytes = nbytes;  in read_low_bytes()
240 static inline void read_extra_bytes(struct pio_buf *pbuf,  in read_extra_bytes()  argument
243 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
244 	pbuf->carry_bytes += nbytes;  in read_extra_bytes()
257 	struct pio_buf *pbuf,  in merge_write8()  argument
264 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));  in merge_write8()
266 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);  in merge_write8()
282 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)  in carry_write8()  argument
284 	if (pbuf->carry_bytes) {  in carry_write8()
286 		writeq(pbuf->carry.val64, dest);  in carry_write8()
303 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,  in seg_pio_copy_start()  argument
306 	void __iomem *dest = pbuf->start + SOP_DISTANCE;  in seg_pio_copy_start()
356 		if (pbuf->end <= dend) {  in seg_pio_copy_start()
357 			while (dest < pbuf->end) {  in seg_pio_copy_start()
363 			dest -= pbuf->sc->size;  in seg_pio_copy_start()
364 			dend -= pbuf->sc->size;  in seg_pio_copy_start()
379 	read_low_bytes(pbuf, from, nbytes & 0x7);  in seg_pio_copy_start()
381 	pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);  in seg_pio_copy_start()
396 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)  in mid_copy_mix()  argument
398 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_mix()
406 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_mix()
419 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_mix()
428 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
453 	if (pbuf->end <= dend) {  in mid_copy_mix()
454 		while (dest < pbuf->end) {  in mid_copy_mix()
455 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
460 		dest -= pbuf->sc->size;  in mid_copy_mix()
461 		dend -= pbuf->sc->size;  in mid_copy_mix()
466 		merge_write8(pbuf, dest, from);  in mid_copy_mix()
471 	pbuf->qw_written += qw_to_write;  in mid_copy_mix()
474 	if (pbuf->carry_bytes + bytes_left >= 8) {  in mid_copy_mix()
478 		nread = 8 - pbuf->carry_bytes;  in mid_copy_mix()
479 		read_extra_bytes(pbuf, from, nread);  in mid_copy_mix()
493 		if (dest >= pbuf->end)  in mid_copy_mix()
494 			dest -= pbuf->sc->size;  in mid_copy_mix()
496 		else if (pbuf->qw_written < PIO_BLOCK_QWS)  in mid_copy_mix()
500 		carry8_write8(pbuf->carry, dest);  in mid_copy_mix()
501 		pbuf->qw_written++;  in mid_copy_mix()
506 		read_low_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
509 		read_extra_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
523 static void mid_copy_straight(struct pio_buf *pbuf,  in mid_copy_straight()  argument
526 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in mid_copy_straight()
532 	if (pbuf->qw_written < PIO_BLOCK_QWS) {  in mid_copy_straight()
545 		send = pbuf->start + PIO_BLOCK_SIZE;  in mid_copy_straight()
579 	if (pbuf->end <= dend) {  in mid_copy_straight()
580 		while (dest < pbuf->end) {  in mid_copy_straight()
586 		dest -= pbuf->sc->size;  in mid_copy_straight()
587 		dend -= pbuf->sc->size;  in mid_copy_straight()
598 	read_low_bytes(pbuf, from, nbytes & 0x7);  in mid_copy_straight()
600 	pbuf->qw_written += nbytes >> 3;  in mid_copy_straight()
612 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)  in seg_pio_copy_mid()  argument
616 	if (pbuf->carry_bytes + nbytes < 8) {  in seg_pio_copy_mid()
618 		read_extra_bytes(pbuf, from, nbytes);  in seg_pio_copy_mid()
635 		if (pbuf->carry_bytes + to_align < 8) {  in seg_pio_copy_mid()
637 			read_extra_bytes(pbuf, from, to_align);  in seg_pio_copy_mid()
642 			unsigned long to_fill = 8 - pbuf->carry_bytes;  in seg_pio_copy_mid()
648 			read_extra_bytes(pbuf, from, to_fill);  in seg_pio_copy_mid()
656 			dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_mid()
667 			if (dest >= pbuf->end)  in seg_pio_copy_mid()
668 				dest -= pbuf->sc->size;  in seg_pio_copy_mid()
670 			else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_mid()
673 			carry8_write8(pbuf->carry, dest);  in seg_pio_copy_mid()
674 			pbuf->qw_written++;  in seg_pio_copy_mid()
678 			read_low_bytes(pbuf, from, extra);  in seg_pio_copy_mid()
695 	if (pbuf->carry_bytes)  in seg_pio_copy_mid()
696 		mid_copy_mix(pbuf, from, nbytes);  in seg_pio_copy_mid()
698 		mid_copy_straight(pbuf, from, nbytes);  in seg_pio_copy_mid()
708 void seg_pio_copy_end(struct pio_buf *pbuf)  in seg_pio_copy_end()  argument
710 	void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));  in seg_pio_copy_end()
720 	if (dest >= pbuf->end)  in seg_pio_copy_end()
721 		dest -= pbuf->sc->size;  in seg_pio_copy_end()
723 	else if (pbuf->qw_written < PIO_BLOCK_QWS)  in seg_pio_copy_end()
727 	if (carry_write8(pbuf, dest)) {  in seg_pio_copy_end()
754 	this_cpu_dec(*pbuf->sc->buffers_allocated);  in seg_pio_copy_end()