275 	wqe->sqe.flags = 0;  in siw_qp_mpa_rts()
276 	wqe->sqe.num_sge = 1;  in siw_qp_mpa_rts()
277 	wqe->sqe.sge[0].length = 0;  in siw_qp_mpa_rts()
278 	wqe->sqe.sge[0].laddr = 0;  in siw_qp_mpa_rts()
279 	wqe->sqe.sge[0].lkey = 0;  in siw_qp_mpa_rts()
284 	wqe->sqe.rkey = 1;  in siw_qp_mpa_rts()
285 	wqe->sqe.raddr = 0;  in siw_qp_mpa_rts()
289 		wqe->sqe.opcode = SIW_OP_WRITE;  in siw_qp_mpa_rts()
293 		wqe->sqe.opcode = SIW_OP_READ;  in siw_qp_mpa_rts()
300 			siw_read_to_orq(rreq, &wqe->sqe);  in siw_qp_mpa_rts()
480 				rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);  in siw_send_terminate()
483 				rreq->sink_stag = htonl(wqe->sqe.rkey);  in siw_send_terminate()
484 				rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);  in siw_send_terminate()
485 				rreq->read_size = htonl(wqe->sqe.sge[0].length);  in siw_send_terminate()
486 				rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);  in siw_send_terminate()
488 					cpu_to_be64(wqe->sqe.sge[0].laddr);  in siw_send_terminate()
869 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)  in siw_read_to_orq()  argument
871 	rreq->id = sqe->id;  in siw_read_to_orq()
872 	rreq->opcode = sqe->opcode;  in siw_read_to_orq()
873 	rreq->sge[0].laddr = sqe->sge[0].laddr;  in siw_read_to_orq()
874 	rreq->sge[0].length = sqe->sge[0].length;  in siw_read_to_orq()
875 	rreq->sge[0].lkey = sqe->sge[0].lkey;  in siw_read_to_orq()
876 	rreq->sge[1].lkey = sqe->sge[1].lkey;  in siw_read_to_orq()
877 	rreq->flags = sqe->flags | SIW_WQE_VALID;  in siw_read_to_orq()
883 	struct siw_sqe *sqe;  in siw_activate_tx_from_sq()  local
887 	sqe = sq_get_next(qp);  in siw_activate_tx_from_sq()
888 	if (!sqe)  in siw_activate_tx_from_sq()
895 	memcpy(&wqe->sqe, sqe, sizeof(*sqe));  in siw_activate_tx_from_sq()
897 	if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {  in siw_activate_tx_from_sq()
901 	if (wqe->sqe.flags & SIW_WQE_INLINE) {  in siw_activate_tx_from_sq()
902 		if (wqe->sqe.opcode != SIW_OP_SEND &&  in siw_activate_tx_from_sq()
903 		    wqe->sqe.opcode != SIW_OP_WRITE) {  in siw_activate_tx_from_sq()
907 		if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {  in siw_activate_tx_from_sq()
911 		wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];  in siw_activate_tx_from_sq()
912 		wqe->sqe.sge[0].lkey = 0;  in siw_activate_tx_from_sq()
913 		wqe->sqe.num_sge = 1;  in siw_activate_tx_from_sq()
915 	if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {  in siw_activate_tx_from_sq()
917 		if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||  in siw_activate_tx_from_sq()
918 			     wqe->sqe.opcode ==  in siw_activate_tx_from_sq()
932 	} else if (wqe->sqe.opcode == SIW_OP_READ ||  in siw_activate_tx_from_sq()
933 		   wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {  in siw_activate_tx_from_sq()
941 		wqe->sqe.num_sge = 1;  in siw_activate_tx_from_sq()
951 			siw_read_to_orq(rreq, &wqe->sqe);  in siw_activate_tx_from_sq()
961 	smp_store_mb(sqe->flags, 0);  in siw_activate_tx_from_sq()
1002 	wqe->sqe.opcode = SIW_OP_READ_RESPONSE;  in siw_activate_tx()
1003 	wqe->sqe.flags = 0;  in siw_activate_tx()
1005 		wqe->sqe.num_sge = 1;  in siw_activate_tx()
1006 		wqe->sqe.sge[0].length = irqe->sge[0].length;  in siw_activate_tx()
1007 		wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;  in siw_activate_tx()
1008 		wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;  in siw_activate_tx()
1010 		wqe->sqe.num_sge = 0;  in siw_activate_tx()
1016 	wqe->sqe.sge[1].length = irqe->sge[1].length;  in siw_activate_tx()
1018 	wqe->sqe.rkey = irqe->rkey;  in siw_activate_tx()
1019 	wqe->sqe.raddr = irqe->raddr;  in siw_activate_tx()
1060 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,  in siw_sqe_complete()  argument
1067 		u32 sqe_flags = sqe->flags;  in siw_sqe_complete()
1080 			cqe->id = sqe->id;  in siw_sqe_complete()
1081 			cqe->opcode = sqe->opcode;  in siw_sqe_complete()
1094 			smp_store_mb(sqe->flags, 0);  in siw_sqe_complete()
1113 		smp_store_mb(sqe->flags, 0);  in siw_sqe_complete()
1190 	struct siw_sqe *sqe;  in siw_sq_flush()  local
1198 		sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];  in siw_sq_flush()
1199 		if (!READ_ONCE(sqe->flags))  in siw_sq_flush()
1202 		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_sq_flush()
1205 		WRITE_ONCE(sqe->flags, 0);  in siw_sq_flush()
1225 			siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,  in siw_sq_flush()
1234 		sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];  in siw_sq_flush()
1235 		if (!READ_ONCE(sqe->flags))  in siw_sq_flush()
1239 		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_sq_flush()
1246 		WRITE_ONCE(sqe->flags, 0);  in siw_sq_flush()
1283 			siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);  in siw_rq_flush()