353 static ssize_t svc_rdma_encode_read_list(struct svc_rdma_send_ctxt *sctxt)  in svc_rdma_encode_read_list()  argument
356 	return xdr_stream_encode_item_absent(&sctxt->sc_stream);  in svc_rdma_encode_read_list()
371 					     struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_encode_write_segment()  argument
379 	p = xdr_reserve_space(&sctxt->sc_stream, len);  in svc_rdma_encode_write_segment()
415 					   struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_encode_write_chunk()  argument
425 	ret = xdr_stream_encode_item_present(&sctxt->sc_stream);  in svc_rdma_encode_write_chunk()
431 	ret = xdr_stream_encode_u32(&sctxt->sc_stream, nsegs);  in svc_rdma_encode_write_chunk()
437 		ret = svc_rdma_encode_write_segment(src, sctxt, &remaining);  in svc_rdma_encode_write_chunk()
468 			   struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_encode_write_list()  argument
473 	ret = svc_rdma_encode_write_chunk(rctxt->rc_write_list, sctxt, length);  in svc_rdma_encode_write_list()
479 	ret = xdr_stream_encode_item_absent(&sctxt->sc_stream);  in svc_rdma_encode_write_list()
502 			    struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_encode_reply_chunk()  argument
505 	return svc_rdma_encode_write_chunk(rctxt->rc_reply_chunk, sctxt,  in svc_rdma_encode_reply_chunk()
556 				    struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_pull_up_needed()  argument
564 	if (sctxt->sc_hdrbuf.len + xdr->len < RPCRDMA_PULLUP_THRESH)  in svc_rdma_pull_up_needed()
609 				      struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_pull_up_reply_msg()  argument
616 	dst = sctxt->sc_xprt_buf + sctxt->sc_hdrbuf.len;  in svc_rdma_pull_up_reply_msg()
652 	sctxt->sc_sges[0].length += xdr->len;  in svc_rdma_pull_up_reply_msg()
653 	trace_svcrdma_send_pullup(sctxt->sc_sges[0].length);  in svc_rdma_pull_up_reply_msg()
669 			   struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_map_reply_msg()  argument
681 	sctxt->sc_send_wr.num_sge = 1;  in svc_rdma_map_reply_msg()
682 	sctxt->sc_sges[0].length = sctxt->sc_hdrbuf.len;  in svc_rdma_map_reply_msg()
693 	if (svc_rdma_pull_up_needed(rdma, sctxt, rctxt, xdr))  in svc_rdma_map_reply_msg()
694 		return svc_rdma_pull_up_reply_msg(rdma, sctxt, rctxt, xdr);  in svc_rdma_map_reply_msg()
696 	++sctxt->sc_cur_sge_no;  in svc_rdma_map_reply_msg()
697 	ret = svc_rdma_dma_map_buf(rdma, sctxt,  in svc_rdma_map_reply_msg()
727 		++sctxt->sc_cur_sge_no;  in svc_rdma_map_reply_msg()
728 		ret = svc_rdma_dma_map_page(rdma, sctxt, *ppages++,  in svc_rdma_map_reply_msg()
741 		++sctxt->sc_cur_sge_no;  in svc_rdma_map_reply_msg()
742 		ret = svc_rdma_dma_map_buf(rdma, sctxt, base, len);  in svc_rdma_map_reply_msg()
788 				   struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_send_reply_msg()  argument
794 	ret = svc_rdma_map_reply_msg(rdma, sctxt, rctxt, &rqstp->rq_res);  in svc_rdma_send_reply_msg()
798 	svc_rdma_save_io_pages(rqstp, sctxt);  in svc_rdma_send_reply_msg()
801 		sctxt->sc_send_wr.opcode = IB_WR_SEND_WITH_INV;  in svc_rdma_send_reply_msg()
802 		sctxt->sc_send_wr.ex.invalidate_rkey = rctxt->rc_inv_rkey;  in svc_rdma_send_reply_msg()
804 		sctxt->sc_send_wr.opcode = IB_WR_SEND;  in svc_rdma_send_reply_msg()
806 	return svc_rdma_send(rdma, sctxt);  in svc_rdma_send_reply_msg()
825 			     struct svc_rdma_send_ctxt *sctxt,  in svc_rdma_send_error_msg()  argument
832 	rpcrdma_set_xdrlen(&sctxt->sc_hdrbuf, 0);  in svc_rdma_send_error_msg()
833 	xdr_init_encode(&sctxt->sc_stream, &sctxt->sc_hdrbuf,  in svc_rdma_send_error_msg()
834 			sctxt->sc_xprt_buf, NULL);  in svc_rdma_send_error_msg()
836 	p = xdr_reserve_space(&sctxt->sc_stream,  in svc_rdma_send_error_msg()
848 		p = xdr_reserve_space(&sctxt->sc_stream, 3 * sizeof(*p));  in svc_rdma_send_error_msg()
858 		p = xdr_reserve_space(&sctxt->sc_stream, sizeof(*p));  in svc_rdma_send_error_msg()
867 	sctxt->sc_send_wr.num_sge = 1;  in svc_rdma_send_error_msg()
868 	sctxt->sc_send_wr.opcode = IB_WR_SEND;  in svc_rdma_send_error_msg()
869 	sctxt->sc_sges[0].length = sctxt->sc_hdrbuf.len;  in svc_rdma_send_error_msg()
870 	if (svc_rdma_send(rdma, sctxt))  in svc_rdma_send_error_msg()
875 	svc_rdma_send_ctxt_put(rdma, sctxt);  in svc_rdma_send_error_msg()
900 	struct svc_rdma_send_ctxt *sctxt;  in svc_rdma_sendto()  local
909 	sctxt = svc_rdma_send_ctxt_get(rdma);  in svc_rdma_sendto()
910 	if (!sctxt)  in svc_rdma_sendto()
913 	p = xdr_reserve_space(&sctxt->sc_stream,  in svc_rdma_sendto()
922 	if (svc_rdma_encode_read_list(sctxt) < 0)  in svc_rdma_sendto()
940 		if (svc_rdma_encode_write_list(rctxt, sctxt, length) < 0)  in svc_rdma_sendto()
943 		if (xdr_stream_encode_item_absent(&sctxt->sc_stream) < 0)  in svc_rdma_sendto()
950 		if (svc_rdma_encode_reply_chunk(rctxt, sctxt, ret) < 0)  in svc_rdma_sendto()
953 		if (xdr_stream_encode_item_absent(&sctxt->sc_stream) < 0)  in svc_rdma_sendto()
957 	ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp);  in svc_rdma_sendto()
969 	svc_rdma_save_io_pages(rqstp, sctxt);  in svc_rdma_sendto()
970 	svc_rdma_send_error_msg(rdma, sctxt, rctxt, ret);  in svc_rdma_sendto()
974 	svc_rdma_send_ctxt_put(rdma, sctxt);  in svc_rdma_sendto()