133 	struct xdr_buf *xdr = &rqst->rq_snd_buf;  in rpcrdma_args_inline()  local
137 	if (xdr->len > ep->re_max_inline_send)  in rpcrdma_args_inline()
140 	if (xdr->page_len) {  in rpcrdma_args_inline()
141 		remaining = xdr->page_len;  in rpcrdma_args_inline()
142 		offset = offset_in_page(xdr->page_base);  in rpcrdma_args_inline()
295 encode_rdma_segment(struct xdr_stream *xdr, struct rpcrdma_mr *mr)  in encode_rdma_segment()  argument
299 	p = xdr_reserve_space(xdr, 4 * sizeof(*p));  in encode_rdma_segment()
308 encode_read_segment(struct xdr_stream *xdr, struct rpcrdma_mr *mr,  in encode_read_segment()  argument
313 	p = xdr_reserve_space(xdr, 6 * sizeof(*p));  in encode_read_segment()
367 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_encode_read_list()  local
390 		if (encode_read_segment(xdr, mr, pos) < 0)  in rpcrdma_encode_read_list()
399 	if (xdr_stream_encode_item_absent(xdr) < 0)  in rpcrdma_encode_read_list()
424 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_encode_write_list()  local
440 	if (xdr_stream_encode_item_present(xdr) < 0)  in rpcrdma_encode_write_list()
442 	segcount = xdr_reserve_space(xdr, sizeof(*segcount));  in rpcrdma_encode_write_list()
453 		if (encode_rdma_segment(xdr, mr) < 0)  in rpcrdma_encode_write_list()
467 	if (xdr_stream_encode_item_absent(xdr) < 0)  in rpcrdma_encode_write_list()
489 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_encode_reply_chunk()  local
496 		if (xdr_stream_encode_item_absent(xdr) < 0)  in rpcrdma_encode_reply_chunk()
506 	if (xdr_stream_encode_item_present(xdr) < 0)  in rpcrdma_encode_reply_chunk()
508 	segcount = xdr_reserve_space(xdr, sizeof(*segcount));  in rpcrdma_encode_reply_chunk()
519 		if (encode_rdma_segment(xdr, mr) < 0)  in rpcrdma_encode_reply_chunk()
613 				     struct xdr_buf *xdr)  in rpcrdma_prepare_pagelist()  argument
621 	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);  in rpcrdma_prepare_pagelist()
622 	page_base = offset_in_page(xdr->page_base);  in rpcrdma_prepare_pagelist()
623 	remaining = xdr->page_len;  in rpcrdma_prepare_pagelist()
653 				     struct xdr_buf *xdr,  in rpcrdma_prepare_tail_iov()  argument
659 	struct page *page = virt_to_page(xdr->tail[0].iov_base);  in rpcrdma_prepare_tail_iov()
680 				    struct xdr_buf *xdr)  in rpcrdma_pullup_tail_iov()  argument
684 	dst = (unsigned char *)xdr->head[0].iov_base;  in rpcrdma_pullup_tail_iov()
685 	dst += xdr->head[0].iov_len + xdr->page_len;  in rpcrdma_pullup_tail_iov()
686 	memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len);  in rpcrdma_pullup_tail_iov()
687 	r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len;  in rpcrdma_pullup_tail_iov()
694 				    struct xdr_buf *xdr)  in rpcrdma_pullup_pagelist()  argument
700 	dst = (unsigned char *)xdr->head[0].iov_base;  in rpcrdma_pullup_pagelist()
701 	dst += xdr->head[0].iov_len;  in rpcrdma_pullup_pagelist()
702 	ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT);  in rpcrdma_pullup_pagelist()
703 	page_base = offset_in_page(xdr->page_base);  in rpcrdma_pullup_pagelist()
704 	remaining = xdr->page_len;  in rpcrdma_pullup_pagelist()
730 					struct xdr_buf *xdr)  in rpcrdma_prepare_noch_pullup()  argument
732 	if (unlikely(xdr->tail[0].iov_len))  in rpcrdma_prepare_noch_pullup()
733 		rpcrdma_pullup_tail_iov(r_xprt, req, xdr);  in rpcrdma_prepare_noch_pullup()
735 	if (unlikely(xdr->page_len))  in rpcrdma_prepare_noch_pullup()
736 		rpcrdma_pullup_pagelist(r_xprt, req, xdr);  in rpcrdma_prepare_noch_pullup()
739 	return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len);  in rpcrdma_prepare_noch_pullup()
744 					struct xdr_buf *xdr)  in rpcrdma_prepare_noch_mapped()  argument
746 	struct kvec *tail = &xdr->tail[0];  in rpcrdma_prepare_noch_mapped()
748 	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))  in rpcrdma_prepare_noch_mapped()
750 	if (xdr->page_len)  in rpcrdma_prepare_noch_mapped()
751 		if (!rpcrdma_prepare_pagelist(req, xdr))  in rpcrdma_prepare_noch_mapped()
754 		if (!rpcrdma_prepare_tail_iov(req, xdr,  in rpcrdma_prepare_noch_mapped()
766 				   struct xdr_buf *xdr)  in rpcrdma_prepare_readch()  argument
768 	if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len))  in rpcrdma_prepare_readch()
776 	if (xdr->tail[0].iov_len > 3) {  in rpcrdma_prepare_readch()
784 		page_base = offset_in_page(xdr->tail[0].iov_base);  in rpcrdma_prepare_readch()
785 		len = xdr->tail[0].iov_len;  in rpcrdma_prepare_readch()
788 		if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len))  in rpcrdma_prepare_readch()
808 				     struct xdr_buf *xdr,  in rpcrdma_prepare_send_sges()  argument
830 		if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr))  in rpcrdma_prepare_send_sges()
834 		if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr))  in rpcrdma_prepare_send_sges()
838 		if (!rpcrdma_prepare_readch(r_xprt, req, xdr))  in rpcrdma_prepare_send_sges()
879 	struct xdr_stream *xdr = &req->rl_stream;  in rpcrdma_marshal_req()  local
893 	xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf),  in rpcrdma_marshal_req()
898 	p = xdr_reserve_space(xdr, 4 * sizeof(*p));  in rpcrdma_marshal_req()
1141 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_is_bcall()  local
1148 	p = xdr_inline_decode(xdr, 0);  in rpcrdma_is_bcall()
1171 	p = xdr_inline_decode(xdr, 3 * sizeof(*p));  in rpcrdma_is_bcall()
1188 static int decode_rdma_segment(struct xdr_stream *xdr, u32 *length)  in decode_rdma_segment()  argument
1194 	p = xdr_inline_decode(xdr, 4 * sizeof(*p));  in decode_rdma_segment()
1203 static int decode_write_chunk(struct xdr_stream *xdr, u32 *length)  in decode_write_chunk()  argument
1208 	p = xdr_inline_decode(xdr, sizeof(*p));  in decode_write_chunk()
1215 		if (decode_rdma_segment(xdr, &seglength))  in decode_write_chunk()
1227 static int decode_read_list(struct xdr_stream *xdr)  in decode_read_list()  argument
1231 	p = xdr_inline_decode(xdr, sizeof(*p));  in decode_read_list()
1241 static int decode_write_list(struct xdr_stream *xdr, u32 *length)  in decode_write_list()  argument
1250 		p = xdr_inline_decode(xdr, sizeof(*p));  in decode_write_list()
1258 		if (decode_write_chunk(xdr, &chunklen))  in decode_write_list()
1266 static int decode_reply_chunk(struct xdr_stream *xdr, u32 *length)  in decode_reply_chunk()  argument
1270 	p = xdr_inline_decode(xdr, sizeof(*p));  in decode_reply_chunk()
1276 		if (decode_write_chunk(xdr, length))  in decode_reply_chunk()
1285 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_decode_msg()  local
1290 	if (decode_read_list(xdr))  in rpcrdma_decode_msg()
1292 	if (decode_write_list(xdr, &writelist))  in rpcrdma_decode_msg()
1294 	if (decode_reply_chunk(xdr, &replychunk))  in rpcrdma_decode_msg()
1302 	base = (char *)xdr_inline_decode(xdr, 0);  in rpcrdma_decode_msg()
1303 	rpclen = xdr_stream_remaining(xdr);  in rpcrdma_decode_msg()
1314 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_decode_nomsg()  local
1318 	if (decode_read_list(xdr))  in rpcrdma_decode_nomsg()
1320 	if (decode_write_list(xdr, &writelist))  in rpcrdma_decode_nomsg()
1322 	if (decode_reply_chunk(xdr, &replychunk))  in rpcrdma_decode_nomsg()
1340 	struct xdr_stream *xdr = &rep->rr_stream;  in rpcrdma_decode_error()  local
1343 	p = xdr_inline_decode(xdr, sizeof(*p));  in rpcrdma_decode_error()
1349 		p = xdr_inline_decode(xdr, 2 * sizeof(*p));  in rpcrdma_decode_error()