180 xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,  in xdr_inline_pages()  argument
183 	struct kvec *head = xdr->head;  in xdr_inline_pages()
184 	struct kvec *tail = xdr->tail;  in xdr_inline_pages()
190 	xdr->pages = pages;  in xdr_inline_pages()
191 	xdr->page_base = base;  in xdr_inline_pages()
192 	xdr->page_len = len;  in xdr_inline_pages()
196 	if ((xdr->page_len & 3) == 0)  in xdr_inline_pages()
199 	xdr->buflen += len;  in xdr_inline_pages()
633 unsigned int xdr_stream_pos(const struct xdr_stream *xdr)  in xdr_stream_pos()  argument
635 	return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;  in xdr_stream_pos()
643 unsigned int xdr_page_pos(const struct xdr_stream *xdr)  in xdr_page_pos()  argument
645 	unsigned int pos = xdr_stream_pos(xdr);  in xdr_page_pos()
647 	WARN_ON(pos < xdr->buf->head[0].iov_len);  in xdr_page_pos()
648 	return pos - xdr->buf->head[0].iov_len;  in xdr_page_pos()
666 void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,  in xdr_init_encode()  argument
672 	xdr_set_scratch_buffer(xdr, NULL, 0);  in xdr_init_encode()
674 	xdr->buf = buf;  in xdr_init_encode()
675 	xdr->iov = iov;  in xdr_init_encode()
676 	xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len);  in xdr_init_encode()
677 	xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len);  in xdr_init_encode()
680 	if (p != xdr->p && p != NULL) {  in xdr_init_encode()
683 		BUG_ON(p < xdr->p || p > xdr->end);  in xdr_init_encode()
684 		len = (char *)p - (char *)xdr->p;  in xdr_init_encode()
685 		xdr->p = p;  in xdr_init_encode()
689 	xdr->rqst = rqst;  in xdr_init_encode()
706 inline void xdr_commit_encode(struct xdr_stream *xdr)  in xdr_commit_encode()  argument
708 	int shift = xdr->scratch.iov_len;  in xdr_commit_encode()
713 	page = page_address(*xdr->page_ptr);  in xdr_commit_encode()
714 	memcpy(xdr->scratch.iov_base, page, shift);  in xdr_commit_encode()
715 	memmove(page, page + shift, (void *)xdr->p - page);  in xdr_commit_encode()
716 	xdr->scratch.iov_len = 0;  in xdr_commit_encode()
720 static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,  in xdr_get_next_encode_buffer()  argument
729 	if (xdr->buf->len + nbytes > xdr->buf->buflen)  in xdr_get_next_encode_buffer()
731 	frag1bytes = (xdr->end - xdr->p) << 2;  in xdr_get_next_encode_buffer()
733 	if (xdr->iov)  in xdr_get_next_encode_buffer()
734 		xdr->iov->iov_len += frag1bytes;  in xdr_get_next_encode_buffer()
736 		xdr->buf->page_len += frag1bytes;  in xdr_get_next_encode_buffer()
737 	xdr->page_ptr++;  in xdr_get_next_encode_buffer()
738 	xdr->iov = NULL;  in xdr_get_next_encode_buffer()
746 	xdr->scratch.iov_base = xdr->p;  in xdr_get_next_encode_buffer()
747 	xdr->scratch.iov_len = frag1bytes;  in xdr_get_next_encode_buffer()
748 	p = page_address(*xdr->page_ptr);  in xdr_get_next_encode_buffer()
753 	xdr->p = (void *)p + frag2bytes;  in xdr_get_next_encode_buffer()
754 	space_left = xdr->buf->buflen - xdr->buf->len;  in xdr_get_next_encode_buffer()
756 		xdr->end = (void *)p + PAGE_SIZE;  in xdr_get_next_encode_buffer()
758 		xdr->end = (void *)p + space_left - frag1bytes;  in xdr_get_next_encode_buffer()
760 	xdr->buf->page_len += frag2bytes;  in xdr_get_next_encode_buffer()
761 	xdr->buf->len += nbytes;  in xdr_get_next_encode_buffer()
764 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_get_next_encode_buffer()
777 __be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes)  in xdr_reserve_space()  argument
779 	__be32 *p = xdr->p;  in xdr_reserve_space()
782 	xdr_commit_encode(xdr);  in xdr_reserve_space()
787 	if (unlikely(q > xdr->end || q < p))  in xdr_reserve_space()
788 		return xdr_get_next_encode_buffer(xdr, nbytes);  in xdr_reserve_space()
789 	xdr->p = q;  in xdr_reserve_space()
790 	if (xdr->iov)  in xdr_reserve_space()
791 		xdr->iov->iov_len += nbytes;  in xdr_reserve_space()
793 		xdr->buf->page_len += nbytes;  in xdr_reserve_space()
794 	xdr->buf->len += nbytes;  in xdr_reserve_space()
811 int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes)  in xdr_reserve_space_vec()  argument
821 	if (xdr->iov == xdr->buf->head) {  in xdr_reserve_space_vec()
822 		xdr->iov = NULL;  in xdr_reserve_space_vec()
823 		xdr->end = xdr->p;  in xdr_reserve_space_vec()
827 		thislen = xdr->buf->page_len % PAGE_SIZE;  in xdr_reserve_space_vec()
830 		p = xdr_reserve_space(xdr, thislen);  in xdr_reserve_space_vec()
863 void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)  in xdr_truncate_encode()  argument
865 	struct xdr_buf *buf = xdr->buf;  in xdr_truncate_encode()
875 	xdr_commit_encode(xdr);  in xdr_truncate_encode()
881 		xdr->p = tail->iov_base + tail->iov_len;  in xdr_truncate_encode()
882 		WARN_ON_ONCE(!xdr->end);  in xdr_truncate_encode()
883 		WARN_ON_ONCE(!xdr->iov);  in xdr_truncate_encode()
893 	xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT);  in xdr_truncate_encode()
896 		xdr->p = page_address(*xdr->page_ptr);  in xdr_truncate_encode()
897 		xdr->end = (void *)xdr->p + PAGE_SIZE;  in xdr_truncate_encode()
898 		xdr->p = (void *)xdr->p + (new % PAGE_SIZE);  in xdr_truncate_encode()
899 		WARN_ON_ONCE(xdr->iov);  in xdr_truncate_encode()
903 		xdr->end = head->iov_base + head->iov_len;  in xdr_truncate_encode()
905 	xdr->page_ptr--;  in xdr_truncate_encode()
908 	xdr->p = head->iov_base + head->iov_len;  in xdr_truncate_encode()
909 	xdr->iov = buf->head;  in xdr_truncate_encode()
925 int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen)  in xdr_restrict_buflen()  argument
927 	struct xdr_buf *buf = xdr->buf;  in xdr_restrict_buflen()
928 	int left_in_this_buf = (void *)xdr->end - (void *)xdr->p;  in xdr_restrict_buflen()
936 		xdr->end = (void *)xdr->end + newbuflen - end_offset;  in xdr_restrict_buflen()
950 void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base,  in xdr_write_pages()  argument
953 	struct xdr_buf *buf = xdr->buf;  in xdr_write_pages()
959 	iov->iov_base = (char *)xdr->p;  in xdr_write_pages()
961 	xdr->iov = iov;  in xdr_write_pages()
966 		BUG_ON(xdr->p >= xdr->end);  in xdr_write_pages()
967 		iov->iov_base = (char *)xdr->p + (len & 3);  in xdr_write_pages()
970 		*xdr->p++ = 0;  in xdr_write_pages()
977 static void xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov,  in xdr_set_iov()  argument
982 	xdr->p = (__be32*)iov->iov_base;  in xdr_set_iov()
983 	xdr->end = (__be32*)(iov->iov_base + len);  in xdr_set_iov()
984 	xdr->iov = iov;  in xdr_set_iov()
985 	xdr->page_ptr = NULL;  in xdr_set_iov()
988 static int xdr_set_page_base(struct xdr_stream *xdr,  in xdr_set_page_base()  argument
997 	maxlen = xdr->buf->page_len;  in xdr_set_page_base()
1004 	base += xdr->buf->page_base;  in xdr_set_page_base()
1007 	xdr->page_ptr = &xdr->buf->pages[pgnr];  in xdr_set_page_base()
1008 	kaddr = page_address(*xdr->page_ptr);  in xdr_set_page_base()
1011 	xdr->p = (__be32*)(kaddr + pgoff);  in xdr_set_page_base()
1016 	xdr->end = (__be32*)(kaddr + pgend);  in xdr_set_page_base()
1017 	xdr->iov = NULL;  in xdr_set_page_base()
1021 static void xdr_set_page(struct xdr_stream *xdr, unsigned int base,  in xdr_set_page()  argument
1024 	if (xdr_set_page_base(xdr, base, len) < 0)  in xdr_set_page()
1025 		xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);  in xdr_set_page()
1028 static void xdr_set_next_page(struct xdr_stream *xdr)  in xdr_set_next_page()  argument
1032 	newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;  in xdr_set_next_page()
1033 	newbase -= xdr->buf->page_base;  in xdr_set_next_page()
1035 	xdr_set_page(xdr, newbase, PAGE_SIZE);  in xdr_set_next_page()
1038 static bool xdr_set_next_buffer(struct xdr_stream *xdr)  in xdr_set_next_buffer()  argument
1040 	if (xdr->page_ptr != NULL)  in xdr_set_next_buffer()
1041 		xdr_set_next_page(xdr);  in xdr_set_next_buffer()
1042 	else if (xdr->iov == xdr->buf->head) {  in xdr_set_next_buffer()
1043 		xdr_set_page(xdr, 0, PAGE_SIZE);  in xdr_set_next_buffer()
1045 	return xdr->p != xdr->end;  in xdr_set_next_buffer()
1055 void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,  in xdr_init_decode()  argument
1058 	xdr->buf = buf;  in xdr_init_decode()
1059 	xdr->scratch.iov_base = NULL;  in xdr_init_decode()
1060 	xdr->scratch.iov_len = 0;  in xdr_init_decode()
1061 	xdr->nwords = XDR_QUADLEN(buf->len);  in xdr_init_decode()
1063 		xdr_set_iov(xdr, buf->head, buf->len);  in xdr_init_decode()
1065 		xdr_set_page_base(xdr, 0, buf->len);  in xdr_init_decode()
1067 		xdr_set_iov(xdr, buf->head, buf->len);  in xdr_init_decode()
1068 	if (p != NULL && p > xdr->p && xdr->end >= p) {  in xdr_init_decode()
1069 		xdr->nwords -= p - xdr->p;  in xdr_init_decode()
1070 		xdr->p = p;  in xdr_init_decode()
1072 	xdr->rqst = rqst;  in xdr_init_decode()
1083 void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,  in xdr_init_decode_pages()  argument
1091 	xdr_init_decode(xdr, buf, NULL, NULL);  in xdr_init_decode_pages()
1095 static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)  in __xdr_inline_decode()  argument
1098 	__be32 *p = xdr->p;  in __xdr_inline_decode()
1101 	if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p))  in __xdr_inline_decode()
1103 	xdr->p = q;  in __xdr_inline_decode()
1104 	xdr->nwords -= nwords;  in __xdr_inline_decode()
1119 void xdr_set_scratch_buffer(struct xdr_stream *xdr, void *buf, size_t buflen)  in xdr_set_scratch_buffer()  argument
1121 	xdr->scratch.iov_base = buf;  in xdr_set_scratch_buffer()
1122 	xdr->scratch.iov_len = buflen;  in xdr_set_scratch_buffer()
1126 static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes)  in xdr_copy_to_scratch()  argument
1129 	char *cpdest = xdr->scratch.iov_base;  in xdr_copy_to_scratch()
1130 	size_t cplen = (char *)xdr->end - (char *)xdr->p;  in xdr_copy_to_scratch()
1132 	if (nbytes > xdr->scratch.iov_len)  in xdr_copy_to_scratch()
1134 	p = __xdr_inline_decode(xdr, cplen);  in xdr_copy_to_scratch()
1138 	if (!xdr_set_next_buffer(xdr))  in xdr_copy_to_scratch()
1142 	p = __xdr_inline_decode(xdr, nbytes);  in xdr_copy_to_scratch()
1146 	return xdr->scratch.iov_base;  in xdr_copy_to_scratch()
1148 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_copy_to_scratch()
1162 __be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)  in xdr_inline_decode()  argument
1167 		return xdr->p;  in xdr_inline_decode()
1168 	if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))  in xdr_inline_decode()
1170 	p = __xdr_inline_decode(xdr, nbytes);  in xdr_inline_decode()
1173 	return xdr_copy_to_scratch(xdr, nbytes);  in xdr_inline_decode()
1175 	trace_rpc_xdr_overflow(xdr, nbytes);  in xdr_inline_decode()
1180 static void xdr_realign_pages(struct xdr_stream *xdr)  in xdr_realign_pages()  argument
1182 	struct xdr_buf *buf = xdr->buf;  in xdr_realign_pages()
1184 	unsigned int cur = xdr_stream_pos(xdr);  in xdr_realign_pages()
1191 		trace_rpc_xdr_alignment(xdr, offset, copied);  in xdr_realign_pages()
1192 		xdr->nwords = XDR_QUADLEN(buf->len - cur);  in xdr_realign_pages()
1196 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_align_pages()  argument
1198 	struct xdr_buf *buf = xdr->buf;  in xdr_align_pages()
1200 	unsigned int cur = xdr_stream_pos(xdr);  in xdr_align_pages()
1203 	if (xdr->nwords == 0)  in xdr_align_pages()
1206 	xdr_realign_pages(xdr);  in xdr_align_pages()
1207 	if (nwords > xdr->nwords) {  in xdr_align_pages()
1208 		nwords = xdr->nwords;  in xdr_align_pages()
1213 	else if (nwords < xdr->nwords) {  in xdr_align_pages()
1217 		trace_rpc_xdr_alignment(xdr, offset, copied);  in xdr_align_pages()
1218 		xdr->nwords = XDR_QUADLEN(buf->len - cur);  in xdr_align_pages()
1234 unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)  in xdr_read_pages()  argument
1236 	struct xdr_buf *buf = xdr->buf;  in xdr_read_pages()
1242 	len = xdr_align_pages(xdr, len);  in xdr_read_pages()
1247 	xdr->iov = iov = buf->tail;  in xdr_read_pages()
1249 	end = ((xdr->nwords - nwords) << 2) + padding;  in xdr_read_pages()
1257 	xdr->p = (__be32 *)((char *)iov->iov_base + padding);  in xdr_read_pages()
1258 	xdr->end = (__be32 *)((char *)iov->iov_base + end);  in xdr_read_pages()
1259 	xdr->page_ptr = NULL;  in xdr_read_pages()
1260 	xdr->nwords = XDR_QUADLEN(end - padding);  in xdr_read_pages()
1265 uint64_t xdr_align_data(struct xdr_stream *xdr, uint64_t offset, uint32_t length)  in xdr_align_data()  argument
1267 	struct xdr_buf *buf = xdr->buf;  in xdr_align_data()
1275 	xdr_realign_pages(xdr);  in xdr_align_data()
1276 	from = xdr_page_pos(xdr);  in xdr_align_data()
1277 	bytes = xdr->nwords << 2;  in xdr_align_data()
1295 	xdr->nwords -= XDR_QUADLEN(length);  in xdr_align_data()
1296 	xdr_set_page(xdr, from + length, PAGE_SIZE);  in xdr_align_data()
1301 uint64_t xdr_expand_hole(struct xdr_stream *xdr, uint64_t offset, uint64_t length)  in xdr_expand_hole()  argument
1303 	struct xdr_buf *buf = xdr->buf;  in xdr_expand_hole()
1312 	xdr_realign_pages(xdr);  in xdr_expand_hole()
1313 	from = xdr_page_pos(xdr);  in xdr_expand_hole()
1314 	bytes = xdr->nwords << 2;  in xdr_expand_hole()
1320 		xdr->nwords -= XDR_QUADLEN(truncated);  in xdr_expand_hole()
1333 	xdr_set_page(xdr, offset + length, PAGE_SIZE);  in xdr_expand_hole()
1348 void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)  in xdr_enter_page()  argument
1350 	len = xdr_align_pages(xdr, len);  in xdr_enter_page()
1356 		xdr_set_page_base(xdr, 0, len);  in xdr_enter_page()
1857 ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size)  in xdr_stream_decode_opaque()  argument
1862 	ret = xdr_stream_decode_opaque_inline(xdr, &p, size);  in xdr_stream_decode_opaque()
1883 ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,  in xdr_stream_decode_opaque_dup()  argument
1889 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);  in xdr_stream_decode_opaque_dup()
1912 ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size)  in xdr_stream_decode_string()  argument
1917 	ret = xdr_stream_decode_opaque_inline(xdr, &p, size);  in xdr_stream_decode_string()
1941 ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,  in xdr_stream_decode_string_dup()  argument
1947 	ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);  in xdr_stream_decode_string_dup()