123 		struct svc_sock *svsk =  in svc_tcp_release_rqst()  local
127 		skb_free_datagram_locked(svsk->sk_sk, skb);  in svc_tcp_release_rqst()
155 	struct svc_sock *svsk =  in svc_set_cmsg_data()  local
157 	switch (svsk->sk_sk->sk_family) {  in svc_set_cmsg_data()
193 static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining)  in svc_one_sock_name()  argument
195 	const struct sock *sk = svsk->sk_sk;  in svc_one_sock_name()
254 	struct svc_sock *svsk =  in svc_tcp_read_msg()  local
264 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_read_msg()
279 	len = sock_recvmsg(svsk->sk_sock, &msg, MSG_DONTWAIT);  in svc_tcp_read_msg()
287 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_read_msg()
295 static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs)  in svc_sock_setbufsize()  argument
297 	unsigned int max_mesg = svsk->sk_xprt.xpt_server->sv_max_mesg;  in svc_sock_setbufsize()
298 	struct socket *sock = svsk->sk_sock;  in svc_sock_setbufsize()
322 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_data_ready()  local
324 	if (svsk) {  in svc_data_ready()
327 		svsk->sk_odata(sk);  in svc_data_ready()
328 		trace_svcsock_data_ready(&svsk->sk_xprt, 0);  in svc_data_ready()
329 		if (!test_and_set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags))  in svc_data_ready()
330 			svc_xprt_enqueue(&svsk->sk_xprt);  in svc_data_ready()
339 	struct svc_sock	*svsk = (struct svc_sock *)(sk->sk_user_data);  in svc_write_space()  local
341 	if (svsk) {  in svc_write_space()
344 		trace_svcsock_write_space(&svsk->sk_xprt, 0);  in svc_write_space()
345 		svsk->sk_owspace(sk);  in svc_write_space()
346 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_write_space()
352 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_has_wspace()  local
356 	return !test_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_tcp_has_wspace()
361 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_kill_temp_xprt()  local
363 	sock_no_linger(svsk->sk_sock->sk);  in svc_tcp_kill_temp_xprt()
433 	struct svc_sock	*svsk =  in svc_udp_recvfrom()  local
435 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_udp_recvfrom()
451 	if (test_and_clear_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags))  in svc_udp_recvfrom()
460 	    svc_sock_setbufsize(svsk, serv->sv_nrthreads + 3);  in svc_udp_recvfrom()
462 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_recvfrom()
463 	err = kernel_recvmsg(svsk->sk_sock, &msg, NULL,  in svc_udp_recvfrom()
467 	skb = skb_recv_udp(svsk->sk_sk, 0, 1, &err);  in svc_udp_recvfrom()
478 	sock_write_timestamp(svsk->sk_sk, skb->tstamp);  in svc_udp_recvfrom()
479 	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); /* there may be more data... */  in svc_udp_recvfrom()
483 	trace_svcsock_udp_recv(&svsk->sk_xprt, len);  in svc_udp_recvfrom()
527 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_recvfrom()
529 	trace_svcsock_udp_recv_err(&svsk->sk_xprt, err);  in svc_udp_recvfrom()
554 	struct svc_sock	*svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_udp_sendto()  local
579 	err = xprt_sock_sendmsg(svsk->sk_sock, &msg, xdr, 0, 0, &sent);  in svc_udp_sendto()
583 		err = xprt_sock_sendmsg(svsk->sk_sock, &msg, xdr, 0, 0, &sent);  in svc_udp_sendto()
600 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_udp_has_wspace()  local
608 	set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_udp_has_wspace()
609 	required = atomic_read(&svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg;  in svc_udp_has_wspace()
610 	if (required*2 > sock_wspace(svsk->sk_sk))  in svc_udp_has_wspace()
612 	clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_udp_has_wspace()
656 static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv)  in svc_udp_init()  argument
658 	svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_udp_class,  in svc_udp_init()
659 		      &svsk->sk_xprt, serv);  in svc_udp_init()
660 	clear_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
661 	svsk->sk_sk->sk_data_ready = svc_data_ready;  in svc_udp_init()
662 	svsk->sk_sk->sk_write_space = svc_write_space;  in svc_udp_init()
668 	svc_sock_setbufsize(svsk, 3);  in svc_udp_init()
671 	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
672 	set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
675 	switch (svsk->sk_sk->sk_family) {  in svc_udp_init()
677 		ip_sock_set_pktinfo(svsk->sk_sock->sk);  in svc_udp_init()
680 		ip6_sock_set_recvpktinfo(svsk->sk_sock->sk);  in svc_udp_init()
693 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_listen_data_ready()  local
695 	if (svsk) {  in svc_tcp_listen_data_ready()
698 		svsk->sk_odata(sk);  in svc_tcp_listen_data_ready()
712 		if (svsk) {  in svc_tcp_listen_data_ready()
713 			set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_listen_data_ready()
714 			svc_xprt_enqueue(&svsk->sk_xprt);  in svc_tcp_listen_data_ready()
724 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_state_change()  local
726 	if (svsk) {  in svc_tcp_state_change()
729 		svsk->sk_ostate(sk);  in svc_tcp_state_change()
730 		trace_svcsock_tcp_state(&svsk->sk_xprt, svsk->sk_sock);  in svc_tcp_state_change()
732 			set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_state_change()
733 			svc_xprt_enqueue(&svsk->sk_xprt);  in svc_tcp_state_change()
743 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_accept()  local
746 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_accept()
747 	struct socket	*sock = svsk->sk_sock;  in svc_tcp_accept()
755 	clear_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_accept()
767 	set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_accept()
777 	newsock->sk->sk_state_change = svsk->sk_ostate;  in svc_tcp_accept()
778 	newsock->sk->sk_data_ready = svsk->sk_odata;  in svc_tcp_accept()
779 	newsock->sk->sk_write_space = svsk->sk_owspace;  in svc_tcp_accept()
811 static size_t svc_tcp_restore_pages(struct svc_sock *svsk,  in svc_tcp_restore_pages()  argument
814 	size_t len = svsk->sk_datalen;  in svc_tcp_restore_pages()
823 		BUG_ON(svsk->sk_pages[i] == NULL);  in svc_tcp_restore_pages()
824 		rqstp->rq_pages[i] = svsk->sk_pages[i];  in svc_tcp_restore_pages()
825 		svsk->sk_pages[i] = NULL;  in svc_tcp_restore_pages()
831 static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)  in svc_tcp_save_pages()  argument
835 	if (svsk->sk_datalen == 0)  in svc_tcp_save_pages()
837 	len = svsk->sk_datalen;  in svc_tcp_save_pages()
840 		svsk->sk_pages[i] = rqstp->rq_pages[i];  in svc_tcp_save_pages()
845 static void svc_tcp_clear_pages(struct svc_sock *svsk)  in svc_tcp_clear_pages()  argument
849 	if (svsk->sk_datalen == 0)  in svc_tcp_clear_pages()
851 	len = svsk->sk_datalen;  in svc_tcp_clear_pages()
854 		if (svsk->sk_pages[i] == NULL) {  in svc_tcp_clear_pages()
858 		put_page(svsk->sk_pages[i]);  in svc_tcp_clear_pages()
859 		svsk->sk_pages[i] = NULL;  in svc_tcp_clear_pages()
862 	svsk->sk_tcplen = 0;  in svc_tcp_clear_pages()
863 	svsk->sk_datalen = 0;  in svc_tcp_clear_pages()
869 static ssize_t svc_tcp_read_marker(struct svc_sock *svsk,  in svc_tcp_read_marker()  argument
877 	if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) {  in svc_tcp_read_marker()
881 		want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;  in svc_tcp_read_marker()
882 		iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen;  in svc_tcp_read_marker()
885 		len = sock_recvmsg(svsk->sk_sock, &msg, MSG_DONTWAIT);  in svc_tcp_read_marker()
888 		svsk->sk_tcplen += len;  in svc_tcp_read_marker()
893 		trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker);  in svc_tcp_read_marker()
894 		if (svc_sock_reclen(svsk) + svsk->sk_datalen >  in svc_tcp_read_marker()
895 		    svsk->sk_xprt.xpt_server->sv_max_mesg)  in svc_tcp_read_marker()
898 	return svc_sock_reclen(svsk);  in svc_tcp_read_marker()
902 			       __func__, svsk->sk_xprt.xpt_server->sv_name,  in svc_tcp_read_marker()
903 			       svc_sock_reclen(svsk));  in svc_tcp_read_marker()
904 	set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_read_marker()
909 static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)  in receive_cb_reply()  argument
911 	struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt;  in receive_cb_reply()
954 static void svc_tcp_fragment_received(struct svc_sock *svsk)  in svc_tcp_fragment_received()  argument
957 	svsk->sk_tcplen = 0;  in svc_tcp_fragment_received()
958 	svsk->sk_marker = xdr_zero;  in svc_tcp_fragment_received()
981 	struct svc_sock	*svsk =  in svc_tcp_recvfrom()  local
983 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_recvfrom()
989 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_recvfrom()
990 	len = svc_tcp_read_marker(svsk, rqstp);  in svc_tcp_recvfrom()
994 	base = svc_tcp_restore_pages(svsk, rqstp);  in svc_tcp_recvfrom()
995 	want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr));  in svc_tcp_recvfrom()
998 		trace_svcsock_tcp_recv(&svsk->sk_xprt, len);  in svc_tcp_recvfrom()
999 		svsk->sk_tcplen += len;  in svc_tcp_recvfrom()
1000 		svsk->sk_datalen += len;  in svc_tcp_recvfrom()
1002 	if (len != want || !svc_sock_final_rec(svsk))  in svc_tcp_recvfrom()
1004 	if (svsk->sk_datalen < 8)  in svc_tcp_recvfrom()
1007 	rqstp->rq_arg.len = svsk->sk_datalen;  in svc_tcp_recvfrom()
1017 	if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags))  in svc_tcp_recvfrom()
1025 		len = receive_cb_reply(svsk, rqstp);  in svc_tcp_recvfrom()
1028 	svsk->sk_datalen = 0;  in svc_tcp_recvfrom()
1029 	svc_tcp_fragment_received(svsk);  in svc_tcp_recvfrom()
1034 	svc_xprt_copy_addrs(rqstp, &svsk->sk_xprt);  in svc_tcp_recvfrom()
1041 	svc_tcp_save_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1045 		svc_tcp_fragment_received(svsk);  in svc_tcp_recvfrom()
1047 		trace_svcsock_tcp_recv_short(&svsk->sk_xprt,  in svc_tcp_recvfrom()
1048 				svc_sock_reclen(svsk),  in svc_tcp_recvfrom()
1049 				svsk->sk_tcplen - sizeof(rpc_fraghdr));  in svc_tcp_recvfrom()
1054 	trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0);  in svc_tcp_recvfrom()
1057 	svsk->sk_datalen = 0;  in svc_tcp_recvfrom()
1059 	trace_svcsock_tcp_recv_err(&svsk->sk_xprt, len);  in svc_tcp_recvfrom()
1060 	set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_recvfrom()
1164 	struct svc_sock	*svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_sendto()  local
1179 	err = svc_tcp_sendmsg(svsk->sk_sock, &msg, xdr, marker, &sent);  in svc_tcp_sendto()
1243 static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv)  in svc_tcp_init()  argument
1245 	struct sock	*sk = svsk->sk_sk;  in svc_tcp_init()
1247 	svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_tcp_class,  in svc_tcp_init()
1248 		      &svsk->sk_xprt, serv);  in svc_tcp_init()
1249 	set_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1250 	set_bit(XPT_CONG_CTRL, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1252 		strcpy(svsk->sk_xprt.xpt_remotebuf, "listener");  in svc_tcp_init()
1253 		set_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1255 		set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1261 		svsk->sk_marker = xdr_zero;  in svc_tcp_init()
1262 		svsk->sk_tcplen = 0;  in svc_tcp_init()
1263 		svsk->sk_datalen = 0;  in svc_tcp_init()
1264 		memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages));  in svc_tcp_init()
1268 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1274 			set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1285 	struct svc_sock *svsk;  in svc_sock_update_bufs()  local
1288 	list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list)  in svc_sock_update_bufs()
1289 		set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags);  in svc_sock_update_bufs()
1301 	struct svc_sock	*svsk;  in svc_setup_socket()  local
1306 	svsk = kzalloc(sizeof(*svsk), GFP_KERNEL);  in svc_setup_socket()
1307 	if (!svsk)  in svc_setup_socket()
1319 		kfree(svsk);  in svc_setup_socket()
1323 	svsk->sk_sock = sock;  in svc_setup_socket()
1324 	svsk->sk_sk = inet;  in svc_setup_socket()
1325 	svsk->sk_ostate = inet->sk_state_change;  in svc_setup_socket()
1326 	svsk->sk_odata = inet->sk_data_ready;  in svc_setup_socket()
1327 	svsk->sk_owspace = inet->sk_write_space;  in svc_setup_socket()
1334 	inet->sk_user_data = svsk;  in svc_setup_socket()
1338 		svc_udp_init(svsk, serv);  in svc_setup_socket()
1340 		svc_tcp_init(svsk, serv);  in svc_setup_socket()
1343 	return svsk;  in svc_setup_socket()
1379 	struct svc_sock *svsk = NULL;  in svc_addsock()  local
1399 	svsk = svc_setup_socket(serv, so, SVC_SOCK_DEFAULTS);  in svc_addsock()
1400 	if (IS_ERR(svsk)) {  in svc_addsock()
1402 		err = PTR_ERR(svsk);  in svc_addsock()
1405 	salen = kernel_getsockname(svsk->sk_sock, sin);  in svc_addsock()
1407 		svc_xprt_set_local(&svsk->sk_xprt, sin, salen);  in svc_addsock()
1408 	svsk->sk_xprt.xpt_cred = get_cred(cred);  in svc_addsock()
1409 	svc_add_new_perm_xprt(serv, &svsk->sk_xprt);  in svc_addsock()
1410 	return svc_one_sock_name(svsk, name_return, len);  in svc_addsock()
1426 	struct svc_sock	*svsk;  in svc_create_socket()  local
1482 	svsk = svc_setup_socket(serv, sock, flags);  in svc_create_socket()
1483 	if (IS_ERR(svsk)) {  in svc_create_socket()
1484 		error = PTR_ERR(svsk);  in svc_create_socket()
1487 	svc_xprt_set_local(&svsk->sk_xprt, newsin, newlen);  in svc_create_socket()
1488 	return (struct svc_xprt *)svsk;  in svc_create_socket()
1500 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_sock_detach()  local
1501 	struct sock *sk = svsk->sk_sk;  in svc_sock_detach()
1505 	sk->sk_state_change = svsk->sk_ostate;  in svc_sock_detach()
1506 	sk->sk_data_ready = svsk->sk_odata;  in svc_sock_detach()
1507 	sk->sk_write_space = svsk->sk_owspace;  in svc_sock_detach()
1517 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_sock_detach()  local
1522 		svc_tcp_clear_pages(svsk);  in svc_tcp_sock_detach()
1523 		kernel_sock_shutdown(svsk->sk_sock, SHUT_RDWR);  in svc_tcp_sock_detach()
1532 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_sock_free()  local
1534 	if (svsk->sk_sock->file)  in svc_sock_free()
1535 		sockfd_put(svsk->sk_sock);  in svc_sock_free()
1537 		sock_release(svsk->sk_sock);  in svc_sock_free()
1538 	kfree(svsk);  in svc_sock_free()