73 static void	 xprt_request_init(struct rpc_task *task);
236 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt()  argument
238 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt()
241 		if (task == xprt->snd_task)  in xprt_reserve_xprt()
247 	xprt->snd_task = task;  in xprt_reserve_xprt()
250 	trace_xprt_reserve_xprt(xprt, task);  in xprt_reserve_xprt()
256 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt()
257 	if  (RPC_IS_SOFT(task))  in xprt_reserve_xprt()
258 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,  in xprt_reserve_xprt()
261 		rpc_sleep_on(&xprt->sending, task, NULL);  in xprt_reserve_xprt()
300 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt_cong()  argument
302 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt_cong()
305 		if (task == xprt->snd_task)  in xprt_reserve_xprt_cong()
310 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
316 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
322 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt_cong()
323 	if (RPC_IS_SOFT(task))  in xprt_reserve_xprt_cong()
324 		rpc_sleep_on_timeout(&xprt->sending, task, NULL,  in xprt_reserve_xprt_cong()
327 		rpc_sleep_on(&xprt->sending, task, NULL);  in xprt_reserve_xprt_cong()
330 	trace_xprt_reserve_cong(xprt, task);  in xprt_reserve_xprt_cong()
335 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_lock_write()  argument
339 	if (test_bit(XPRT_LOCKED, &xprt->state) && xprt->snd_task == task)  in xprt_lock_write()
342 	retval = xprt->ops->reserve_xprt(xprt, task);  in xprt_lock_write()
347 static bool __xprt_lock_write_func(struct rpc_task *task, void *data)  in __xprt_lock_write_func()  argument
351 	xprt->snd_task = task;  in __xprt_lock_write_func()
390 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt()  argument
392 	if (xprt->snd_task == task) {  in xprt_release_xprt()
396 	trace_xprt_release_xprt(xprt, task);  in xprt_release_xprt()
408 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt_cong()  argument
410 	if (xprt->snd_task == task) {  in xprt_release_xprt_cong()
414 	trace_xprt_release_cong(xprt, task);  in xprt_release_xprt_cong()
418 static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_write()  argument
420 	if (xprt->snd_task != task)  in xprt_release_write()
423 	xprt->ops->release_xprt(xprt, task);  in xprt_release_write()
489 void xprt_release_rqst_cong(struct rpc_task *task)  in xprt_release_rqst_cong()  argument
491 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_release_rqst_cong()
533 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)  in xprt_adjust_cwnd()  argument
535 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_adjust_cwnd()
649 static void xprt_init_majortimeo(struct rpc_task *task, struct rpc_rqst *req)  in xprt_init_majortimeo()  argument
657 		time_init = xprt_abs_ktime_to_jiffies(task->tk_start);  in xprt_init_majortimeo()
658 	req->rq_timeout = task->tk_client->cl_timeout->to_initval;  in xprt_init_majortimeo()
772 xprt_request_retransmit_after_disconnect(struct rpc_task *task)  in xprt_request_retransmit_after_disconnect()  argument
774 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_retransmit_after_disconnect()
835 		struct rpc_task *task,  in xprt_lock_connect()  argument
843 	if (xprt->snd_task != task)  in xprt_lock_connect()
876 void xprt_connect(struct rpc_task *task)  in xprt_connect()  argument
878 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_connect()
883 		task->tk_status = -EAGAIN;  in xprt_connect()
886 	if (!xprt_lock_write(xprt, task))  in xprt_connect()
890 		task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;  in xprt_connect()
891 		rpc_sleep_on_timeout(&xprt->pending, task, NULL,  in xprt_connect()
892 				xprt_request_timeout(task->tk_rqstp));  in xprt_connect()
901 			xprt->ops->connect(xprt, task);  in xprt_connect()
904 			task->tk_status = 0;  in xprt_connect()
905 			rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_connect()
908 	xprt_release_write(xprt, task);  in xprt_connect()
1080 xprt_request_data_received(struct rpc_task *task)  in xprt_request_data_received()  argument
1082 	return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&  in xprt_request_data_received()
1083 		READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) != 0;  in xprt_request_data_received()
1087 xprt_request_need_enqueue_receive(struct rpc_task *task, struct rpc_rqst *req)  in xprt_request_need_enqueue_receive()  argument
1089 	return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&  in xprt_request_need_enqueue_receive()
1090 		READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) == 0;  in xprt_request_need_enqueue_receive()
1099 xprt_request_enqueue_receive(struct rpc_task *task)  in xprt_request_enqueue_receive()  argument
1101 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_enqueue_receive()
1104 	if (!xprt_request_need_enqueue_receive(task, req))  in xprt_request_enqueue_receive()
1107 	xprt_request_prepare(task->tk_rqstp);  in xprt_request_enqueue_receive()
1116 	set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate);  in xprt_request_enqueue_receive()
1130 xprt_request_dequeue_receive_locked(struct rpc_task *task)  in xprt_request_dequeue_receive_locked()  argument
1132 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_receive_locked()
1134 	if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))  in xprt_request_dequeue_receive_locked()
1144 void xprt_update_rtt(struct rpc_task *task)  in xprt_update_rtt()  argument
1146 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_update_rtt()
1147 	struct rpc_rtt *rtt = task->tk_client->cl_rtt;  in xprt_update_rtt()
1148 	unsigned int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_update_rtt()
1166 void xprt_complete_rqst(struct rpc_task *task, int copied)  in xprt_complete_rqst()  argument
1168 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_complete_rqst()
1178 	xprt_request_dequeue_receive_locked(task);  in xprt_complete_rqst()
1179 	rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_complete_rqst()
1183 static void xprt_timer(struct rpc_task *task)  in xprt_timer()  argument
1185 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_timer()
1188 	if (task->tk_status != -ETIMEDOUT)  in xprt_timer()
1191 	trace_xprt_timer(xprt, req->rq_xid, task->tk_status);  in xprt_timer()
1194 			xprt->ops->timer(xprt, task);  in xprt_timer()
1196 		task->tk_status = 0;  in xprt_timer()
1208 void xprt_wait_for_reply_request_def(struct rpc_task *task)  in xprt_wait_for_reply_request_def()  argument
1210 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_reply_request_def()
1212 	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,  in xprt_wait_for_reply_request_def()
1224 void xprt_wait_for_reply_request_rtt(struct rpc_task *task)  in xprt_wait_for_reply_request_rtt()  argument
1226 	int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_wait_for_reply_request_rtt()
1227 	struct rpc_clnt *clnt = task->tk_client;  in xprt_wait_for_reply_request_rtt()
1229 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_reply_request_rtt()
1237 	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,  in xprt_wait_for_reply_request_rtt()
1247 void xprt_request_wait_receive(struct rpc_task *task)  in xprt_request_wait_receive()  argument
1249 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_wait_receive()
1252 	if (!test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))  in xprt_request_wait_receive()
1260 	if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) {  in xprt_request_wait_receive()
1261 		xprt->ops->wait_for_reply_request(task);  in xprt_request_wait_receive()
1267 		if (xprt_request_retransmit_after_disconnect(task))  in xprt_request_wait_receive()
1269 					task, -ENOTCONN);  in xprt_request_wait_receive()
1275 xprt_request_need_enqueue_transmit(struct rpc_task *task, struct rpc_rqst *req)  in xprt_request_need_enqueue_transmit()  argument
1277 	return !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);  in xprt_request_need_enqueue_transmit()
1287 xprt_request_enqueue_transmit(struct rpc_task *task)  in xprt_request_enqueue_transmit()  argument
1289 	struct rpc_rqst *pos, *req = task->tk_rqstp;  in xprt_request_enqueue_transmit()
1292 	if (xprt_request_need_enqueue_transmit(task, req)) {  in xprt_request_enqueue_transmit()
1311 				if (pos->rq_task->tk_owner != task->tk_owner)  in xprt_request_enqueue_transmit()
1321 		set_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);  in xprt_request_enqueue_transmit()
1334 xprt_request_dequeue_transmit_locked(struct rpc_task *task)  in xprt_request_dequeue_transmit_locked()  argument
1336 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_transmit_locked()
1338 	if (!test_and_clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_request_dequeue_transmit_locked()
1359 xprt_request_dequeue_transmit(struct rpc_task *task)  in xprt_request_dequeue_transmit()  argument
1361 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_request_dequeue_transmit()
1365 	xprt_request_dequeue_transmit_locked(task);  in xprt_request_dequeue_transmit()
1377 xprt_request_dequeue_xprt(struct rpc_task *task)  in xprt_request_dequeue_xprt()  argument
1379 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_dequeue_xprt()
1382 	if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate) ||  in xprt_request_dequeue_xprt()
1383 	    test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) ||  in xprt_request_dequeue_xprt()
1386 		xprt_request_dequeue_transmit_locked(task);  in xprt_request_dequeue_xprt()
1387 		xprt_request_dequeue_receive_locked(task);  in xprt_request_dequeue_xprt()
1389 			set_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);  in xprt_request_dequeue_xprt()
1393 			clear_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);  in xprt_request_dequeue_xprt()
1422 xprt_request_need_retransmit(struct rpc_task *task)  in xprt_request_need_retransmit()  argument
1424 	return xprt_request_retransmit_after_disconnect(task);  in xprt_request_need_retransmit()
1432 bool xprt_prepare_transmit(struct rpc_task *task)  in xprt_prepare_transmit()  argument
1434 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_prepare_transmit()
1437 	if (!xprt_lock_write(xprt, task)) {  in xprt_prepare_transmit()
1439 		if (!test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_prepare_transmit()
1441 					task, 0);  in xprt_prepare_transmit()
1448 void xprt_end_transmit(struct rpc_task *task)  in xprt_end_transmit()  argument
1450 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_end_transmit()
1453 	xprt_release_write(xprt, task);  in xprt_end_transmit()
1470 	struct rpc_task *task = req->rq_task;  in xprt_request_transmit()  local
1472 	int is_retrans = RPC_WAS_SENT(task);  in xprt_request_transmit()
1476 		if (xprt_request_data_received(task)) {  in xprt_request_transmit()
1481 		if (rpcauth_xmit_need_reencode(task)) {  in xprt_request_transmit()
1485 		if (RPC_SIGNALLED(task)) {  in xprt_request_transmit()
1498 	trace_rpc_xdr_sendto(task, &req->rq_snd_buf);  in xprt_request_transmit()
1508 		task->tk_client->cl_stats->rpcretrans++;  in xprt_request_transmit()
1512 	task->tk_flags |= RPC_TASK_SENT;  in xprt_request_transmit()
1525 	xprt_request_dequeue_transmit(task);  in xprt_request_transmit()
1526 	rpc_wake_up_queued_task_set_status(&xprt->sending, task, status);  in xprt_request_transmit()
1540 xprt_transmit(struct rpc_task *task)  in xprt_transmit()  argument
1542 	struct rpc_rqst *next, *req = task->tk_rqstp;  in xprt_transmit()
1554 		status = xprt_request_transmit(next, task);  in xprt_transmit()
1560 			if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_transmit()
1561 				task->tk_status = status;  in xprt_transmit()
1565 		if (xprt_request_data_received(task) &&  in xprt_transmit()
1566 		    !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))  in xprt_transmit()
1573 static void xprt_complete_request_init(struct rpc_task *task)  in xprt_complete_request_init()  argument
1575 	if (task->tk_rqstp)  in xprt_complete_request_init()
1576 		xprt_request_init(task);  in xprt_complete_request_init()
1579 void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_add_backlog()  argument
1582 	rpc_sleep_on(&xprt->backlog, task, xprt_complete_request_init);  in xprt_add_backlog()
1586 static bool __xprt_set_rq(struct rpc_task *task, void *data)  in __xprt_set_rq()  argument
1590 	if (task->tk_rqstp == NULL) {  in __xprt_set_rq()
1592 		task->tk_rqstp = req;  in __xprt_set_rq()
1608 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_throttle_congested()  argument
1616 		xprt_add_backlog(xprt, task);  in xprt_throttle_congested()
1655 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_alloc_slot()  argument
1672 		task->tk_status = -ENOMEM;  in xprt_alloc_slot()
1675 		xprt_add_backlog(xprt, task);  in xprt_alloc_slot()
1679 		task->tk_status = -EAGAIN;  in xprt_alloc_slot()
1688 	task->tk_status = 0;  in xprt_alloc_slot()
1689 	task->tk_rqstp = req;  in xprt_alloc_slot()
1783 xprt_request_init(struct rpc_task *task)  in xprt_request_init()  argument
1785 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_request_init()
1786 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_init()
1788 	req->rq_task	= task;  in xprt_request_init()
1800 	xprt_init_majortimeo(task, req);  in xprt_request_init()
1806 xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_do_reserve()  argument
1808 	xprt->ops->alloc_slot(xprt, task);  in xprt_do_reserve()
1809 	if (task->tk_rqstp != NULL)  in xprt_do_reserve()
1810 		xprt_request_init(task);  in xprt_do_reserve()
1821 void xprt_reserve(struct rpc_task *task)  in xprt_reserve()  argument
1823 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_reserve()
1825 	task->tk_status = 0;  in xprt_reserve()
1826 	if (task->tk_rqstp != NULL)  in xprt_reserve()
1829 	task->tk_status = -EAGAIN;  in xprt_reserve()
1830 	if (!xprt_throttle_congested(xprt, task))  in xprt_reserve()
1831 		xprt_do_reserve(xprt, task);  in xprt_reserve()
1843 void xprt_retry_reserve(struct rpc_task *task)  in xprt_retry_reserve()  argument
1845 	struct rpc_xprt *xprt = task->tk_xprt;  in xprt_retry_reserve()
1847 	task->tk_status = 0;  in xprt_retry_reserve()
1848 	if (task->tk_rqstp != NULL)  in xprt_retry_reserve()
1851 	task->tk_status = -EAGAIN;  in xprt_retry_reserve()
1852 	xprt_do_reserve(xprt, task);  in xprt_retry_reserve()
1860 void xprt_release(struct rpc_task *task)  in xprt_release()  argument
1863 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_release()
1866 		if (task->tk_client) {  in xprt_release()
1867 			xprt = task->tk_xprt;  in xprt_release()
1868 			xprt_release_write(xprt, task);  in xprt_release()
1874 	xprt_request_dequeue_xprt(task);  in xprt_release()
1876 	xprt->ops->release_xprt(xprt, task);  in xprt_release()
1878 		xprt->ops->release_request(task);  in xprt_release()
1882 		xprt->ops->buf_free(task);  in xprt_release()
1890 	task->tk_rqstp = NULL;  in xprt_release()
1899 xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task)  in xprt_init_bc_request()  argument
1903 	task->tk_rqstp = req;  in xprt_init_bc_request()
1904 	req->rq_task = task;  in xprt_init_bc_request()