77 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt);
78 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt);
79 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
81 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
82 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
84 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
85 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
86 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
99 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_drain()  argument
101 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_drain()
150 void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc)  in rpcrdma_flush_disconnect()  argument
153 		rpcrdma_force_disconnect(r_xprt->rx_ep);  in rpcrdma_flush_disconnect()
167 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_send()  local
171 	rpcrdma_sendctx_put_locked(r_xprt, sc);  in rpcrdma_wc_send()
172 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_send()
186 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_receive()  local
190 	--r_xprt->rx_ep->re_receive_count;  in rpcrdma_wc_receive()
207 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_receive()
311 static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt,  in rpcrdma_create_id()  argument
315 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_create_id()
395 static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_ep_create()  argument
406 	ep->re_xprt = &r_xprt->rx_xprt;  in rpcrdma_ep_create()
409 	id = rpcrdma_create_id(r_xprt, ep);  in rpcrdma_ep_create()
418 	ep->re_max_requests = r_xprt->rx_xprt.max_reqs;  in rpcrdma_ep_create()
425 	r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests);  in rpcrdma_ep_create()
447 	ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
456 	ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
508 	r_xprt->rx_ep = ep;  in rpcrdma_ep_create()
523 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_connect()  argument
525 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_xprt_connect()
529 	rc = rpcrdma_ep_create(r_xprt);  in rpcrdma_xprt_connect()
532 	ep = r_xprt->rx_ep;  in rpcrdma_xprt_connect()
535 	rpcrdma_reset_cwnd(r_xprt);  in rpcrdma_xprt_connect()
541 	rpcrdma_post_recvs(r_xprt, 1, true);  in rpcrdma_xprt_connect()
556 	rc = rpcrdma_sendctxs_create(r_xprt);  in rpcrdma_xprt_connect()
562 	rc = rpcrdma_reqs_setup(r_xprt);  in rpcrdma_xprt_connect()
567 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_xprt_connect()
570 	trace_xprtrdma_connect(r_xprt, rc);  in rpcrdma_xprt_connect()
584 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_disconnect()  argument
586 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_disconnect()
595 	trace_xprtrdma_disconnect(r_xprt, rc);  in rpcrdma_xprt_disconnect()
597 	rpcrdma_xprt_drain(r_xprt);  in rpcrdma_xprt_disconnect()
598 	rpcrdma_reps_unmap(r_xprt);  in rpcrdma_xprt_disconnect()
599 	rpcrdma_reqs_reset(r_xprt);  in rpcrdma_xprt_disconnect()
600 	rpcrdma_mrs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
601 	rpcrdma_sendctxs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
606 	r_xprt->rx_ep = NULL;  in rpcrdma_xprt_disconnect()
626 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_destroy()  argument
628 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_destroy()
652 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_create()  argument
654 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_create()
663 	i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS;  in rpcrdma_sendctxs_create()
670 		sc = rpcrdma_sendctx_create(r_xprt->rx_ep);  in rpcrdma_sendctxs_create()
705 struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctx_get_locked()  argument
707 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_get_locked()
731 	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_get_locked()
732 	r_xprt->rx_stats.empty_sendctx_q++;  in rpcrdma_sendctx_get_locked()
746 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,  in rpcrdma_sendctx_put_locked()  argument
749 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_put_locked()
767 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_put_locked()
771 rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_create()  argument
773 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_create()
774 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_create()
785 		rc = frwr_mr_init(r_xprt, mr);  in rpcrdma_mrs_create()
797 	r_xprt->rx_stats.mrs_allocated += count;  in rpcrdma_mrs_create()
798 	trace_xprtrdma_createmrs(r_xprt, count);  in rpcrdma_mrs_create()
806 	struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,  in rpcrdma_mr_refresh_worker()  local
809 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_mr_refresh_worker()
810 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_mr_refresh_worker()
818 void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_refresh()  argument
820 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_refresh()
821 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_refresh()
843 struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,  in rpcrdma_req_create()  argument
846 	struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;  in rpcrdma_req_create()
883 int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_req_setup()  argument
890 		     r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz;  in rpcrdma_req_setup()
897 	if (!__rpcrdma_regbuf_dma_map(r_xprt, rb))  in rpcrdma_req_setup()
915 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_setup()  argument
917 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_setup()
922 		rc = rpcrdma_req_setup(r_xprt, req);  in rpcrdma_reqs_setup()
948 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_reset()  argument
950 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_reset()
961 struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,  in rpcrdma_rep_create()  argument
970 	rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,  in rpcrdma_rep_create()
975 	if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf))  in rpcrdma_rep_create()
981 	rep->rr_rxprt = r_xprt;  in rpcrdma_rep_create()
987 	list_add(&rep->rr_all, &r_xprt->rx_buf.rb_all_reps);  in rpcrdma_rep_create()
1025 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reps_unmap()  argument
1027 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reps_unmap()
1050 int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_buffer_create()  argument
1052 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_buffer_create()
1066 	for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) {  in rpcrdma_buffer_create()
1069 		req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2,  in rpcrdma_buffer_create()
1120 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_destroy()  argument
1122 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_destroy()
1173 rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mr_get()  argument
1175 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mr_get()
1191 	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;  in rpcrdma_mr_put()  local
1195 		ib_dma_unmap_sg(r_xprt->rx_ep->re_id->device,  in rpcrdma_mr_put()
1322 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,  in __rpcrdma_regbuf_dma_map()  argument
1325 	struct ib_device *device = r_xprt->rx_ep->re_id->device;  in __rpcrdma_regbuf_dma_map()
1338 	rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey;  in __rpcrdma_regbuf_dma_map()
1371 int rpcrdma_post_sends(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_post_sends()  argument
1374 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_sends()
1386 	rc = frwr_send(r_xprt, req);  in rpcrdma_post_sends()
1399 void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)  in rpcrdma_post_recvs()  argument
1401 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_post_recvs()
1402 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_recvs()
1425 			rep = rpcrdma_rep_create(r_xprt, temp);  in rpcrdma_post_recvs()
1441 	trace_xprtrdma_post_recvs(r_xprt, count, rc);  in rpcrdma_post_recvs()