Lines Matching +full:rpc +full:- +full:if
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9 * COPYING in the main directory of this source tree, or the BSD-type
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * This file contains the top-level implementation of an RPC RDMA
49 * transport switch. All others are RPC RDMA internal.
63 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
77 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
85 static unsigned int max_memreg = RPCRDMA_LAST - 1;
165 snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); in xprt_rdma_format_addresses4()
166 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses4()
168 xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA; in xprt_rdma_format_addresses4()
177 snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); in xprt_rdma_format_addresses6()
178 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses6()
180 xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA6; in xprt_rdma_format_addresses6()
188 switch (sap->sa_family) { in xprt_rdma_format_addresses()
201 xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses()
204 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses()
207 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses()
209 xprt->address_strings[RPC_DISPLAY_PROTO] = "rdma"; in xprt_rdma_format_addresses()
223 kfree(xprt->address_strings[i]); in xprt_rdma_free_addresses()
228 * xprt_rdma_connect_worker - establish connection in the background
232 * transport while a fresh connection is being established. RPC tasks
240 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in xprt_rdma_connect_worker()
245 if (!rc) { in xprt_rdma_connect_worker()
246 xprt->connect_cookie++; in xprt_rdma_connect_worker()
247 xprt->stat.connect_count++; in xprt_rdma_connect_worker()
248 xprt->stat.connect_time += (long)jiffies - in xprt_rdma_connect_worker()
249 xprt->stat.connect_start; in xprt_rdma_connect_worker()
251 rc = -EAGAIN; in xprt_rdma_connect_worker()
259 * xprt_rdma_inject_disconnect - inject a connection fault
262 * If @xprt is connected, disconnect it to simulate spurious
273 rdma_disconnect(r_xprt->rx_ep->re_id); in xprt_rdma_inject_disconnect()
277 * xprt_rdma_destroy - Full tear down of transport
288 cancel_delayed_work_sync(&r_xprt->rx_connect_worker); in xprt_rdma_destroy()
291 rpcrdma_buffer_destroy(&r_xprt->rx_buf); in xprt_rdma_destroy()
306 * xprt_setup_rdma - Set up transport to use RDMA
308 * @args: rpc transport arguments
318 if (args->addrlen > sizeof(xprt->addr)) in xprt_setup_rdma()
319 return ERR_PTR(-EBADF); in xprt_setup_rdma()
321 if (!try_module_get(THIS_MODULE)) in xprt_setup_rdma()
322 return ERR_PTR(-EIO); in xprt_setup_rdma()
324 xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, in xprt_setup_rdma()
326 if (!xprt) { in xprt_setup_rdma()
328 return ERR_PTR(-ENOMEM); in xprt_setup_rdma()
331 xprt->timeout = &xprt_rdma_default_timeout; in xprt_setup_rdma()
332 xprt->connect_timeout = xprt->timeout->to_initval; in xprt_setup_rdma()
333 xprt->max_reconnect_timeout = xprt->timeout->to_maxval; in xprt_setup_rdma()
334 xprt->bind_timeout = RPCRDMA_BIND_TO; in xprt_setup_rdma()
335 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; in xprt_setup_rdma()
336 xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO; in xprt_setup_rdma()
338 xprt->resvport = 0; /* privileged port not needed */ in xprt_setup_rdma()
339 xprt->ops = &xprt_rdma_procs; in xprt_setup_rdma()
342 * Set up RDMA-specific connect data. in xprt_setup_rdma()
344 sap = args->dstaddr; in xprt_setup_rdma()
346 /* Ensure xprt->addr holds valid server TCP (not RDMA) in xprt_setup_rdma()
348 xprt->prot = IPPROTO_TCP; in xprt_setup_rdma()
349 xprt->addrlen = args->addrlen; in xprt_setup_rdma()
350 memcpy(&xprt->addr, sap, xprt->addrlen); in xprt_setup_rdma()
352 if (rpc_get_port(sap)) in xprt_setup_rdma()
358 if (rc) { in xprt_setup_rdma()
365 INIT_DELAYED_WORK(&new_xprt->rx_connect_worker, in xprt_setup_rdma()
368 xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; in xprt_setup_rdma()
374 * xprt_rdma_close - close a transport connection
388 xprt->reestablish_timeout = 0; in xprt_rdma_close()
389 ++xprt->connect_cookie; in xprt_rdma_close()
394 * xprt_rdma_set_port - update server port with rpcbind result
395 * @xprt: controlling RPC transport
403 struct sockaddr *sap = (struct sockaddr *)&xprt->addr; in xprt_rdma_set_port()
408 kfree(xprt->address_strings[RPC_DISPLAY_PORT]); in xprt_rdma_set_port()
410 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_set_port()
412 kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); in xprt_rdma_set_port()
414 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_set_port()
418 * xprt_rdma_timer - invoked when an RPC times out
419 * @xprt: controlling RPC transport
420 * @task: RPC task that timed out
422 * Invoked when the transport is still connected, but an RPC
425 * Since RDMA connections don't have a keep-alive, forcibly
437 * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
451 spin_lock(&xprt->transport_lock); in xprt_rdma_set_connect_timeout()
453 if (connect_timeout < xprt->connect_timeout) { in xprt_rdma_set_connect_timeout()
457 to = *xprt->timeout; in xprt_rdma_set_connect_timeout()
459 if (initval < RPCRDMA_INIT_REEST_TO << 1) in xprt_rdma_set_connect_timeout()
463 r_xprt->rx_timeout = to; in xprt_rdma_set_connect_timeout()
464 xprt->timeout = &r_xprt->rx_timeout; in xprt_rdma_set_connect_timeout()
465 xprt->connect_timeout = connect_timeout; in xprt_rdma_set_connect_timeout()
468 if (reconnect_timeout < xprt->max_reconnect_timeout) in xprt_rdma_set_connect_timeout()
469 xprt->max_reconnect_timeout = reconnect_timeout; in xprt_rdma_set_connect_timeout()
471 spin_unlock(&xprt->transport_lock); in xprt_rdma_set_connect_timeout()
475 * xprt_rdma_connect - schedule an attempt to reconnect
477 * @task: RPC scheduler context (unused)
484 struct rpcrdma_ep *ep = r_xprt->rx_ep; in xprt_rdma_connect()
490 if (ep && ep->re_connect_status != 0) { in xprt_rdma_connect()
495 queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker, in xprt_rdma_connect()
500 * xprt_rdma_alloc_slot - allocate an rpc_rqst
501 * @xprt: controlling RPC transport
502 * @task: RPC task requesting a fresh rpc_rqst
505 * %0 if task->tk_rqstp points to a fresh rpc_rqst
506 * %-EAGAIN if no rpc_rqst is available; queued on backlog
514 req = rpcrdma_buffer_get(&r_xprt->rx_buf); in xprt_rdma_alloc_slot()
515 if (!req) in xprt_rdma_alloc_slot()
517 task->tk_rqstp = &req->rl_slot; in xprt_rdma_alloc_slot()
518 task->tk_status = 0; in xprt_rdma_alloc_slot()
522 task->tk_status = -ENOMEM; in xprt_rdma_alloc_slot()
527 * xprt_rdma_free_slot - release an rpc_rqst
528 * @xprt: controlling RPC transport
538 rpcrdma_reply_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); in xprt_rdma_free_slot()
539 if (!xprt_wake_up_backlog(xprt, rqst)) { in xprt_rdma_free_slot()
541 rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); in xprt_rdma_free_slot()
549 if (unlikely(rdmab_length(rb) < size)) { in rpcrdma_check_regbuf()
550 if (!rpcrdma_regbuf_realloc(rb, size, flags)) in rpcrdma_check_regbuf()
552 r_xprt->rx_stats.hardway_register_count += size; in rpcrdma_check_regbuf()
558 * xprt_rdma_allocate - allocate transport resources for an RPC
559 * @task: RPC task
562 * 0: Success; rq_buffer points to RPC buffer to use
569 struct rpc_rqst *rqst = task->tk_rqstp; in xprt_rdma_allocate()
570 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); in xprt_rdma_allocate()
575 if (RPC_IS_ASYNC(task)) in xprt_rdma_allocate()
577 if (RPC_IS_SWAPPER(task)) in xprt_rdma_allocate()
580 if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize, in xprt_rdma_allocate()
583 if (!rpcrdma_check_regbuf(r_xprt, req->rl_recvbuf, rqst->rq_rcvsize, in xprt_rdma_allocate()
587 rqst->rq_buffer = rdmab_data(req->rl_sendbuf); in xprt_rdma_allocate()
588 rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf); in xprt_rdma_allocate()
592 return -ENOMEM; in xprt_rdma_allocate()
596 * xprt_rdma_free - release resources allocated by xprt_rdma_allocate
597 * @task: RPC task
599 * Caller guarantees rqst->rq_buffer is non-NULL.
604 struct rpc_rqst *rqst = task->tk_rqstp; in xprt_rdma_free()
605 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); in xprt_rdma_free()
608 if (!list_empty(&req->rl_registered)) in xprt_rdma_free()
611 /* XXX: If the RPC is completing because of a signal and in xprt_rdma_free()
619 * xprt_rdma_send_request - marshal and send an RPC request
620 * @rqst: RPC message in rq_snd_buf
625 * %0 if the RPC message has been sent
626 * %-ENOTCONN if the caller should reconnect and call again
627 * %-EAGAIN if the caller should call again
628 * %-ENOBUFS if the caller should call again after a delay
629 * %-EMSGSIZE if encoding ran out of buffer space. The request
631 * %-EIO if an I/O error occurred. The request was not sent.
637 struct rpc_xprt *xprt = rqst->rq_xprt; in xprt_rdma_send_request()
642 #if defined(CONFIG_SUNRPC_BACKCHANNEL) in xprt_rdma_send_request()
643 if (unlikely(!rqst->rq_buffer)) in xprt_rdma_send_request()
647 if (!xprt_connected(xprt)) in xprt_rdma_send_request()
648 return -ENOTCONN; in xprt_rdma_send_request()
650 if (!xprt_request_get_cong(xprt, rqst)) in xprt_rdma_send_request()
651 return -EBADSLT; in xprt_rdma_send_request()
654 if (rc < 0) in xprt_rdma_send_request()
658 if (rqst->rq_connect_cookie == xprt->connect_cookie) in xprt_rdma_send_request()
660 rqst->rq_xtime = ktime_get(); in xprt_rdma_send_request()
662 if (rpcrdma_post_sends(r_xprt, req)) in xprt_rdma_send_request()
665 rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len; in xprt_rdma_send_request()
667 /* An RPC with no reply will throw off credit accounting, in xprt_rdma_send_request()
670 if (!rpc_reply_expected(rqst->rq_task)) in xprt_rdma_send_request()
675 if (rc != -ENOTCONN) in xprt_rdma_send_request()
679 return -ENOTCONN; in xprt_rdma_send_request()
687 if (xprt_connected(xprt)) in xprt_rdma_print_stats()
688 idle_time = (long)(jiffies - xprt->last_used) / HZ; in xprt_rdma_print_stats()
693 xprt->stat.bind_count, in xprt_rdma_print_stats()
694 xprt->stat.connect_count, in xprt_rdma_print_stats()
695 xprt->stat.connect_time / HZ, in xprt_rdma_print_stats()
697 xprt->stat.sends, in xprt_rdma_print_stats()
698 xprt->stat.recvs, in xprt_rdma_print_stats()
699 xprt->stat.bad_xids, in xprt_rdma_print_stats()
700 xprt->stat.req_u, in xprt_rdma_print_stats()
701 xprt->stat.bklog_u); in xprt_rdma_print_stats()
703 r_xprt->rx_stats.read_chunk_count, in xprt_rdma_print_stats()
704 r_xprt->rx_stats.write_chunk_count, in xprt_rdma_print_stats()
705 r_xprt->rx_stats.reply_chunk_count, in xprt_rdma_print_stats()
706 r_xprt->rx_stats.total_rdma_request, in xprt_rdma_print_stats()
707 r_xprt->rx_stats.total_rdma_reply, in xprt_rdma_print_stats()
708 r_xprt->rx_stats.pullup_copy_count, in xprt_rdma_print_stats()
709 r_xprt->rx_stats.fixup_copy_count, in xprt_rdma_print_stats()
710 r_xprt->rx_stats.hardway_register_count, in xprt_rdma_print_stats()
711 r_xprt->rx_stats.failed_marshal_count, in xprt_rdma_print_stats()
712 r_xprt->rx_stats.bad_reply_count, in xprt_rdma_print_stats()
713 r_xprt->rx_stats.nomsg_call_count); in xprt_rdma_print_stats()
715 r_xprt->rx_stats.mrs_recycled, in xprt_rdma_print_stats()
716 r_xprt->rx_stats.mrs_orphaned, in xprt_rdma_print_stats()
717 r_xprt->rx_stats.mrs_allocated, in xprt_rdma_print_stats()
718 r_xprt->rx_stats.local_inv_needed, in xprt_rdma_print_stats()
719 r_xprt->rx_stats.empty_sendctx_q, in xprt_rdma_print_stats()
720 r_xprt->rx_stats.reply_waits_for_send); in xprt_rdma_print_stats()
735 * Plumbing for rpc transport switch and kernel module
759 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
779 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) in xprt_rdma_cleanup()
780 if (sunrpc_table_header) { in xprt_rdma_cleanup()
795 if (rc) in xprt_rdma_init()
799 if (rc) { in xprt_rdma_init()
804 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) in xprt_rdma_init()
805 if (!sunrpc_table_header) in xprt_rdma_init()