185 	return to_clt_con(sess->s.con[id]);  in rtrs_permit_to_clt_con()
303 static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)  in rtrs_rdma_error_recovery()  argument
305 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_rdma_error_recovery()
334 	struct rtrs_clt_con *con = cq->cq_context;  in rtrs_clt_fast_reg_done()  local
337 		rtrs_err(con->c.sess, "Failed IB_WR_REG_MR: %s\n",  in rtrs_clt_fast_reg_done()
339 		rtrs_rdma_error_recovery(con);  in rtrs_clt_fast_reg_done()
354 	struct rtrs_clt_con *con = cq->cq_context;  in rtrs_clt_inv_rkey_done()  local
357 		rtrs_err(con->c.sess, "Failed IB_WR_LOCAL_INV: %s\n",  in rtrs_clt_inv_rkey_done()
359 		rtrs_rdma_error_recovery(con);  in rtrs_clt_inv_rkey_done()
371 	struct rtrs_clt_con *con = req->con;  in rtrs_inv_rkey()  local
380 	return ib_post_send(con->c.qp, &wr, NULL);  in rtrs_inv_rkey()
386 	struct rtrs_clt_con *con = req->con;  in complete_rdma_req()  local
392 	if (WARN_ON(!req->con))  in complete_rdma_req()
394 	sess = to_clt_sess(con->c.sess);  in complete_rdma_req()
423 				rtrs_err(con->c.sess, "Send INV WR key=%#x: %d\n",  in complete_rdma_req()
444 	req->con = NULL;  in complete_rdma_req()
450 static int rtrs_post_send_rdma(struct rtrs_clt_con *con,  in rtrs_post_send_rdma()  argument
455 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_post_send_rdma()
460 		rtrs_wrn(con->c.sess,  in rtrs_post_send_rdma()
474 	flags = atomic_inc_return(&con->io_cnt) % sess->queue_depth ?  in rtrs_post_send_rdma()
480 	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, &sge, 1,  in rtrs_post_send_rdma()
499 static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc)  in rtrs_clt_recv_done()  argument
503 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_clt_recv_done()
508 	err = rtrs_iu_post_recv(&con->c, iu);  in rtrs_clt_recv_done()
510 		rtrs_err(con->c.sess, "post iu failed %d\n", err);  in rtrs_clt_recv_done()
511 		rtrs_rdma_error_recovery(con);  in rtrs_clt_recv_done()
515 static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc)  in rtrs_clt_rkey_rsp_done()  argument
517 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_clt_rkey_rsp_done()
530 		rtrs_err(con->c.sess, "rkey response is malformed: size %d\n",  in rtrs_clt_rkey_rsp_done()
561 	return rtrs_clt_recv_done(con, wc);  in rtrs_clt_rkey_rsp_done()
563 	rtrs_rdma_error_recovery(con);  in rtrs_clt_rkey_rsp_done()
576 static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe)  in rtrs_post_recv_empty_x2()  argument
590 	return ib_post_recv(con->qp, wr, NULL);  in rtrs_post_recv_empty_x2()
595 	struct rtrs_clt_con *con = cq->cq_context;  in rtrs_clt_rdma_done()  local
596 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_clt_rdma_done()
605 			rtrs_rdma_error_recovery(con);  in rtrs_clt_rdma_done()
609 	rtrs_clt_update_wc_stats(con);  in rtrs_clt_rdma_done()
630 			WARN_ON(con->c.cid);  in rtrs_clt_rdma_done()
633 				return  rtrs_clt_recv_done(con, wc);  in rtrs_clt_rdma_done()
635 			WARN_ON(con->c.cid);  in rtrs_clt_rdma_done()
638 				return  rtrs_clt_recv_done(con, wc);  in rtrs_clt_rdma_done()
640 			rtrs_wrn(con->c.sess, "Unknown IMM type %u\n",  in rtrs_clt_rdma_done()
648 			err = rtrs_post_recv_empty_x2(&con->c, &io_comp_cqe);  in rtrs_clt_rdma_done()
650 			err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);  in rtrs_clt_rdma_done()
652 			rtrs_err(con->c.sess, "rtrs_post_recv_empty(): %d\n",  in rtrs_clt_rdma_done()
654 			rtrs_rdma_error_recovery(con);  in rtrs_clt_rdma_done()
667 				return  rtrs_clt_recv_done(con, wc);  in rtrs_clt_rdma_done()
669 			return  rtrs_clt_rkey_rsp_done(con, wc);  in rtrs_clt_rdma_done()
684 static int post_recv_io(struct rtrs_clt_con *con, size_t q_size)  in post_recv_io()  argument
687 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in post_recv_io()
691 			struct rtrs_iu *iu = &con->rsp_ius[i];  in post_recv_io()
693 			err = rtrs_iu_post_recv(&con->c, iu);  in post_recv_io()
695 			err = rtrs_post_recv_empty(&con->c, &io_comp_cqe);  in post_recv_io()
721 		err = post_recv_io(to_clt_con(sess->s.con[cid]), q_size);  in post_recv_sess()
899 	req->con = rtrs_permit_to_clt_con(sess, permit);  in rtrs_clt_init_req()
946 static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con,  in rtrs_post_rdma_write_sg()  argument
951 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_post_rdma_write_sg()
973 	flags = atomic_inc_return(&con->io_cnt) % sess->queue_depth ?  in rtrs_post_rdma_write_sg()
979 	return rtrs_iu_post_rdma_write_imm(&con->c, req->iu, sge, num_sge,  in rtrs_post_rdma_write_sg()
986 	struct rtrs_clt_con *con = req->con;  in rtrs_clt_write_req()  local
987 	struct rtrs_sess *s = con->c.sess;  in rtrs_clt_write_req()
1028 	ret = rtrs_post_rdma_write_sg(req->con, req, rbuf,  in rtrs_clt_write_req()
1060 	struct rtrs_clt_con *con = req->con;  in rtrs_clt_read_req()  local
1061 	struct rtrs_sess *s = con->c.sess;  in rtrs_clt_read_req()
1150 	ret = rtrs_post_send_rdma(req->con, req, &sess->rbufs[buf_id],  in rtrs_clt_read_req()
1389 	struct rtrs_clt_con *con = container_of(c, typeof(*con), c);  in rtrs_clt_hb_err_handler()  local
1391 	rtrs_rdma_error_recovery(con);  in rtrs_clt_hb_err_handler()
1432 	sess->s.con = kcalloc(con_num, sizeof(*sess->s.con), GFP_KERNEL);  in alloc_sess()
1433 	if (!sess->s.con)  in alloc_sess()
1482 	kfree(sess->s.con);  in alloc_sess()
1493 	kfree(sess->s.con);  in free_sess()
1500 	struct rtrs_clt_con *con;  in create_con()  local
1502 	con = kzalloc(sizeof(*con), GFP_KERNEL);  in create_con()
1503 	if (!con)  in create_con()
1507 	con->cpu  = (cid ? cid - 1 : 0) % nr_cpu_ids;  in create_con()
1508 	con->c.cid = cid;  in create_con()
1509 	con->c.sess = &sess->s;  in create_con()
1510 	atomic_set(&con->io_cnt, 0);  in create_con()
1512 	sess->s.con[cid] = &con->c;  in create_con()
1517 static void destroy_con(struct rtrs_clt_con *con)  in destroy_con()  argument
1519 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in destroy_con()
1521 	sess->s.con[con->c.cid] = NULL;  in destroy_con()
1522 	kfree(con);  in destroy_con()
1525 static int create_con_cq_qp(struct rtrs_clt_con *con)  in create_con_cq_qp()  argument
1527 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in create_con_cq_qp()
1541 	if (con->c.cid == 0) {  in create_con_cq_qp()
1559 		sess->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device,  in create_con_cq_qp()
1590 	if (sess->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) {  in create_con_cq_qp()
1591 		con->rsp_ius = rtrs_iu_alloc(max_recv_wr, sizeof(*rsp),  in create_con_cq_qp()
1595 		if (!con->rsp_ius)  in create_con_cq_qp()
1597 		con->queue_size = max_recv_wr;  in create_con_cq_qp()
1600 	cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors;  in create_con_cq_qp()
1601 	err = rtrs_cq_qp_create(&sess->s, &con->c, sess->max_send_sge,  in create_con_cq_qp()
1611 static void destroy_con_cq_qp(struct rtrs_clt_con *con)  in destroy_con_cq_qp()  argument
1613 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in destroy_con_cq_qp()
1620 	rtrs_cq_qp_destroy(&con->c);  in destroy_con_cq_qp()
1621 	if (con->rsp_ius) {  in destroy_con_cq_qp()
1622 		rtrs_iu_free(con->rsp_ius, sess->s.dev->ib_dev, con->queue_size);  in destroy_con_cq_qp()
1623 		con->rsp_ius = NULL;  in destroy_con_cq_qp()
1624 		con->queue_size = 0;  in destroy_con_cq_qp()
1632 static void stop_cm(struct rtrs_clt_con *con)  in stop_cm()  argument
1634 	rdma_disconnect(con->c.cm_id);  in stop_cm()
1635 	if (con->c.qp)  in stop_cm()
1636 		ib_drain_qp(con->c.qp);  in stop_cm()
1639 static void destroy_cm(struct rtrs_clt_con *con)  in destroy_cm()  argument
1641 	rdma_destroy_id(con->c.cm_id);  in destroy_cm()
1642 	con->c.cm_id = NULL;  in destroy_cm()
1645 static int rtrs_rdma_addr_resolved(struct rtrs_clt_con *con)  in rtrs_rdma_addr_resolved()  argument
1647 	struct rtrs_sess *s = con->c.sess;  in rtrs_rdma_addr_resolved()
1650 	err = create_con_cq_qp(con);  in rtrs_rdma_addr_resolved()
1655 	err = rdma_resolve_route(con->c.cm_id, RTRS_CONNECT_TIMEOUT_MS);  in rtrs_rdma_addr_resolved()
1662 static int rtrs_rdma_route_resolved(struct rtrs_clt_con *con)  in rtrs_rdma_route_resolved()  argument
1664 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_rdma_route_resolved()
1681 		.cid = cpu_to_le16(con->c.cid),  in rtrs_rdma_route_resolved()
1689 	err = rdma_connect_locked(con->c.cm_id, &param);  in rtrs_rdma_route_resolved()
1696 static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,  in rtrs_rdma_conn_established()  argument
1699 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_rdma_conn_established()
1728 	if (con->c.cid == 0) {  in rtrs_rdma_conn_established()
1773 		sess->hca_port = con->c.cm_id->port_num;  in rtrs_rdma_conn_established()
1776 		sess->s.src_addr = con->c.cm_id->route.addr.src_addr;  in rtrs_rdma_conn_established()
1784 static inline void flag_success_on_conn(struct rtrs_clt_con *con)  in flag_success_on_conn()  argument
1786 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in flag_success_on_conn()
1789 	con->cm_err = 1;  in flag_success_on_conn()
1792 static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,  in rtrs_rdma_conn_rejected()  argument
1795 	struct rtrs_sess *s = con->c.sess;  in rtrs_rdma_conn_rejected()
1802 	rej_msg = rdma_reject_msg(con->c.cm_id, status);  in rtrs_rdma_conn_rejected()
1803 	msg = rdma_consumer_reject_data(con->c.cm_id, ev, &data_len);  in rtrs_rdma_conn_rejected()
1831 static inline void flag_error_on_conn(struct rtrs_clt_con *con, int cm_err)  in flag_error_on_conn()  argument
1833 	if (con->cm_err == 1) {  in flag_error_on_conn()
1836 		sess = to_clt_sess(con->c.sess);  in flag_error_on_conn()
1841 	con->cm_err = cm_err;  in flag_error_on_conn()
1847 	struct rtrs_clt_con *con = cm_id->context;  in rtrs_clt_rdma_cm_handler()  local
1848 	struct rtrs_sess *s = con->c.sess;  in rtrs_clt_rdma_cm_handler()
1854 		cm_err = rtrs_rdma_addr_resolved(con);  in rtrs_clt_rdma_cm_handler()
1857 		cm_err = rtrs_rdma_route_resolved(con);  in rtrs_clt_rdma_cm_handler()
1860 		cm_err = rtrs_rdma_conn_established(con, ev);  in rtrs_clt_rdma_cm_handler()
1866 			flag_success_on_conn(con);  in rtrs_clt_rdma_cm_handler()
1872 		cm_err = rtrs_rdma_conn_rejected(con, ev);  in rtrs_clt_rdma_cm_handler()
1905 		flag_error_on_conn(con, cm_err);  in rtrs_clt_rdma_cm_handler()
1906 		rtrs_rdma_error_recovery(con);  in rtrs_clt_rdma_cm_handler()
1912 static int create_cm(struct rtrs_clt_con *con)  in create_cm()  argument
1914 	struct rtrs_sess *s = con->c.sess;  in create_cm()
1919 	cm_id = rdma_create_id(&init_net, rtrs_clt_rdma_cm_handler, con,  in create_cm()
1928 	con->c.cm_id = cm_id;  in create_cm()
1929 	con->cm_err = 0;  in create_cm()
1950 			con->cm_err || sess->state != RTRS_CLT_CONNECTING,  in create_cm()
1958 	if (con->cm_err < 0) {  in create_cm()
1959 		err = con->cm_err;  in create_cm()
1971 	stop_cm(con);  in create_cm()
1973 	destroy_con_cq_qp(con);  in create_cm()
1975 	destroy_cm(con);  in create_cm()
2028 	struct rtrs_clt_con *con;  in rtrs_clt_stop_and_destroy_conns()  local
2056 		if (!sess->s.con[cid])  in rtrs_clt_stop_and_destroy_conns()
2058 		con = to_clt_con(sess->s.con[cid]);  in rtrs_clt_stop_and_destroy_conns()
2059 		stop_cm(con);  in rtrs_clt_stop_and_destroy_conns()
2077 		if (!sess->s.con[cid])  in rtrs_clt_stop_and_destroy_conns()
2079 		con = to_clt_con(sess->s.con[cid]);  in rtrs_clt_stop_and_destroy_conns()
2080 		destroy_con_cq_qp(con);  in rtrs_clt_stop_and_destroy_conns()
2081 		destroy_cm(con);  in rtrs_clt_stop_and_destroy_conns()
2082 		destroy_con(con);  in rtrs_clt_stop_and_destroy_conns()
2228 		err = create_cm(to_clt_con(sess->s.con[cid]));  in init_conns()
2230 			destroy_con(to_clt_con(sess->s.con[cid]));  in init_conns()
2244 		struct rtrs_clt_con *con = to_clt_con(sess->s.con[cid]);  in init_conns()  local
2246 		stop_cm(con);  in init_conns()
2247 		destroy_con_cq_qp(con);  in init_conns()
2248 		destroy_cm(con);  in init_conns()
2249 		destroy_con(con);  in init_conns()
2263 	struct rtrs_clt_con *con = cq->cq_context;  in rtrs_clt_info_req_done()  local
2264 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_clt_info_req_done()
2277 	rtrs_clt_update_wc_stats(con);  in rtrs_clt_info_req_done()
2346 	struct rtrs_clt_con *con = cq->cq_context;  in rtrs_clt_info_rsp_done()  local
2347 	struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);  in rtrs_clt_info_rsp_done()
2356 	WARN_ON(con->c.cid);  in rtrs_clt_info_rsp_done()
2396 	rtrs_clt_update_wc_stats(con);  in rtrs_clt_info_rsp_done()
2403 	struct rtrs_clt_con *usr_con = to_clt_con(sess->s.con[0]);  in rtrs_send_sess_info()