147 	struct rtrs_srv *srv = sess->srv;  in rtrs_srv_free_ops_ids()  local
152 		for (i = 0; i < srv->queue_depth; i++)  in rtrs_srv_free_ops_ids()
167 	struct rtrs_srv *srv = sess->srv;  in rtrs_srv_alloc_ops_ids()  local
171 	sess->ops_ids = kcalloc(srv->queue_depth, sizeof(*sess->ops_ids),  in rtrs_srv_alloc_ops_ids()
176 	for (i = 0; i < srv->queue_depth; ++i) {  in rtrs_srv_alloc_ops_ids()
234 	struct rtrs_srv *srv = sess->srv;  in rdma_write_sg()  local
302 	flags = (atomic_inc_return(&id->con->wr_cnt) % srv->queue_depth) ?  in rdma_write_sg()
380 	struct rtrs_srv *srv = sess->srv;  in send_io_resp_imm()  local
429 	flags = (atomic_inc_return(&con->wr_cnt) % srv->queue_depth) ?  in send_io_resp_imm()
573 void rtrs_srv_set_sess_priv(struct rtrs_srv *srv, void *priv)  in rtrs_srv_set_sess_priv()  argument
575 	srv->priv = priv;  in rtrs_srv_set_sess_priv()
598 	struct rtrs_srv *srv = sess->srv;  in map_cont_bufs()  local
613 		mrs_num = srv->queue_depth;  in map_cont_bufs()
617 		mrs_num = DIV_ROUND_UP(srv->queue_depth, chunks_per_mr);  in map_cont_bufs()
618 		chunks_per_mr = DIV_ROUND_UP(srv->queue_depth, mrs_num);  in map_cont_bufs()
637 					      srv->queue_depth - chunks);  in map_cont_bufs()
644 			sg_set_page(s, srv->chunks[chunks + i],  in map_cont_bufs()
704 	chunk_bits = ilog2(srv->queue_depth - 1) + 1;  in map_cont_bufs()
755 	struct rtrs_srv *srv = sess->srv;  in rtrs_srv_sess_up()  local
756 	struct rtrs_srv_ctx *ctx = srv->ctx;  in rtrs_srv_sess_up()
759 	mutex_lock(&srv->paths_ev_mutex);  in rtrs_srv_sess_up()
760 	up = ++srv->paths_up;  in rtrs_srv_sess_up()
762 		ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_CONNECTED, NULL);  in rtrs_srv_sess_up()
763 	mutex_unlock(&srv->paths_ev_mutex);  in rtrs_srv_sess_up()
771 	struct rtrs_srv *srv = sess->srv;  in rtrs_srv_sess_down()  local
772 	struct rtrs_srv_ctx *ctx = srv->ctx;  in rtrs_srv_sess_down()
778 	mutex_lock(&srv->paths_ev_mutex);  in rtrs_srv_sess_down()
779 	WARN_ON(!srv->paths_up);  in rtrs_srv_sess_down()
780 	if (--srv->paths_up == 0)  in rtrs_srv_sess_down()
781 		ctx->ops.link_ev(srv, RTRS_SRV_LINK_EV_DISCONNECTED, srv->priv);  in rtrs_srv_sess_down()
782 	mutex_unlock(&srv->paths_ev_mutex);  in rtrs_srv_sess_down()
848 	get_device(&sess->srv->dev);  in process_info_req()
958 	struct rtrs_srv *srv = sess->srv;  in post_recv_sess()  local
967 			q_size = srv->queue_depth;  in post_recv_sess()
985 	struct rtrs_srv *srv = sess->srv;  in process_read()  local
986 	struct rtrs_srv_ctx *ctx = srv->ctx;  in process_read()
1013 	data = page_address(srv->chunks[buf_id]);  in process_read()
1014 	ret = ctx->ops.rdma_ev(srv, srv->priv, id, READ, data, data_len,  in process_read()
1043 	struct rtrs_srv *srv = sess->srv;  in process_write()  local
1044 	struct rtrs_srv_ctx *ctx = srv->ctx;  in process_write()
1066 	data = page_address(srv->chunks[buf_id]);  in process_write()
1067 	ret = ctx->ops.rdma_ev(srv, srv->priv, id, WRITE, data, data_len,  in process_write()
1129 	struct rtrs_srv *srv = sess->srv;  in rtrs_srv_inv_rkey_done()  local
1140 	data = page_address(srv->chunks[msg_id]) + off;  in rtrs_srv_inv_rkey_done()
1186 	struct rtrs_srv *srv = sess->srv;  in rtrs_srv_rdma_done()  local
1223 			if (unlikely(msg_id >= srv->queue_depth ||  in rtrs_srv_rdma_done()
1243 				data = page_address(srv->chunks[msg_id]) + off;  in rtrs_srv_rdma_done()
1261 		atomic_add(srv->queue_depth, &con->sq_wr_avail);  in rtrs_srv_rdma_done()
1279 int rtrs_srv_get_sess_name(struct rtrs_srv *srv, char *sessname, size_t len)  in rtrs_srv_get_sess_name()  argument
1284 	mutex_lock(&srv->paths_mutex);  in rtrs_srv_get_sess_name()
1285 	list_for_each_entry(sess, &srv->paths_list, s.entry) {  in rtrs_srv_get_sess_name()
1293 	mutex_unlock(&srv->paths_mutex);  in rtrs_srv_get_sess_name()
1303 int rtrs_srv_get_queue_depth(struct rtrs_srv *srv)  in rtrs_srv_get_queue_depth()  argument
1305 	return srv->queue_depth;  in rtrs_srv_get_queue_depth()
1329 	struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev);  in rtrs_srv_dev_release()  local
1331 	kfree(srv);  in rtrs_srv_dev_release()
1334 static void free_srv(struct rtrs_srv *srv)  in free_srv()  argument
1338 	WARN_ON(refcount_read(&srv->refcount));  in free_srv()
1339 	for (i = 0; i < srv->queue_depth; i++)  in free_srv()
1340 		mempool_free(srv->chunks[i], chunk_pool);  in free_srv()
1341 	kfree(srv->chunks);  in free_srv()
1342 	mutex_destroy(&srv->paths_mutex);  in free_srv()
1343 	mutex_destroy(&srv->paths_ev_mutex);  in free_srv()
1345 	put_device(&srv->dev);  in free_srv()
1352 	struct rtrs_srv *srv;  in get_or_create_srv()  local
1356 	list_for_each_entry(srv, &ctx->srv_list, ctx_list) {  in get_or_create_srv()
1357 		if (uuid_equal(&srv->paths_uuid, paths_uuid) &&  in get_or_create_srv()
1358 		    refcount_inc_not_zero(&srv->refcount)) {  in get_or_create_srv()
1360 			return srv;  in get_or_create_srv()
1372 	srv = kzalloc(sizeof(*srv), GFP_KERNEL);  in get_or_create_srv()
1373 	if  (!srv)  in get_or_create_srv()
1376 	INIT_LIST_HEAD(&srv->paths_list);  in get_or_create_srv()
1377 	mutex_init(&srv->paths_mutex);  in get_or_create_srv()
1378 	mutex_init(&srv->paths_ev_mutex);  in get_or_create_srv()
1379 	uuid_copy(&srv->paths_uuid, paths_uuid);  in get_or_create_srv()
1380 	srv->queue_depth = sess_queue_depth;  in get_or_create_srv()
1381 	srv->ctx = ctx;  in get_or_create_srv()
1382 	device_initialize(&srv->dev);  in get_or_create_srv()
1383 	srv->dev.release = rtrs_srv_dev_release;  in get_or_create_srv()
1385 	srv->chunks = kcalloc(srv->queue_depth, sizeof(*srv->chunks),  in get_or_create_srv()
1387 	if (!srv->chunks)  in get_or_create_srv()
1390 	for (i = 0; i < srv->queue_depth; i++) {  in get_or_create_srv()
1391 		srv->chunks[i] = mempool_alloc(chunk_pool, GFP_KERNEL);  in get_or_create_srv()
1392 		if (!srv->chunks[i])  in get_or_create_srv()
1395 	refcount_set(&srv->refcount, 1);  in get_or_create_srv()
1397 	list_add(&srv->ctx_list, &ctx->srv_list);  in get_or_create_srv()
1400 	return srv;  in get_or_create_srv()
1404 		mempool_free(srv->chunks[i], chunk_pool);  in get_or_create_srv()
1405 	kfree(srv->chunks);  in get_or_create_srv()
1408 	kfree(srv);  in get_or_create_srv()
1412 static void put_srv(struct rtrs_srv *srv)  in put_srv()  argument
1414 	if (refcount_dec_and_test(&srv->refcount)) {  in put_srv()
1415 		struct rtrs_srv_ctx *ctx = srv->ctx;  in put_srv()
1417 		WARN_ON(srv->dev.kobj.state_in_sysfs);  in put_srv()
1420 		list_del(&srv->ctx_list);  in put_srv()
1422 		free_srv(srv);  in put_srv()
1426 static void __add_path_to_srv(struct rtrs_srv *srv,  in __add_path_to_srv()  argument
1429 	list_add_tail(&sess->s.entry, &srv->paths_list);  in __add_path_to_srv()
1430 	srv->paths_num++;  in __add_path_to_srv()
1431 	WARN_ON(srv->paths_num >= MAX_PATHS_NUM);  in __add_path_to_srv()
1436 	struct rtrs_srv *srv = sess->srv;  in del_path_from_srv()  local
1438 	if (WARN_ON(!srv))  in del_path_from_srv()
1441 	mutex_lock(&srv->paths_mutex);  in del_path_from_srv()
1443 	WARN_ON(!srv->paths_num);  in del_path_from_srv()
1444 	srv->paths_num--;  in del_path_from_srv()
1445 	mutex_unlock(&srv->paths_mutex);  in del_path_from_srv()
1472 static bool __is_path_w_addr_exists(struct rtrs_srv *srv,  in __is_path_w_addr_exists()  argument
1477 	list_for_each_entry(sess, &srv->paths_list, s.entry)  in __is_path_w_addr_exists()
1536 	put_srv(sess->srv);  in rtrs_srv_close_work()
1537 	sess->srv = NULL;  in rtrs_srv_close_work()
1548 	struct rtrs_srv *srv = sess->srv;  in rtrs_rdma_do_accept()  local
1562 		.queue_depth = cpu_to_le16(srv->queue_depth),  in rtrs_rdma_do_accept()
1597 __find_sess(struct rtrs_srv *srv, const uuid_t *sess_uuid)  in __find_sess()  argument
1601 	list_for_each_entry(sess, &srv->paths_list, s.entry) {  in __find_sess()
1613 	struct rtrs_srv *srv = sess->srv;  in create_con()  local
1653 				      srv->queue_depth * (1 + 4) + 1);  in create_con()
1657 				      srv->queue_depth * (1 + 2) + 1);  in create_con()
1659 		max_recv_wr = srv->queue_depth + 1;  in create_con()
1704 static struct rtrs_srv_sess *__alloc_sess(struct rtrs_srv *srv,  in __alloc_sess()  argument
1713 	if (srv->paths_num >= MAX_PATHS_NUM) {  in __alloc_sess()
1717 	if (__is_path_w_addr_exists(srv, &cm_id->route.addr)) {  in __alloc_sess()
1732 	sess->dma_addr = kcalloc(srv->queue_depth, sizeof(*sess->dma_addr),  in __alloc_sess()
1742 	sess->srv = srv;  in __alloc_sess()
1766 	__add_path_to_srv(srv, sess);  in __alloc_sess()
1792 	struct rtrs_srv *srv;  in rtrs_rdma_connect()  local
1825 	srv = get_or_create_srv(ctx, &msg->paths_uuid, msg->first_conn);  in rtrs_rdma_connect()
1826 	if (IS_ERR(srv)) {  in rtrs_rdma_connect()
1827 		err = PTR_ERR(srv);  in rtrs_rdma_connect()
1830 	mutex_lock(&srv->paths_mutex);  in rtrs_rdma_connect()
1831 	sess = __find_sess(srv, &msg->sess_uuid);  in rtrs_rdma_connect()
1836 		put_srv(srv);  in rtrs_rdma_connect()
1841 			mutex_unlock(&srv->paths_mutex);  in rtrs_rdma_connect()
1850 			mutex_unlock(&srv->paths_mutex);  in rtrs_rdma_connect()
1856 			mutex_unlock(&srv->paths_mutex);  in rtrs_rdma_connect()
1860 		sess = __alloc_sess(srv, cm_id, con_num, recon_cnt,  in rtrs_rdma_connect()
1863 			mutex_unlock(&srv->paths_mutex);  in rtrs_rdma_connect()
1864 			put_srv(srv);  in rtrs_rdma_connect()
1891 	mutex_unlock(&srv->paths_mutex);  in rtrs_rdma_connect()
1902 	mutex_unlock(&srv->paths_mutex);  in rtrs_rdma_connect()
2153 static void close_sessions(struct rtrs_srv *srv)  in close_sessions()  argument
2157 	mutex_lock(&srv->paths_mutex);  in close_sessions()
2158 	list_for_each_entry(sess, &srv->paths_list, s.entry)  in close_sessions()
2160 	mutex_unlock(&srv->paths_mutex);  in close_sessions()
2165 	struct rtrs_srv *srv;  in close_ctx()  local
2168 	list_for_each_entry(srv, &ctx->srv_list, ctx_list)  in close_ctx()
2169 		close_sessions(srv);  in close_ctx()