Lines Matching refs:clt
47 static inline bool rtrs_clt_is_connected(const struct rtrs_clt *clt) in rtrs_clt_is_connected() argument
53 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) in rtrs_clt_is_connected()
61 __rtrs_get_permit(struct rtrs_clt *clt, enum rtrs_clt_con_type con_type) in __rtrs_get_permit() argument
63 size_t max_depth = clt->queue_depth; in __rtrs_get_permit()
75 bit = find_first_zero_bit(clt->permits_map, max_depth); in __rtrs_get_permit()
78 } while (unlikely(test_and_set_bit_lock(bit, clt->permits_map))); in __rtrs_get_permit()
80 permit = get_permit(clt, bit); in __rtrs_get_permit()
88 static inline void __rtrs_put_permit(struct rtrs_clt *clt, in __rtrs_put_permit() argument
91 clear_bit_unlock(permit->mem_id, clt->permits_map); in __rtrs_put_permit()
108 struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *clt, in rtrs_clt_get_permit() argument
115 permit = __rtrs_get_permit(clt, con_type); in rtrs_clt_get_permit()
120 prepare_to_wait(&clt->permits_wait, &wait, in rtrs_clt_get_permit()
122 permit = __rtrs_get_permit(clt, con_type); in rtrs_clt_get_permit()
129 finish_wait(&clt->permits_wait, &wait); in rtrs_clt_get_permit()
143 void rtrs_clt_put_permit(struct rtrs_clt *clt, struct rtrs_permit *permit) in rtrs_clt_put_permit() argument
145 if (WARN_ON(!test_bit(permit->mem_id, clt->permits_map))) in rtrs_clt_put_permit()
148 __rtrs_put_permit(clt, permit); in rtrs_clt_put_permit()
157 if (waitqueue_active(&clt->permits_wait)) in rtrs_clt_put_permit()
158 wake_up(&clt->permits_wait); in rtrs_clt_put_permit()
310 struct rtrs_clt *clt = sess->clt; in rtrs_rdma_error_recovery() local
316 delay_ms = clt->reconnect_delay_sec * 1000; in rtrs_rdma_error_recovery()
440 if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT) in complete_rdma_req()
538 rtrs_err(sess->clt, "rkey response is malformed: type %d\n", in rtrs_clt_rkey_rsp_done()
603 rtrs_err(sess->clt, "RDMA failed: %s\n", in rtrs_clt_rdma_done()
679 rtrs_wrn(sess->clt, "Unexpected WC type: %d\n", wc->opcode); in rtrs_clt_rdma_done()
723 rtrs_err(sess->clt, "post_recv_io(), err: %d\n", err); in post_recv_sess()
734 struct rtrs_clt *clt; member
771 struct rtrs_clt *clt; in get_next_path_rr() local
773 clt = it->clt; in get_next_path_rr()
781 ppcpu_path = this_cpu_ptr(clt->pcpu_path); in get_next_path_rr()
784 path = list_first_or_null_rcu(&clt->paths_list, in get_next_path_rr()
787 path = list_next_or_null_rr_rcu(&clt->paths_list, in get_next_path_rr()
808 struct rtrs_clt *clt = it->clt; in get_next_path_min_inflight() local
813 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) { in get_next_path_min_inflight()
838 static inline void path_it_init(struct path_it *it, struct rtrs_clt *clt) in path_it_init() argument
841 it->clt = clt; in path_it_init()
844 if (clt->mp_policy == MP_POLICY_RR) in path_it_init()
1033 if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT) in rtrs_clt_write_req()
1154 if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT) in rtrs_clt_read_req()
1170 static int rtrs_clt_failover_req(struct rtrs_clt *clt, in rtrs_clt_failover_req() argument
1179 for (path_it_init(&it, clt); in rtrs_clt_failover_req()
1180 (alive_sess = it.next_path(&it)) && it.i < it.clt->paths_num; in rtrs_clt_failover_req()
1206 struct rtrs_clt *clt = sess->clt; in fail_all_outstanding_reqs() local
1224 err = rtrs_clt_failover_req(clt, req); in fail_all_outstanding_reqs()
1252 struct rtrs_clt *clt = sess->clt; in alloc_sess_reqs() local
1269 req->sge = kmalloc_array(clt->max_segments + 1, in alloc_sess_reqs()
1295 static int alloc_permits(struct rtrs_clt *clt) in alloc_permits() argument
1300 clt->permits_map = kcalloc(BITS_TO_LONGS(clt->queue_depth), in alloc_permits()
1302 if (!clt->permits_map) { in alloc_permits()
1306 clt->permits = kcalloc(clt->queue_depth, permit_size(clt), GFP_KERNEL); in alloc_permits()
1307 if (!clt->permits) { in alloc_permits()
1311 chunk_bits = ilog2(clt->queue_depth - 1) + 1; in alloc_permits()
1312 for (i = 0; i < clt->queue_depth; i++) { in alloc_permits()
1315 permit = get_permit(clt, i); in alloc_permits()
1323 kfree(clt->permits_map); in alloc_permits()
1324 clt->permits_map = NULL; in alloc_permits()
1329 static void free_permits(struct rtrs_clt *clt) in free_permits() argument
1331 if (clt->permits_map) { in free_permits()
1332 size_t sz = clt->queue_depth; in free_permits()
1334 wait_event(clt->permits_wait, in free_permits()
1335 find_first_bit(clt->permits_map, sz) >= sz); in free_permits()
1337 kfree(clt->permits_map); in free_permits()
1338 clt->permits_map = NULL; in free_permits()
1339 kfree(clt->permits); in free_permits()
1340 clt->permits = NULL; in free_permits()
1416 static struct rtrs_clt_sess *alloc_sess(struct rtrs_clt *clt, in alloc_sess() argument
1453 strlcpy(sess->s.sessname, clt->sessname, sizeof(sess->s.sessname)); in alloc_sess()
1455 sess->clt = clt; in alloc_sess()
1562 rtrs_wrn(sess->clt, in create_con_cq_qp()
1665 struct rtrs_clt *clt = sess->clt; in rtrs_rdma_route_resolved() local
1687 uuid_copy(&msg.paths_uuid, &clt->paths_uuid); in rtrs_rdma_route_resolved()
1691 rtrs_err(clt, "rdma_connect_locked(): %d\n", err); in rtrs_rdma_route_resolved()
1700 struct rtrs_clt *clt = sess->clt; in rtrs_rdma_conn_established() local
1709 rtrs_err(clt, "Invalid RTRS connection response\n"); in rtrs_rdma_conn_established()
1713 rtrs_err(clt, "Invalid RTRS magic\n"); in rtrs_rdma_conn_established()
1718 rtrs_err(clt, "Unsupported major RTRS version: %d, expected %d\n", in rtrs_rdma_conn_established()
1724 rtrs_err(clt, "Invalid RTRS message: errno %d\n", in rtrs_rdma_conn_established()
1732 rtrs_err(clt, "Error: queue depth changed\n"); in rtrs_rdma_conn_established()
1738 rtrs_err(clt, in rtrs_rdma_conn_established()
1764 mutex_lock(&clt->paths_mutex); in rtrs_rdma_conn_established()
1765 clt->queue_depth = sess->queue_depth; in rtrs_rdma_conn_established()
1766 clt->max_io_size = min_not_zero(sess->max_io_size, in rtrs_rdma_conn_established()
1767 clt->max_io_size); in rtrs_rdma_conn_established()
1768 mutex_unlock(&clt->paths_mutex); in rtrs_rdma_conn_established()
1982 struct rtrs_clt *clt = sess->clt; in rtrs_clt_sess_up() local
1992 mutex_lock(&clt->paths_ev_mutex); in rtrs_clt_sess_up()
1993 up = ++clt->paths_up; in rtrs_clt_sess_up()
1999 if (up > MAX_PATHS_NUM && up == MAX_PATHS_NUM + clt->paths_num) in rtrs_clt_sess_up()
2000 clt->paths_up = clt->paths_num; in rtrs_clt_sess_up()
2002 clt->link_ev(clt->priv, RTRS_CLT_LINK_EV_RECONNECTED); in rtrs_clt_sess_up()
2003 mutex_unlock(&clt->paths_ev_mutex); in rtrs_clt_sess_up()
2013 struct rtrs_clt *clt = sess->clt; in rtrs_clt_sess_down() local
2019 mutex_lock(&clt->paths_ev_mutex); in rtrs_clt_sess_down()
2020 WARN_ON(!clt->paths_up); in rtrs_clt_sess_down()
2021 if (--clt->paths_up == 0) in rtrs_clt_sess_down()
2022 clt->link_ev(clt->priv, RTRS_CLT_LINK_EV_DISCONNECTED); in rtrs_clt_sess_down()
2023 mutex_unlock(&clt->paths_ev_mutex); in rtrs_clt_sess_down()
2099 struct rtrs_clt *clt = sess->clt; in rtrs_clt_remove_path_from_arr() local
2104 mutex_lock(&clt->paths_mutex); in rtrs_clt_remove_path_from_arr()
2139 clt->paths_num--; in rtrs_clt_remove_path_from_arr()
2146 next = list_next_or_null_rr_rcu(&clt->paths_list, &sess->s.entry, in rtrs_clt_remove_path_from_arr()
2157 ppcpu_path = per_cpu_ptr(clt->pcpu_path, cpu); in rtrs_clt_remove_path_from_arr()
2159 lockdep_is_held(&clt->paths_mutex)) != sess) in rtrs_clt_remove_path_from_arr()
2184 mutex_unlock(&clt->paths_mutex); in rtrs_clt_remove_path_from_arr()
2190 struct rtrs_clt *clt = sess->clt; in rtrs_clt_add_path_to_arr() local
2192 mutex_lock(&clt->paths_mutex); in rtrs_clt_add_path_to_arr()
2193 clt->paths_num++; in rtrs_clt_add_path_to_arr()
2195 list_add_tail_rcu(&sess->s.entry, &clt->paths_list); in rtrs_clt_add_path_to_arr()
2196 mutex_unlock(&clt->paths_mutex); in rtrs_clt_add_path_to_arr()
2271 rtrs_err(sess->clt, "Sess info request send failed: %s\n", in rtrs_clt_info_req_done()
2296 rtrs_err(sess->clt, in process_info_rsp()
2302 rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n", in process_info_rsp()
2319 rtrs_err(sess->clt, "Incorrect [%d].len %d\n", sgi, in process_info_rsp()
2333 rtrs_err(sess->clt, "Incorrect sg vector, not fully mapped\n"); in process_info_rsp()
2337 rtrs_err(sess->clt, "Incorrect total_len %d\n", total_len); in process_info_rsp()
2359 rtrs_err(sess->clt, "Sess info response recv failed: %s\n", in rtrs_clt_info_rsp_done()
2366 rtrs_err(sess->clt, "Sess info response is malformed: size %d\n", in rtrs_clt_info_rsp_done()
2374 rtrs_err(sess->clt, "Sess info response is malformed: type %d\n", in rtrs_clt_info_rsp_done()
2381 rtrs_err(sess->clt, "Sess info response is malformed: size %d\n", in rtrs_clt_info_rsp_done()
2424 rtrs_err(sess->clt, "rtrs_iu_post_recv(), err: %d\n", err); in rtrs_send_sess_info()
2439 rtrs_err(sess->clt, "rtrs_iu_post_send(), err: %d\n", err); in rtrs_send_sess_info()
2482 rtrs_err(sess->clt, "init_conns(), err: %d\n", err); in init_sess()
2487 rtrs_err(sess->clt, "rtrs_send_sess_info(), err: %d\n", err); in init_sess()
2500 struct rtrs_clt *clt; in rtrs_clt_reconnect_work() local
2506 clt = sess->clt; in rtrs_clt_reconnect_work()
2511 if (sess->reconnect_attempts >= clt->max_reconnect_attempts) { in rtrs_clt_reconnect_work()
2532 delay_ms = clt->reconnect_delay_sec * 1000; in rtrs_clt_reconnect_work()
2542 struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev); in rtrs_clt_dev_release() local
2544 mutex_destroy(&clt->paths_ev_mutex); in rtrs_clt_dev_release()
2545 mutex_destroy(&clt->paths_mutex); in rtrs_clt_dev_release()
2546 kfree(clt); in rtrs_clt_dev_release()
2558 struct rtrs_clt *clt; in alloc_clt() local
2564 if (strlen(sessname) >= sizeof(clt->sessname)) in alloc_clt()
2567 clt = kzalloc(sizeof(*clt), GFP_KERNEL); in alloc_clt()
2568 if (!clt) in alloc_clt()
2571 clt->pcpu_path = alloc_percpu(typeof(*clt->pcpu_path)); in alloc_clt()
2572 if (!clt->pcpu_path) { in alloc_clt()
2573 kfree(clt); in alloc_clt()
2577 clt->dev.class = rtrs_clt_dev_class; in alloc_clt()
2578 clt->dev.release = rtrs_clt_dev_release; in alloc_clt()
2579 uuid_gen(&clt->paths_uuid); in alloc_clt()
2580 INIT_LIST_HEAD_RCU(&clt->paths_list); in alloc_clt()
2581 clt->paths_num = paths_num; in alloc_clt()
2582 clt->paths_up = MAX_PATHS_NUM; in alloc_clt()
2583 clt->port = port; in alloc_clt()
2584 clt->pdu_sz = pdu_sz; in alloc_clt()
2585 clt->max_segments = max_segments; in alloc_clt()
2586 clt->max_segment_size = max_segment_size; in alloc_clt()
2587 clt->reconnect_delay_sec = reconnect_delay_sec; in alloc_clt()
2588 clt->max_reconnect_attempts = max_reconnect_attempts; in alloc_clt()
2589 clt->priv = priv; in alloc_clt()
2590 clt->link_ev = link_ev; in alloc_clt()
2591 clt->mp_policy = MP_POLICY_MIN_INFLIGHT; in alloc_clt()
2592 strlcpy(clt->sessname, sessname, sizeof(clt->sessname)); in alloc_clt()
2593 init_waitqueue_head(&clt->permits_wait); in alloc_clt()
2594 mutex_init(&clt->paths_ev_mutex); in alloc_clt()
2595 mutex_init(&clt->paths_mutex); in alloc_clt()
2596 device_initialize(&clt->dev); in alloc_clt()
2598 err = dev_set_name(&clt->dev, "%s", sessname); in alloc_clt()
2606 dev_set_uevent_suppress(&clt->dev, true); in alloc_clt()
2607 err = device_add(&clt->dev); in alloc_clt()
2611 clt->kobj_paths = kobject_create_and_add("paths", &clt->dev.kobj); in alloc_clt()
2612 if (!clt->kobj_paths) { in alloc_clt()
2616 err = rtrs_clt_create_sysfs_root_files(clt); in alloc_clt()
2618 kobject_del(clt->kobj_paths); in alloc_clt()
2619 kobject_put(clt->kobj_paths); in alloc_clt()
2622 dev_set_uevent_suppress(&clt->dev, false); in alloc_clt()
2623 kobject_uevent(&clt->dev.kobj, KOBJ_ADD); in alloc_clt()
2625 return clt; in alloc_clt()
2627 device_del(&clt->dev); in alloc_clt()
2629 free_percpu(clt->pcpu_path); in alloc_clt()
2630 put_device(&clt->dev); in alloc_clt()
2634 static void free_clt(struct rtrs_clt *clt) in free_clt() argument
2636 free_percpu(clt->pcpu_path); in free_clt()
2641 device_unregister(&clt->dev); in free_clt()
2673 struct rtrs_clt *clt; in rtrs_clt_open() local
2676 clt = alloc_clt(sessname, paths_num, port, pdu_sz, ops->priv, in rtrs_clt_open()
2680 if (IS_ERR(clt)) { in rtrs_clt_open()
2681 err = PTR_ERR(clt); in rtrs_clt_open()
2687 sess = alloc_sess(clt, &paths[i], nr_cpu_ids, in rtrs_clt_open()
2695 list_add_tail_rcu(&sess->s.entry, &clt->paths_list); in rtrs_clt_open()
2717 err = alloc_permits(clt); in rtrs_clt_open()
2721 return clt; in rtrs_clt_open()
2724 list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) { in rtrs_clt_open()
2729 rtrs_clt_destroy_sysfs_root_files(clt); in rtrs_clt_open()
2730 rtrs_clt_destroy_sysfs_root_folders(clt); in rtrs_clt_open()
2731 free_clt(clt); in rtrs_clt_open()
2742 void rtrs_clt_close(struct rtrs_clt *clt) in rtrs_clt_close() argument
2747 rtrs_clt_destroy_sysfs_root_files(clt); in rtrs_clt_close()
2748 rtrs_clt_destroy_sysfs_root_folders(clt); in rtrs_clt_close()
2751 list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) { in rtrs_clt_close()
2756 free_permits(clt); in rtrs_clt_close()
2757 free_clt(clt); in rtrs_clt_close()
2825 void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value) in rtrs_clt_set_max_reconnect_attempts() argument
2827 clt->max_reconnect_attempts = (unsigned int)value; in rtrs_clt_set_max_reconnect_attempts()
2830 int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt) in rtrs_clt_get_max_reconnect_attempts() argument
2832 return (int)clt->max_reconnect_attempts; in rtrs_clt_get_max_reconnect_attempts()
2860 struct rtrs_clt *clt, struct rtrs_permit *permit, in rtrs_clt_request() argument
2886 for (path_it_init(&it, clt); in rtrs_clt_request()
2887 (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { in rtrs_clt_request()
2892 rtrs_wrn_rl(sess->clt, in rtrs_clt_request()
2928 int rtrs_clt_query(struct rtrs_clt *clt, struct rtrs_attrs *attr) in rtrs_clt_query() argument
2930 if (!rtrs_clt_is_connected(clt)) in rtrs_clt_query()
2933 attr->queue_depth = clt->queue_depth; in rtrs_clt_query()
2934 attr->max_io_size = clt->max_io_size; in rtrs_clt_query()
2935 attr->sess_kobj = &clt->dev.kobj; in rtrs_clt_query()
2936 strlcpy(attr->sessname, clt->sessname, sizeof(attr->sessname)); in rtrs_clt_query()
2942 int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt, in rtrs_clt_create_path_from_sysfs() argument
2948 sess = alloc_sess(clt, addr, nr_cpu_ids, clt->max_segments, in rtrs_clt_create_path_from_sysfs()
2949 clt->max_segment_size); in rtrs_clt_create_path_from_sysfs()