2934 					  struct rvt_qp *sqp)  in loopback_qp_drop()  argument
2941 	return sqp->ibqp.qp_type == IB_QPT_RC ?  in loopback_qp_drop()
2955 void rvt_ruc_loopback(struct rvt_qp *sqp)  in rvt_ruc_loopback()  argument
2958 	struct rvt_dev_info *rdi = ib_to_rvt(sqp->ibqp.device);  in rvt_ruc_loopback()
2973 	rvp = rdi->ports[sqp->port_num - 1];  in rvt_ruc_loopback()
2980 	qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), rvp,  in rvt_ruc_loopback()
2981 			    sqp->remote_qpn);  in rvt_ruc_loopback()
2983 	spin_lock_irqsave(&sqp->s_lock, flags);  in rvt_ruc_loopback()
2986 	if ((sqp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT)) ||  in rvt_ruc_loopback()
2987 	    !(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_OR_FLUSH_SEND))  in rvt_ruc_loopback()
2990 	sqp->s_flags |= RVT_S_BUSY;  in rvt_ruc_loopback()
2993 	if (sqp->s_last == READ_ONCE(sqp->s_head))  in rvt_ruc_loopback()
2995 	wqe = rvt_get_swqe_ptr(sqp, sqp->s_last);  in rvt_ruc_loopback()
2998 	if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_NEXT_SEND_OK)) {  in rvt_ruc_loopback()
2999 		if (!(ib_rvt_state_ops[sqp->state] & RVT_FLUSH_SEND))  in rvt_ruc_loopback()
3011 	if (sqp->s_last == sqp->s_cur) {  in rvt_ruc_loopback()
3012 		if (++sqp->s_cur >= sqp->s_size)  in rvt_ruc_loopback()
3013 			sqp->s_cur = 0;  in rvt_ruc_loopback()
3015 	spin_unlock_irqrestore(&sqp->s_lock, flags);  in rvt_ruc_loopback()
3018 		send_status = loopback_qp_drop(rvp, sqp);  in rvt_ruc_loopback()
3023 	    qp->ibqp.qp_type != sqp->ibqp.qp_type) {  in rvt_ruc_loopback()
3024 		send_status = loopback_qp_drop(rvp, sqp);  in rvt_ruc_loopback()
3032 	sqp->s_sge.sge = wqe->sg_list[0];  in rvt_ruc_loopback()
3033 	sqp->s_sge.sg_list = wqe->sg_list + 1;  in rvt_ruc_loopback()
3034 	sqp->s_sge.num_sge = wqe->wr.num_sge;  in rvt_ruc_loopback()
3035 	sqp->s_len = wqe->length;  in rvt_ruc_loopback()
3042 			if (rvt_invalidate_rkey(sqp,  in rvt_ruc_loopback()
3109 		if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length,  in rvt_ruc_loopback()
3115 		sqp->s_sge.sg_list = NULL;  in rvt_ruc_loopback()
3116 		sqp->s_sge.num_sge = 1;  in rvt_ruc_loopback()
3137 		*(u64 *)sqp->s_sge.sge.vaddr =  in rvt_ruc_loopback()
3151 	sge = &sqp->s_sge.sge;  in rvt_ruc_loopback()
3152 	while (sqp->s_len) {  in rvt_ruc_loopback()
3153 		u32 len = rvt_get_sge_length(sge, sqp->s_len);  in rvt_ruc_loopback()
3158 		rvt_update_sge(&sqp->s_sge, len, !release);  in rvt_ruc_loopback()
3159 		sqp->s_len -= len;  in rvt_ruc_loopback()
3184 	spin_lock_irqsave(&sqp->s_lock, flags);  in rvt_ruc_loopback()
3187 	sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;  in rvt_ruc_loopback()
3188 	spin_lock(&sqp->r_lock);  in rvt_ruc_loopback()
3189 	rvt_send_complete(sqp, wqe, send_status);  in rvt_ruc_loopback()
3190 	spin_unlock(&sqp->r_lock);  in rvt_ruc_loopback()
3192 		atomic_dec(&sqp->local_ops_pending);  in rvt_ruc_loopback()
3206 	if (sqp->s_rnr_retry == 0) {  in rvt_ruc_loopback()
3210 	if (sqp->s_rnr_retry_cnt < 7)  in rvt_ruc_loopback()
3211 		sqp->s_rnr_retry--;  in rvt_ruc_loopback()
3213 	spin_lock_irqsave(&sqp->s_lock, flags);  in rvt_ruc_loopback()
3214 	if (!(ib_rvt_state_ops[sqp->state] & RVT_PROCESS_RECV_OK))  in rvt_ruc_loopback()
3216 	rvt_add_rnr_timer(sqp, qp->r_min_rnr_timer <<  in rvt_ruc_loopback()
3227 		sqp->ibqp.qp_type == IB_QPT_RC ?  in rvt_ruc_loopback()
3243 	spin_lock_irqsave(&sqp->s_lock, flags);  in rvt_ruc_loopback()
3244 	spin_lock(&sqp->r_lock);  in rvt_ruc_loopback()
3245 	rvt_send_complete(sqp, wqe, send_status);  in rvt_ruc_loopback()
3246 	spin_unlock(&sqp->r_lock);  in rvt_ruc_loopback()
3247 	if (sqp->ibqp.qp_type == IB_QPT_RC) {  in rvt_ruc_loopback()
3250 		spin_lock(&sqp->r_lock);  in rvt_ruc_loopback()
3251 		lastwqe = rvt_error_qp(sqp, IB_WC_WR_FLUSH_ERR);  in rvt_ruc_loopback()
3252 		spin_unlock(&sqp->r_lock);  in rvt_ruc_loopback()
3254 		sqp->s_flags &= ~RVT_S_BUSY;  in rvt_ruc_loopback()
3255 		spin_unlock_irqrestore(&sqp->s_lock, flags);  in rvt_ruc_loopback()
3259 			ev.device = sqp->ibqp.device;  in rvt_ruc_loopback()
3260 			ev.element.qp = &sqp->ibqp;  in rvt_ruc_loopback()
3262 			sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context);  in rvt_ruc_loopback()
3267 	sqp->s_flags &= ~RVT_S_BUSY;  in rvt_ruc_loopback()
3269 	spin_unlock_irqrestore(&sqp->s_lock, flags);  in rvt_ruc_loopback()