Lines Matching refs:wqe
377 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_priv_init() local
385 priv->tid_req.e.swqe = wqe; in hfi1_qp_priv_init()
386 wqe->priv = priv; in hfi1_qp_priv_init()
415 struct rvt_swqe *wqe; in hfi1_qp_priv_tid_free() local
420 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_priv_tid_free()
421 kfree(wqe->priv); in hfi1_qp_priv_tid_free()
422 wqe->priv = NULL; in hfi1_qp_priv_tid_free()
1614 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) in __trdma_clean_swqe() argument
1616 struct hfi1_swqe_priv *p = wqe->priv; in __trdma_clean_swqe()
1696 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, in hfi1_build_tid_rdma_read_packet() argument
1700 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_read_packet()
1704 struct hfi1_swqe_priv *wpriv = wqe->priv; in hfi1_build_tid_rdma_read_packet()
1741 rreq->reth.vaddr = cpu_to_be64(wqe->rdma_wr.remote_addr + in hfi1_build_tid_rdma_read_packet()
1743 rreq->reth.rkey = cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_build_tid_rdma_read_packet()
1780 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_read_req() argument
1786 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_read_req()
1793 trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_build_tid_rdma_read_req()
1794 wqe->lpsn, req); in hfi1_build_tid_rdma_read_req()
1822 restart_sge(&qp->s_sge, wqe, req->s_next_psn, in hfi1_build_tid_rdma_read_req()
1872 hdwords = hfi1_build_tid_rdma_read_packet(wqe, ohdr, bth1, bth2, len); in hfi1_build_tid_rdma_read_req()
2411 struct rvt_swqe *wqe; in find_tid_request() local
2419 wqe = rvt_get_swqe_ptr(qp, i); in find_tid_request()
2420 if (cmp_psn(psn, wqe->psn) >= 0 && in find_tid_request()
2421 cmp_psn(psn, wqe->lpsn) <= 0) { in find_tid_request()
2422 if (wqe->wr.opcode == opcode) in find_tid_request()
2423 req = wqe_to_tid_req(wqe); in find_tid_request()
2575 struct rvt_swqe *wqe; in hfi1_kern_read_tid_flow_free() local
2582 wqe = rvt_get_swqe_ptr(qp, n); in hfi1_kern_read_tid_flow_free()
2583 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in hfi1_kern_read_tid_flow_free()
2584 req = wqe_to_tid_req(wqe); in hfi1_kern_read_tid_flow_free()
2623 struct rvt_qp *qp, struct rvt_swqe *wqe) in restart_tid_rdma_read_req() argument
2630 req = wqe_to_tid_req(wqe); in restart_tid_rdma_read_req()
2656 struct rvt_swqe *wqe; in handle_read_kdeth_eflags() local
2682 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in handle_read_kdeth_eflags()
2686 while ((int)delta_psn(ack_psn, wqe->lpsn) >= 0) { in handle_read_kdeth_eflags()
2692 if (wqe->wr.opcode == IB_WR_RDMA_READ || in handle_read_kdeth_eflags()
2693 wqe->wr.opcode == IB_WR_TID_RDMA_READ || in handle_read_kdeth_eflags()
2694 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in handle_read_kdeth_eflags()
2695 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { in handle_read_kdeth_eflags()
2699 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in handle_read_kdeth_eflags()
2701 wqe); in handle_read_kdeth_eflags()
2721 wqe = do_rc_completion(qp, wqe, ibp); in handle_read_kdeth_eflags()
2730 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) in handle_read_kdeth_eflags()
2733 req = wqe_to_tid_req(wqe); in handle_read_kdeth_eflags()
2734 trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn, in handle_read_kdeth_eflags()
2735 wqe->lpsn, req); in handle_read_kdeth_eflags()
2801 wqe); in handle_read_kdeth_eflags()
3037 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_tid_rdma_restart_req() argument
3040 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_tid_rdma_restart_req()
3047 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in hfi1_tid_rdma_restart_req()
3054 trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, in hfi1_tid_rdma_restart_req()
3055 wqe->psn, wqe->lpsn, in hfi1_tid_rdma_restart_req()
3065 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_tid_rdma_restart_req()
3097 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { in hfi1_tid_rdma_restart_req()
3116 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) in hfi1_tid_rdma_restart_req()
3123 trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_tid_rdma_restart_req()
3124 wqe->lpsn, req); in hfi1_tid_rdma_restart_req()
3126 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { in hfi1_tid_rdma_restart_req()
3143 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_tid_rdma_restart_req()
3144 } while (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE); in hfi1_tid_rdma_restart_req()
3145 req = wqe_to_tid_req(wqe); in hfi1_tid_rdma_restart_req()
3172 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); in hfi1_qp_kern_exp_rcv_clear_all() local
3177 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) in hfi1_qp_kern_exp_rcv_clear_all()
3180 struct hfi1_swqe_priv *priv = wqe->priv; in hfi1_qp_kern_exp_rcv_clear_all()
3201 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) in hfi1_tid_rdma_wqe_interlock() argument
3211 switch (wqe->wr.opcode) { in hfi1_tid_rdma_wqe_interlock()
3270 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe) in setup_tid_rdma_wqe() argument
3273 struct hfi1_swqe_priv *priv = wqe->priv; in setup_tid_rdma_wqe()
3294 if (wqe->wr.opcode == IB_WR_RDMA_READ) { in setup_tid_rdma_wqe()
3295 if (hfi1_check_sge_align(qp, &wqe->sg_list[0], in setup_tid_rdma_wqe()
3296 wqe->wr.num_sge)) { in setup_tid_rdma_wqe()
3300 } else if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { in setup_tid_rdma_wqe()
3307 if (!(wqe->rdma_wr.remote_addr & ~PAGE_MASK) && in setup_tid_rdma_wqe()
3308 !(wqe->length & ~PAGE_MASK)) { in setup_tid_rdma_wqe()
3317 wqe->wr.opcode = new_opcode; in setup_tid_rdma_wqe()
3319 min_t(u32, remote->max_len, wqe->length); in setup_tid_rdma_wqe()
3321 DIV_ROUND_UP(wqe->length, priv->tid_req.seg_len); in setup_tid_rdma_wqe()
3323 wqe->lpsn = wqe->psn; in setup_tid_rdma_wqe()
3324 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { in setup_tid_rdma_wqe()
3327 wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1; in setup_tid_rdma_wqe()
3329 wqe->lpsn += priv->tid_req.total_segs - 1; in setup_tid_rdma_wqe()
3344 trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode, in setup_tid_rdma_wqe()
3345 wqe->psn, wqe->lpsn, in setup_tid_rdma_wqe()
3354 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_write_req() argument
3359 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_write_req()
3374 cpu_to_be64(wqe->rdma_wr.remote_addr + (wqe->length - *len)); in hfi1_build_tid_rdma_write_req()
3376 cpu_to_be32(wqe->rdma_wr.rkey); in hfi1_build_tid_rdma_write_req()
4032 struct rvt_swqe *wqe; in hfi1_rc_rcv_tid_rdma_write_resp() local
4069 wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); in hfi1_rc_rcv_tid_rdma_write_resp()
4070 if (unlikely(wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)) in hfi1_rc_rcv_tid_rdma_write_resp()
4073 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_write_resp()
4108 (wqe->length - (req->comp_seg * req->seg_len))); in hfi1_rc_rcv_tid_rdma_write_resp()
4143 trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_write_resp()
4144 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_write_resp()
4149 if (!cmp_psn(psn, wqe->psn)) { in hfi1_rc_rcv_tid_rdma_write_resp()
4150 req->r_last_acked = mask_psn(wqe->psn - 1); in hfi1_rc_rcv_tid_rdma_write_resp()
4171 wqe = rvt_get_swqe_ptr(qp, i); in hfi1_rc_rcv_tid_rdma_write_resp()
4174 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in hfi1_rc_rcv_tid_rdma_write_resp()
4193 bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, in hfi1_build_tid_rdma_packet() argument
4197 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_packet()
4209 hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR); in hfi1_build_tid_rdma_packet()
4490 struct rvt_swqe *wqe; in hfi1_rc_rcv_tid_rdma_ack() local
4525 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_rcv_tid_rdma_ack()
4527 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in hfi1_rc_rcv_tid_rdma_ack()
4530 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4531 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4532 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_ack()
4548 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4549 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_ack()
4552 wqe = do_rc_completion(qp, wqe, in hfi1_rc_rcv_tid_rdma_ack()
4559 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) in hfi1_rc_rcv_tid_rdma_ack()
4561 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4567 trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_rc_rcv_tid_rdma_ack()
4568 wqe->lpsn, req); in hfi1_rc_rcv_tid_rdma_ack()
4575 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && in hfi1_rc_rcv_tid_rdma_ack()
4622 if (delta_psn(ack_psn, wqe->lpsn)) in hfi1_rc_rcv_tid_rdma_ack()
4623 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_rcv_tid_rdma_ack()
4624 req = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4692 wqe = rvt_get_swqe_ptr(qp, last_acked); in hfi1_rc_rcv_tid_rdma_ack()
4693 rptr = wqe_to_tid_req(wqe); in hfi1_rc_rcv_tid_rdma_ack()
4790 struct rvt_swqe *wqe; in hfi1_tid_retry_timeout() local
4805 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_tid_retry_timeout()
4806 hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in hfi1_tid_retry_timeout()
4809 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_tid_retry_timeout()
4810 req = wqe_to_tid_req(wqe); in hfi1_tid_retry_timeout()
4812 qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req); in hfi1_tid_retry_timeout()
4831 u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, in hfi1_build_tid_rdma_resync() argument
4837 struct tid_rdma_request *req = wqe_to_tid_req(wqe); in hfi1_build_tid_rdma_resync()
4976 struct rvt_swqe *wqe; in update_tid_tail() local
4988 wqe = rvt_get_swqe_ptr(qp, i); in update_tid_tail()
4989 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) in update_tid_tail()
5000 struct rvt_swqe *wqe; in hfi1_make_tid_rdma_pkt() local
5056 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); in hfi1_make_tid_rdma_pkt()
5057 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5058 trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn, in hfi1_make_tid_rdma_pkt()
5059 wqe->lpsn, req); in hfi1_make_tid_rdma_pkt()
5063 priv->tid_ss.sge = wqe->sg_list[0]; in hfi1_make_tid_rdma_pkt()
5064 priv->tid_ss.sg_list = wqe->sg_list + 1; in hfi1_make_tid_rdma_pkt()
5065 priv->tid_ss.num_sge = wqe->wr.num_sge; in hfi1_make_tid_rdma_pkt()
5066 priv->tid_ss.total_len = wqe->length; in hfi1_make_tid_rdma_pkt()
5069 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_tid_rdma_pkt()
5089 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); in hfi1_make_tid_rdma_pkt()
5090 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5091 len = wqe->length; in hfi1_make_tid_rdma_pkt()
5096 trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, in hfi1_make_tid_rdma_pkt()
5097 wqe->psn, wqe->lpsn, req); in hfi1_make_tid_rdma_pkt()
5098 last = hfi1_build_tid_rdma_packet(wqe, ohdr, &bth1, &bth2, in hfi1_make_tid_rdma_pkt()
5124 wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur); in hfi1_make_tid_rdma_pkt()
5125 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5128 wqe = rvt_get_swqe_ptr(qp, in hfi1_make_tid_rdma_pkt()
5131 req = wqe_to_tid_req(wqe); in hfi1_make_tid_rdma_pkt()
5133 hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1, in hfi1_make_tid_rdma_pkt()