Lines Matching refs:wr
104 static void set_eth_seg(const struct ib_send_wr *wr, struct mlx5_ib_qp *qp, in set_eth_seg() argument
111 if (wr->send_flags & IB_SEND_IP_CSUM) in set_eth_seg()
115 if (wr->opcode == IB_WR_LSO) { in set_eth_seg()
116 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr); in set_eth_seg()
153 const struct ib_send_wr *wr) in set_datagram_seg() argument
155 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av)); in set_datagram_seg()
157 cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV); in set_datagram_seg()
158 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey); in set_datagram_seg()
321 const struct ib_send_wr *wr) in set_reg_umr_segment() argument
323 const struct mlx5_umr_wr *umrwr = umr_wr(wr); in set_reg_umr_segment()
328 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE) in set_reg_umr_segment()
337 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) { in set_reg_umr_segment()
344 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION) in set_reg_umr_segment()
346 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) { in set_reg_umr_segment()
353 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR) in set_reg_umr_segment()
355 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR) in set_reg_umr_segment()
358 if (!wr->num_sge) in set_reg_umr_segment()
403 const struct ib_send_wr *wr) in set_reg_mkey_segment() argument
405 const struct mlx5_umr_wr *umrwr = umr_wr(wr); in set_reg_mkey_segment()
408 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR) in set_reg_mkey_segment()
427 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION && in set_reg_mkey_segment()
449 static __be32 send_ieth(const struct ib_send_wr *wr) in send_ieth() argument
451 switch (wr->opcode) { in send_ieth()
454 return wr->ex.imm_data; in send_ieth()
457 return cpu_to_be32(wr->ex.invalidate_rkey); in send_ieth()
481 static int set_data_inl_seg(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in set_data_inl_seg() argument
493 for (i = 0; i < wr->num_sge; i++) { in set_data_inl_seg()
494 size_t len = wr->sg_list[i].length; in set_data_inl_seg()
495 void *addr = (void *)(unsigned long)(wr->sg_list[i].addr); in set_data_inl_seg()
782 const struct ib_reg_wr *wr = reg_wr(send_wr); in set_pi_umr_wr() local
783 struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr); in set_pi_umr_wr()
791 unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || in set_pi_umr_wr()
814 set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len, in set_pi_umr_wr()
820 ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size, in set_pi_umr_wr()
857 const struct ib_reg_wr *wr, in set_reg_wr() argument
861 struct mlx5_ib_mr *mr = to_mmr(wr->mr); in set_reg_wr()
866 bool atomic = wr->access & IB_ACCESS_REMOTE_ATOMIC; in set_reg_wr()
870 if (!mlx5_ib_can_reconfig_with_umr(dev, 0, wr->access)) { in set_reg_wr()
877 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) { in set_reg_wr()
893 set_reg_mkey_seg(*seg, mr, wr->key, wr->access); in set_reg_wr()
944 const struct ib_send_wr *wr, unsigned int *idx, in __begin_wqe() argument
955 (*ctrl)->imm = send_ieth(wr); in __begin_wqe()
969 const struct ib_send_wr *wr, unsigned int *idx, int *size, in begin_wqe() argument
972 return __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq, in begin_wqe()
973 wr->send_flags & IB_SEND_SIGNALED, in begin_wqe()
974 wr->send_flags & IB_SEND_SOLICITED); in begin_wqe()
1008 static void handle_rdma_op(const struct ib_send_wr *wr, void **seg, int *size) in handle_rdma_op() argument
1010 set_raddr_seg(*seg, rdma_wr(wr)->remote_addr, rdma_wr(wr)->rkey); in handle_rdma_op()
1015 static void handle_local_inv(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in handle_local_inv() argument
1020 (*ctrl)->imm = cpu_to_be32(wr->ex.invalidate_rkey); in handle_local_inv()
1024 static int handle_reg_mr(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in handle_reg_mr() argument
1029 (*ctrl)->imm = cpu_to_be32(reg_wr(wr)->key); in handle_reg_mr()
1030 return set_reg_wr(qp, reg_wr(wr), seg, size, cur_edge, true); in handle_reg_mr()
1034 const struct ib_send_wr *wr, in handle_psv() argument
1045 err = __begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq, in handle_psv()
1057 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq, in handle_psv()
1066 const struct ib_send_wr *wr, in handle_reg_mr_integrity() argument
1081 mr = to_mmr(reg_wr(wr)->mr); in handle_reg_mr_integrity()
1089 reg_pi_wr.access = reg_wr(wr)->access; in handle_reg_mr_integrity()
1098 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, in handle_reg_mr_integrity()
1101 err = begin_wqe(qp, seg, ctrl, wr, idx, size, cur_edge, nreq); in handle_reg_mr_integrity()
1125 err = set_pi_umr_wr(wr, qp, seg, size, cur_edge); in handle_reg_mr_integrity()
1130 finish_wqe(qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq, in handle_reg_mr_integrity()
1134 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, in handle_reg_mr_integrity()
1140 err = handle_psv(dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, in handle_reg_mr_integrity()
1153 const struct ib_send_wr *wr, in handle_qpt_rc() argument
1160 switch (wr->opcode) { in handle_qpt_rc()
1164 handle_rdma_op(wr, seg, size); in handle_qpt_rc()
1175 handle_local_inv(qp, wr, ctrl, seg, size, cur_edge, *idx); in handle_qpt_rc()
1180 err = handle_reg_mr(qp, wr, ctrl, seg, size, cur_edge, *idx); in handle_qpt_rc()
1187 err = handle_reg_mr_integrity(dev, qp, wr, ctrl, seg, size, in handle_qpt_rc()
1203 static void handle_qpt_uc(const struct ib_send_wr *wr, void **seg, int *size) in handle_qpt_uc() argument
1205 switch (wr->opcode) { in handle_qpt_uc()
1208 handle_rdma_op(wr, seg, size); in handle_qpt_uc()
1216 const struct ib_send_wr *wr, void **seg, in handle_qpt_hw_gsi() argument
1219 set_datagram_seg(*seg, wr); in handle_qpt_hw_gsi()
1225 static void handle_qpt_ud(struct mlx5_ib_qp *qp, const struct ib_send_wr *wr, in handle_qpt_ud() argument
1228 set_datagram_seg(*seg, wr); in handle_qpt_ud()
1241 set_eth_seg(wr, qp, seg, size, cur_edge); in handle_qpt_ud()
1247 const struct ib_send_wr *wr, in handle_qpt_reg_umr() argument
1253 if (unlikely(wr->opcode != MLX5_IB_WR_UMR)) { in handle_qpt_reg_umr()
1255 mlx5_ib_warn(dev, "bad opcode %d\n", wr->opcode); in handle_qpt_reg_umr()
1260 (*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey); in handle_qpt_reg_umr()
1261 err = set_reg_umr_segment(dev, *seg, wr); in handle_qpt_reg_umr()
1267 set_reg_mkey_segment(dev, *seg, wr); in handle_qpt_reg_umr()
1275 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, in mlx5_ib_post_send() argument
1298 *bad_wr = wr; in mlx5_ib_post_send()
1303 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr); in mlx5_ib_post_send()
1310 for (nreq = 0; wr; nreq++, wr = wr->next) { in mlx5_ib_post_send()
1311 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) { in mlx5_ib_post_send()
1314 *bad_wr = wr; in mlx5_ib_post_send()
1318 num_sge = wr->num_sge; in mlx5_ib_post_send()
1322 *bad_wr = wr; in mlx5_ib_post_send()
1326 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge, in mlx5_ib_post_send()
1331 *bad_wr = wr; in mlx5_ib_post_send()
1335 if (wr->opcode == IB_WR_REG_MR || in mlx5_ib_post_send()
1336 wr->opcode == IB_WR_REG_MR_INTEGRITY) { in mlx5_ib_post_send()
1340 if (wr->send_flags & IB_SEND_FENCE) { in mlx5_ib_post_send()
1357 err = handle_qpt_rc(dev, qp, wr, &ctrl, &seg, &size, in mlx5_ib_post_send()
1361 *bad_wr = wr; in mlx5_ib_post_send()
1363 } else if (wr->opcode == IB_WR_REG_MR_INTEGRITY) { in mlx5_ib_post_send()
1369 handle_qpt_uc(wr, &seg, &size); in mlx5_ib_post_send()
1375 *bad_wr = wr; in mlx5_ib_post_send()
1380 handle_qpt_hw_gsi(qp, wr, &seg, &size, &cur_edge); in mlx5_ib_post_send()
1383 handle_qpt_ud(qp, wr, &seg, &size, &cur_edge); in mlx5_ib_post_send()
1386 err = handle_qpt_reg_umr(dev, qp, wr, &ctrl, &seg, in mlx5_ib_post_send()
1396 if (wr->send_flags & IB_SEND_INLINE && num_sge) { in mlx5_ib_post_send()
1397 err = set_data_inl_seg(qp, wr, &seg, &size, &cur_edge); in mlx5_ib_post_send()
1400 *bad_wr = wr; in mlx5_ib_post_send()
1407 if (unlikely(!wr->sg_list[i].length)) in mlx5_ib_post_send()
1412 wr->sg_list + i); in mlx5_ib_post_send()
1419 finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq, in mlx5_ib_post_send()
1420 fence, mlx5_ib_opcode[wr->opcode]); in mlx5_ib_post_send()
1459 int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, in mlx5_ib_post_recv() argument
1475 *bad_wr = wr; in mlx5_ib_post_recv()
1480 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr); in mlx5_ib_post_recv()
1486 for (nreq = 0; wr; nreq++, wr = wr->next) { in mlx5_ib_post_recv()
1489 *bad_wr = wr; in mlx5_ib_post_recv()
1493 if (unlikely(wr->num_sge > qp->rq.max_gs)) { in mlx5_ib_post_recv()
1495 *bad_wr = wr; in mlx5_ib_post_recv()
1503 for (i = 0; i < wr->num_sge; i++) in mlx5_ib_post_recv()
1504 set_data_ptr_seg(scat + i, wr->sg_list + i); in mlx5_ib_post_recv()
1517 qp->rq.wrid[ind] = wr->wr_id; in mlx5_ib_post_recv()