1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2*4882a593Smuzhiyun
3*4882a593Smuzhiyun /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4*4882a593Smuzhiyun /* Copyright (c) 2008-2019, IBM Corporation */
5*4882a593Smuzhiyun
6*4882a593Smuzhiyun #include <linux/errno.h>
7*4882a593Smuzhiyun #include <linux/types.h>
8*4882a593Smuzhiyun
9*4882a593Smuzhiyun #include <rdma/ib_verbs.h>
10*4882a593Smuzhiyun
11*4882a593Smuzhiyun #include "siw.h"
12*4882a593Smuzhiyun
13*4882a593Smuzhiyun static int map_wc_opcode[SIW_NUM_OPCODES] = {
14*4882a593Smuzhiyun [SIW_OP_WRITE] = IB_WC_RDMA_WRITE,
15*4882a593Smuzhiyun [SIW_OP_SEND] = IB_WC_SEND,
16*4882a593Smuzhiyun [SIW_OP_SEND_WITH_IMM] = IB_WC_SEND,
17*4882a593Smuzhiyun [SIW_OP_READ] = IB_WC_RDMA_READ,
18*4882a593Smuzhiyun [SIW_OP_READ_LOCAL_INV] = IB_WC_RDMA_READ,
19*4882a593Smuzhiyun [SIW_OP_COMP_AND_SWAP] = IB_WC_COMP_SWAP,
20*4882a593Smuzhiyun [SIW_OP_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
21*4882a593Smuzhiyun [SIW_OP_INVAL_STAG] = IB_WC_LOCAL_INV,
22*4882a593Smuzhiyun [SIW_OP_REG_MR] = IB_WC_REG_MR,
23*4882a593Smuzhiyun [SIW_OP_RECEIVE] = IB_WC_RECV,
24*4882a593Smuzhiyun [SIW_OP_READ_RESPONSE] = -1 /* not used */
25*4882a593Smuzhiyun };
26*4882a593Smuzhiyun
27*4882a593Smuzhiyun static struct {
28*4882a593Smuzhiyun enum siw_wc_status siw;
29*4882a593Smuzhiyun enum ib_wc_status ib;
30*4882a593Smuzhiyun } map_cqe_status[SIW_NUM_WC_STATUS] = {
31*4882a593Smuzhiyun { SIW_WC_SUCCESS, IB_WC_SUCCESS },
32*4882a593Smuzhiyun { SIW_WC_LOC_LEN_ERR, IB_WC_LOC_LEN_ERR },
33*4882a593Smuzhiyun { SIW_WC_LOC_PROT_ERR, IB_WC_LOC_PROT_ERR },
34*4882a593Smuzhiyun { SIW_WC_LOC_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
35*4882a593Smuzhiyun { SIW_WC_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
36*4882a593Smuzhiyun { SIW_WC_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
37*4882a593Smuzhiyun { SIW_WC_LOC_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
38*4882a593Smuzhiyun { SIW_WC_REM_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
39*4882a593Smuzhiyun { SIW_WC_REM_INV_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
40*4882a593Smuzhiyun { SIW_WC_GENERAL_ERR, IB_WC_GENERAL_ERR }
41*4882a593Smuzhiyun };
42*4882a593Smuzhiyun
43*4882a593Smuzhiyun /*
44*4882a593Smuzhiyun * Reap one CQE from the CQ. Only used by kernel clients
45*4882a593Smuzhiyun * during CQ normal operation. Might be called during CQ
46*4882a593Smuzhiyun * flush for user mapped CQE array as well.
47*4882a593Smuzhiyun */
siw_reap_cqe(struct siw_cq * cq,struct ib_wc * wc)48*4882a593Smuzhiyun int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc)
49*4882a593Smuzhiyun {
50*4882a593Smuzhiyun struct siw_cqe *cqe;
51*4882a593Smuzhiyun unsigned long flags;
52*4882a593Smuzhiyun
53*4882a593Smuzhiyun spin_lock_irqsave(&cq->lock, flags);
54*4882a593Smuzhiyun
55*4882a593Smuzhiyun cqe = &cq->queue[cq->cq_get % cq->num_cqe];
56*4882a593Smuzhiyun if (READ_ONCE(cqe->flags) & SIW_WQE_VALID) {
57*4882a593Smuzhiyun memset(wc, 0, sizeof(*wc));
58*4882a593Smuzhiyun wc->wr_id = cqe->id;
59*4882a593Smuzhiyun wc->status = map_cqe_status[cqe->status].ib;
60*4882a593Smuzhiyun wc->opcode = map_wc_opcode[cqe->opcode];
61*4882a593Smuzhiyun wc->byte_len = cqe->bytes;
62*4882a593Smuzhiyun
63*4882a593Smuzhiyun /*
64*4882a593Smuzhiyun * During CQ flush, also user land CQE's may get
65*4882a593Smuzhiyun * reaped here, which do not hold a QP reference
66*4882a593Smuzhiyun * and do not qualify for memory extension verbs.
67*4882a593Smuzhiyun */
68*4882a593Smuzhiyun if (likely(rdma_is_kernel_res(&cq->base_cq.res))) {
69*4882a593Smuzhiyun if (cqe->flags & SIW_WQE_REM_INVAL) {
70*4882a593Smuzhiyun wc->ex.invalidate_rkey = cqe->inval_stag;
71*4882a593Smuzhiyun wc->wc_flags = IB_WC_WITH_INVALIDATE;
72*4882a593Smuzhiyun }
73*4882a593Smuzhiyun wc->qp = cqe->base_qp;
74*4882a593Smuzhiyun siw_dbg_cq(cq,
75*4882a593Smuzhiyun "idx %u, type %d, flags %2x, id 0x%pK\n",
76*4882a593Smuzhiyun cq->cq_get % cq->num_cqe, cqe->opcode,
77*4882a593Smuzhiyun cqe->flags, (void *)(uintptr_t)cqe->id);
78*4882a593Smuzhiyun }
79*4882a593Smuzhiyun WRITE_ONCE(cqe->flags, 0);
80*4882a593Smuzhiyun cq->cq_get++;
81*4882a593Smuzhiyun
82*4882a593Smuzhiyun spin_unlock_irqrestore(&cq->lock, flags);
83*4882a593Smuzhiyun
84*4882a593Smuzhiyun return 1;
85*4882a593Smuzhiyun }
86*4882a593Smuzhiyun spin_unlock_irqrestore(&cq->lock, flags);
87*4882a593Smuzhiyun
88*4882a593Smuzhiyun return 0;
89*4882a593Smuzhiyun }
90*4882a593Smuzhiyun
91*4882a593Smuzhiyun /*
92*4882a593Smuzhiyun * siw_cq_flush()
93*4882a593Smuzhiyun *
94*4882a593Smuzhiyun * Flush all CQ elements.
95*4882a593Smuzhiyun */
siw_cq_flush(struct siw_cq * cq)96*4882a593Smuzhiyun void siw_cq_flush(struct siw_cq *cq)
97*4882a593Smuzhiyun {
98*4882a593Smuzhiyun struct ib_wc wc;
99*4882a593Smuzhiyun
100*4882a593Smuzhiyun while (siw_reap_cqe(cq, &wc))
101*4882a593Smuzhiyun ;
102*4882a593Smuzhiyun }
103