1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright 2022-2023 HiSilicon Limited.
4 * Kunpeng hardware accelerator queue management module.
5 */
6 #include "hisi_qm.h"
7
8 #define QM_FVT_CFG_RDY_BIT 0x1
9 /* Doorbell */
10 #define QM_DOORBELL_SQ_CQ_BASE 0x1000
11 #define QM_DB_CMD_SHIFT 12
12 #define QM_DB_RAND_DATA_SHIFT 16
13 #define QM_DB_INDEX_SHIFT 32
14 #define QM_DB_PRIORITY_SHIFT 48
15 #define QM_DB_RAND_DATA 0x5a
16 #define QM_DOORBELL_CMD_SQ 0
17 #define QM_DOORBELL_CMD_CQ 1
18 /* Mailbox */
19 #define QM_MAILBOX_BASE 0x300
20 #define QM_MAILBOX_DATA_ADDR_L 0x304
21 #define QM_MAILBOX_DATA_ADDR_H 0x308
22 #define QM_MB_BUSY_SHIFT 13
23 #define QM_MB_BUSY_BIT BIT32(QM_MB_BUSY_SHIFT)
24 #define QM_MB_OP_SHIFT 14
25 #define QM_MB_OP_WR 0
26 #define QM_MB_OP_RD 1
27 #define QM_MB_STATUS_MASK GENMASK_32(12, 9)
28 #define QM_MB_WAIT_READY_CNT 10
29 #define QM_MB_WAIT_MAX_CNT 21000
30 #define QM_MB_WAIT_PERIOD 200
31 /* XQC_VFT */
32 #define QM_VFT_CFG_OP_ENABLE 0x100054
33 #define QM_VFT_CFG_OP_WR 0x100058
34 #define QM_VFT_CFG_TYPE 0x10005c
35 #define QM_VFT_CFG_ADDRESS 0x100060
36 #define QM_VFT_CFG_DATA_L 0x100064
37 #define QM_VFT_CFG_DATA_H 0x100068
38 #define QM_VFT_CFG_RDY 0x10006c
39 #define QM_SQC_VFT 0
40 #define QM_CQC_VFT 1
41 #define QM_SQC_VFT_START_SQN_SHIFT 28
42 #define QM_SQC_VFT_VALID BIT64(44)
43 #define QM_SQC_VFT_SQ_NUM_SHIFT 45
44 #define QM_CQC_VFT_VALID BIT(28)
45 #define QM_VFT_WRITE 0
46 #define QM_VFT_READ 1
47 #define QM_SQC_VFT_BASE_MASK 0x3ff
48 #define QM_SQC_VFT_NUM_MASK 0x3ff
49 /* QM INIT */
50 #define QM_MEM_START_INIT 0x100040
51 #define QM_MEM_INIT_DONE 0x100044
52 #define QM_VF_AEQ_INT_MASK 0x4
53 #define QM_VF_AEQ_INT_MASK_EN 0x1
54 #define QM_VF_EQ_INT_MASK 0xc
55 #define QM_VF_EQ_INT_MASK_EN 0x1
56 #define QM_ARUSER_M_CFG_1 0x100088
57 #define QM_ARUSER_M_CFG_ENABLE 0x100090
58 #define QM_AWUSER_M_CFG_1 0x100098
59 #define QM_AWUSER_M_CFG_ENABLE 0x1000a0
60 #define QM_AXUSER_CFG 0x40001070
61 #define AXUSER_M_CFG_ENABLE 0x7ffffc
62 #define QM_AXI_M_CFG 0x1000ac
63 #define AXI_M_CFG 0xffff
64 #define QM_PEH_AXUSER_CFG 0x1000cc
65 #define PEH_AXUSER_CFG 0x400801
66 #define QM_CACHE_CTL 0x100050
67 #define QM_CACHE_CFG 0x4893
68 #define QM_CACHE_WB_START 0x204
69 #define QM_CACHE_WB_DONE 0x208
70 #define QM_PM_CTRL0 0x100148
71 #define QM_IDLE_DISABLE BIT(9)
72 #define QM_DB_TIMEOUT_CFG 0x100074
73 #define QM_DB_TIMEOUT_SET 0x1fffff
74 /* XQC shift */
75 #define QM_SQ_SQE_SIZE_SHIFT 12
76 #define QM_SQ_ORDER_SHIFT 4
77 #define QM_SQ_TYPE_SHIFT 8
78 #define QM_CQE_SIZE 4
79 #define QM_CQ_CQE_SIZE_SHIFT 12
80 /* CQE */
81 #define QM_CQE_PHASE(cqe) (((cqe)->w7) & QM_FVT_CFG_RDY_BIT)
82
83 enum qm_mailbox_common_cmd {
84 QM_MB_CMD_SQC = 0x0,
85 QM_MB_CMD_CQC,
86 QM_MB_CMD_EQC,
87 QM_MB_CMD_AEQC,
88 QM_MB_CMD_SQC_BT,
89 QM_MB_CMD_CQC_BT,
90 QM_MB_CMD_SQC_VFT,
91 };
92
93 enum qm_mailbox_cmd_v3 {
94 QM_MB_CM_CLOSE_QM = 0x7,
95 QM_MB_CMD_CLOSE_QP,
96 QM_MB_CMD_FLUSH_QM,
97 QM_MB_CMD_FLUSH_QP,
98 QM_MB_CMD_SRC = 0xc,
99 QM_MB_CMD_DST,
100 QM_MB_CMD_STOP_QM,
101 };
102
103 struct qm_mailbox {
104 union {
105 struct {
106 uint16_t w0;
107 uint16_t queue;
108 uint32_t base_l;
109 uint32_t base_h;
110 uint32_t token;
111 };
112 uint64_t x[2];
113 };
114 };
115
116 struct qm_dfx_registers {
117 const char *reg_name;
118 uint32_t reg_offset;
119 };
120
121 static const struct qm_dfx_registers qm_dfx_regs[] = {
122 { .reg_name = "QM_ECC_1BIT_CNT ", .reg_offset = 0x104000 },
123 { .reg_name = "QM_ECC_MBIT_CNT ", .reg_offset = 0x104008 },
124 { .reg_name = "QM_DFX_MB_CNT ", .reg_offset = 0x104018 },
125 { .reg_name = "QM_DFX_DB_CNT ", .reg_offset = 0x104028 },
126 { .reg_name = "QM_DFX_SQE_CNT ", .reg_offset = 0x104038 },
127 { .reg_name = "QM_DFX_CQE_CNT ", .reg_offset = 0x104048 },
128 { .reg_name = "QM_DFX_SEND_SQE_TO_ACC_CNT", .reg_offset = 0x104050 },
129 { .reg_name = "QM_DFX_WB_SQE_FROM_ACC_CNT", .reg_offset = 0x104058 },
130 { .reg_name = "QM_DFX_ACC_FINISH_CNT ", .reg_offset = 0x104060 },
131 { .reg_name = "QM_DFX_CQE_ERR_CNT ", .reg_offset = 0x1040b4 },
132 { }
133 };
134
hisi_qm_get_version(struct hisi_qm * qm)135 void hisi_qm_get_version(struct hisi_qm *qm)
136 {
137 qm->version = io_read32(qm->io_base + HISI_QM_REVISON_ID_BASE) &
138 HISI_QM_REVISON_ID_MASK;
139 }
140
qm_db(struct hisi_qm * qm,uint16_t qn,uint8_t cmd,uint16_t index,uint8_t priority)141 static void qm_db(struct hisi_qm *qm, uint16_t qn, uint8_t cmd, uint16_t index,
142 uint8_t priority)
143 {
144 uint64_t doorbell = 0;
145
146 doorbell = qn | SHIFT_U64(cmd, QM_DB_CMD_SHIFT) |
147 SHIFT_U64(QM_DB_RAND_DATA, QM_DB_RAND_DATA_SHIFT) |
148 SHIFT_U64(index, QM_DB_INDEX_SHIFT) |
149 SHIFT_U64(priority, QM_DB_PRIORITY_SHIFT);
150
151 io_write64(qm->io_base + QM_DOORBELL_SQ_CQ_BASE, doorbell);
152 }
153
qm_mb_write(struct hisi_qm * qm,struct qm_mailbox * mb)154 static void qm_mb_write(struct hisi_qm *qm, struct qm_mailbox *mb)
155 {
156 vaddr_t dst = qm->io_base + QM_MAILBOX_BASE;
157
158 write_64bit_pair(dst, mb->x[1], mb->x[0]);
159 dsb_osh();
160 }
161
qm_mb_read(struct hisi_qm * qm,struct qm_mailbox * mb)162 static void qm_mb_read(struct hisi_qm *qm, struct qm_mailbox *mb)
163 {
164 vaddr_t mb_base = qm->io_base + QM_MAILBOX_BASE;
165
166 read_64bit_pair(mb_base, mb->x + 1, mb->x);
167 dsb_osh();
168 }
169
qm_wait_mb_ready(struct hisi_qm * qm)170 static enum hisi_drv_status qm_wait_mb_ready(struct hisi_qm *qm)
171 {
172 struct qm_mailbox mb = { };
173 uint32_t timeout = 0;
174
175 timeout = timeout_init_us(QM_MB_WAIT_PERIOD * QM_MB_WAIT_READY_CNT);
176 while (!timeout_elapsed(timeout)) {
177 /* 128 bits should be read from hardware at one time*/
178 qm_mb_read(qm, &mb);
179 if (!(mb.w0 & QM_MB_BUSY_BIT))
180 return HISI_QM_DRVCRYPT_NO_ERR;
181 }
182
183 EMSG("QM mailbox is busy to start!");
184
185 return HISI_QM_DRVCRYPT_EBUSY;
186 }
187
qm_wait_mb_finish(struct hisi_qm * qm,struct qm_mailbox * mb)188 static enum hisi_drv_status qm_wait_mb_finish(struct hisi_qm *qm,
189 struct qm_mailbox *mb)
190 {
191 uint32_t timeout = 0;
192
193 timeout = timeout_init_us(QM_MB_WAIT_PERIOD * QM_MB_WAIT_MAX_CNT);
194 while (!timeout_elapsed(timeout)) {
195 qm_mb_read(qm, mb);
196 if (!(mb->w0 & QM_MB_BUSY_BIT)) {
197 if (mb->w0 & QM_MB_STATUS_MASK) {
198 EMSG("QM mailbox operation failed!");
199 return HISI_QM_DRVCRYPT_EIO;
200 } else {
201 return HISI_QM_DRVCRYPT_NO_ERR;
202 }
203 }
204 }
205
206 return HISI_QM_DRVCRYPT_ETMOUT;
207 }
208
qm_mb_init(struct qm_mailbox * mb,uint8_t cmd,uint64_t base,uint16_t qnum,uint8_t op)209 static void qm_mb_init(struct qm_mailbox *mb, uint8_t cmd, uint64_t base,
210 uint16_t qnum, uint8_t op)
211 {
212 mb->w0 = cmd | SHIFT_U32(op, QM_MB_OP_SHIFT) | QM_MB_BUSY_BIT;
213 mb->queue = qnum;
214 reg_pair_from_64(base, &mb->base_h, &mb->base_l);
215 mb->token = 0;
216 }
217
qm_mb_nolock(struct hisi_qm * qm,struct qm_mailbox * mb)218 static enum hisi_drv_status qm_mb_nolock(struct hisi_qm *qm,
219 struct qm_mailbox *mb)
220 {
221 if (qm_wait_mb_ready(qm))
222 return HISI_QM_DRVCRYPT_EBUSY;
223
224 qm_mb_write(qm, mb);
225
226 return qm_wait_mb_finish(qm, mb);
227 }
228
hisi_qm_mb_write(struct hisi_qm * qm,uint8_t cmd,uintptr_t dma_addr,uint16_t qnum)229 static enum hisi_drv_status hisi_qm_mb_write(struct hisi_qm *qm, uint8_t cmd,
230 uintptr_t dma_addr, uint16_t qnum)
231 {
232 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
233 struct qm_mailbox mb = { };
234
235 qm_mb_init(&mb, cmd, dma_addr, qnum, QM_MB_OP_WR);
236 mutex_lock(&qm->mailbox_lock);
237 ret = qm_mb_nolock(qm, &mb);
238 mutex_unlock(&qm->mailbox_lock);
239
240 return ret;
241 }
242
hisi_qm_mb_read(struct hisi_qm * qm,uint64_t * base,uint8_t cmd,uint16_t qnum)243 static enum hisi_drv_status hisi_qm_mb_read(struct hisi_qm *qm, uint64_t *base,
244 uint8_t cmd, uint16_t qnum)
245 {
246 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
247 struct qm_mailbox mb = { };
248
249 qm_mb_init(&mb, cmd, 0, qnum, QM_MB_OP_RD);
250 mutex_lock(&qm->mailbox_lock);
251 ret = qm_mb_nolock(qm, &mb);
252 mutex_unlock(&qm->mailbox_lock);
253 if (ret)
254 return ret;
255
256 reg_pair_from_64(*base, &mb.base_h, &mb.base_l);
257
258 return HISI_QM_DRVCRYPT_NO_ERR;
259 }
260
qm_cfg_vft_data(struct hisi_qm * qm,uint8_t vft_type,uint32_t base,uint32_t number)261 static void qm_cfg_vft_data(struct hisi_qm *qm, uint8_t vft_type,
262 uint32_t base, uint32_t number)
263 {
264 uint32_t data_h = 0;
265 uint32_t data_l = 0;
266 uint64_t data = 0;
267
268 switch (vft_type) {
269 case QM_SQC_VFT:
270 data = SHIFT_U64(base, QM_SQC_VFT_START_SQN_SHIFT) |
271 QM_SQC_VFT_VALID |
272 SHIFT_U64((number - 1), QM_SQC_VFT_SQ_NUM_SHIFT);
273 break;
274 case QM_CQC_VFT:
275 data = QM_CQC_VFT_VALID;
276 break;
277 default:
278 panic("Invalid vft type");
279 }
280
281 reg_pair_from_64(data, &data_h, &data_l);
282 io_write32(qm->io_base + QM_VFT_CFG_DATA_L, data_l);
283 io_write32(qm->io_base + QM_VFT_CFG_DATA_H, data_h);
284 }
285
qm_set_vft_common(struct hisi_qm * qm,uint8_t vft_type,uint32_t function,uint32_t base,uint32_t num)286 static enum hisi_drv_status qm_set_vft_common(struct hisi_qm *qm,
287 uint8_t vft_type,
288 uint32_t function,
289 uint32_t base,
290 uint32_t num)
291 {
292 uint32_t val = 0;
293
294 if (IO_READ32_POLL_TIMEOUT(qm->io_base + QM_VFT_CFG_RDY, val,
295 val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD,
296 POLL_TIMEOUT)) {
297 EMSG("QM VFT is not ready");
298 return HISI_QM_DRVCRYPT_EBUSY;
299 }
300
301 io_write32(qm->io_base + QM_VFT_CFG_OP_WR, QM_VFT_WRITE);
302 io_write32(qm->io_base + QM_VFT_CFG_TYPE, vft_type);
303 io_write32(qm->io_base + QM_VFT_CFG_ADDRESS, function);
304 qm_cfg_vft_data(qm, vft_type, base, num);
305 io_write32(qm->io_base + QM_VFT_CFG_RDY, 0x0);
306 io_write32(qm->io_base + QM_VFT_CFG_OP_ENABLE, QM_FVT_CFG_RDY_BIT);
307
308 if (IO_READ32_POLL_TIMEOUT(qm->io_base + QM_VFT_CFG_RDY, val,
309 val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD,
310 POLL_TIMEOUT)) {
311 EMSG("QM VFT is not ready");
312 return HISI_QM_DRVCRYPT_EBUSY;
313 }
314
315 return HISI_QM_DRVCRYPT_NO_ERR;
316 }
317
qm_set_xqc_vft(struct hisi_qm * qm,uint32_t function,uint32_t base,uint32_t num)318 static enum hisi_drv_status qm_set_xqc_vft(struct hisi_qm *qm,
319 uint32_t function,
320 uint32_t base, uint32_t num)
321 {
322 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
323 int i = 0;
324
325 for (i = QM_SQC_VFT; i <= QM_CQC_VFT; i++) {
326 ret = qm_set_vft_common(qm, i, function, base, num);
327 if (ret) {
328 EMSG("QM set type %d fail", i);
329 return ret;
330 }
331 }
332
333 return HISI_QM_DRVCRYPT_NO_ERR;
334 }
335
qm_get_vft(struct hisi_qm * qm,uint32_t * base,uint32_t * num)336 static enum hisi_drv_status qm_get_vft(struct hisi_qm *qm, uint32_t *base,
337 uint32_t *num)
338 {
339 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
340 uint64_t sqc_vft = 0;
341
342 ret = hisi_qm_mb_read(qm, &sqc_vft, QM_MB_CMD_SQC_VFT, 0);
343 if (ret)
344 return ret;
345
346 *base = (sqc_vft >> QM_SQC_VFT_START_SQN_SHIFT) & QM_SQC_VFT_BASE_MASK;
347 *num = ((sqc_vft >> QM_SQC_VFT_SQ_NUM_SHIFT) & QM_SQC_VFT_NUM_MASK) + 1;
348
349 return HISI_QM_DRVCRYPT_NO_ERR;
350 }
351
qp_free(struct hisi_qm * qm,uint32_t id)352 static void qp_free(struct hisi_qm *qm, uint32_t id)
353 {
354 struct hisi_qp *qp = &qm->qp_array[id];
355
356 free(qp->sqe);
357 free(qp->cqe);
358 }
359
qp_alloc(struct hisi_qm * qm,uint32_t id)360 static enum hisi_drv_status qp_alloc(struct hisi_qm *qm, uint32_t id)
361 {
362 size_t sq_size = qm->sqe_size * HISI_QM_Q_DEPTH;
363 size_t cq_size = sizeof(struct qm_cqe) * HISI_QM_Q_DEPTH;
364 struct hisi_qp *qp = &qm->qp_array[id];
365 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
366
367 qp->sqe = memalign(HISI_QM_ALIGN128, sq_size);
368 if (!qp->sqe) {
369 EMSG("Fail to malloc sq[%"PRIu32"]", id);
370 return HISI_QM_DRVCRYPT_ENOMEM;
371 }
372 qp->sqe_dma = virt_to_phys(qp->sqe);
373 qp->cqe = memalign(HISI_QM_ALIGN32, cq_size);
374 if (!qp->cqe) {
375 EMSG("Fail to malloc cq[%"PRIu32"]", id);
376 ret = HISI_QM_DRVCRYPT_ENOMEM;
377 goto free_sqe;
378 }
379 qp->cqe_dma = virt_to_phys(qp->cqe);
380
381 qp->qp_id = id;
382 qp->qm = qm;
383 return HISI_QM_DRVCRYPT_NO_ERR;
384
385 free_sqe:
386 free(qp->sqe);
387 return ret;
388 }
389
hisi_qm_free_xqc(struct qm_xqc * xqc)390 static void hisi_qm_free_xqc(struct qm_xqc *xqc)
391 {
392 free(xqc->cqc);
393 free(xqc->sqc);
394 }
395
qm_free(struct hisi_qm * qm)396 static void qm_free(struct hisi_qm *qm)
397 {
398 unsigned int i = 0;
399
400 for (i = 0; i < qm->qp_num; i++)
401 qp_free(qm, i);
402
403 free(qm->qp_array);
404 hisi_qm_free_xqc(&qm->xqc);
405 hisi_qm_free_xqc(&qm->cfg_xqc);
406 }
407
hisi_qm_alloc_xqc(struct qm_xqc * xqc,uint32_t qp_num)408 static enum hisi_drv_status hisi_qm_alloc_xqc(struct qm_xqc *xqc,
409 uint32_t qp_num)
410 {
411 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
412 size_t sqc_size = 0;
413 size_t cqc_size = 0;
414
415 sqc_size = sizeof(struct qm_sqc) * qp_num;
416 cqc_size = sizeof(struct qm_cqc) * qp_num;
417
418 xqc->sqc = memalign(HISI_QM_ALIGN32, sqc_size);
419 if (!xqc->sqc) {
420 EMSG("Fail to malloc sqc");
421 return HISI_QM_DRVCRYPT_ENOMEM;
422 }
423 memset(xqc->sqc, 0, sqc_size);
424 xqc->sqc_dma = virt_to_phys(xqc->sqc);
425
426 xqc->cqc = memalign(HISI_QM_ALIGN32, cqc_size);
427 if (!xqc->cqc) {
428 EMSG("Fail to malloc cqc");
429 ret = HISI_QM_DRVCRYPT_ENOMEM;
430 goto free_sqc;
431 }
432 memset(xqc->cqc, 0, cqc_size);
433 xqc->cqc_dma = virt_to_phys(xqc->cqc);
434
435 return HISI_QM_DRVCRYPT_NO_ERR;
436
437 free(xqc->cqc);
438 free_sqc:
439 free(xqc->sqc);
440 return ret;
441 }
442
qm_alloc(struct hisi_qm * qm)443 static enum hisi_drv_status qm_alloc(struct hisi_qm *qm)
444 {
445 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
446 int32_t j;
447 uint32_t i;
448
449 ret = hisi_qm_alloc_xqc(&qm->xqc, qm->qp_num);
450 if (ret)
451 return ret;
452
453 ret = hisi_qm_alloc_xqc(&qm->cfg_xqc, 1);
454 if (ret)
455 goto free_xqc;
456
457 qm->qp_array = calloc(qm->qp_num, sizeof(struct hisi_qp));
458 if (!qm->qp_array) {
459 EMSG("Fail to malloc qp_array");
460 ret = HISI_QM_DRVCRYPT_ENOMEM;
461 goto free_cfg_xqc;
462 }
463
464 for (i = 0; i < qm->qp_num; i++) {
465 ret = qp_alloc(qm, i);
466 if (ret)
467 goto free_qp_mem;
468 }
469
470 return HISI_QM_DRVCRYPT_NO_ERR;
471
472 free_qp_mem:
473 for (j = (int)i - 1; j >= 0; j--)
474 qp_free(qm, j);
475 free(qm->qp_array);
476 free_cfg_xqc:
477 hisi_qm_free_xqc(&qm->cfg_xqc);
478 free_xqc:
479 hisi_qm_free_xqc(&qm->xqc);
480 return ret;
481 }
482
hisi_qm_init(struct hisi_qm * qm)483 enum hisi_drv_status hisi_qm_init(struct hisi_qm *qm)
484 {
485 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
486
487 if (qm->fun_type == HISI_QM_HW_VF) {
488 ret = qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
489 if (ret) {
490 EMSG("Fail to get function vft config");
491 return ret;
492 }
493 }
494
495 if (!qm->qp_num || !qm->sqe_size) {
496 EMSG("Invalid QM parameters");
497 return HISI_QM_DRVCRYPT_EINVAL;
498 }
499
500 ret = qm_alloc(qm);
501 if (ret)
502 return ret;
503
504 qm->qp_in_used = 0;
505 qm->qp_idx = 0;
506 mutex_init(&qm->qp_lock);
507 mutex_init(&qm->mailbox_lock);
508
509 return HISI_QM_DRVCRYPT_NO_ERR;
510 }
511
qm_cache_writeback(struct hisi_qm * qm)512 static void qm_cache_writeback(struct hisi_qm *qm)
513 {
514 uint32_t val = 0;
515
516 io_write32(qm->io_base + QM_CACHE_WB_START, QM_FVT_CFG_RDY_BIT);
517
518 if (IO_READ32_POLL_TIMEOUT(qm->io_base + QM_CACHE_WB_DONE, val,
519 val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD,
520 POLL_TIMEOUT))
521 panic("QM writeback sqc cache fail");
522 }
523
hisi_qm_uninit(struct hisi_qm * qm)524 void hisi_qm_uninit(struct hisi_qm *qm)
525 {
526 qm_cache_writeback(qm);
527 qm_free(qm);
528 mutex_destroy(&qm->qp_lock);
529 mutex_destroy(&qm->mailbox_lock);
530 }
531
qm_hw_mem_reset(struct hisi_qm * qm)532 static enum hisi_drv_status qm_hw_mem_reset(struct hisi_qm *qm)
533 {
534 uint32_t val = 0;
535
536 io_write32(qm->io_base + QM_MEM_START_INIT, QM_FVT_CFG_RDY_BIT);
537
538 if (IO_READ32_POLL_TIMEOUT(qm->io_base + QM_MEM_INIT_DONE, val,
539 val & QM_FVT_CFG_RDY_BIT, POLL_PERIOD,
540 POLL_TIMEOUT))
541 return HISI_QM_DRVCRYPT_EBUSY;
542
543 return HISI_QM_DRVCRYPT_NO_ERR;
544 }
545
qm_func_vft_cfg(struct hisi_qm * qm)546 static enum hisi_drv_status qm_func_vft_cfg(struct hisi_qm *qm)
547 {
548 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
549 uint32_t q_base = qm->qp_num;
550 uint32_t act_q_num = 0;
551 unsigned int i = 0;
552 unsigned int j = 0;
553
554 if (!qm->vfs_num)
555 return HISI_QM_DRVCRYPT_NO_ERR;
556
557 if (qm->vfs_num > HISI_QM_MAX_VFS_NUM) {
558 EMSG("Invalid QM vfs_num");
559 return HISI_QM_DRVCRYPT_EINVAL;
560 }
561
562 for (i = 1; i <= qm->vfs_num; i++) {
563 act_q_num = HISI_QM_VF_Q_NUM;
564 ret = qm_set_xqc_vft(qm, i, q_base, act_q_num);
565 if (ret) {
566 for (j = 1; j < i; j++)
567 (void)qm_set_xqc_vft(qm, j, 0, 0);
568 return ret;
569 }
570 q_base += act_q_num;
571 }
572
573 return HISI_QM_DRVCRYPT_NO_ERR;
574 }
575
hisi_qm_start(struct hisi_qm * qm)576 enum hisi_drv_status hisi_qm_start(struct hisi_qm *qm)
577 {
578 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
579
580 if (qm->fun_type == HISI_QM_HW_PF) {
581 ret = qm_hw_mem_reset(qm);
582 if (ret) {
583 EMSG("Fail to reset QM hardware mem");
584 return ret;
585 }
586
587 ret = qm_set_xqc_vft(qm, 0, qm->qp_base, qm->qp_num);
588 if (ret) {
589 EMSG("Fail to set PF xqc_vft");
590 return ret;
591 }
592
593 ret = qm_func_vft_cfg(qm);
594 if (ret) {
595 EMSG("Fail to set VF xqc_vft");
596 return ret;
597 }
598 }
599
600 ret = hisi_qm_mb_write(qm, QM_MB_CMD_SQC_BT, qm->xqc.sqc_dma, 0);
601 if (ret) {
602 EMSG("Fail to set sqc_bt");
603 return ret;
604 }
605
606 ret = hisi_qm_mb_write(qm, QM_MB_CMD_CQC_BT, qm->xqc.cqc_dma, 0);
607 if (ret) {
608 EMSG("Fail to set cqc_bt");
609 return ret;
610 }
611
612 /* Security mode does not support msi */
613 io_write32(qm->io_base + QM_VF_AEQ_INT_MASK, QM_VF_AEQ_INT_MASK_EN);
614 io_write32(qm->io_base + QM_VF_EQ_INT_MASK, QM_VF_EQ_INT_MASK_EN);
615
616 return HISI_QM_DRVCRYPT_NO_ERR;
617 }
618
qm_disable_clock_gate(struct hisi_qm * qm)619 static void qm_disable_clock_gate(struct hisi_qm *qm)
620
621 {
622 if (qm->version == HISI_QM_HW_V2)
623 return;
624
625 io_setbits32(qm->io_base + QM_PM_CTRL0, QM_IDLE_DISABLE);
626 }
627
hisi_qm_dev_init(struct hisi_qm * qm)628 void hisi_qm_dev_init(struct hisi_qm *qm)
629 {
630 if (qm->fun_type == HISI_QM_HW_VF)
631 return;
632
633 qm_disable_clock_gate(qm);
634
635 /* QM user domain */
636 io_write32(qm->io_base + QM_ARUSER_M_CFG_1, QM_AXUSER_CFG);
637 io_write32(qm->io_base + QM_ARUSER_M_CFG_ENABLE, AXUSER_M_CFG_ENABLE);
638 io_write32(qm->io_base + QM_AWUSER_M_CFG_1, QM_AXUSER_CFG);
639 io_write32(qm->io_base + QM_AWUSER_M_CFG_ENABLE, AXUSER_M_CFG_ENABLE);
640 /* QM cache */
641 io_write32(qm->io_base + QM_AXI_M_CFG, AXI_M_CFG);
642
643 if (qm->version == HISI_QM_HW_V2) {
644 /* Disable FLR triggered by BME(bus master enable) */
645 io_write32(qm->io_base + QM_PEH_AXUSER_CFG, PEH_AXUSER_CFG);
646 /* Set sec sqc and cqc cache wb threshold 4 */
647 io_write32(qm->io_base + QM_CACHE_CTL, QM_CACHE_CFG);
648 }
649 /* Disable QM ras */
650 io_write32(qm->io_base + HISI_QM_ABNML_INT_MASK,
651 HISI_QM_ABNML_INT_MASK_CFG);
652 /* Set doorbell timeout to QM_DB_TIMEOUT_SET ns */
653 io_write32(qm->io_base + QM_DB_TIMEOUT_CFG, QM_DB_TIMEOUT_SET);
654 }
655
qm_sqc_cfg(struct hisi_qp * qp)656 static enum hisi_drv_status qm_sqc_cfg(struct hisi_qp *qp)
657 {
658 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
659 struct hisi_qm *qm = qp->qm;
660 struct qm_sqc *sqc = qm->cfg_xqc.sqc;
661 struct qm_mailbox mb = { };
662
663 mutex_lock(&qm->mailbox_lock);
664 memset(sqc, 0, sizeof(struct qm_sqc));
665 reg_pair_from_64(qp->sqe_dma, &sqc->base_h, &sqc->base_l);
666 sqc->dw3 = (HISI_QM_Q_DEPTH - 1) |
667 SHIFT_U32(qm->sqe_log2_size, QM_SQ_SQE_SIZE_SHIFT);
668 sqc->rand_data = QM_DB_RAND_DATA;
669 sqc->cq_num = qp->qp_id;
670 sqc->w13 = BIT32(QM_SQ_ORDER_SHIFT) |
671 SHIFT_U32(qp->sq_type, QM_SQ_TYPE_SHIFT);
672
673 qm_mb_init(&mb, QM_MB_CMD_SQC, qm->cfg_xqc.sqc_dma, qp->qp_id,
674 QM_MB_OP_WR);
675 ret = qm_mb_nolock(qm, &mb);
676 mutex_unlock(&qm->mailbox_lock);
677
678 return ret;
679 }
680
qm_cqc_cfg(struct hisi_qp * qp)681 static enum hisi_drv_status qm_cqc_cfg(struct hisi_qp *qp)
682 {
683 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
684 struct hisi_qm *qm = qp->qm;
685 struct qm_cqc *cqc = qm->cfg_xqc.cqc;
686 struct qm_mailbox mb = { };
687
688 mutex_lock(&qm->mailbox_lock);
689 memset(cqc, 0, sizeof(struct qm_cqc));
690 reg_pair_from_64(qp->cqe_dma, &cqc->base_h, &cqc->base_l);
691 cqc->dw3 = (HISI_QM_Q_DEPTH - 1) |
692 SHIFT_U32(QM_CQE_SIZE, QM_CQ_CQE_SIZE_SHIFT);
693 cqc->rand_data = QM_DB_RAND_DATA;
694 cqc->dw6 = PHASE_DEFAULT_VAL;
695
696 qm_mb_init(&mb, QM_MB_CMD_CQC, qm->cfg_xqc.cqc_dma, qp->qp_id,
697 QM_MB_OP_WR);
698 ret = qm_mb_nolock(qm, &mb);
699 mutex_unlock(&qm->mailbox_lock);
700
701 return ret;
702 }
703
hisi_qm_create_qp(struct hisi_qm * qm,uint8_t sq_type)704 struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, uint8_t sq_type)
705 {
706 struct hisi_qp *qp = NULL;
707 int cur_idx = 0;
708 uint32_t i = 0;
709
710 mutex_lock(&qm->qp_lock);
711 if (qm->qp_in_used == qm->qp_num) {
712 EMSG("All %"PRIu32" queues of QM are busy", qm->qp_num);
713 goto err_proc;
714 }
715
716 for (i = 0; i < qm->qp_num; i++) {
717 cur_idx = (qm->qp_idx + i) % qm->qp_num;
718 if (!qm->qp_array[cur_idx].used) {
719 qm->qp_array[cur_idx].used = true;
720 qm->qp_idx = cur_idx + 1;
721 break;
722 }
723 }
724
725 qp = qm->qp_array + cur_idx;
726 memset(qp->cqe, 0, sizeof(struct qm_cqe) * HISI_QM_Q_DEPTH);
727 qp->sq_type = sq_type;
728 qp->sq_tail = 0;
729 qp->cq_head = 0;
730 qp->cqc_phase = true;
731
732 if (qm_sqc_cfg(qp)) {
733 EMSG("Fail to set qp[%"PRIu32"] sqc", qp->qp_id);
734 goto err_qp_release;
735 }
736
737 if (qm_cqc_cfg(qp)) {
738 EMSG("Fail to set qp[%"PRIu32"] cqc", qp->qp_id);
739 goto err_qp_release;
740 }
741
742 qm->qp_in_used++;
743 mutex_unlock(&qm->qp_lock);
744 return qp;
745
746 err_qp_release:
747 qp->used = false;
748 err_proc:
749 qp->sq_type = 0;
750 qp->cqc_phase = false;
751 mutex_unlock(&qm->qp_lock);
752 return NULL;
753 }
754
hisi_qm_release_qp(struct hisi_qp * qp)755 void hisi_qm_release_qp(struct hisi_qp *qp)
756 {
757 struct hisi_qm *qm = NULL;
758
759 if (!qp) {
760 EMSG("QP is NULL");
761 return;
762 }
763
764 qm = qp->qm;
765 mutex_lock(&qm->qp_lock);
766 qm->qp_in_used--;
767 qp->used = false;
768 mutex_unlock(&qm->qp_lock);
769 }
770
qm_sq_tail_update(struct hisi_qp * qp)771 static void qm_sq_tail_update(struct hisi_qp *qp)
772 {
773 if (qp->sq_tail == HISI_QM_Q_DEPTH - 1)
774 qp->sq_tail = 0;
775 else
776 qp->sq_tail++;
777 }
778
779 /*
780 * One task thread will just bind to one hardware queue, and
781 * hardware does not support msi. So we have no lock here.
782 */
hisi_qp_send(struct hisi_qp * qp,void * msg)783 enum hisi_drv_status hisi_qp_send(struct hisi_qp *qp, void *msg)
784 {
785 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
786 struct hisi_qm *qm = NULL;
787 void *sqe = NULL;
788
789 if (!qp) {
790 EMSG("QP is NULL");
791 return HISI_QM_DRVCRYPT_EINVAL;
792 }
793
794 qm = qp->qm;
795 ret = qm->dev_status_check(qm);
796 if (ret)
797 return ret;
798
799 sqe = (void *)((vaddr_t)qp->sqe + qm->sqe_size * qp->sq_tail);
800 memset(sqe, 0, qm->sqe_size);
801
802 ret = qp->fill_sqe(sqe, msg);
803 if (ret) {
804 EMSG("Fail to fill sqe");
805 return ret;
806 }
807
808 qm_sq_tail_update(qp);
809
810 dsb();
811 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_SQ, qp->sq_tail, 0);
812
813 return HISI_QM_DRVCRYPT_NO_ERR;
814 }
815
qm_cq_head_update(struct hisi_qp * qp)816 static void qm_cq_head_update(struct hisi_qp *qp)
817 {
818 if (qp->cq_head == HISI_QM_Q_DEPTH - 1) {
819 qp->cqc_phase = !qp->cqc_phase;
820 qp->cq_head = 0;
821 } else {
822 qp->cq_head++;
823 }
824 }
825
hisi_qp_recv(struct hisi_qp * qp,void * msg)826 static enum hisi_drv_status hisi_qp_recv(struct hisi_qp *qp, void *msg)
827 {
828 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
829 struct hisi_qm *qm = qp->qm;
830 struct qm_cqe *cqe = NULL;
831 void *sqe = NULL;
832
833 ret = qm->dev_status_check(qm);
834 if (ret)
835 return ret;
836
837 cqe = qp->cqe + qp->cq_head;
838 if (QM_CQE_PHASE(cqe) == qp->cqc_phase) {
839 dsb_osh();
840 sqe = (void *)((vaddr_t)qp->sqe + qm->sqe_size * cqe->sq_head);
841 ret = qp->parse_sqe(sqe, msg);
842 qm_cq_head_update(qp);
843 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->cq_head, 0);
844 if (ret) {
845 EMSG("Fail to parse sqe");
846 return ret;
847 }
848 } else {
849 return HISI_QM_DRVCRYPT_NO_ERR;
850 }
851
852 return HISI_QM_DRVCRYPT_RECV_DONE;
853 }
854
qm_dfx_dump(struct hisi_qm * qm)855 static void qm_dfx_dump(struct hisi_qm *qm)
856 {
857 const struct qm_dfx_registers *regs = qm_dfx_regs;
858 __maybe_unused uint32_t val = 0;
859
860 if (qm->fun_type == HISI_QM_HW_VF)
861 return;
862
863 while (regs->reg_name) {
864 val = io_read32(qm->io_base + regs->reg_offset);
865 EMSG("%s= 0x%" PRIx32, regs->reg_name, val);
866 regs++;
867 }
868 }
869
hisi_qp_recv_sync(struct hisi_qp * qp,void * msg)870 enum hisi_drv_status hisi_qp_recv_sync(struct hisi_qp *qp, void *msg)
871 {
872 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
873 uint32_t timeout = 0;
874
875 if (!qp || !qp->qm || !msg) {
876 EMSG("Invalid qp recv sync parameters");
877 return HISI_QM_DRVCRYPT_EINVAL;
878 }
879
880 timeout = timeout_init_us(QM_SINGLE_WAIT_TIME *
881 HISI_QM_RECV_SYNC_TIMEOUT);
882 while (!timeout_elapsed(timeout)) {
883 ret = hisi_qp_recv(qp, msg);
884 if (ret) {
885 if (ret != HISI_QM_DRVCRYPT_RECV_DONE) {
886 EMSG("QM recv task error");
887 qm_dfx_dump(qp->qm);
888 return ret;
889 } else {
890 return HISI_QM_DRVCRYPT_NO_ERR;
891 }
892 }
893 }
894
895 EMSG("QM recv task timeout");
896 qm_dfx_dump(qp->qm);
897 return HISI_QM_DRVCRYPT_ETMOUT;
898 }
899