1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright 2024 HiSilicon Limited.
4 * Kunpeng hardware accelerator HPRE module init.
5 */
6
7 #include <initcall.h>
8
9 #include "hpre_main.h"
10
11 /* base config */
12 #define HPRE_COMMON_CNT_CLR_CE 0x301000
13 #define HPRE_CFG_AXCACHE 0x301010
14 #define HPRE_RDCHN_INI_CFG 0x301014
15 #define HPRE_BD_ENDIAN 0x301020
16 #define HPRE_ECC_BYPASS 0x301024
17 #define HPRE_POISON_BYPASS 0x30102c
18 #define HPRE_BD_ARUSR_CFG 0x301030
19 #define HPRE_BD_AWUSR_CFG 0x301034
20 #define HPRE_TYPES_ENB 0x301038
21 #define HPRE_DATA_RUSER_CFG 0x30103c
22 #define HPRE_DATA_WUSER_CFG 0x301040
23 #define HPRE_HAC_INT_MASK 0x301400
24 #define HPRE_RAS_ECC_1BIT_TH 0x30140c
25 #define HPRE_RAS_CE_ENB 0x301410
26 #define HPRE_RAS_NFE_ENB 0x301414
27 #define HPRE_RAS_FE_ENB 0x301418
28 #define HPRE_HAC_INT_SRC 0x301600
29 #define HPRE_RDCHN_INI_ST 0x301a00
30 #define HPRE_OOO_SHUTDOWN_SEL 0x301a3c
31 #define HPRE_CORE_ENB 0x302004
32 #define HPRE_CORE_INI_CFG 0x302020
33 #define HPRE_CORE_INI_STATUS 0x302080
34 /* clock gate */
35 #define HPRE_CLKGATE_CTL 0x301a10
36 #define HPRE_PEH_CFG_AUTO_GATE 0x301a2c
37 #define HPRE_CLUSTER_DYN_CTL 0x302010
38 #define HPRE_CORE_SHB_CFG 0x302088
39 #define HPRE_CORE_GATE_ENABLE GENMASK_32(31, 30)
40
41 #define HPRE_AXCACHE_MASK 0xff
42 #define HPRE_HAC_INT_DISABLE 0x1ffffff
43 #define HPRE_RAS_CE_MASK 0x1
44 #define HPRE_RAS_NFE_MASK 0x1fffffe
45 #define HPRE_RAS_FE_MASK 0
46 #define HPRE_BD_LITTLE_ENDIAN 0
47 #define HPRE_RSA_ENB BIT(0)
48 #define HPRE_ECC_ENB BIT(1)
49 #define HPRE_BD_ARUSR_MASK 0x2
50 #define HPRE_BD_AWUSR_MASK 0x102
51 #define HPRE_DATA_USR_MASK 0x32
52 #define HPRE_CLUSTER_CORE_MASK GENMASK_32(9, 0)
53
54 static SLIST_HEAD(, acc_device) hpre_list = SLIST_HEAD_INITIALIZER(hpre_list);
55
hpre_create_qp(uint8_t sq_type)56 struct hisi_qp *hpre_create_qp(uint8_t sq_type)
57 {
58 struct acc_device *hpre_dev = NULL;
59 struct acc_device *cur_dev = NULL;
60 struct hisi_qm *qm = NULL;
61 uint32_t free_qp_num = 0;
62 uint32_t max_qp_num = 0;
63
64 /* Find the HPRE device with the most remaining qp numbers */
65 SLIST_FOREACH(cur_dev, &hpre_list, link) {
66 qm = &cur_dev->qm;
67 if (qm->fun_type == HISI_QM_HW_PF)
68 free_qp_num = HISI_QM_PF_Q_NUM - qm->qp_in_used;
69 else
70 free_qp_num = HISI_QM_VF_Q_NUM - qm->qp_in_used;
71 if (free_qp_num > max_qp_num) {
72 max_qp_num = free_qp_num;
73 hpre_dev = cur_dev;
74 }
75 }
76
77 if (!hpre_dev) {
78 EMSG("No available hpre device");
79 return NULL;
80 }
81
82 return hisi_qm_create_qp(&hpre_dev->qm, sq_type);
83 }
84
hpre_bin_from_crypto_bin(uint8_t * dst,const uint8_t * src,uint32_t bsize,uint32_t dsize)85 enum hisi_drv_status hpre_bin_from_crypto_bin(uint8_t *dst, const uint8_t *src,
86 uint32_t bsize, uint32_t dsize)
87 {
88 if (!src || !dst || !dsize || !bsize) {
89 EMSG("parameter error");
90 return HISI_QM_DRVCRYPT_EINVAL;
91 }
92
93 if (bsize < dsize) {
94 EMSG("dsize is too long");
95 return HISI_QM_DRVCRYPT_EINVAL;
96 }
97
98 if (src == dst && bsize == dsize)
99 return HISI_QM_DRVCRYPT_NO_ERR;
100
101 /*
102 * Copying non-zero data and padding with zeroes in high-bits
103 * (eg: 1 2 3 0 0 -> 0 0 1 2 3)
104 */
105 memmove(dst + bsize - dsize, src, dsize);
106 memset(dst, 0, bsize - dsize);
107
108 return HISI_QM_DRVCRYPT_NO_ERR;
109 }
110
hpre_bin_to_crypto_bin(uint8_t * dst,const uint8_t * src,uint32_t bsize,uint32_t dsize)111 enum hisi_drv_status hpre_bin_to_crypto_bin(uint8_t *dst, const uint8_t *src,
112 uint32_t bsize, uint32_t dsize)
113 {
114 if (!dst || !src || !bsize || !dsize) {
115 EMSG("parameter error");
116 return HISI_QM_DRVCRYPT_EINVAL;
117 }
118
119 if (bsize < dsize) {
120 EMSG("dsize is too long");
121 return HISI_QM_DRVCRYPT_EINVAL;
122 }
123
124 if (src == dst && bsize == dsize)
125 return HISI_QM_DRVCRYPT_NO_ERR;
126 /*
127 * Copying non-zero data and padding with zeroes in low-bits
128 * (eg: 0 0 1 2 3 -> 1 2 3 0 0)
129 */
130 memmove(dst, src + bsize - dsize, dsize);
131 memset(dst + dsize, 0, bsize - dsize);
132
133 return HISI_QM_DRVCRYPT_NO_ERR;
134 }
135
hpre_set_cluster(struct hisi_qm * qm)136 static enum hisi_drv_status hpre_set_cluster(struct hisi_qm *qm)
137 {
138 uint32_t val = 0;
139
140 io_write32(qm->io_base + HPRE_CORE_ENB, HPRE_CLUSTER_CORE_MASK);
141 io_write32(qm->io_base + HPRE_CORE_INI_CFG, 0x1);
142
143 if (IO_READ32_POLL_TIMEOUT(qm->io_base + HPRE_CORE_INI_STATUS, val,
144 (val & HPRE_CLUSTER_CORE_MASK) ==
145 HPRE_CLUSTER_CORE_MASK, POLL_PERIOD,
146 POLL_TIMEOUT))
147 return HISI_QM_DRVCRYPT_EBUSY;
148 return HISI_QM_DRVCRYPT_NO_ERR;
149 }
150
hpre_disable_clock_gate(struct hisi_qm * qm)151 static void hpre_disable_clock_gate(struct hisi_qm *qm)
152 {
153 io_write32(qm->io_base + HPRE_CLKGATE_CTL, 0x0);
154 io_write32(qm->io_base + HPRE_PEH_CFG_AUTO_GATE, 0x0);
155 io_write32(qm->io_base + HPRE_CLUSTER_DYN_CTL, 0x0);
156 io_clrbits32(qm->io_base + HPRE_CORE_SHB_CFG, HPRE_CORE_GATE_ENABLE);
157 }
158
hpre_enable_clock_gate(struct hisi_qm * qm)159 static void hpre_enable_clock_gate(struct hisi_qm *qm)
160 {
161 io_write32(qm->io_base + HPRE_CLKGATE_CTL, 0x1);
162 io_write32(qm->io_base + HPRE_PEH_CFG_AUTO_GATE, 0x1);
163 io_write32(qm->io_base + HPRE_CLUSTER_DYN_CTL, 0x1);
164 io_setbits32(qm->io_base + HPRE_CORE_SHB_CFG, HPRE_CORE_GATE_ENABLE);
165 }
166
hpre_engine_init(struct acc_device * hpre_dev)167 static TEE_Result hpre_engine_init(struct acc_device *hpre_dev)
168 {
169 struct hisi_qm *qm = &hpre_dev->qm;
170 uint32_t val = 0;
171 int32_t ret = 0;
172
173 if (qm->fun_type == HISI_QM_HW_VF)
174 return TEE_SUCCESS;
175
176 hpre_disable_clock_gate(qm);
177 hisi_qm_dev_init(qm);
178
179 io_write32(qm->io_base + HPRE_CFG_AXCACHE, HPRE_AXCACHE_MASK);
180 io_write32(qm->io_base + HPRE_BD_ENDIAN, HPRE_BD_LITTLE_ENDIAN);
181 io_write32(qm->io_base + HPRE_RAS_CE_ENB, HPRE_RAS_CE_MASK);
182 io_write32(qm->io_base + HPRE_RAS_NFE_ENB, HPRE_RAS_NFE_MASK);
183 io_write32(qm->io_base + HPRE_RAS_FE_ENB, HPRE_RAS_FE_MASK);
184 io_write32(qm->io_base + HPRE_HAC_INT_MASK, HPRE_HAC_INT_DISABLE);
185 io_write32(qm->io_base + HPRE_POISON_BYPASS, 0x0);
186 io_write32(qm->io_base + HPRE_COMMON_CNT_CLR_CE, 0x0);
187 io_write32(qm->io_base + HPRE_ECC_BYPASS, 0x0);
188 /* cmd_type is controlled by hac subctrl */
189 io_write32(qm->io_base + HPRE_BD_ARUSR_CFG, HPRE_BD_ARUSR_MASK);
190 io_write32(qm->io_base + HPRE_BD_AWUSR_CFG, HPRE_BD_AWUSR_MASK);
191 io_write32(qm->io_base + HPRE_DATA_RUSER_CFG, HPRE_DATA_USR_MASK);
192 io_write32(qm->io_base + HPRE_DATA_WUSER_CFG, HPRE_DATA_USR_MASK);
193 io_write32(qm->io_base + HPRE_TYPES_ENB, HPRE_RSA_ENB | HPRE_ECC_ENB);
194 io_write32(qm->io_base + HPRE_RDCHN_INI_CFG, 0x1);
195 ret = IO_READ32_POLL_TIMEOUT(qm->io_base + HPRE_RDCHN_INI_ST, val,
196 val & 0x1, POLL_PERIOD, POLL_TIMEOUT);
197 if (ret) {
198 EMSG("Fail to init rd channel");
199 return TEE_ERROR_BUSY;
200 }
201
202 ret = hpre_set_cluster(qm);
203 if (ret) {
204 EMSG("Fail to init hpre cluster cores");
205 return TEE_ERROR_BUSY;
206 }
207
208 hpre_enable_clock_gate(qm);
209
210 return TEE_SUCCESS;
211 }
212
hpre_dev_status_check(struct hisi_qm * qm)213 static enum hisi_drv_status hpre_dev_status_check(struct hisi_qm *qm)
214 {
215 uint32_t val = 0;
216
217 val = io_read32(qm->io_base + HPRE_HAC_INT_SRC);
218 if (val & HPRE_RAS_NFE_MASK) {
219 EMSG("HPRE NFE RAS happened, need to reset");
220 return HISI_QM_DRVCRYPT_HW_EACCESS;
221 }
222
223 val = io_read32(qm->io_base + HISI_QM_ABNML_INT_SRC);
224 if (val) {
225 if (val & HISI_QM_HPRE_NFE_INT_MASK)
226 EMSG("QM NFE RAS happened, need to reset");
227
228 if (val & HISI_QM_INVALID_DB) {
229 EMSG("QM invalid db happened, please check");
230 io_write32(qm->io_base + HISI_QM_ABNML_INT_SRC,
231 HISI_QM_INVALID_DB);
232 }
233
234 return HISI_QM_DRVCRYPT_HW_EACCESS;
235 }
236
237 return HISI_QM_DRVCRYPT_NO_ERR;
238 }
239
hpre_qm_init(struct acc_device * hpre_dev)240 static enum hisi_drv_status hpre_qm_init(struct acc_device *hpre_dev)
241 {
242 struct hisi_qm *qm = &hpre_dev->qm;
243
244 if (cpu_mmu_enabled()) {
245 qm->io_base = (uintptr_t)phys_to_virt_io(hpre_dev->io_base,
246 hpre_dev->io_size);
247 if (!qm->io_base) {
248 EMSG("Fail to get qm io_base");
249 return HISI_QM_DRVCRYPT_EFAULT;
250 }
251 } else {
252 qm->io_base = hpre_dev->io_base;
253 }
254
255 qm->vfs_num = hpre_dev->vfs_num;
256 qm->fun_type = hpre_dev->fun_type;
257 qm->sqe_size = HPRE_SQE_SIZE;
258 qm->sqe_log2_size = HPRE_SQE_LOG2_SIZE;
259 if (qm->fun_type == HISI_QM_HW_PF) {
260 hisi_qm_get_version(qm);
261 DMSG("HPRE hardware version is 0x%"PRIx32, qm->version);
262 qm->qp_base = HISI_QM_PF_Q_BASE;
263 qm->qp_num = HISI_QM_PF_Q_NUM;
264 qm->dev_status_check = hpre_dev_status_check;
265 }
266
267 return hisi_qm_init(qm);
268 }
269
hpre_pre_init(void)270 static struct acc_device *hpre_pre_init(void)
271 {
272 struct acc_device *hpre_dev = NULL;
273
274 hpre_dev = calloc(1, sizeof(*hpre_dev));
275 if (!hpre_dev) {
276 EMSG("Fail to alloc hpre_dev");
277 return NULL;
278 }
279
280 hpre_dev->io_base = HPRE_BAR_BASE;
281 hpre_dev->io_size = HPRE_BAR_SIZE;
282 hpre_dev->fun_type = HISI_QM_HW_PF;
283 SLIST_INSERT_HEAD(&hpre_list, hpre_dev, link);
284
285 return hpre_dev;
286 }
287
hpre_probe(void)288 static TEE_Result hpre_probe(void)
289 {
290 TEE_Result ret = TEE_ERROR_GENERIC;
291 struct acc_device *hpre_dev = NULL;
292 struct hisi_qm *qm = NULL;
293
294 DMSG("HPRE driver init start");
295 hpre_dev = hpre_pre_init();
296 if (!hpre_dev)
297 return TEE_ERROR_OUT_OF_MEMORY;
298
299 qm = &hpre_dev->qm;
300 if (hpre_qm_init(hpre_dev)) {
301 EMSG("Fail to init hpre qm");
302 goto err_with_pre_init;
303 }
304
305 ret = hpre_engine_init(hpre_dev);
306 if (ret) {
307 EMSG("Fail to init engine");
308 goto err_with_qm_init;
309 }
310
311 if (hisi_qm_start(qm)) {
312 EMSG("Fail to start qm");
313 ret = TEE_ERROR_BAD_STATE;
314 goto err_with_qm_init;
315 }
316
317 DMSG("HPRE driver init done");
318 return TEE_SUCCESS;
319
320 err_with_qm_init:
321 hisi_qm_uninit(qm);
322 err_with_pre_init:
323 SLIST_REMOVE_HEAD(&hpre_list, link);
324 free(hpre_dev);
325
326 return ret;
327 }
328
329 driver_init(hpre_probe);
330