1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright 2024 HiSilicon Limited. 4 * Kunpeng hardware accelerator HPRE module init. 5 */ 6 7 #include <initcall.h> 8 9 #include "hpre_main.h" 10 11 /* base config */ 12 #define HPRE_COMMON_CNT_CLR_CE 0x301000 13 #define HPRE_CFG_AXCACHE 0x301010 14 #define HPRE_RDCHN_INI_CFG 0x301014 15 #define HPRE_BD_ENDIAN 0x301020 16 #define HPRE_ECC_BYPASS 0x301024 17 #define HPRE_POISON_BYPASS 0x30102c 18 #define HPRE_BD_ARUSR_CFG 0x301030 19 #define HPRE_BD_AWUSR_CFG 0x301034 20 #define HPRE_TYPES_ENB 0x301038 21 #define HPRE_DATA_RUSER_CFG 0x30103c 22 #define HPRE_DATA_WUSER_CFG 0x301040 23 #define HPRE_HAC_INT_MASK 0x301400 24 #define HPRE_RAS_ECC_1BIT_TH 0x30140c 25 #define HPRE_RAS_CE_ENB 0x301410 26 #define HPRE_RAS_NFE_ENB 0x301414 27 #define HPRE_RAS_FE_ENB 0x301418 28 #define HPRE_HAC_INT_SRC 0x301600 29 #define HPRE_RDCHN_INI_ST 0x301a00 30 #define HPRE_OOO_SHUTDOWN_SEL 0x301a3c 31 #define HPRE_CORE_ENB 0x302004 32 #define HPRE_CORE_INI_CFG 0x302020 33 #define HPRE_CORE_INI_STATUS 0x302080 34 /* clock gate */ 35 #define HPRE_CLKGATE_CTL 0x301a10 36 #define HPRE_PEH_CFG_AUTO_GATE 0x301a2c 37 #define HPRE_CLUSTER_DYN_CTL 0x302010 38 #define HPRE_CORE_SHB_CFG 0x302088 39 #define HPRE_CORE_GATE_ENABLE GENMASK_32(31, 30) 40 41 #define HPRE_AXCACHE_MASK 0xff 42 #define HPRE_HAC_INT_DISABLE 0x1ffffff 43 #define HPRE_RAS_CE_MASK 0x1 44 #define HPRE_RAS_NFE_MASK 0x1fffffe 45 #define HPRE_RAS_FE_MASK 0 46 #define HPRE_BD_LITTLE_ENDIAN 0 47 #define HPRE_RSA_ENB BIT(0) 48 #define HPRE_ECC_ENB BIT(1) 49 #define HPRE_BD_ARUSR_MASK 0x2 50 #define HPRE_BD_AWUSR_MASK 0x102 51 #define HPRE_DATA_USR_MASK 0x32 52 #define HPRE_CLUSTER_CORE_MASK GENMASK_32(9, 0) 53 54 static SLIST_HEAD(, acc_device) hpre_list = SLIST_HEAD_INITIALIZER(hpre_list); 55 56 struct hisi_qp *hpre_create_qp(uint8_t sq_type) 57 { 58 struct acc_device *hpre_dev = NULL; 59 struct acc_device *cur_dev = NULL; 60 struct hisi_qm *qm = NULL; 61 uint32_t free_qp_num = 0; 62 uint32_t max_qp_num = 0; 63 64 /* Find the HPRE device with the most remaining qp numbers */ 65 SLIST_FOREACH(cur_dev, &hpre_list, link) { 66 qm = &cur_dev->qm; 67 if (qm->fun_type == HISI_QM_HW_PF) 68 free_qp_num = HISI_QM_PF_Q_NUM - qm->qp_in_used; 69 else 70 free_qp_num = HISI_QM_VF_Q_NUM - qm->qp_in_used; 71 if (free_qp_num > max_qp_num) { 72 max_qp_num = free_qp_num; 73 hpre_dev = cur_dev; 74 } 75 } 76 77 if (!hpre_dev) { 78 EMSG("No available hpre device"); 79 return NULL; 80 } 81 82 return hisi_qm_create_qp(&hpre_dev->qm, sq_type); 83 } 84 85 enum hisi_drv_status hpre_bin_from_crypto_bin(uint8_t *dst, const uint8_t *src, 86 uint32_t bsize, uint32_t dsize) 87 { 88 if (!src || !dst || !dsize || !bsize) { 89 EMSG("parameter error"); 90 return HISI_QM_DRVCRYPT_EINVAL; 91 } 92 93 if (bsize < dsize) { 94 EMSG("dsize is too long"); 95 return HISI_QM_DRVCRYPT_EINVAL; 96 } 97 98 if (src == dst && bsize == dsize) 99 return HISI_QM_DRVCRYPT_NO_ERR; 100 101 /* 102 * Copying non-zero data and padding with zeroes in high-bits 103 * (eg: 1 2 3 0 0 -> 0 0 1 2 3) 104 */ 105 memmove(dst + bsize - dsize, src, dsize); 106 memset(dst, 0, bsize - dsize); 107 108 return HISI_QM_DRVCRYPT_NO_ERR; 109 } 110 111 enum hisi_drv_status hpre_bin_to_crypto_bin(uint8_t *dst, const uint8_t *src, 112 uint32_t bsize, uint32_t dsize) 113 { 114 if (!dst || !src || !bsize || !dsize) { 115 EMSG("parameter error"); 116 return HISI_QM_DRVCRYPT_EINVAL; 117 } 118 119 if (bsize < dsize) { 120 EMSG("dsize is too long"); 121 return HISI_QM_DRVCRYPT_EINVAL; 122 } 123 124 if (src == dst && bsize == dsize) 125 return HISI_QM_DRVCRYPT_NO_ERR; 126 /* 127 * Copying non-zero data and padding with zeroes in low-bits 128 * (eg: 0 0 1 2 3 -> 1 2 3 0 0) 129 */ 130 memmove(dst, src + bsize - dsize, dsize); 131 memset(dst + dsize, 0, bsize - dsize); 132 133 return HISI_QM_DRVCRYPT_NO_ERR; 134 } 135 136 static enum hisi_drv_status hpre_set_cluster(struct hisi_qm *qm) 137 { 138 uint32_t val = 0; 139 140 io_write32(qm->io_base + HPRE_CORE_ENB, HPRE_CLUSTER_CORE_MASK); 141 io_write32(qm->io_base + HPRE_CORE_INI_CFG, 0x1); 142 143 if (IO_READ32_POLL_TIMEOUT(qm->io_base + HPRE_CORE_INI_STATUS, val, 144 (val & HPRE_CLUSTER_CORE_MASK) == 145 HPRE_CLUSTER_CORE_MASK, POLL_PERIOD, 146 POLL_TIMEOUT)) 147 return HISI_QM_DRVCRYPT_EBUSY; 148 return HISI_QM_DRVCRYPT_NO_ERR; 149 } 150 151 static void hpre_disable_clock_gate(struct hisi_qm *qm) 152 { 153 io_write32(qm->io_base + HPRE_CLKGATE_CTL, 0x0); 154 io_write32(qm->io_base + HPRE_PEH_CFG_AUTO_GATE, 0x0); 155 io_write32(qm->io_base + HPRE_CLUSTER_DYN_CTL, 0x0); 156 io_clrbits32(qm->io_base + HPRE_CORE_SHB_CFG, HPRE_CORE_GATE_ENABLE); 157 } 158 159 static void hpre_enable_clock_gate(struct hisi_qm *qm) 160 { 161 io_write32(qm->io_base + HPRE_CLKGATE_CTL, 0x1); 162 io_write32(qm->io_base + HPRE_PEH_CFG_AUTO_GATE, 0x1); 163 io_write32(qm->io_base + HPRE_CLUSTER_DYN_CTL, 0x1); 164 io_setbits32(qm->io_base + HPRE_CORE_SHB_CFG, HPRE_CORE_GATE_ENABLE); 165 } 166 167 static TEE_Result hpre_engine_init(struct acc_device *hpre_dev) 168 { 169 struct hisi_qm *qm = &hpre_dev->qm; 170 uint32_t val = 0; 171 int32_t ret = 0; 172 173 if (qm->fun_type == HISI_QM_HW_VF) 174 return TEE_SUCCESS; 175 176 hpre_disable_clock_gate(qm); 177 hisi_qm_dev_init(qm); 178 179 io_write32(qm->io_base + HPRE_CFG_AXCACHE, HPRE_AXCACHE_MASK); 180 io_write32(qm->io_base + HPRE_BD_ENDIAN, HPRE_BD_LITTLE_ENDIAN); 181 io_write32(qm->io_base + HPRE_RAS_CE_ENB, HPRE_RAS_CE_MASK); 182 io_write32(qm->io_base + HPRE_RAS_NFE_ENB, HPRE_RAS_NFE_MASK); 183 io_write32(qm->io_base + HPRE_RAS_FE_ENB, HPRE_RAS_FE_MASK); 184 io_write32(qm->io_base + HPRE_HAC_INT_MASK, HPRE_HAC_INT_DISABLE); 185 io_write32(qm->io_base + HPRE_POISON_BYPASS, 0x0); 186 io_write32(qm->io_base + HPRE_COMMON_CNT_CLR_CE, 0x0); 187 io_write32(qm->io_base + HPRE_ECC_BYPASS, 0x0); 188 /* cmd_type is controlled by hac subctrl */ 189 io_write32(qm->io_base + HPRE_BD_ARUSR_CFG, HPRE_BD_ARUSR_MASK); 190 io_write32(qm->io_base + HPRE_BD_AWUSR_CFG, HPRE_BD_AWUSR_MASK); 191 io_write32(qm->io_base + HPRE_DATA_RUSER_CFG, HPRE_DATA_USR_MASK); 192 io_write32(qm->io_base + HPRE_DATA_WUSER_CFG, HPRE_DATA_USR_MASK); 193 io_write32(qm->io_base + HPRE_TYPES_ENB, HPRE_RSA_ENB | HPRE_ECC_ENB); 194 io_write32(qm->io_base + HPRE_RDCHN_INI_CFG, 0x1); 195 ret = IO_READ32_POLL_TIMEOUT(qm->io_base + HPRE_RDCHN_INI_ST, val, 196 val & 0x1, POLL_PERIOD, POLL_TIMEOUT); 197 if (ret) { 198 EMSG("Fail to init rd channel"); 199 return TEE_ERROR_BUSY; 200 } 201 202 ret = hpre_set_cluster(qm); 203 if (ret) { 204 EMSG("Fail to init hpre cluster cores"); 205 return TEE_ERROR_BUSY; 206 } 207 208 hpre_enable_clock_gate(qm); 209 210 return TEE_SUCCESS; 211 } 212 213 static enum hisi_drv_status hpre_dev_status_check(struct hisi_qm *qm) 214 { 215 uint32_t val = 0; 216 217 val = io_read32(qm->io_base + HPRE_HAC_INT_SRC); 218 if (val & HPRE_RAS_NFE_MASK) { 219 EMSG("HPRE NFE RAS happened, need to reset"); 220 return HISI_QM_DRVCRYPT_HW_EACCESS; 221 } 222 223 val = io_read32(qm->io_base + HISI_QM_ABNML_INT_SRC); 224 if (val) { 225 if (val & HISI_QM_HPRE_NFE_INT_MASK) 226 EMSG("QM NFE RAS happened, need to reset"); 227 228 if (val & HISI_QM_INVALID_DB) { 229 EMSG("QM invalid db happened, please check"); 230 io_write32(qm->io_base + HISI_QM_ABNML_INT_SRC, 231 HISI_QM_INVALID_DB); 232 } 233 234 return HISI_QM_DRVCRYPT_HW_EACCESS; 235 } 236 237 return HISI_QM_DRVCRYPT_NO_ERR; 238 } 239 240 static enum hisi_drv_status hpre_qm_init(struct acc_device *hpre_dev) 241 { 242 struct hisi_qm *qm = &hpre_dev->qm; 243 244 if (cpu_mmu_enabled()) { 245 qm->io_base = (uintptr_t)phys_to_virt_io(hpre_dev->io_base, 246 hpre_dev->io_size); 247 if (!qm->io_base) { 248 EMSG("Fail to get qm io_base"); 249 return HISI_QM_DRVCRYPT_EFAULT; 250 } 251 } else { 252 qm->io_base = hpre_dev->io_base; 253 } 254 255 qm->vfs_num = hpre_dev->vfs_num; 256 qm->fun_type = hpre_dev->fun_type; 257 qm->sqe_size = HPRE_SQE_SIZE; 258 qm->sqe_log2_size = HPRE_SQE_LOG2_SIZE; 259 if (qm->fun_type == HISI_QM_HW_PF) { 260 hisi_qm_get_version(qm); 261 DMSG("HPRE hardware version is 0x%"PRIx32, qm->version); 262 qm->qp_base = HISI_QM_PF_Q_BASE; 263 qm->qp_num = HISI_QM_PF_Q_NUM; 264 qm->dev_status_check = hpre_dev_status_check; 265 } 266 267 return hisi_qm_init(qm); 268 } 269 270 static struct acc_device *hpre_pre_init(void) 271 { 272 struct acc_device *hpre_dev = NULL; 273 274 hpre_dev = calloc(1, sizeof(*hpre_dev)); 275 if (!hpre_dev) { 276 EMSG("Fail to alloc hpre_dev"); 277 return NULL; 278 } 279 280 hpre_dev->io_base = HPRE_BAR_BASE; 281 hpre_dev->io_size = HPRE_BAR_SIZE; 282 hpre_dev->fun_type = HISI_QM_HW_PF; 283 SLIST_INSERT_HEAD(&hpre_list, hpre_dev, link); 284 285 return hpre_dev; 286 } 287 288 static TEE_Result hpre_probe(void) 289 { 290 TEE_Result ret = TEE_ERROR_GENERIC; 291 struct acc_device *hpre_dev = NULL; 292 struct hisi_qm *qm = NULL; 293 294 DMSG("HPRE driver init start"); 295 hpre_dev = hpre_pre_init(); 296 if (!hpre_dev) 297 return TEE_ERROR_OUT_OF_MEMORY; 298 299 qm = &hpre_dev->qm; 300 if (hpre_qm_init(hpre_dev)) { 301 EMSG("Fail to init hpre qm"); 302 goto err_with_pre_init; 303 } 304 305 ret = hpre_engine_init(hpre_dev); 306 if (ret) { 307 EMSG("Fail to init engine"); 308 goto err_with_qm_init; 309 } 310 311 if (hisi_qm_start(qm)) { 312 EMSG("Fail to start qm"); 313 ret = TEE_ERROR_BAD_STATE; 314 goto err_with_qm_init; 315 } 316 317 DMSG("HPRE driver init done"); 318 return TEE_SUCCESS; 319 320 err_with_qm_init: 321 hisi_qm_uninit(qm); 322 err_with_pre_init: 323 SLIST_REMOVE_HEAD(&hpre_list, link); 324 free(hpre_dev); 325 326 return ret; 327 } 328 329 driver_init(hpre_probe); 330