1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2022-2024, HiSilicon Technologies Co., Ltd. 4 * Kunpeng hardware accelerator hpre montgomery algorithm implementation. 5 */ 6 #include <drvcrypt.h> 7 #include <drvcrypt_acipher.h> 8 #include <initcall.h> 9 #include <malloc.h> 10 #include <rng_support.h> 11 #include <stdlib_ext.h> 12 #include <string.h> 13 #include <string_ext.h> 14 #include <trace.h> 15 16 #include "hpre_main.h" 17 #include "hpre_montgomery.h" 18 19 #define X25519_CURVE_INDEX 0 20 #define X448_CURVE_INDEX 1 21 22 struct hpre_mgm_curve { 23 uint32_t key_bits; 24 const uint8_t *p; 25 const uint8_t *a; 26 const uint8_t *x; 27 }; 28 29 /* NID_X25519 */ 30 /* p = (2 ^ 255 - 19) big endian */ 31 static const uint8_t g_x25519_p[] = { 32 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 33 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 34 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xED 35 }; 36 37 /* a = (486662 - 2) / 4 = 121665 big endian */ 38 static const uint8_t g_x25519_a[] = { 39 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 40 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 41 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xDB, 0x41 42 }; 43 44 /* big endian */ 45 static const uint8_t g_x25519_gx[] = { 46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 48 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09 49 }; 50 51 /* NID_X448 */ 52 /* p = (2 ^ 448 - 2 ^ 224 - 1) big endian */ 53 static const uint8_t g_x448_p[] = { 54 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 55 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 56 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 57 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 58 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF 59 }; 60 61 /* a = (156326 - 2) / 4 = 39081 big endian */ 62 static const uint8_t g_x448_a[] = { 63 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 64 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 65 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 66 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 67 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0xA9 68 }; 69 70 /* big endian */ 71 static const uint8_t g_x448_gx[] = { 72 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 73 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 74 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 75 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 76 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05 77 }; 78 79 static const struct hpre_mgm_curve g_curve_list[] = { 80 { 81 .key_bits = 256, 82 .p = g_x25519_p, 83 .a = g_x25519_a, 84 .x = g_x25519_gx, 85 }, { 86 .key_bits = 448, 87 .p = g_x448_p, 88 .a = g_x448_a, 89 .x = g_x448_gx, 90 } 91 }; 92 93 static TEE_Result 94 hpre_montgomery_alloc_keypair(struct montgomery_keypair *key, 95 size_t size_bits) 96 { 97 size_t key_size = BITS_TO_BYTES(size_bits); 98 99 if (!key || (size_bits != X25519_KEY_BITS && 100 size_bits != X448_KEY_BITS)) { 101 EMSG("Invalid input parameter"); 102 return TEE_ERROR_BAD_PARAMETERS; 103 } 104 105 key->priv = calloc(1, key_size); 106 if (!key->priv) 107 goto priv_err; 108 109 key->pub = calloc(1, key_size); 110 if (!key->pub) 111 goto pub_err; 112 113 return TEE_SUCCESS; 114 pub_err: 115 free(key->priv); 116 key->priv = NULL; 117 priv_err: 118 EMSG("HPRE montgomery alloc key pair fail"); 119 120 return TEE_ERROR_OUT_OF_MEMORY; 121 } 122 123 static TEE_Result gen_random_privkey(uint8_t *priv, size_t key_bits) 124 { 125 size_t key_size = BITS_TO_BYTES(key_bits); 126 TEE_Result ret = TEE_SUCCESS; 127 128 if (!priv) { 129 EMSG("Privkey param is NULL"); 130 return TEE_ERROR_BAD_PARAMETERS; 131 } 132 133 ret = hw_get_random_bytes(priv, key_size); 134 if (ret) { 135 EMSG("Fail to fill privkey"); 136 return TEE_ERROR_NO_DATA; 137 } 138 139 return ret; 140 } 141 142 static enum hisi_drv_status 143 hpre_montgomery_params_alloc(struct hpre_montgomery_msg *msg) 144 { 145 uint32_t size = HPRE_MONTGOMERY_TOTAL_BUF_SIZE(msg->key_bytes); 146 147 msg->key = calloc(1, size); 148 if (!msg->key) { 149 EMSG("Fail to alloc montgomery key buf"); 150 return HISI_QM_DRVCRYPT_ENOMEM; 151 } 152 153 msg->key_dma = virt_to_phys(msg->key); 154 msg->in = msg->key + HPRE_X_KEY_SIZE(msg->key_bytes); 155 msg->in_dma = msg->key_dma + HPRE_X_KEY_SIZE(msg->key_bytes); 156 msg->out = msg->in + msg->key_bytes; 157 msg->out_dma = msg->in_dma + msg->key_bytes; 158 159 return HISI_QM_DRVCRYPT_NO_ERR; 160 } 161 162 static void hpre_montgomery_params_free(struct hpre_montgomery_msg *msg) 163 { 164 if (msg->key) { 165 memzero_explicit(msg->key, HPRE_X_KEY_SIZE(msg->key_bytes)); 166 free(msg->key); 167 msg->key = NULL; 168 } 169 } 170 171 static enum hisi_drv_status 172 hpre_montgomery_params_pretreatment(struct hpre_montgomery_msg *msg) 173 { 174 uint8_t *p = msg->key; 175 uint8_t *a = p + msg->key_bytes; 176 uint8_t *k = a + msg->key_bytes; 177 uint8_t *u = msg->in; 178 uint8_t *dst = k; 179 uint32_t bsize = msg->key_bytes; 180 uint32_t dsize = msg->curve_bytes; 181 /* 182 * It is a constraint of HPRE hardware that key_bytes will be set 183 * to 72 when curve_bytes is between 48 and 72, and the high-order 184 * bits will be set to 0. 185 */ 186 uint32_t offset = bsize - dsize; 187 int ret = 0; 188 189 /* 190 * This is a pretreatment of X25519 with a 32-byte integer, 191 * as described in RFC 7748: 192 * Set the three LSB of the first byte and MSB of the last 193 * to zero, set the second MSB of the last byte to 1. 194 * When receiving u-array, set MSB of last byte to zero. 195 * HPRE hardware module uses big-endian mode, so the bytes to be 196 * set are reversed compared to RFC 7748 197 */ 198 if (msg->key_bytes == BITS_TO_BYTES(X25519_KEY_BITS)) { 199 dst[31] &= 0xF8; 200 dst[0] &= 0x7F; 201 dst[0] |= 0x40; 202 u[0] &= 0x7F; 203 } else { 204 /* 205 * This is a pretreatment of X448 with a 56-byte integer, 206 * as described in RFC 7748: 207 * For X448, set the two LSB of the first byte to 0, and MSB of the 208 * last byte to 1. 209 * HPRE hardware module uses big-endian mode, so the bytes to be 210 * set are reversed compared to RFC 7748 211 */ 212 dst[55 + offset] &= 0xFC; 213 dst[0 + offset] |= 0x80; 214 } 215 216 ret = memcmp(u + offset, p + offset, dsize); 217 if (ret >= 0) { 218 EMSG("u >= p"); 219 return HISI_QM_DRVCRYPT_EINVAL; 220 } 221 222 return HISI_QM_DRVCRYPT_NO_ERR; 223 } 224 225 static enum hisi_drv_status 226 hpre_montgomery_params_fill(const struct hpre_mgm_curve *curve, 227 struct hpre_montgomery_msg *msg, 228 uint8_t *privkey, uint8_t *pubkey) 229 { 230 uint8_t *p = msg->key; 231 uint8_t *a = p + msg->key_bytes; 232 uint8_t *k = a + msg->key_bytes; 233 uint8_t *x = msg->in; 234 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; 235 236 memcpy(p, curve->p, msg->curve_bytes); 237 memcpy(a, curve->a, msg->curve_bytes); 238 memcpy(k, privkey, msg->curve_bytes); 239 msg->x_bytes = msg->curve_bytes; 240 if (!pubkey) 241 memcpy(x, curve->x, msg->x_bytes); 242 else 243 memcpy(x, pubkey, msg->x_bytes); 244 245 ret = hpre_bin_from_crypto_bin(p, p, msg->key_bytes, msg->curve_bytes); 246 if (ret) { 247 EMSG("Fail to transfer montgomery p from crypto_bin to hpre_bin"); 248 return ret; 249 } 250 251 ret = hpre_bin_from_crypto_bin(a, a, msg->key_bytes, msg->curve_bytes); 252 if (ret) { 253 EMSG("Fail to transfer montgomery a from crypto_bin to hpre_bin"); 254 return ret; 255 } 256 257 ret = hpre_bin_from_crypto_bin(k, k, msg->key_bytes, msg->curve_bytes); 258 if (ret) { 259 EMSG("Fail to transfer montgomery k from crypto_bin to hpre_bin"); 260 return ret; 261 } 262 263 ret = hpre_bin_from_crypto_bin(x, x, msg->key_bytes, msg->x_bytes); 264 if (ret) { 265 EMSG("Fail to transfer montgomery x from crypto_bin to hpre_bin"); 266 return ret; 267 } 268 269 return hpre_montgomery_params_pretreatment(msg); 270 } 271 272 static TEE_Result 273 hpre_montgomery_request_init(const struct hpre_mgm_curve *curve, 274 struct hpre_montgomery_msg *msg, 275 uint8_t *privkey, 276 uint8_t *pubkey) 277 { 278 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; 279 280 msg->alg_type = HPRE_ALG_X_DH_MULTIPLY; 281 msg->curve_bytes = BITS_TO_BYTES(curve->key_bits); 282 283 if (curve->key_bits == X25519_KEY_BITS) { 284 msg->key_bytes = BITS_TO_BYTES(HPRE_HW_X25519_KBITS); 285 } else if (curve->key_bits == X448_KEY_BITS) { 286 msg->key_bytes = BITS_TO_BYTES(HPRE_HW_X448_KBITS); 287 } else { 288 EMSG("Curve key bits param error"); 289 return TEE_ERROR_BAD_PARAMETERS; 290 } 291 292 ret = hpre_montgomery_params_alloc(msg); 293 if (ret) 294 return TEE_ERROR_OUT_OF_MEMORY; 295 296 ret = hpre_montgomery_params_fill(curve, msg, privkey, pubkey); 297 if (ret) { 298 hpre_montgomery_params_free(msg); 299 return TEE_ERROR_BAD_STATE; 300 } 301 302 return TEE_SUCCESS; 303 } 304 305 static void hpre_montgomery_request_deinit(struct hpre_montgomery_msg *msg) 306 { 307 hpre_montgomery_params_free(msg); 308 } 309 310 static enum hisi_drv_status hpre_montgomery_fill_sqe(void *bd, void *info) 311 { 312 struct hpre_montgomery_msg *msg = info; 313 struct hpre_sqe *sqe = bd; 314 315 sqe->w0 = msg->alg_type | SHIFT_U32(0x1, HPRE_DONE_SHIFT); 316 sqe->task_len1 = TASK_LENGTH(msg->key_bytes); 317 sqe->key = msg->key_dma; 318 sqe->in = msg->in_dma; 319 sqe->out = msg->out_dma; 320 321 return HISI_QM_DRVCRYPT_NO_ERR; 322 } 323 324 static enum hisi_drv_status hpre_montgomery_parse_sqe(void *bd, void *info) 325 { 326 struct hpre_montgomery_msg *msg = info; 327 struct hpre_sqe *sqe = bd; 328 uint8_t *rx = msg->out; 329 uint16_t err = 0; 330 uint16_t err1 = 0; 331 uint16_t done = 0; 332 333 err = HPRE_TASK_ETYPE(sqe->w0); 334 err1 = HPRE_TASK_ETYPE1(sqe->w0); 335 done = HPRE_TASK_DONE(sqe->w0); 336 if (done != HPRE_HW_TASK_DONE || err || err1) { 337 EMSG("HPRE do x_dh fail! done=0x%"PRIX16", etype=0x%"PRIX16",etype1=0x%"PRIX16, 338 done, err, err1); 339 if (done == HPRE_HW_TASK_INIT) { 340 msg->result = HISI_QM_DRVCRYPT_ENOPROC; 341 return HISI_QM_DRVCRYPT_ENOPROC; 342 } 343 344 msg->result = HISI_QM_DRVCRYPT_IN_EPARA; 345 return HISI_QM_DRVCRYPT_IN_EPARA; 346 } 347 348 if (hpre_bin_to_crypto_bin(rx, rx, msg->key_bytes, msg->curve_bytes)) { 349 EMSG("Fail to transfer x_dh out from hpre_bin to crypto_bin"); 350 msg->result = HISI_QM_DRVCRYPT_EINVAL; 351 return HISI_QM_DRVCRYPT_EINVAL; 352 } 353 354 return HISI_QM_DRVCRYPT_NO_ERR; 355 } 356 357 static TEE_Result hpre_montgomery_do_task(struct hpre_montgomery_msg *msg) 358 { 359 struct hisi_qp *montgomery_qp = NULL; 360 TEE_Result res = TEE_SUCCESS; 361 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; 362 363 montgomery_qp = hpre_create_qp(HISI_QM_CHANNEL_TYPE1); 364 if (!montgomery_qp) { 365 EMSG("Fail to create montgomery qp"); 366 return TEE_ERROR_BUSY; 367 } 368 369 montgomery_qp->fill_sqe = hpre_montgomery_fill_sqe; 370 montgomery_qp->parse_sqe = hpre_montgomery_parse_sqe; 371 ret = hisi_qp_send(montgomery_qp, msg); 372 if (ret) { 373 EMSG("Fail to send task, ret=%d", ret); 374 res = TEE_ERROR_BAD_STATE; 375 goto done; 376 } 377 378 ret = hisi_qp_recv_sync(montgomery_qp, msg); 379 if (ret) { 380 EMSG("Recv task error, ret=%d", ret); 381 res = TEE_ERROR_BAD_STATE; 382 } 383 384 done: 385 hisi_qm_release_qp(montgomery_qp); 386 387 return res; 388 } 389 390 static TEE_Result hpre_montgomery_gen_keypair(struct montgomery_keypair *key, 391 size_t size_bits) 392 { 393 struct hpre_montgomery_msg msg = { }; 394 const struct hpre_mgm_curve *curve = NULL; 395 TEE_Result ret = TEE_SUCCESS; 396 397 if (!key || !key->priv || !key->pub) { 398 EMSG("Invalid montgomery_gen_keypair input parameters"); 399 return TEE_ERROR_BAD_PARAMETERS; 400 } 401 402 if (size_bits == X25519_KEY_BITS) 403 curve = &g_curve_list[X25519_CURVE_INDEX]; 404 else if (size_bits == X448_KEY_BITS) 405 curve = &g_curve_list[X448_CURVE_INDEX]; 406 else 407 return TEE_ERROR_BAD_PARAMETERS; 408 409 ret = gen_random_privkey(key->priv, size_bits); 410 if (ret) { 411 EMSG("Fail to gen privkey"); 412 return ret; 413 } 414 415 ret = hpre_montgomery_request_init(curve, &msg, key->priv, NULL); 416 if (ret) { 417 EMSG("Fail to init montgomery key pair"); 418 return ret; 419 } 420 421 ret = hpre_montgomery_do_task(&msg); 422 if (ret) { 423 EMSG("Fail to do montgomery key pair task ret = 0x%"PRIX32, ret); 424 goto done; 425 } 426 memcpy(key->pub, msg.out, msg.curve_bytes); 427 428 done: 429 hpre_montgomery_request_deinit(&msg); 430 431 return ret; 432 } 433 434 static TEE_Result 435 hpre_montgomery_do_shared_secret(struct drvcrypt_secret_data *sdata) 436 { 437 struct hpre_montgomery_msg msg = { }; 438 const struct hpre_mgm_curve *curve = NULL; 439 struct montgomery_keypair *key = NULL; 440 uint8_t *pubkey = NULL; 441 TEE_Result ret = TEE_SUCCESS; 442 443 if (!sdata || !sdata->key_priv || !sdata->key_pub) { 444 EMSG("Invalid montgomery_do_shared_secret input parameters"); 445 return TEE_ERROR_BAD_PARAMETERS; 446 } 447 448 key = sdata->key_priv; 449 pubkey = sdata->key_pub; 450 if (sdata->size_sec == BITS_TO_BYTES(X25519_KEY_BITS)) 451 curve = &g_curve_list[X25519_CURVE_INDEX]; 452 else if (sdata->size_sec == BITS_TO_BYTES(X448_KEY_BITS)) 453 curve = &g_curve_list[X448_CURVE_INDEX]; 454 else 455 return TEE_ERROR_BAD_PARAMETERS; 456 457 ret = hpre_montgomery_request_init(curve, &msg, key->priv, pubkey); 458 if (ret) { 459 EMSG("Fail to init montgomery shared secret"); 460 return ret; 461 } 462 463 ret = hpre_montgomery_do_task(&msg); 464 if (ret) { 465 EMSG("Fail to do montgomery shared secret task! ret = 0x%"PRIX32, 466 ret); 467 goto done; 468 } 469 memcpy(sdata->secret.data, msg.out, msg.curve_bytes); 470 sdata->secret.length = msg.curve_bytes; 471 memzero_explicit(msg.out, msg.curve_bytes); 472 473 done: 474 hpre_montgomery_request_deinit(&msg); 475 476 return ret; 477 } 478 479 static struct drvcrypt_montgomery driver_x25519 = { 480 .alloc_keypair = hpre_montgomery_alloc_keypair, 481 .gen_keypair = hpre_montgomery_gen_keypair, 482 .shared_secret = hpre_montgomery_do_shared_secret, 483 }; 484 485 static struct drvcrypt_montgomery driver_x448 = { 486 .alloc_keypair = hpre_montgomery_alloc_keypair, 487 .gen_keypair = hpre_montgomery_gen_keypair, 488 .shared_secret = hpre_montgomery_do_shared_secret, 489 }; 490 491 static TEE_Result hpre_montgomery_init(void) 492 { 493 TEE_Result ret = TEE_SUCCESS; 494 495 ret = drvcrypt_register_x25519(&driver_x25519); 496 if (ret != TEE_SUCCESS) { 497 EMSG("Hpre x25519 register to crypto fail"); 498 return ret; 499 } 500 501 ret = drvcrypt_register_x448(&driver_x448); 502 if (ret != TEE_SUCCESS) { 503 EMSG("Hpre x448 register to crypto fail"); 504 return ret; 505 } 506 507 return ret; 508 } 509 510 driver_init(hpre_montgomery_init); 511