1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright (c) 2022-2024, HiSilicon Technologies Co., Ltd.
4 * Kunpeng hardware accelerator hpre montgomery algorithm implementation.
5 */
6 #include <drvcrypt.h>
7 #include <drvcrypt_acipher.h>
8 #include <initcall.h>
9 #include <malloc.h>
10 #include <rng_support.h>
11 #include <stdlib_ext.h>
12 #include <string.h>
13 #include <string_ext.h>
14 #include <trace.h>
15
16 #include "hpre_main.h"
17 #include "hpre_montgomery.h"
18
19 #define X25519_CURVE_INDEX 0
20 #define X448_CURVE_INDEX 1
21
22 struct hpre_mgm_curve {
23 uint32_t key_bits;
24 const uint8_t *p;
25 const uint8_t *a;
26 const uint8_t *x;
27 };
28
29 /* NID_X25519 */
30 /* p = (2 ^ 255 - 19) big endian */
31 static const uint8_t g_x25519_p[] = {
32 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
33 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
34 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xED
35 };
36
37 /* a = (486662 - 2) / 4 = 121665 big endian */
38 static const uint8_t g_x25519_a[] = {
39 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
40 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
41 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xDB, 0x41
42 };
43
44 /* big endian */
45 static const uint8_t g_x25519_gx[] = {
46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
48 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09
49 };
50
51 /* NID_X448 */
52 /* p = (2 ^ 448 - 2 ^ 224 - 1) big endian */
53 static const uint8_t g_x448_p[] = {
54 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
55 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
56 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
57 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
58 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
59 };
60
61 /* a = (156326 - 2) / 4 = 39081 big endian */
62 static const uint8_t g_x448_a[] = {
63 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
64 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
65 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
66 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
67 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0xA9
68 };
69
70 /* big endian */
71 static const uint8_t g_x448_gx[] = {
72 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
73 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
74 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
75 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
76 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05
77 };
78
79 static const struct hpre_mgm_curve g_curve_list[] = {
80 {
81 .key_bits = 256,
82 .p = g_x25519_p,
83 .a = g_x25519_a,
84 .x = g_x25519_gx,
85 }, {
86 .key_bits = 448,
87 .p = g_x448_p,
88 .a = g_x448_a,
89 .x = g_x448_gx,
90 }
91 };
92
93 static TEE_Result
hpre_montgomery_alloc_keypair(struct montgomery_keypair * key,size_t size_bits)94 hpre_montgomery_alloc_keypair(struct montgomery_keypair *key,
95 size_t size_bits)
96 {
97 size_t key_size = BITS_TO_BYTES(size_bits);
98
99 if (!key || (size_bits != X25519_KEY_BITS &&
100 size_bits != X448_KEY_BITS)) {
101 EMSG("Invalid input parameter");
102 return TEE_ERROR_BAD_PARAMETERS;
103 }
104
105 key->priv = calloc(1, key_size);
106 if (!key->priv)
107 goto priv_err;
108
109 key->pub = calloc(1, key_size);
110 if (!key->pub)
111 goto pub_err;
112
113 return TEE_SUCCESS;
114 pub_err:
115 free(key->priv);
116 key->priv = NULL;
117 priv_err:
118 EMSG("HPRE montgomery alloc key pair fail");
119
120 return TEE_ERROR_OUT_OF_MEMORY;
121 }
122
gen_random_privkey(uint8_t * priv,size_t key_bits)123 static TEE_Result gen_random_privkey(uint8_t *priv, size_t key_bits)
124 {
125 size_t key_size = BITS_TO_BYTES(key_bits);
126 TEE_Result ret = TEE_SUCCESS;
127
128 if (!priv) {
129 EMSG("Privkey param is NULL");
130 return TEE_ERROR_BAD_PARAMETERS;
131 }
132
133 ret = hw_get_random_bytes(priv, key_size);
134 if (ret) {
135 EMSG("Fail to fill privkey");
136 return TEE_ERROR_NO_DATA;
137 }
138
139 return ret;
140 }
141
142 static enum hisi_drv_status
hpre_montgomery_params_alloc(struct hpre_montgomery_msg * msg)143 hpre_montgomery_params_alloc(struct hpre_montgomery_msg *msg)
144 {
145 uint32_t size = HPRE_MONTGOMERY_TOTAL_BUF_SIZE(msg->key_bytes);
146
147 msg->key = calloc(1, size);
148 if (!msg->key) {
149 EMSG("Fail to alloc montgomery key buf");
150 return HISI_QM_DRVCRYPT_ENOMEM;
151 }
152
153 msg->key_dma = virt_to_phys(msg->key);
154 msg->in = msg->key + HPRE_X_KEY_SIZE(msg->key_bytes);
155 msg->in_dma = msg->key_dma + HPRE_X_KEY_SIZE(msg->key_bytes);
156 msg->out = msg->in + msg->key_bytes;
157 msg->out_dma = msg->in_dma + msg->key_bytes;
158
159 return HISI_QM_DRVCRYPT_NO_ERR;
160 }
161
hpre_montgomery_params_free(struct hpre_montgomery_msg * msg)162 static void hpre_montgomery_params_free(struct hpre_montgomery_msg *msg)
163 {
164 if (msg->key) {
165 memzero_explicit(msg->key, HPRE_X_KEY_SIZE(msg->key_bytes));
166 free(msg->key);
167 msg->key = NULL;
168 }
169 }
170
171 static enum hisi_drv_status
hpre_montgomery_params_pretreatment(struct hpre_montgomery_msg * msg)172 hpre_montgomery_params_pretreatment(struct hpre_montgomery_msg *msg)
173 {
174 uint8_t *p = msg->key;
175 uint8_t *a = p + msg->key_bytes;
176 uint8_t *k = a + msg->key_bytes;
177 uint8_t *u = msg->in;
178 uint8_t *dst = k;
179 uint32_t bsize = msg->key_bytes;
180 uint32_t dsize = msg->curve_bytes;
181 /*
182 * It is a constraint of HPRE hardware that key_bytes will be set
183 * to 72 when curve_bytes is between 48 and 72, and the high-order
184 * bits will be set to 0.
185 */
186 uint32_t offset = bsize - dsize;
187 int ret = 0;
188
189 /*
190 * This is a pretreatment of X25519 with a 32-byte integer,
191 * as described in RFC 7748:
192 * Set the three LSB of the first byte and MSB of the last
193 * to zero, set the second MSB of the last byte to 1.
194 * When receiving u-array, set MSB of last byte to zero.
195 * HPRE hardware module uses big-endian mode, so the bytes to be
196 * set are reversed compared to RFC 7748
197 */
198 if (msg->key_bytes == BITS_TO_BYTES(X25519_KEY_BITS)) {
199 dst[31] &= 0xF8;
200 dst[0] &= 0x7F;
201 dst[0] |= 0x40;
202 u[0] &= 0x7F;
203 } else {
204 /*
205 * This is a pretreatment of X448 with a 56-byte integer,
206 * as described in RFC 7748:
207 * For X448, set the two LSB of the first byte to 0, and MSB of the
208 * last byte to 1.
209 * HPRE hardware module uses big-endian mode, so the bytes to be
210 * set are reversed compared to RFC 7748
211 */
212 dst[55 + offset] &= 0xFC;
213 dst[0 + offset] |= 0x80;
214 }
215
216 ret = memcmp(u + offset, p + offset, dsize);
217 if (ret >= 0) {
218 EMSG("u >= p");
219 return HISI_QM_DRVCRYPT_EINVAL;
220 }
221
222 return HISI_QM_DRVCRYPT_NO_ERR;
223 }
224
225 static enum hisi_drv_status
hpre_montgomery_params_fill(const struct hpre_mgm_curve * curve,struct hpre_montgomery_msg * msg,uint8_t * privkey,uint8_t * pubkey)226 hpre_montgomery_params_fill(const struct hpre_mgm_curve *curve,
227 struct hpre_montgomery_msg *msg,
228 uint8_t *privkey, uint8_t *pubkey)
229 {
230 uint8_t *p = msg->key;
231 uint8_t *a = p + msg->key_bytes;
232 uint8_t *k = a + msg->key_bytes;
233 uint8_t *x = msg->in;
234 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
235
236 memcpy(p, curve->p, msg->curve_bytes);
237 memcpy(a, curve->a, msg->curve_bytes);
238 memcpy(k, privkey, msg->curve_bytes);
239 msg->x_bytes = msg->curve_bytes;
240 if (!pubkey)
241 memcpy(x, curve->x, msg->x_bytes);
242 else
243 memcpy(x, pubkey, msg->x_bytes);
244
245 ret = hpre_bin_from_crypto_bin(p, p, msg->key_bytes, msg->curve_bytes);
246 if (ret) {
247 EMSG("Fail to transfer montgomery p from crypto_bin to hpre_bin");
248 return ret;
249 }
250
251 ret = hpre_bin_from_crypto_bin(a, a, msg->key_bytes, msg->curve_bytes);
252 if (ret) {
253 EMSG("Fail to transfer montgomery a from crypto_bin to hpre_bin");
254 return ret;
255 }
256
257 ret = hpre_bin_from_crypto_bin(k, k, msg->key_bytes, msg->curve_bytes);
258 if (ret) {
259 EMSG("Fail to transfer montgomery k from crypto_bin to hpre_bin");
260 return ret;
261 }
262
263 ret = hpre_bin_from_crypto_bin(x, x, msg->key_bytes, msg->x_bytes);
264 if (ret) {
265 EMSG("Fail to transfer montgomery x from crypto_bin to hpre_bin");
266 return ret;
267 }
268
269 return hpre_montgomery_params_pretreatment(msg);
270 }
271
272 static TEE_Result
hpre_montgomery_request_init(const struct hpre_mgm_curve * curve,struct hpre_montgomery_msg * msg,uint8_t * privkey,uint8_t * pubkey)273 hpre_montgomery_request_init(const struct hpre_mgm_curve *curve,
274 struct hpre_montgomery_msg *msg,
275 uint8_t *privkey,
276 uint8_t *pubkey)
277 {
278 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
279
280 msg->alg_type = HPRE_ALG_X_DH_MULTIPLY;
281 msg->curve_bytes = BITS_TO_BYTES(curve->key_bits);
282
283 if (curve->key_bits == X25519_KEY_BITS) {
284 msg->key_bytes = BITS_TO_BYTES(HPRE_HW_X25519_KBITS);
285 } else if (curve->key_bits == X448_KEY_BITS) {
286 msg->key_bytes = BITS_TO_BYTES(HPRE_HW_X448_KBITS);
287 } else {
288 EMSG("Curve key bits param error");
289 return TEE_ERROR_BAD_PARAMETERS;
290 }
291
292 ret = hpre_montgomery_params_alloc(msg);
293 if (ret)
294 return TEE_ERROR_OUT_OF_MEMORY;
295
296 ret = hpre_montgomery_params_fill(curve, msg, privkey, pubkey);
297 if (ret) {
298 hpre_montgomery_params_free(msg);
299 return TEE_ERROR_BAD_STATE;
300 }
301
302 return TEE_SUCCESS;
303 }
304
hpre_montgomery_request_deinit(struct hpre_montgomery_msg * msg)305 static void hpre_montgomery_request_deinit(struct hpre_montgomery_msg *msg)
306 {
307 hpre_montgomery_params_free(msg);
308 }
309
hpre_montgomery_fill_sqe(void * bd,void * info)310 static enum hisi_drv_status hpre_montgomery_fill_sqe(void *bd, void *info)
311 {
312 struct hpre_montgomery_msg *msg = info;
313 struct hpre_sqe *sqe = bd;
314
315 sqe->w0 = msg->alg_type | SHIFT_U32(0x1, HPRE_DONE_SHIFT);
316 sqe->task_len1 = TASK_LENGTH(msg->key_bytes);
317 sqe->key = msg->key_dma;
318 sqe->in = msg->in_dma;
319 sqe->out = msg->out_dma;
320
321 return HISI_QM_DRVCRYPT_NO_ERR;
322 }
323
hpre_montgomery_parse_sqe(void * bd,void * info)324 static enum hisi_drv_status hpre_montgomery_parse_sqe(void *bd, void *info)
325 {
326 struct hpre_montgomery_msg *msg = info;
327 struct hpre_sqe *sqe = bd;
328 uint8_t *rx = msg->out;
329 uint16_t err = 0;
330 uint16_t err1 = 0;
331 uint16_t done = 0;
332
333 err = HPRE_TASK_ETYPE(sqe->w0);
334 err1 = HPRE_TASK_ETYPE1(sqe->w0);
335 done = HPRE_TASK_DONE(sqe->w0);
336 if (done != HPRE_HW_TASK_DONE || err || err1) {
337 EMSG("HPRE do x_dh fail! done=0x%"PRIX16", etype=0x%"PRIX16",etype1=0x%"PRIX16,
338 done, err, err1);
339 if (done == HPRE_HW_TASK_INIT) {
340 msg->result = HISI_QM_DRVCRYPT_ENOPROC;
341 return HISI_QM_DRVCRYPT_ENOPROC;
342 }
343
344 msg->result = HISI_QM_DRVCRYPT_IN_EPARA;
345 return HISI_QM_DRVCRYPT_IN_EPARA;
346 }
347
348 if (hpre_bin_to_crypto_bin(rx, rx, msg->key_bytes, msg->curve_bytes)) {
349 EMSG("Fail to transfer x_dh out from hpre_bin to crypto_bin");
350 msg->result = HISI_QM_DRVCRYPT_EINVAL;
351 return HISI_QM_DRVCRYPT_EINVAL;
352 }
353
354 return HISI_QM_DRVCRYPT_NO_ERR;
355 }
356
hpre_montgomery_do_task(struct hpre_montgomery_msg * msg)357 static TEE_Result hpre_montgomery_do_task(struct hpre_montgomery_msg *msg)
358 {
359 struct hisi_qp *montgomery_qp = NULL;
360 TEE_Result res = TEE_SUCCESS;
361 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR;
362
363 montgomery_qp = hpre_create_qp(HISI_QM_CHANNEL_TYPE1);
364 if (!montgomery_qp) {
365 EMSG("Fail to create montgomery qp");
366 return TEE_ERROR_BUSY;
367 }
368
369 montgomery_qp->fill_sqe = hpre_montgomery_fill_sqe;
370 montgomery_qp->parse_sqe = hpre_montgomery_parse_sqe;
371 ret = hisi_qp_send(montgomery_qp, msg);
372 if (ret) {
373 EMSG("Fail to send task, ret=%d", ret);
374 res = TEE_ERROR_BAD_STATE;
375 goto done;
376 }
377
378 ret = hisi_qp_recv_sync(montgomery_qp, msg);
379 if (ret) {
380 EMSG("Recv task error, ret=%d", ret);
381 res = TEE_ERROR_BAD_STATE;
382 }
383
384 done:
385 hisi_qm_release_qp(montgomery_qp);
386
387 return res;
388 }
389
hpre_montgomery_gen_keypair(struct montgomery_keypair * key,size_t size_bits)390 static TEE_Result hpre_montgomery_gen_keypair(struct montgomery_keypair *key,
391 size_t size_bits)
392 {
393 struct hpre_montgomery_msg msg = { };
394 const struct hpre_mgm_curve *curve = NULL;
395 TEE_Result ret = TEE_SUCCESS;
396
397 if (!key || !key->priv || !key->pub) {
398 EMSG("Invalid montgomery_gen_keypair input parameters");
399 return TEE_ERROR_BAD_PARAMETERS;
400 }
401
402 if (size_bits == X25519_KEY_BITS)
403 curve = &g_curve_list[X25519_CURVE_INDEX];
404 else if (size_bits == X448_KEY_BITS)
405 curve = &g_curve_list[X448_CURVE_INDEX];
406 else
407 return TEE_ERROR_BAD_PARAMETERS;
408
409 ret = gen_random_privkey(key->priv, size_bits);
410 if (ret) {
411 EMSG("Fail to gen privkey");
412 return ret;
413 }
414
415 ret = hpre_montgomery_request_init(curve, &msg, key->priv, NULL);
416 if (ret) {
417 EMSG("Fail to init montgomery key pair");
418 return ret;
419 }
420
421 ret = hpre_montgomery_do_task(&msg);
422 if (ret) {
423 EMSG("Fail to do montgomery key pair task ret = 0x%"PRIX32, ret);
424 goto done;
425 }
426 memcpy(key->pub, msg.out, msg.curve_bytes);
427
428 done:
429 hpre_montgomery_request_deinit(&msg);
430
431 return ret;
432 }
433
434 static TEE_Result
hpre_montgomery_do_shared_secret(struct drvcrypt_secret_data * sdata)435 hpre_montgomery_do_shared_secret(struct drvcrypt_secret_data *sdata)
436 {
437 struct hpre_montgomery_msg msg = { };
438 const struct hpre_mgm_curve *curve = NULL;
439 struct montgomery_keypair *key = NULL;
440 uint8_t *pubkey = NULL;
441 TEE_Result ret = TEE_SUCCESS;
442
443 if (!sdata || !sdata->key_priv || !sdata->key_pub) {
444 EMSG("Invalid montgomery_do_shared_secret input parameters");
445 return TEE_ERROR_BAD_PARAMETERS;
446 }
447
448 key = sdata->key_priv;
449 pubkey = sdata->key_pub;
450 if (sdata->size_sec == BITS_TO_BYTES(X25519_KEY_BITS))
451 curve = &g_curve_list[X25519_CURVE_INDEX];
452 else if (sdata->size_sec == BITS_TO_BYTES(X448_KEY_BITS))
453 curve = &g_curve_list[X448_CURVE_INDEX];
454 else
455 return TEE_ERROR_BAD_PARAMETERS;
456
457 ret = hpre_montgomery_request_init(curve, &msg, key->priv, pubkey);
458 if (ret) {
459 EMSG("Fail to init montgomery shared secret");
460 return ret;
461 }
462
463 ret = hpre_montgomery_do_task(&msg);
464 if (ret) {
465 EMSG("Fail to do montgomery shared secret task! ret = 0x%"PRIX32,
466 ret);
467 goto done;
468 }
469 memcpy(sdata->secret.data, msg.out, msg.curve_bytes);
470 sdata->secret.length = msg.curve_bytes;
471 memzero_explicit(msg.out, msg.curve_bytes);
472
473 done:
474 hpre_montgomery_request_deinit(&msg);
475
476 return ret;
477 }
478
479 static struct drvcrypt_montgomery driver_x25519 = {
480 .alloc_keypair = hpre_montgomery_alloc_keypair,
481 .gen_keypair = hpre_montgomery_gen_keypair,
482 .shared_secret = hpre_montgomery_do_shared_secret,
483 };
484
485 static struct drvcrypt_montgomery driver_x448 = {
486 .alloc_keypair = hpre_montgomery_alloc_keypair,
487 .gen_keypair = hpre_montgomery_gen_keypair,
488 .shared_secret = hpre_montgomery_do_shared_secret,
489 };
490
hpre_montgomery_init(void)491 static TEE_Result hpre_montgomery_init(void)
492 {
493 TEE_Result ret = TEE_SUCCESS;
494
495 ret = drvcrypt_register_x25519(&driver_x25519);
496 if (ret != TEE_SUCCESS) {
497 EMSG("Hpre x25519 register to crypto fail");
498 return ret;
499 }
500
501 ret = drvcrypt_register_x448(&driver_x448);
502 if (ret != TEE_SUCCESS) {
503 EMSG("Hpre x448 register to crypto fail");
504 return ret;
505 }
506
507 return ret;
508 }
509
510 driver_init(hpre_montgomery_init);
511