1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright (C) Foundries Ltd. 2022.
4 * Author: Jorge Ramirez <jorge@foundries.io>
5 */
6
7 #include <assert.h>
8 #include <crypto/crypto.h>
9 #include <crypto/crypto_impl.h>
10 #include <crypto/internal_aes-gcm.h>
11 #include <drvcrypt.h>
12 #include <drvcrypt_authenc.h>
13 #include <initcall.h>
14 #include <ipi.h>
15 #include <kernel/boot.h>
16 #include <kernel/dt.h>
17 #include <kernel/panic.h>
18 #include <kernel/spinlock.h>
19 #include <libfdt.h>
20 #include <mm/core_memprot.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <string_ext.h>
24 #include <tee_api_types.h>
25 #include <tee/cache.h>
26 #include <utee_defines.h>
27 #include <util.h>
28
29 /*
30 * This driver does not queue/pad non-aligned data.
31 *
32 * Allow debug information for future PLM work: if the PLM can not implement
33 * the required changes, we might be able to do it in OP-TEE.
34 */
35 #define DEBUG_VERSAL_AES 0
36
37 #define GCM_TAG_LEN 16
38
39 #define XSECURE_AES_KEY_SIZE_128 0 /* Key Length = 32 bytes = 256 bits */
40 #define XSECURE_AES_KEY_SIZE_256 2 /* Key Length = 16 bytes = 128 bits */
41
42 #define XSECURE_ENCRYPT 0
43 #define XSECURE_DECRYPT 1
44
45 enum versal_aes_err {
46 AES_GCM_TAG_MISMATCH = 0x40,
47 AES_KEY_CLEAR_ERROR,
48 AES_DPA_CM_NOT_SUPPORTED,
49 AES_KAT_WRITE_KEY_FAILED_ERROR,
50 AES_KAT_DECRYPT_INIT_FAILED_ERROR,
51 AES_KAT_GCM_TAG_MISMATCH_ERROR,
52 AES_KAT_DATA_MISMATCH_ERROR,
53 AES_KAT_FAILED_ERROR,
54 AESDPACM_KAT_WRITE_KEY_FAILED_ERROR,
55 AESDPACM_KAT_KEYLOAD_FAILED_ERROR,
56 AESDPACM_SSS_CFG_FAILED_ERROR,
57 AESDPACM_KAT_FAILED_ERROR,
58 AESDPACM_KAT_CHECK1_FAILED_ERROR,
59 AESDPACM_KAT_CHECK2_FAILED_ERROR,
60 AESDPACM_KAT_CHECK3_FAILED_ERROR,
61 AESDPACM_KAT_CHECK4_FAILED_ERROR,
62 AESDPACM_KAT_CHECK5_FAILED_ERROR,
63 AES_INVALID_PARAM,
64 AESKAT_INVALID_PARAM,
65 AES_STATE_MISMATCH_ERROR,
66 AES_DEVICE_KEY_NOT_ALLOWED,
67 };
68
69 #define VERSAL_AES_ERROR(m) { . error = (m), .name = TO_STR(m) }
70
versal_aes_error(uint8_t err)71 static const char *versal_aes_error(uint8_t err)
72 {
73 const struct {
74 enum versal_aes_err error;
75 const char *name;
76 } elist[] = {
77 VERSAL_AES_ERROR(AES_GCM_TAG_MISMATCH),
78 VERSAL_AES_ERROR(AES_KEY_CLEAR_ERROR),
79 VERSAL_AES_ERROR(AES_DPA_CM_NOT_SUPPORTED),
80 VERSAL_AES_ERROR(AES_KAT_WRITE_KEY_FAILED_ERROR),
81 VERSAL_AES_ERROR(AES_KAT_DECRYPT_INIT_FAILED_ERROR),
82 VERSAL_AES_ERROR(AES_KAT_GCM_TAG_MISMATCH_ERROR),
83 VERSAL_AES_ERROR(AES_KAT_DATA_MISMATCH_ERROR),
84 VERSAL_AES_ERROR(AES_KAT_FAILED_ERROR),
85 VERSAL_AES_ERROR(AESDPACM_KAT_WRITE_KEY_FAILED_ERROR),
86 VERSAL_AES_ERROR(AESDPACM_KAT_KEYLOAD_FAILED_ERROR),
87 VERSAL_AES_ERROR(AESDPACM_SSS_CFG_FAILED_ERROR),
88 VERSAL_AES_ERROR(AESDPACM_KAT_FAILED_ERROR),
89 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK1_FAILED_ERROR),
90 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK2_FAILED_ERROR),
91 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK3_FAILED_ERROR),
92 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK4_FAILED_ERROR),
93 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK5_FAILED_ERROR),
94 VERSAL_AES_ERROR(AES_INVALID_PARAM),
95 VERSAL_AES_ERROR(AESKAT_INVALID_PARAM),
96 VERSAL_AES_ERROR(AES_STATE_MISMATCH_ERROR),
97 VERSAL_AES_ERROR(AES_DEVICE_KEY_NOT_ALLOWED),
98 };
99
100 if (err >= AES_GCM_TAG_MISMATCH && err <= AES_DEVICE_KEY_NOT_ALLOWED) {
101 if (elist[err - AES_GCM_TAG_MISMATCH].name)
102 return elist[err - AES_GCM_TAG_MISMATCH].name;
103
104 return "Invalid";
105 }
106
107 return "Unknown";
108 }
109
110 enum aes_key_src {
111 XSECURE_AES_BBRAM_KEY = 0, /* BBRAM Key */
112 XSECURE_AES_BBRAM_RED_KEY, /* BBRAM Red Key */
113 XSECURE_AES_BH_KEY, /* BH Key */
114 XSECURE_AES_BH_RED_KEY, /* BH Red Key */
115 XSECURE_AES_EFUSE_KEY, /* eFUSE Key */
116 XSECURE_AES_EFUSE_RED_KEY, /* eFUSE Red Key */
117 XSECURE_AES_EFUSE_USER_KEY_0, /* eFUSE User Key 0 */
118 XSECURE_AES_EFUSE_USER_KEY_1, /* eFUSE User Key 1 */
119 XSECURE_AES_EFUSE_USER_RED_KEY_0, /* eFUSE User Red Key 0 */
120 XSECURE_AES_EFUSE_USER_RED_KEY_1, /* eFUSE User Red Key 1 */
121 XSECURE_AES_KUP_KEY, /* KUP key */
122 XSECURE_AES_PUF_KEY, /* PUF key */
123 XSECURE_AES_USER_KEY_0, /* User Key 0 */
124 XSECURE_AES_USER_KEY_1, /* User Key 1 */
125 XSECURE_AES_USER_KEY_2, /* User Key 2 */
126 XSECURE_AES_USER_KEY_3, /* User Key 3 */
127 XSECURE_AES_USER_KEY_4, /* User Key 4 */
128 XSECURE_AES_USER_KEY_5, /* User Key 5 */
129 XSECURE_AES_USER_KEY_6, /* User Key 6 */
130 XSECURE_AES_USER_KEY_7, /* User Key 7 */
131 XSECURE_AES_EXPANDED_KEYS, /* Expanded keys */
132 XSECURE_AES_ALL_KEYS, /* AES All keys */
133 };
134
135 struct versal_payload {
136 struct versal_mbox_mem input_cmd;
137 struct versal_mbox_mem src;
138 struct versal_mbox_mem dst;
139 bool encrypt;
140 };
141
142 struct versal_aad {
143 struct versal_mbox_mem mem;
144 };
145
146 struct versal_node {
147 struct versal_payload payload;
148 struct versal_aad aad;
149 bool is_aad;
150 STAILQ_ENTRY(versal_node) link;
151 };
152
153 struct versal_init {
154 uint32_t key_len;
155 uint32_t operation;
156 struct versal_mbox_mem key;
157 struct versal_mbox_mem nonce;
158 struct versal_mbox_mem init_buf;
159 };
160
161 struct versal_ae_ctx {
162 struct crypto_authenc_ctx a_ctx;
163 };
164
165 struct versal_context_node {
166 struct versal_ae_ctx *ctx;
167 STAILQ_ENTRY(versal_context_node) link;
168 };
169
170 enum engine_state {
171 READY = 1, INIT = 2, FINALIZED = 3,
172 };
173
174 static struct mutex engine_lock = MUTEX_INITIALIZER;
175
176 static struct versal_engine {
177 enum aes_key_src key_src;
178 enum engine_state state;
179 struct versal_init init;
180 struct refcount refc;
181 struct mutex *lock; /* protect the HW instance */
182 STAILQ_HEAD(authenc_context_list, versal_context_node) context_list;
183 STAILQ_HEAD(authenc_replay_list, versal_node) replay_list;
184 } engine = {
185 .key_src = XSECURE_AES_USER_KEY_0,
186 .lock = &engine_lock,
187 };
188
to_versal_ctx(struct crypto_authenc_ctx * ctx)189 static struct versal_ae_ctx *to_versal_ctx(struct crypto_authenc_ctx *ctx)
190 {
191 assert(ctx);
192
193 return container_of(ctx, struct versal_ae_ctx, a_ctx);
194 }
195
replay_init(void)196 static TEE_Result replay_init(void)
197 {
198 struct versal_cmd_args arg = { };
199 uint32_t err = 0;
200
201 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) {
202 EMSG("AES_INIT error");
203 return TEE_ERROR_GENERIC;
204 }
205
206 arg.data[arg.dlen++] = engine.init.key_len;
207 arg.data[arg.dlen++] = engine.key_src;
208 arg.ibuf[0].mem = engine.init.key;
209
210 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) {
211 EMSG("AES_WRITE_KEY error");
212 return TEE_ERROR_GENERIC;
213 }
214
215 memset(&arg, 0, sizeof(arg));
216
217 arg.ibuf[0].mem = engine.init.init_buf;
218 arg.ibuf[1].mem = engine.init.nonce;
219 arg.ibuf[1].only_cache = true;
220
221 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) {
222 EMSG("AES_OP_INIT error");
223 return TEE_ERROR_GENERIC;
224 }
225
226 return TEE_SUCCESS;
227 }
228
replay_aad(struct versal_aad * p)229 static TEE_Result replay_aad(struct versal_aad *p)
230 {
231 struct versal_cmd_args arg = { };
232 uint32_t err = 0;
233
234 arg.data[arg.dlen++] = p->mem.len % 16 ? p->mem.alloc_len : p->mem.len;
235 arg.ibuf[0].mem = p->mem;
236
237 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) {
238 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err));
239 return TEE_ERROR_GENERIC;
240 }
241
242 return TEE_SUCCESS;
243 }
244
replay_payload(struct versal_payload * p)245 static TEE_Result replay_payload(struct versal_payload *p)
246 {
247 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE;
248 struct versal_cmd_args arg = { };
249 uint32_t err = 0;
250
251 arg.ibuf[0].mem = p->input_cmd;
252 arg.ibuf[1].mem = p->dst;
253 arg.ibuf[2].mem = p->src;
254
255 if (p->encrypt)
256 id = VERSAL_AES_ENCRYPT_UPDATE;
257
258 if (versal_crypto_request(id, &arg, &err)) {
259 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err));
260 return TEE_ERROR_GENERIC;
261 }
262
263 return TEE_SUCCESS;
264 }
265
do_replay(void)266 static TEE_Result do_replay(void)
267 {
268 struct versal_node *node = NULL;
269 TEE_Result ret = TEE_SUCCESS;
270
271 ret = replay_init();
272 if (ret)
273 return ret;
274
275 STAILQ_FOREACH(node, &engine.replay_list, link) {
276 if (node->is_aad) {
277 ret = replay_aad(&node->aad);
278 if (ret)
279 return ret;
280 } else {
281 ret = replay_payload(&node->payload);
282 if (ret)
283 return ret;
284 }
285 }
286
287 /* Engine has been init */
288 engine.state = INIT;
289
290 return TEE_SUCCESS;
291 }
292
engine_in_use(void)293 static bool engine_in_use(void)
294 {
295 if (STAILQ_EMPTY(&engine.context_list))
296 return false;
297
298 return true;
299 }
300
context_allowed(struct crypto_authenc_ctx * ctx)301 static bool context_allowed(struct crypto_authenc_ctx *ctx)
302 {
303 struct versal_context_node *node = NULL;
304
305 STAILQ_FOREACH(node, &engine.context_list, link) {
306 if (node->ctx == to_versal_ctx(ctx))
307 return true;
308 }
309
310 return false;
311 }
312
do_init(struct drvcrypt_authenc_init * dinit)313 static TEE_Result do_init(struct drvcrypt_authenc_init *dinit)
314 {
315 uint32_t key_len = XSECURE_AES_KEY_SIZE_128;
316 struct versal_context_node *node = NULL;
317 struct versal_aes_init *init = NULL;
318 struct versal_mbox_mem init_buf = { };
319 struct versal_mbox_mem key = { };
320 struct versal_mbox_mem nonce = { };
321 struct versal_cmd_args arg = { };
322 TEE_Result ret = TEE_SUCCESS;
323 uint32_t err = 0;
324
325 if (engine_in_use()) {
326 EMSG("Versal: AES-GCM Engine busy");
327 return TEE_ERROR_BUSY;
328 }
329
330 if (dinit->key.length != 32 && dinit->key.length != 16)
331 return TEE_ERROR_BAD_PARAMETERS;
332
333 if (dinit->key.length == 32)
334 key_len = XSECURE_AES_KEY_SIZE_256;
335
336 if (engine.state != READY)
337 return TEE_ERROR_BAD_STATE;
338
339 /* Initialize the AES engine */
340 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) {
341 EMSG("AES_INIT error: %s", versal_aes_error(err));
342 return TEE_ERROR_GENERIC;
343 }
344
345 /* Write the key */
346 ret = versal_mbox_alloc(dinit->key.length, dinit->key.data, &key);
347 if (ret)
348 return ret;
349
350 arg.data[arg.dlen++] = key_len;
351 arg.data[arg.dlen++] = engine.key_src;
352 arg.ibuf[0].mem = key;
353
354 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) {
355 EMSG("AES_WRITE_KEY error: %s", versal_aes_error(err));
356 ret = TEE_ERROR_GENERIC;
357 goto error;
358 }
359
360 memset(&arg, 0, sizeof(arg));
361
362 /* Send the initialization structure */
363 ret = versal_mbox_alloc(sizeof(*init), NULL, &init_buf);
364 if (ret)
365 goto error;
366 ret = versal_mbox_alloc(dinit->nonce.length, dinit->nonce.data, &nonce);
367 if (ret)
368 goto error;
369
370 init = init_buf.buf;
371 init->iv_addr = virt_to_phys(nonce.buf);
372 init->operation = dinit->encrypt ? XSECURE_ENCRYPT : XSECURE_DECRYPT;
373 init->key_src = engine.key_src;
374 init->key_len = key_len;
375
376 arg.ibuf[0].mem = init_buf;
377 arg.ibuf[1].mem = nonce;
378 arg.ibuf[1].only_cache = true;
379
380 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) {
381 EMSG("AES_OP_INIT error: %s", versal_aes_error(err));
382 ret = TEE_ERROR_GENERIC;
383 goto error;
384 }
385
386 node = calloc(1, sizeof(*node));
387 if (!node) {
388 ret = TEE_ERROR_OUT_OF_MEMORY;
389 goto error;
390 }
391
392 /* Save key context */
393 engine.init.operation = dinit->encrypt ?
394 XSECURE_ENCRYPT : XSECURE_DECRYPT;
395 engine.init.key_len = key_len;
396 engine.init.init_buf = init_buf;
397 engine.init.nonce = nonce;
398 engine.init.key = key;
399
400 /* Active context */
401 node->ctx = to_versal_ctx(dinit->ctx);
402 STAILQ_INSERT_TAIL(&engine.context_list, node, link);
403
404 /* Engine has been init*/
405 engine.state = INIT;
406
407 return TEE_SUCCESS;
408 error:
409 versal_mbox_free(&nonce);
410 versal_mbox_free(&init_buf);
411 versal_mbox_free(&key);
412
413 return ret;
414 }
415
do_update_aad(struct drvcrypt_authenc_update_aad * dupdate)416 static TEE_Result do_update_aad(struct drvcrypt_authenc_update_aad *dupdate)
417 {
418 struct versal_cmd_args arg = { };
419 struct versal_mbox_mem p = { };
420 TEE_Result ret = TEE_SUCCESS;
421 struct versal_node *node = NULL;
422 uint32_t err = 0;
423
424 /* This context has not been inited */
425 if (!context_allowed(dupdate->ctx))
426 return TEE_ERROR_BUSY;
427
428 /* There is a copy of the context: don't allow updates, only finalize */
429 if (refcount_val(&engine.refc) > 1)
430 return TEE_ERROR_BUSY;
431
432 /* There was a copy of the context and it was finalized, then replay */
433 if (engine.state == FINALIZED)
434 do_replay();
435
436 ret = versal_mbox_alloc(dupdate->aad.length, dupdate->aad.data, &p);
437 if (ret)
438 return ret;
439
440 arg.data[arg.dlen++] = p.len % 16 ? p.alloc_len : p.len;
441 arg.ibuf[0].mem = p;
442
443 #if DEBUG_VERSAL_AES
444 IMSG("versal: aad length - requested: %zu, sent to plm: %"PRIu32,
445 dupdate->aad.length, arg.data[0]);
446 #endif
447 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) {
448 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err));
449 ret = TEE_ERROR_GENERIC;
450 goto error;
451 }
452
453 node = calloc(1, sizeof(*node));
454 if (!node) {
455 ret = TEE_ERROR_OUT_OF_MEMORY;
456 goto error;
457 }
458
459 /* Save the context */
460 node->aad.mem = p;
461 node->is_aad = true;
462 STAILQ_INSERT_TAIL(&engine.replay_list, node, link);
463
464 return TEE_SUCCESS;
465 error:
466 versal_mbox_free(&p);
467 return ret;
468 }
469
470 static TEE_Result
update_payload(struct drvcrypt_authenc_update_payload * dupdate,bool is_last)471 update_payload(struct drvcrypt_authenc_update_payload *dupdate, bool is_last)
472 {
473 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE;
474 struct versal_aes_input_param *input = NULL;
475 struct versal_mbox_mem input_cmd = { };
476 struct versal_mbox_mem p = { };
477 struct versal_mbox_mem q = { };
478 TEE_Result ret = TEE_SUCCESS;
479 struct versal_cmd_args arg = { };
480 struct versal_node *node = NULL;
481 uint32_t err = 0;
482
483 if (!context_allowed(dupdate->ctx))
484 return TEE_ERROR_BUSY;
485
486 if (!dupdate->src.length) {
487 EMSG("Versal AES payload length must be non-zero");
488 return TEE_ERROR_BAD_PARAMETERS;
489 }
490 #if !defined(PLATFORM_FLAVOR_net)
491 if ((!is_last && dupdate->src.length % 4) ||
492 (is_last && dupdate->src.length % 16)) {
493 EMSG("Versal AES payload length not properly aligned (len = %zu)",
494 dupdate->src.length);
495 return TEE_ERROR_BAD_PARAMETERS;
496 }
497 #endif
498
499 ret = versal_mbox_alloc(dupdate->src.length, dupdate->src.data, &p);
500 if (ret)
501 return ret;
502 ret = versal_mbox_alloc(dupdate->dst.length, NULL, &q);
503 if (ret)
504 goto out;
505 ret = versal_mbox_alloc(sizeof(*input), NULL, &input_cmd);
506 if (ret)
507 goto out;
508
509 input = input_cmd.buf;
510 input->input_addr = virt_to_phys(p.buf);
511 input->input_len = p.len % 16 ? p.alloc_len : p.len;
512 input->is_last = is_last;
513
514 arg.ibuf[0].mem = input_cmd;
515 arg.ibuf[1].mem = q;
516 arg.ibuf[2].mem = p;
517
518 if (dupdate->encrypt)
519 id = VERSAL_AES_ENCRYPT_UPDATE;
520
521 #if DEBUG_VERSAL_AES
522 IMSG("versal: payload length - requested %zu, sent to plm: %"PRIu32,
523 dupdate->src.length, input->input_len);
524 IMSG("versal: destination length - %zu ", dupdate->dst.length);
525 #endif
526 if (versal_crypto_request(id, &arg, &err)) {
527 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err));
528 ret = TEE_ERROR_GENERIC;
529 goto out;
530 }
531
532 if (dupdate->dst.data)
533 memcpy(dupdate->dst.data, q.buf, dupdate->dst.length);
534
535 if (!is_last) {
536 node = calloc(1, sizeof(*node));
537 if (!node) {
538 ret = TEE_ERROR_OUT_OF_MEMORY;
539 goto out;
540 }
541
542 node->is_aad = false;
543 node->payload.dst = q;
544 node->payload.src = p;
545 node->payload.input_cmd = input_cmd;
546 node->payload.encrypt = dupdate->encrypt;
547 STAILQ_INSERT_TAIL(&engine.replay_list, node, link);
548
549 return TEE_SUCCESS;
550 }
551 out:
552 versal_mbox_free(&input_cmd);
553 versal_mbox_free(&q);
554 versal_mbox_free(&p);
555
556 return ret;
557 }
558
do_update_payload(struct drvcrypt_authenc_update_payload * p)559 static TEE_Result do_update_payload(struct drvcrypt_authenc_update_payload *p)
560 {
561 TEE_Result ret = TEE_SUCCESS;
562
563 if (!context_allowed(p->ctx))
564 return TEE_ERROR_BUSY;
565
566 /*
567 * If there is a copy, we don't allow updates until one of the copies
568 * has been deleted
569 */
570 if (refcount_val(&engine.refc) > 1)
571 return TEE_ERROR_BUSY;
572
573 /*
574 * If there was a copy and it was finalized, we need to replay before
575 * we can update; do not clear the list so the state can be copied
576 */
577 if (engine.state == FINALIZED) {
578 ret = do_replay();
579 if (ret)
580 return ret;
581 }
582
583 return update_payload(p, false);
584 }
585
do_enc_final(struct drvcrypt_authenc_final * dfinal)586 static TEE_Result do_enc_final(struct drvcrypt_authenc_final *dfinal)
587 {
588 struct drvcrypt_authenc_update_payload last = { };
589 struct versal_cmd_args arg = { };
590 struct versal_mbox_mem p = { };
591 TEE_Result ret = TEE_SUCCESS;
592 uint32_t err = 0;
593
594 if (!context_allowed(dfinal->ctx))
595 return TEE_ERROR_BUSY;
596
597 if (engine.state == FINALIZED) {
598 DMSG("Operation was already finalized");
599 ret = do_replay();
600 if (ret)
601 return ret;
602 }
603
604 if (engine.state != INIT)
605 panic();
606
607 last.ctx = dfinal->ctx;
608 last.dst = dfinal->dst;
609 last.encrypt = true;
610 last.src = dfinal->src;
611
612 ret = update_payload(&last, true);
613 if (ret)
614 return ret;
615
616 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length);
617
618 ret = versal_mbox_alloc(GCM_TAG_LEN, NULL, &p);
619 if (ret)
620 return ret;
621
622 arg.ibuf[0].mem = p;
623 if (versal_crypto_request(VERSAL_AES_ENCRYPT_FINAL, &arg, &err)) {
624 EMSG("AES_ENCRYPT_FINAL error: %s", versal_aes_error(err));
625 ret = TEE_ERROR_GENERIC;
626 goto out;
627 }
628
629 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN);
630 dfinal->tag.length = GCM_TAG_LEN;
631 out:
632 versal_mbox_free(&p);
633
634 if (refcount_val(&engine.refc) > 1)
635 engine.state = FINALIZED;
636 else
637 engine.state = READY;
638
639 return ret;
640 }
641
do_dec_final(struct drvcrypt_authenc_final * dfinal)642 static TEE_Result do_dec_final(struct drvcrypt_authenc_final *dfinal)
643 {
644 struct drvcrypt_authenc_update_payload last = { };
645 struct versal_cmd_args arg = { };
646 struct versal_mbox_mem p = { };
647 TEE_Result ret = TEE_SUCCESS;
648 uint32_t err = 0;
649
650 if (!context_allowed(dfinal->ctx))
651 return TEE_ERROR_BUSY;
652
653 if (engine.state == FINALIZED) {
654 DMSG("Operation was already finalized");
655 ret = do_replay();
656 if (ret)
657 return ret;
658 }
659
660 if (engine.state != INIT)
661 panic();
662
663 last.encrypt = false;
664 last.ctx = dfinal->ctx;
665 last.dst = dfinal->dst;
666 last.src = dfinal->src;
667
668 ret = update_payload(&last, true);
669 if (ret)
670 return ret;
671
672 ret = versal_mbox_alloc(dfinal->tag.length, dfinal->tag.data, &p);
673 if (ret)
674 return ret;
675
676 arg.ibuf[0].mem = p;
677
678 if (versal_crypto_request(VERSAL_AES_DECRYPT_FINAL, &arg, &err)) {
679 EMSG("AES_DECRYPT_FINAL error: %s", versal_aes_error(err));
680 ret = TEE_ERROR_GENERIC;
681 goto out;
682 }
683
684 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length);
685 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN);
686 dfinal->tag.length = GCM_TAG_LEN;
687 out:
688 versal_mbox_free(&p);
689
690 if (refcount_val(&engine.refc) > 1)
691 engine.state = FINALIZED;
692 else
693 engine.state = READY;
694
695 return ret;
696 }
697
do_final(void * ctx __unused)698 static void do_final(void *ctx __unused)
699 {
700 }
701
do_free(void * ctx)702 static void do_free(void *ctx)
703 {
704 struct versal_ae_ctx *c = to_versal_ctx(ctx);
705 struct versal_node *next = NULL;
706 struct versal_node *node = NULL;
707 struct versal_context_node *ctx_next = NULL;
708 struct versal_context_node *ctx_node = NULL;
709 bool release = false;
710
711 if (refcount_dec(&engine.refc)) {
712 /* this is a final release */
713 release = true;
714 refcount_set(&engine.refc, 1);
715 engine.state = READY;
716 versal_mbox_free(&engine.init.init_buf);
717 versal_mbox_free(&engine.init.nonce);
718 versal_mbox_free(&engine.init.key);
719 memset(&engine.init, 0, sizeof(engine.init));
720 STAILQ_FOREACH_SAFE(node, &engine.replay_list, link, next) {
721 STAILQ_REMOVE(&engine.replay_list, node,
722 versal_node, link);
723 if (node->is_aad) {
724 versal_mbox_free(&node->aad.mem);
725 } else {
726 versal_mbox_free(&node->payload.dst);
727 versal_mbox_free(&node->payload.src);
728 versal_mbox_free(&node->payload.input_cmd);
729 }
730 free(node);
731 }
732 }
733
734 STAILQ_FOREACH_SAFE(ctx_node, &engine.context_list, link, ctx_next) {
735 if (c == ctx_node->ctx) {
736 STAILQ_REMOVE(&engine.context_list, ctx_node,
737 versal_context_node, link);
738 free(ctx_node);
739 }
740 }
741
742 if (release && engine_in_use())
743 panic();
744
745 free(c);
746 }
747
do_copy_state(void * dst_ctx,void * src_ctx)748 static void do_copy_state(void *dst_ctx, void *src_ctx)
749 {
750 struct versal_context_node *node = NULL;
751
752 STAILQ_FOREACH(node, &engine.context_list, link) {
753 if (node->ctx != to_versal_ctx(src_ctx))
754 continue;
755 /*
756 * The running context has been copied: from now on we can only
757 * finalize any of the contexts until there is only one active
758 * again.
759 */
760 node = calloc(1, sizeof(*node));
761 if (!node)
762 panic();
763
764 node->ctx = to_versal_ctx(dst_ctx);
765 STAILQ_INSERT_TAIL(&engine.context_list, node, link);
766
767 /* number of active contexts */
768 refcount_inc(&engine.refc);
769
770 return;
771 }
772
773 panic();
774 }
775
do_allocate(void ** ctx,uint32_t algo)776 static TEE_Result do_allocate(void **ctx, uint32_t algo)
777 {
778 struct versal_ae_ctx *c = NULL;
779
780 if (algo != TEE_ALG_AES_GCM)
781 return TEE_ERROR_NOT_IMPLEMENTED;
782
783 c = calloc(1, sizeof(*c));
784 if (!c)
785 return TEE_ERROR_OUT_OF_MEMORY;
786
787 *ctx = &c->a_ctx;
788
789 return TEE_SUCCESS;
790 }
791
792 static TEE_Result
do_update_payload_locked(struct drvcrypt_authenc_update_payload * p)793 do_update_payload_locked(struct drvcrypt_authenc_update_payload *p)
794 {
795 TEE_Result ret = TEE_SUCCESS;
796
797 mutex_lock(engine.lock);
798 ret = do_update_payload(p);
799 mutex_unlock(engine.lock);
800 return ret;
801 }
802
803 static TEE_Result
do_update_aad_locked(struct drvcrypt_authenc_update_aad * p)804 do_update_aad_locked(struct drvcrypt_authenc_update_aad *p)
805 {
806 TEE_Result ret = TEE_SUCCESS;
807
808 mutex_lock(engine.lock);
809 ret = do_update_aad(p);
810 mutex_unlock(engine.lock);
811 return ret;
812 }
813
do_copy_state_locked(void * dst,void * src)814 static void do_copy_state_locked(void *dst, void *src)
815 {
816 mutex_lock(engine.lock);
817 do_copy_state(dst, src);
818 mutex_unlock(engine.lock);
819 }
820
do_enc_final_locked(struct drvcrypt_authenc_final * p)821 static TEE_Result do_enc_final_locked(struct drvcrypt_authenc_final *p)
822 {
823 TEE_Result ret = TEE_SUCCESS;
824
825 mutex_lock(engine.lock);
826 ret = do_enc_final(p);
827 mutex_unlock(engine.lock);
828 return ret;
829 }
830
do_dec_final_locked(struct drvcrypt_authenc_final * p)831 static TEE_Result do_dec_final_locked(struct drvcrypt_authenc_final *p)
832 {
833 TEE_Result ret = TEE_SUCCESS;
834
835 mutex_lock(engine.lock);
836 ret = do_dec_final(p);
837 mutex_unlock(engine.lock);
838 return ret;
839 }
840
do_free_locked(void * p)841 static void do_free_locked(void *p)
842 {
843 mutex_lock(engine.lock);
844 do_free(p);
845 mutex_unlock(engine.lock);
846 }
847
do_init_locked(struct drvcrypt_authenc_init * p)848 static TEE_Result do_init_locked(struct drvcrypt_authenc_init *p)
849 {
850 TEE_Result ret = TEE_SUCCESS;
851
852 mutex_lock(engine.lock);
853 ret = do_init(p);
854 mutex_unlock(engine.lock);
855 return ret;
856 }
857
858 static struct drvcrypt_authenc versal_authenc = {
859 .update_payload = do_update_payload_locked,
860 .update_aad = do_update_aad_locked,
861 .copy_state = do_copy_state_locked,
862 .enc_final = do_enc_final_locked,
863 .dec_final = do_dec_final_locked,
864 .free_ctx = do_free_locked,
865 .alloc_ctx = do_allocate,
866 .init = do_init_locked,
867 .final = do_final,
868 };
869
enable_secure_status(void)870 static TEE_Result enable_secure_status(void)
871 {
872 /* Once Linux has support, we need to reserve the device */
873 return TEE_SUCCESS;
874 }
875
versal_register_authenc(void)876 static TEE_Result versal_register_authenc(void)
877 {
878 TEE_Result ret = TEE_SUCCESS;
879
880 ret = drvcrypt_register_authenc(&versal_authenc);
881 if (ret)
882 return ret;
883
884 if (engine.key_src < XSECURE_AES_USER_KEY_0 ||
885 engine.key_src > XSECURE_AES_USER_KEY_7)
886 return TEE_ERROR_GENERIC;
887
888 engine.state = READY;
889 STAILQ_INIT(&engine.replay_list);
890 STAILQ_INIT(&engine.context_list);
891 refcount_set(&engine.refc, 1);
892
893 return enable_secure_status();
894 }
895
896 driver_init_late(versal_register_authenc);
897