1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright 2024 NXP
4 */
5 #include <caam_ae.h>
6 #include <caam_common.h>
7 #include <caam_io.h>
8 #include <caam_jr.h>
9 #include <caam_status.h>
10 #include <caam_utils_mem.h>
11 #include <caam_utils_status.h>
12 #include <drvcrypt.h>
13 #include <drvcrypt_authenc.h>
14 #include <mm/core_memprot.h>
15 #include <tee_api_defines.h>
16 #include <tee_api_types.h>
17 #include <tee/cache.h>
18 #include <utee_defines.h>
19 #include <utee_types.h>
20
21 #include "local.h"
22
23 #define MAX_DESC_ENTRIES 64
24
25 /*
26 * Constants definition of the AES algorithm
27 */
28 static const struct cipheralg aes_alg[] = {
29 #if defined(CFG_NXP_CAAM_AE_CCM_DRV)
30 [TEE_CHAIN_MODE_CCM] = {
31 .type = OP_ALGO(AES) | ALGO_AAI(AES_CCM),
32 .size_block = TEE_AES_BLOCK_SIZE,
33 .size_ctx = 7 * sizeof(uint64_t),
34 .ctx_offset = 0,
35 .def_key = { .min = 16, .max = 32, .mod = 8 },
36 .initialize = caam_ae_initialize_ccm,
37 .final = caam_ae_final_ccm,
38 },
39 #endif
40 #if defined(CFG_NXP_CAAM_AE_GCM_DRV)
41 [TEE_CHAIN_MODE_GCM] = {
42 .type = OP_ALGO(AES) | ALGO_AAI(AES_GCM),
43 .size_block = TEE_AES_BLOCK_SIZE,
44 .size_ctx = 8 * sizeof(uint64_t),
45 .ctx_offset = 0,
46 .def_key = { .min = 16, .max = 32, .mod = 8 },
47 .initialize = caam_ae_initialize_gcm,
48 .final = caam_ae_final_gcm,
49 },
50 #endif
51 };
52
53 /*
54 * Checks if the algorithm @algo is supported and returns the
55 * local algorithm entry in the corresponding cipher array
56 */
get_cipheralgo(uint32_t algo)57 static const struct cipheralg *get_cipheralgo(uint32_t algo)
58 {
59 unsigned int algo_id = TEE_ALG_GET_MAIN_ALG(algo);
60 unsigned int algo_md = TEE_ALG_GET_CHAIN_MODE(algo);
61 const struct cipheralg *ca = NULL;
62
63 AE_TRACE("Algo id:%u md:%u", algo_id, algo_md);
64
65 switch (algo_id) {
66 case TEE_MAIN_ALGO_AES:
67 if (algo_md < ARRAY_SIZE(aes_alg))
68 ca = &aes_alg[algo_md];
69 break;
70
71 default:
72 break;
73 }
74
75 if (ca && ca->type)
76 return ca;
77
78 return NULL;
79 }
80
81 /*
82 * Allocate the SW cipher data context
83 *
84 * @ctx [out] Caller context variable
85 * @algo Algorithm ID of the context
86 */
caam_ae_allocate(void ** ctx,uint32_t algo)87 static TEE_Result caam_ae_allocate(void **ctx, uint32_t algo)
88 {
89 TEE_Result ret = TEE_ERROR_GENERIC;
90 struct caam_ae_ctx *caam_ctx = NULL;
91 const struct cipheralg *alg = NULL;
92
93 assert(ctx);
94
95 alg = get_cipheralgo(algo);
96 if (!alg) {
97 AE_TRACE("Algorithm not implemented");
98 return TEE_ERROR_NOT_IMPLEMENTED;
99 }
100
101 caam_ctx = caam_calloc(sizeof(*caam_ctx));
102 if (!caam_ctx)
103 return TEE_ERROR_OUT_OF_MEMORY;
104
105 caam_ctx->descriptor = caam_calloc_desc(MAX_DESC_ENTRIES);
106 if (!caam_ctx->descriptor) {
107 ret = TEE_ERROR_OUT_OF_MEMORY;
108 goto err;
109 }
110
111 /* Setup the Algorithm pointer */
112 caam_ctx->alg = alg;
113 /* Initialize the block buffer */
114 caam_ctx->blockbuf.max = caam_ctx->alg->size_block;
115
116 *ctx = caam_ctx;
117
118 return TEE_SUCCESS;
119 err:
120 caam_free_desc(&caam_ctx->descriptor);
121 caam_free(caam_ctx);
122
123 return ret;
124 }
125
126 /*
127 * Free the internal cipher data context
128 *
129 * @ctx Caller context variable or NULL
130 */
caam_ae_free(void * ctx)131 static void caam_ae_free(void *ctx)
132 {
133 struct caam_ae_ctx *caam_ctx = ctx;
134
135 assert(ctx);
136
137 caam_free_desc(&caam_ctx->descriptor);
138 caam_free_buf(&caam_ctx->key);
139 caam_free_buf(&caam_ctx->nonce);
140 caam_free_buf(&caam_ctx->ctx);
141 caam_free_buf(&caam_ctx->initial_ctx);
142 caam_free_buf(&caam_ctx->buf_aad.buf);
143 caam_free_buf(&caam_ctx->blockbuf.buf);
144 caam_free(caam_ctx);
145 }
146
147 /*
148 * Initialization of the cipher operation
149 *
150 * @dinit Data initialization object
151 */
caam_ae_initialize(struct drvcrypt_authenc_init * dinit)152 static TEE_Result caam_ae_initialize(struct drvcrypt_authenc_init *dinit)
153 {
154 TEE_Result ret = TEE_ERROR_GENERIC;
155 enum caam_status retstatus = CAAM_FAILURE;
156 struct caam_ae_ctx *caam_ctx = NULL;
157
158 assert(dinit);
159
160 if (dinit->aad_len >= AAD_LENGTH_OVERFLOW)
161 return TEE_ERROR_NOT_SUPPORTED;
162
163 caam_ctx = dinit->ctx;
164 if (!caam_ctx)
165 return TEE_ERROR_BAD_PARAMETERS;
166
167 caam_ctx->encrypt = dinit->encrypt;
168 caam_ctx->aad_length = dinit->aad_len;
169 caam_ctx->payload_length = dinit->payload_len;
170 caam_ctx->tag_length = dinit->tag_len;
171
172 if (dinit->key.data && dinit->key.length) {
173 retstatus = caam_cpy_buf(&caam_ctx->key, dinit->key.data,
174 dinit->key.length);
175 AE_TRACE("Copy key returned %d", retstatus);
176 if (retstatus) {
177 ret = caam_status_to_tee_result(retstatus);
178 goto err;
179 }
180 }
181
182 caam_ctx->blockbuf.filled = 0;
183 caam_ctx->buf_aad.filled = 0;
184
185 ret = caam_ctx->alg->initialize(dinit);
186 if (ret)
187 goto err;
188
189 return TEE_SUCCESS;
190 err:
191 caam_free_buf(&caam_ctx->key);
192
193 return ret;
194 }
195
196 /*
197 * Update Additional Authenticated Data part of the authenc operation
198 *
199 * @dupdate Additional Authenticated Data update object
200 */
201 static TEE_Result
caam_ae_update_aad(struct drvcrypt_authenc_update_aad * dupdate)202 caam_ae_update_aad(struct drvcrypt_authenc_update_aad *dupdate)
203 {
204 TEE_Result ret = TEE_ERROR_GENERIC;
205 enum caam_status retstatus = CAAM_FAILURE;
206 struct caam_ae_ctx *caam_ctx = NULL;
207 struct caambuf aad = { };
208
209 assert(dupdate);
210
211 caam_ctx = dupdate->ctx;
212 if (!caam_ctx)
213 return TEE_ERROR_BAD_PARAMETERS;
214
215 if (dupdate->aad.data) {
216 retstatus = caam_cpy_buf(&aad, dupdate->aad.data,
217 dupdate->aad.length);
218 if (retstatus) {
219 ret = caam_status_to_tee_result(retstatus);
220 goto out;
221 }
222
223 /* Initialize the AAD buffer if not already done */
224 if (!caam_ctx->buf_aad.max)
225 caam_ctx->buf_aad.max = dupdate->aad.length;
226
227 retstatus = caam_cpy_block_src(&caam_ctx->buf_aad, &aad, 0);
228 if (retstatus) {
229 ret = caam_status_to_tee_result(retstatus);
230 goto out;
231 }
232 }
233
234 ret = TEE_SUCCESS;
235 out:
236 caam_free_buf(&aad);
237 return ret;
238 }
239
240 /*
241 * Update of the cipher operation. Call the algorithm update
242 * function associated.
243 *
244 * @dupdate Data update object
245 */
246 static TEE_Result
caam_ae_update_payload(struct drvcrypt_authenc_update_payload * dupdate)247 caam_ae_update_payload(struct drvcrypt_authenc_update_payload *dupdate)
248 {
249 struct caam_ae_ctx *caam_ctx = NULL;
250
251 assert(dupdate);
252
253 caam_ctx = dupdate->ctx;
254 if (!caam_ctx)
255 return TEE_ERROR_BAD_PARAMETERS;
256
257 return caam_ae_do_update(caam_ctx, &dupdate->src, &dupdate->dst, false);
258 }
259
260 /*
261 * Last cipher update operation. Call the algorithm final
262 * function associated.
263 *
264 * @dfinal Data final object
265 */
caam_ae_final(struct drvcrypt_authenc_final * dfinal)266 static TEE_Result caam_ae_final(struct drvcrypt_authenc_final *dfinal)
267 {
268 struct caam_ae_ctx *caam_ctx = NULL;
269 TEE_Result ret = TEE_ERROR_BAD_PARAMETERS;
270
271 assert(dfinal);
272
273 caam_ctx = dfinal->ctx;
274 if (!caam_ctx)
275 return ret;
276
277 ret = caam_ctx->alg->final(dfinal);
278
279 caam_free_buf(&caam_ctx->nonce);
280 caam_free_buf(&caam_ctx->ctx);
281 caam_free_buf(&caam_ctx->initial_ctx);
282 caam_free_buf(&caam_ctx->buf_aad.buf);
283 caam_free_buf(&caam_ctx->blockbuf.buf);
284
285 return ret;
286 }
287
288 /*
289 * Finalize of the cipher operation
290 *
291 * @ctx Caller context variable or NULL
292 */
caam_ae_finalize(void * ctx __unused)293 static void caam_ae_finalize(void *ctx __unused)
294 {
295 }
296
297 /*
298 * Copy software Context
299 *
300 * @dst_ctx [out] Reference the context destination
301 * @src_ctx Reference the context source
302 */
caam_ae_copy_state(void * dst_ctx,void * src_ctx)303 static void caam_ae_copy_state(void *dst_ctx, void *src_ctx)
304 {
305 struct caam_ae_ctx *dst = dst_ctx;
306 struct caam_ae_ctx *src = src_ctx;
307
308 if (!dst || !src)
309 return;
310
311 AE_TRACE("Copy State context (%p) to (%p)", src_ctx, dst_ctx);
312
313 dst->alg = src->alg;
314 dst->encrypt = src->encrypt;
315 dst->aad_length = src->aad_length;
316 dst->tag_length = src->tag_length;
317 dst->payload_length = src->payload_length;
318 dst->buf_aad.max = src->buf_aad.max;
319 dst->do_block = src->do_block;
320
321 caam_free_buf(&dst->key);
322 caam_free_buf(&dst->nonce);
323 caam_free_buf(&dst->ctx);
324 caam_free_buf(&dst->initial_ctx);
325 caam_free_buf(&dst->buf_aad.buf);
326 caam_free_buf(&dst->blockbuf.buf);
327 dst->buf_aad.filled = 0;
328 dst->blockbuf.filled = 0;
329
330 if (src->blockbuf.filled) {
331 struct caambuf srcdata = {
332 .data = src->blockbuf.buf.data,
333 .length = src->blockbuf.filled
334 };
335
336 caam_cpy_block_src(&dst->blockbuf, &srcdata, 0);
337 }
338
339 if (src->buf_aad.filled) {
340 struct caambuf srcdata = {
341 .data = src->buf_aad.buf.data,
342 .length = src->buf_aad.filled
343 };
344
345 caam_cpy_block_src(&dst->buf_aad, &srcdata, 0);
346 }
347
348 if (src->key.length)
349 caam_cpy_buf(&dst->key, src->key.data, src->key.length);
350
351 if (src->ctx.length)
352 caam_cpy_buf(&dst->ctx, src->ctx.data, src->ctx.length);
353
354 if (src->initial_ctx.length)
355 caam_cpy_buf(&dst->initial_ctx, src->initial_ctx.data,
356 src->initial_ctx.length);
357
358 if (src->nonce.length)
359 caam_cpy_buf(&dst->nonce, src->nonce.data,
360 src->nonce.length);
361 }
362
363 /*
364 * Registration of the Authentication Encryption Driver
365 */
366 static struct drvcrypt_authenc driver_ae = {
367 .alloc_ctx = &caam_ae_allocate,
368 .free_ctx = &caam_ae_free,
369 .init = &caam_ae_initialize,
370 .update_aad = &caam_ae_update_aad,
371 .update_payload = &caam_ae_update_payload,
372 .enc_final = &caam_ae_final,
373 .dec_final = &caam_ae_final,
374 .final = &caam_ae_finalize,
375 .copy_state = &caam_ae_copy_state,
376 };
377
378 /*
379 * Init descriptor with a cipher key
380 *
381 * @caam_ctx Reference the AE cipher context
382 */
init_descriptor(struct caam_ae_ctx * caam_ctx)383 static void init_descriptor(struct caam_ae_ctx *caam_ctx)
384 {
385 uint32_t *desc = NULL;
386
387 assert(caam_ctx);
388
389 desc = caam_ctx->descriptor;
390
391 caam_desc_init(desc);
392 caam_desc_add_word(desc, DESC_HEADER(0));
393
394 /* Build the descriptor */
395 caam_desc_add_word(desc,
396 LD_KEY_PLAIN(CLASS_1, REG, caam_ctx->key.length));
397 caam_desc_add_ptr(desc, caam_ctx->key.paddr);
398 }
399
400 /*
401 * Init descriptor with an initial context
402 *
403 * @caam_ctx Reference the AE cipher context
404 */
add_initial_context(struct caam_ae_ctx * caam_ctx)405 static void add_initial_context(struct caam_ae_ctx *caam_ctx)
406 {
407 uint32_t *desc = NULL;
408 size_t length = 0;
409
410 assert(caam_ctx);
411
412 desc = caam_ctx->descriptor;
413 length = caam_ctx->initial_ctx.length;
414
415 if (length) {
416 caam_desc_add_word(desc,
417 LD_NOIMM_OFF(CLASS_1, REG_CTX, length, 0));
418 caam_desc_add_ptr(desc, caam_ctx->initial_ctx.paddr);
419
420 /* Ensure Context register data are not in cache */
421 cache_operation(TEE_CACHECLEAN, caam_ctx->initial_ctx.data,
422 length);
423 }
424 }
425
426 /*
427 * Set descriptor with a saved CAAM context
428 *
429 * @caam_ctx Reference the AE cipher context
430 */
load_context(struct caam_ae_ctx * caam_ctx)431 static void load_context(struct caam_ae_ctx *caam_ctx)
432 {
433 uint32_t *desc = NULL;
434
435 assert(caam_ctx);
436
437 desc = caam_ctx->descriptor;
438
439 caam_desc_add_word(desc,
440 LD_NOIMM_OFF(CLASS_1, REG_CTX, caam_ctx->ctx.length,
441 caam_ctx->alg->ctx_offset));
442 caam_desc_add_ptr(desc, caam_ctx->ctx.paddr);
443 }
444
445 /*
446 * Set descriptor to saved CAAM context
447 *
448 * @caam_ctx Reference the AE cipher context
449 */
store_context(struct caam_ae_ctx * caam_ctx)450 static void store_context(struct caam_ae_ctx *caam_ctx)
451 {
452 uint32_t *desc = NULL;
453
454 assert(caam_ctx);
455
456 desc = caam_ctx->descriptor;
457
458 /* Store the context */
459 caam_desc_add_word(desc,
460 ST_NOIMM_OFF(CLASS_1, REG_CTX, caam_ctx->ctx.length,
461 caam_ctx->alg->ctx_offset));
462 caam_desc_add_ptr(desc, caam_ctx->ctx.paddr);
463
464 /* Ensure Context register data are not in cache */
465 cache_operation(TEE_CACHECLEAN, caam_ctx->ctx.data,
466 caam_ctx->ctx.length);
467 }
468
469 /*
470 * Cipher operation and generates a message authentication
471 *
472 * @caam_ctx AE Cipher context
473 * @encrypt Encrypt or decrypt direction
474 * @src Source data to encrypt/decrypt
475 * @dst [out] Destination data encrypted/decrypted
476 * @aad Additional Authenticated data
477 */
caam_ae_do_oneshot(struct caam_ae_ctx * caam_ctx,bool encrypt,struct caamdmaobj * src,struct caamdmaobj * dst,struct caamdmaobj * aad)478 static enum caam_status caam_ae_do_oneshot(struct caam_ae_ctx *caam_ctx,
479 bool encrypt, struct caamdmaobj *src,
480 struct caamdmaobj *dst,
481 struct caamdmaobj *aad)
482 {
483 enum caam_status retstatus = CAAM_FAILURE;
484 struct caam_jobctx jobctx = { };
485 uint32_t *desc = NULL;
486
487 assert(caam_ctx);
488
489 desc = caam_ctx->descriptor;
490
491 init_descriptor(caam_ctx);
492
493 add_initial_context(caam_ctx);
494
495 AE_TRACE("Init/Final operation");
496
497 /* Operation with the direction */
498 caam_desc_add_word(desc,
499 CIPHER_INITFINAL(caam_ctx->alg->type, encrypt));
500
501 if (!caam_ctx->ctx.data) {
502 retstatus = caam_alloc_align_buf(&caam_ctx->ctx,
503 caam_ctx->alg->size_ctx);
504 if (retstatus)
505 return retstatus;
506 }
507
508 if (caam_ctx->nonce.data) {
509 if (!src && !aad)
510 caam_desc_add_word(desc,
511 FIFO_LD(CLASS_1, IV, LAST_C1,
512 caam_ctx->nonce.length));
513 else
514 caam_desc_add_word(desc,
515 FIFO_LD(CLASS_1, IV, FLUSH,
516 caam_ctx->nonce.length));
517 caam_desc_add_ptr(desc, caam_ctx->nonce.paddr);
518
519 /* Ensure Nonce data are not in cache */
520 cache_operation(TEE_CACHECLEAN, caam_ctx->nonce.data,
521 caam_ctx->nonce.length);
522 }
523
524 if (aad) {
525 if (!src)
526 caam_desc_fifo_load(desc, aad, CLASS_1, AAD, LAST_C1);
527 else
528 caam_desc_fifo_load(desc, aad, CLASS_1, AAD, FLUSH);
529 caam_dmaobj_cache_push(aad);
530 }
531
532 /* Load the source data if any */
533 if (src) {
534 caam_desc_fifo_load(desc, src, CLASS_1, MSG, LAST_C1);
535 caam_dmaobj_cache_push(src);
536 }
537
538 /* Store the output data if any */
539 if (dst) {
540 caam_desc_fifo_store(desc, dst, MSG_DATA);
541 caam_dmaobj_cache_push(dst);
542 }
543
544 store_context(caam_ctx);
545
546 AE_DUMPDESC(desc);
547
548 jobctx.desc = desc;
549 retstatus = caam_jr_enqueue(&jobctx, NULL);
550 if (retstatus) {
551 AE_TRACE("CAAM return 0x%08x Status 0x%08" PRIx32,
552 retstatus, jobctx.status);
553 retstatus = CAAM_FAILURE;
554 }
555
556 /* Ensure Context register data are not in cache */
557 cache_operation(TEE_CACHEINVALIDATE, caam_ctx->ctx.data,
558 caam_ctx->ctx.length);
559
560 return retstatus;
561 }
562
563 /*
564 * Init cipher operation
565 *
566 * @caam_ctx AE Cipher context
567 * @encrypt Encrypt or decrypt direction
568 * @aad Additional Authenticated data
569 */
caam_ae_do_init(struct caam_ae_ctx * caam_ctx,bool encrypt,struct caamdmaobj * aad)570 static enum caam_status caam_ae_do_init(struct caam_ae_ctx *caam_ctx,
571 bool encrypt, struct caamdmaobj *aad)
572 {
573 enum caam_status retstatus = CAAM_FAILURE;
574 struct caam_jobctx jobctx = { };
575 uint32_t *desc = NULL;
576
577 assert(caam_ctx);
578
579 desc = caam_ctx->descriptor;
580
581 init_descriptor(caam_ctx);
582
583 add_initial_context(caam_ctx);
584
585 AE_TRACE("Init operation");
586
587 /* Operation with the direction */
588 caam_desc_add_word(desc, CIPHER_INIT(caam_ctx->alg->type, encrypt));
589
590 if (!caam_ctx->ctx.data) {
591 retstatus = caam_alloc_align_buf(&caam_ctx->ctx,
592 caam_ctx->alg->size_ctx);
593 if (retstatus)
594 return retstatus;
595 }
596
597 if (caam_ctx->nonce.data) {
598 if (!aad)
599 caam_desc_add_word(desc,
600 FIFO_LD(CLASS_1, IV, LAST_C1,
601 caam_ctx->nonce.length));
602 else
603 caam_desc_add_word(desc,
604 FIFO_LD(CLASS_1, IV, FLUSH,
605 caam_ctx->nonce.length));
606 caam_desc_add_ptr(desc, caam_ctx->nonce.paddr);
607
608 /* Ensure Nonce data are not in cache */
609 cache_operation(TEE_CACHECLEAN, caam_ctx->nonce.data,
610 caam_ctx->nonce.length);
611 }
612
613 if (aad) {
614 caam_desc_fifo_load(desc, aad, CLASS_1, AAD, LAST_C1);
615 caam_dmaobj_cache_push(aad);
616 } else if (!caam_ctx->nonce.data) {
617 /* Required for null aad (initialize nonce only) */
618 caam_desc_add_word(desc, FIFO_LD_IMM(CLASS_1, AAD, LAST_C1, 0));
619 }
620
621 store_context(caam_ctx);
622
623 AE_DUMPDESC(desc);
624
625 jobctx.desc = desc;
626 retstatus = caam_jr_enqueue(&jobctx, NULL);
627 if (retstatus) {
628 AE_TRACE("CAAM return 0x%08x Status 0x%08" PRIx32,
629 retstatus, jobctx.status);
630 retstatus = CAAM_FAILURE;
631 }
632
633 /* Ensure Context register data are not in cache */
634 cache_operation(TEE_CACHEINVALIDATE, caam_ctx->ctx.data,
635 caam_ctx->ctx.length);
636
637 return retstatus;
638 }
639
640 /*
641 * Update cipher operation and generates a message authentication
642 * on the last update
643 *
644 * @caam_ctx AE Cipher context
645 * @savectx Save or not the context
646 * @encrypt Encrypt or decrypt direction
647 * @src Source data to encrypt/decrypt
648 * @dst [out] Destination data encrypted/decrypted
649 * @final Final AES block flag
650 */
caam_ae_do_block(struct caam_ae_ctx * caam_ctx,bool savectx,bool encrypt,struct caamdmaobj * src,struct caamdmaobj * dst,bool final)651 static enum caam_status caam_ae_do_block(struct caam_ae_ctx *caam_ctx,
652 bool savectx, bool encrypt,
653 struct caamdmaobj *src,
654 struct caamdmaobj *dst, bool final)
655 {
656 enum caam_status retstatus = CAAM_FAILURE;
657 struct caam_jobctx jobctx = { };
658 uint32_t *desc = NULL;
659
660 assert(caam_ctx);
661
662 desc = caam_ctx->descriptor;
663
664 if (!caam_ctx->ctx.length)
665 return CAAM_NOT_INIT;
666
667 init_descriptor(caam_ctx);
668
669 load_context(caam_ctx);
670
671 if (!caam_ctx->do_block ||
672 !caam_ctx->do_block(caam_ctx, encrypt, src, dst, final)) {
673 if (final)
674 caam_desc_add_word(desc,
675 CIPHER_FINAL(caam_ctx->alg->type,
676 encrypt));
677 else
678 caam_desc_add_word(desc,
679 CIPHER_UPDATE(caam_ctx->alg->type,
680 encrypt));
681
682 /* Load the source data if any */
683 if (src) {
684 caam_desc_fifo_load(desc, src, CLASS_1, MSG, LAST_C1);
685 caam_dmaobj_cache_push(src);
686 } else {
687 /*
688 * Add the input data of 0 bytes to start
689 * algorithm by setting the input data size
690 */
691 caam_desc_add_word(desc,
692 FIFO_LD(CLASS_1, MSG, LAST_C1, 0));
693 caam_desc_add_ptr(desc, 0);
694 }
695
696 /* Store the output data if any */
697 if (dst) {
698 caam_desc_fifo_store(desc, dst, MSG_DATA);
699 caam_dmaobj_cache_push(dst);
700 }
701 }
702
703 if (savectx)
704 store_context(caam_ctx);
705
706 AE_DUMPDESC(desc);
707
708 jobctx.desc = desc;
709 retstatus = caam_jr_enqueue(&jobctx, NULL);
710 if (retstatus) {
711 AE_TRACE("CAAM return 0x%08x Status 0x%08" PRIx32,
712 retstatus, jobctx.status);
713 retstatus = CAAM_FAILURE;
714 }
715
716 /* Ensure Context register data are not in cache */
717 if (savectx)
718 cache_operation(TEE_CACHEINVALIDATE, caam_ctx->ctx.data,
719 caam_ctx->ctx.length);
720
721 return retstatus;
722 }
723
caam_ae_do_update(struct caam_ae_ctx * caam_ctx,struct drvcrypt_buf * src,struct drvcrypt_buf * dst,bool last)724 TEE_Result caam_ae_do_update(struct caam_ae_ctx *caam_ctx,
725 struct drvcrypt_buf *src, struct drvcrypt_buf *dst,
726 bool last)
727 {
728 TEE_Result ret = TEE_ERROR_GENERIC;
729 enum caam_status retstatus = CAAM_FAILURE;
730 struct caamdmaobj caam_src = { };
731 struct caamdmaobj caam_dst = { };
732 struct caamdmaobj caam_aad = { };
733 struct caamdmaobj *caam_aad_ptr = NULL;
734 struct caamblock trash_bck = { };
735 size_t full_size = 0;
736 size_t size_topost = 0;
737 size_t size_todo = 0;
738 size_t size_done = 0;
739 size_t size_inmade = 0;
740 size_t offset = 0;
741 bool do_init = false;
742
743 if (!caam_ctx || !src || !dst)
744 return TEE_ERROR_BAD_PARAMETERS;
745
746 AE_TRACE("Length=%zu - %s", src->length,
747 caam_ctx->encrypt ? "Encrypt" : "Decrypt");
748
749 do_init = (caam_ctx->ctx.length == 0);
750
751 /*
752 * According to the TEE API function TEE_AEUpdateAAD
753 * Additional Authenticated data buffer could only be loaded
754 * at Init state
755 */
756 if (do_init && caam_ctx->buf_aad.filled) {
757 size_t aad_length = caam_ctx->buf_aad.filled;
758
759 ret = caam_dmaobj_init_input(&caam_aad,
760 caam_ctx->buf_aad.buf.data,
761 aad_length);
762 if (ret)
763 goto end_cipher;
764
765 ret = caam_dmaobj_prepare(&caam_aad, NULL, aad_length);
766 if (ret)
767 goto end_cipher;
768
769 ret = caam_dmaobj_sgtbuf_build(&caam_aad, &aad_length, 0,
770 aad_length);
771 if (ret)
772 goto end_cipher;
773
774 if (aad_length != caam_ctx->buf_aad.filled) {
775 ret = TEE_ERROR_GENERIC;
776 goto end_cipher;
777 }
778
779 caam_aad_ptr = &caam_aad;
780 }
781
782 /*
783 * Calculate the total data to be handled
784 * which is data saved to complete the previous buffer
785 * plus actual buffer length
786 */
787 full_size = caam_ctx->blockbuf.filled + src->length;
788 if (!last) {
789 if (full_size < caam_ctx->alg->size_block) {
790 size_topost = src->length;
791 dst->length = 0;
792 goto end_cipher_post;
793 } else {
794 size_topost = full_size % caam_ctx->alg->size_block;
795 size_inmade = src->length - size_topost;
796 /* Total size that is a cipher block multiple */
797 size_todo = full_size - size_topost;
798 }
799 } else {
800 /* Last total size that is the remaining data */
801 size_todo = full_size;
802 }
803
804 AE_TRACE("FullSize %zu - posted %zu - todo %zu", full_size,
805 size_topost, size_todo);
806
807 if (!size_todo) {
808 if (!last) {
809 ret = TEE_SUCCESS;
810 goto end_cipher_post;
811 } else if (do_init) {
812 retstatus = caam_ae_do_oneshot(caam_ctx,
813 caam_ctx->encrypt, NULL,
814 NULL, caam_aad_ptr);
815
816 ret = caam_status_to_tee_result(retstatus);
817
818 /* Nothing to post on last update operation */
819 goto end_cipher;
820 } else {
821 retstatus = caam_ae_do_block(caam_ctx, true,
822 caam_ctx->encrypt, NULL,
823 NULL, true);
824
825 ret = caam_status_to_tee_result(retstatus);
826
827 /* Nothing to post on last update operation */
828 goto end_cipher;
829 }
830 }
831
832 if (src->length) {
833 ret = caam_dmaobj_init_input(&caam_src, src->data, src->length);
834 if (ret)
835 goto end_cipher;
836 } else {
837 /* Init the buffer with saved data */
838 ret = caam_dmaobj_init_input(&caam_src,
839 caam_ctx->blockbuf.buf.data,
840 caam_ctx->blockbuf.filled);
841 if (ret)
842 goto end_cipher;
843
844 caam_ctx->blockbuf.filled = 0;
845 }
846
847 ret = caam_dmaobj_init_output(&caam_dst, dst->data, dst->length,
848 size_todo);
849 if (ret)
850 goto end_cipher;
851
852 ret = caam_dmaobj_prepare(&caam_src, &caam_dst, size_todo);
853 if (ret)
854 goto end_cipher;
855
856 /* Check if there is some data saved to complete the buffer */
857 if (caam_ctx->blockbuf.filled) {
858 ret = caam_dmaobj_add_first_block(&caam_src,
859 &caam_ctx->blockbuf);
860 if (ret)
861 goto end_cipher;
862
863 ret = caam_dmaobj_add_first_block(&caam_dst,
864 &caam_ctx->blockbuf);
865 if (ret)
866 goto end_cipher;
867
868 caam_ctx->blockbuf.filled = 0;
869 }
870
871 if (do_init) {
872 retstatus = caam_ae_do_init(caam_ctx, caam_ctx->encrypt,
873 caam_aad_ptr);
874
875 if (retstatus) {
876 ret = caam_status_to_tee_result(retstatus);
877 goto end_cipher;
878 }
879 do_init = false;
880 }
881
882 size_done = size_todo;
883 dst->length = 0;
884 for (offset = 0; size_todo;
885 offset += size_done, size_todo -= size_done) {
886 AE_TRACE("Do input %zu bytes, offset %zu", size_done, offset);
887
888 ret = caam_dmaobj_sgtbuf_inout_build(&caam_src, &caam_dst,
889 &size_done, offset,
890 size_todo);
891 if (ret)
892 goto end_cipher;
893
894 /* is it last update and last block ? */
895 if (last && size_todo == size_done)
896 retstatus = caam_ae_do_block(caam_ctx, true,
897 caam_ctx->encrypt,
898 &caam_src, &caam_dst,
899 true);
900 else
901 retstatus = caam_ae_do_block(caam_ctx, true,
902 caam_ctx->encrypt,
903 &caam_src, &caam_dst,
904 false);
905
906 if (retstatus) {
907 ret = caam_status_to_tee_result(retstatus);
908 goto end_cipher;
909 }
910
911 dst->length += caam_dmaobj_copy_to_orig(&caam_dst);
912 }
913
914 end_cipher_post:
915 if (size_topost) {
916 /*
917 * Save the input data in the block buffer for next operation
918 * and prepare the source DMA Object with the overall saved
919 * data to generate destination bytes.
920 */
921 struct caambuf cpysrc = { .data = src->data,
922 .length = src->length };
923
924 caam_dmaobj_free(&caam_src);
925 caam_dmaobj_free(&caam_dst);
926 AE_TRACE("Save input data %zu bytes (done %zu) - off %zu",
927 size_topost, size_inmade, offset);
928
929 size_todo = size_topost + caam_ctx->blockbuf.filled;
930
931 /*
932 * Prepare the destination DMA Object:
933 * - Use given destination parameter bytes to return
934 * - If the previous operation saved data, use a trash
935 * buffer to do the operation but don't use unneeded data.
936 */
937 ret = caam_dmaobj_init_output(&caam_dst,
938 dst->data + size_inmade,
939 size_topost, size_topost);
940 if (ret)
941 goto end_cipher;
942
943 ret = caam_dmaobj_prepare(NULL, &caam_dst,
944 caam_ctx->alg->size_block);
945 if (ret)
946 goto end_cipher;
947
948 if (caam_ctx->blockbuf.filled) {
949 /*
950 * Because there are some bytes to trash, use
951 * a block buffer that will be added to the
952 * destination SGT/Buffer structure to do the
953 * cipher operation.
954 */
955 ret = caam_alloc_align_buf(&trash_bck.buf,
956 caam_ctx->blockbuf.filled);
957 if (ret != CAAM_NO_ERROR) {
958 AE_TRACE("Allocation Trash Block error");
959 goto end_cipher;
960 }
961 trash_bck.filled = caam_ctx->blockbuf.filled;
962
963 ret = caam_dmaobj_add_first_block(&caam_dst,
964 &trash_bck);
965 if (ret)
966 goto end_cipher;
967 }
968
969 retstatus = caam_cpy_block_src(&caam_ctx->blockbuf, &cpysrc,
970 size_inmade);
971 if (retstatus) {
972 ret = caam_status_to_tee_result(retstatus);
973 goto end_cipher;
974 }
975
976 ret = caam_dmaobj_init_input(&caam_src,
977 caam_ctx->blockbuf.buf.data,
978 caam_ctx->blockbuf.filled);
979 if (ret)
980 goto end_cipher;
981
982 ret = caam_dmaobj_prepare(&caam_src, NULL,
983 caam_ctx->alg->size_block);
984 if (ret)
985 goto end_cipher;
986
987 /*
988 * Build input and output DMA Object with the same size.
989 */
990 size_done = size_todo;
991 ret = caam_dmaobj_sgtbuf_inout_build(&caam_src, &caam_dst,
992 &size_done, 0, size_todo);
993 if (ret)
994 goto end_cipher;
995
996 if (size_todo != size_done) {
997 AE_TRACE("Invalid end streaming size %zu vs %zu",
998 size_done, size_todo);
999 ret = TEE_ERROR_GENERIC;
1000 goto end_cipher;
1001 }
1002
1003 if (do_init) {
1004 retstatus = caam_ae_do_init(caam_ctx, caam_ctx->encrypt,
1005 caam_aad_ptr);
1006
1007 if (retstatus) {
1008 ret = caam_status_to_tee_result(retstatus);
1009 goto end_cipher;
1010 }
1011 }
1012
1013 retstatus = caam_ae_do_block(caam_ctx, false, caam_ctx->encrypt,
1014 &caam_src, &caam_dst, false);
1015
1016 if (retstatus) {
1017 ret = caam_status_to_tee_result(retstatus);
1018 goto end_cipher;
1019 }
1020
1021 dst->length += caam_dmaobj_copy_to_orig(&caam_dst);
1022
1023 AE_DUMPBUF("Source", caam_ctx->blockbuf.buf.data,
1024 caam_ctx->blockbuf.filled);
1025 AE_DUMPBUF("Result", dst->data + size_inmade, size_topost);
1026 }
1027
1028 ret = TEE_SUCCESS;
1029
1030 end_cipher:
1031 caam_dmaobj_free(&caam_src);
1032 caam_dmaobj_free(&caam_dst);
1033 caam_dmaobj_free(&caam_aad);
1034
1035 /* Free Trash block buffer */
1036 caam_free_buf(&trash_bck.buf);
1037
1038 return ret;
1039 }
1040
1041 /*
1042 * Initialize the authenticated encryption cipher module
1043 *
1044 * @ctrl_addr Controller base address
1045 */
caam_ae_init(vaddr_t ctrl_addr __unused)1046 enum caam_status caam_ae_init(vaddr_t ctrl_addr __unused)
1047 {
1048 enum caam_status retstatus = CAAM_FAILURE;
1049
1050 if (drvcrypt_register_authenc(&driver_ae) == TEE_SUCCESS)
1051 retstatus = CAAM_NO_ERROR;
1052
1053 return retstatus;
1054 }
1055