1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (C) Foundries Ltd. 2022. 4 * Author: Jorge Ramirez <jorge@foundries.io> 5 */ 6 7 #include <assert.h> 8 #include <crypto/crypto.h> 9 #include <crypto/crypto_impl.h> 10 #include <crypto/internal_aes-gcm.h> 11 #include <drvcrypt.h> 12 #include <drvcrypt_authenc.h> 13 #include <initcall.h> 14 #include <ipi.h> 15 #include <kernel/boot.h> 16 #include <kernel/dt.h> 17 #include <kernel/panic.h> 18 #include <kernel/spinlock.h> 19 #include <libfdt.h> 20 #include <mm/core_memprot.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <string_ext.h> 24 #include <tee_api_types.h> 25 #include <tee/cache.h> 26 #include <utee_defines.h> 27 #include <util.h> 28 29 /* 30 * This driver does not queue/pad non-aligned data. 31 * 32 * Allow debug information for future PLM work: if the PLM can not implement 33 * the required changes, we might be able to do it in OP-TEE. 34 */ 35 #define DEBUG_VERSAL_AES 0 36 37 #define GCM_TAG_LEN 16 38 39 #define XSECURE_AES_KEY_SIZE_128 0 /* Key Length = 32 bytes = 256 bits */ 40 #define XSECURE_AES_KEY_SIZE_256 2 /* Key Length = 16 bytes = 128 bits */ 41 42 #define XSECURE_ENCRYPT 0 43 #define XSECURE_DECRYPT 1 44 45 enum versal_aes_err { 46 AES_GCM_TAG_MISMATCH = 0x40, 47 AES_KEY_CLEAR_ERROR, 48 AES_DPA_CM_NOT_SUPPORTED, 49 AES_KAT_WRITE_KEY_FAILED_ERROR, 50 AES_KAT_DECRYPT_INIT_FAILED_ERROR, 51 AES_KAT_GCM_TAG_MISMATCH_ERROR, 52 AES_KAT_DATA_MISMATCH_ERROR, 53 AES_KAT_FAILED_ERROR, 54 AESDPACM_KAT_WRITE_KEY_FAILED_ERROR, 55 AESDPACM_KAT_KEYLOAD_FAILED_ERROR, 56 AESDPACM_SSS_CFG_FAILED_ERROR, 57 AESDPACM_KAT_FAILED_ERROR, 58 AESDPACM_KAT_CHECK1_FAILED_ERROR, 59 AESDPACM_KAT_CHECK2_FAILED_ERROR, 60 AESDPACM_KAT_CHECK3_FAILED_ERROR, 61 AESDPACM_KAT_CHECK4_FAILED_ERROR, 62 AESDPACM_KAT_CHECK5_FAILED_ERROR, 63 AES_INVALID_PARAM, 64 AESKAT_INVALID_PARAM, 65 AES_STATE_MISMATCH_ERROR, 66 AES_DEVICE_KEY_NOT_ALLOWED, 67 }; 68 69 #define VERSAL_AES_ERROR(m) { . error = (m), .name = TO_STR(m) } 70 71 static const char *versal_aes_error(uint8_t err) 72 { 73 const struct { 74 enum versal_aes_err error; 75 const char *name; 76 } elist[] = { 77 VERSAL_AES_ERROR(AES_GCM_TAG_MISMATCH), 78 VERSAL_AES_ERROR(AES_KEY_CLEAR_ERROR), 79 VERSAL_AES_ERROR(AES_DPA_CM_NOT_SUPPORTED), 80 VERSAL_AES_ERROR(AES_KAT_WRITE_KEY_FAILED_ERROR), 81 VERSAL_AES_ERROR(AES_KAT_DECRYPT_INIT_FAILED_ERROR), 82 VERSAL_AES_ERROR(AES_KAT_GCM_TAG_MISMATCH_ERROR), 83 VERSAL_AES_ERROR(AES_KAT_DATA_MISMATCH_ERROR), 84 VERSAL_AES_ERROR(AES_KAT_FAILED_ERROR), 85 VERSAL_AES_ERROR(AESDPACM_KAT_WRITE_KEY_FAILED_ERROR), 86 VERSAL_AES_ERROR(AESDPACM_KAT_KEYLOAD_FAILED_ERROR), 87 VERSAL_AES_ERROR(AESDPACM_SSS_CFG_FAILED_ERROR), 88 VERSAL_AES_ERROR(AESDPACM_KAT_FAILED_ERROR), 89 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK1_FAILED_ERROR), 90 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK2_FAILED_ERROR), 91 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK3_FAILED_ERROR), 92 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK4_FAILED_ERROR), 93 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK5_FAILED_ERROR), 94 VERSAL_AES_ERROR(AES_INVALID_PARAM), 95 VERSAL_AES_ERROR(AESKAT_INVALID_PARAM), 96 VERSAL_AES_ERROR(AES_STATE_MISMATCH_ERROR), 97 VERSAL_AES_ERROR(AES_DEVICE_KEY_NOT_ALLOWED), 98 }; 99 100 if (err >= AES_GCM_TAG_MISMATCH && err <= AES_DEVICE_KEY_NOT_ALLOWED) { 101 if (elist[err - AES_GCM_TAG_MISMATCH].name) 102 return elist[err - AES_GCM_TAG_MISMATCH].name; 103 104 return "Invalid"; 105 } 106 107 return "Unknown"; 108 } 109 110 enum aes_key_src { 111 XSECURE_AES_BBRAM_KEY = 0, /* BBRAM Key */ 112 XSECURE_AES_BBRAM_RED_KEY, /* BBRAM Red Key */ 113 XSECURE_AES_BH_KEY, /* BH Key */ 114 XSECURE_AES_BH_RED_KEY, /* BH Red Key */ 115 XSECURE_AES_EFUSE_KEY, /* eFUSE Key */ 116 XSECURE_AES_EFUSE_RED_KEY, /* eFUSE Red Key */ 117 XSECURE_AES_EFUSE_USER_KEY_0, /* eFUSE User Key 0 */ 118 XSECURE_AES_EFUSE_USER_KEY_1, /* eFUSE User Key 1 */ 119 XSECURE_AES_EFUSE_USER_RED_KEY_0, /* eFUSE User Red Key 0 */ 120 XSECURE_AES_EFUSE_USER_RED_KEY_1, /* eFUSE User Red Key 1 */ 121 XSECURE_AES_KUP_KEY, /* KUP key */ 122 XSECURE_AES_PUF_KEY, /* PUF key */ 123 XSECURE_AES_USER_KEY_0, /* User Key 0 */ 124 XSECURE_AES_USER_KEY_1, /* User Key 1 */ 125 XSECURE_AES_USER_KEY_2, /* User Key 2 */ 126 XSECURE_AES_USER_KEY_3, /* User Key 3 */ 127 XSECURE_AES_USER_KEY_4, /* User Key 4 */ 128 XSECURE_AES_USER_KEY_5, /* User Key 5 */ 129 XSECURE_AES_USER_KEY_6, /* User Key 6 */ 130 XSECURE_AES_USER_KEY_7, /* User Key 7 */ 131 XSECURE_AES_EXPANDED_KEYS, /* Expanded keys */ 132 XSECURE_AES_ALL_KEYS, /* AES All keys */ 133 }; 134 135 struct versal_payload { 136 struct versal_mbox_mem input_cmd; 137 struct versal_mbox_mem src; 138 struct versal_mbox_mem dst; 139 bool encrypt; 140 }; 141 142 struct versal_aad { 143 struct versal_mbox_mem mem; 144 }; 145 146 struct versal_node { 147 struct versal_payload payload; 148 struct versal_aad aad; 149 bool is_aad; 150 STAILQ_ENTRY(versal_node) link; 151 }; 152 153 struct versal_init { 154 uint32_t key_len; 155 uint32_t operation; 156 struct versal_mbox_mem key; 157 struct versal_mbox_mem nonce; 158 struct versal_mbox_mem init_buf; 159 }; 160 161 struct versal_ae_ctx { 162 struct crypto_authenc_ctx a_ctx; 163 }; 164 165 struct versal_context_node { 166 struct versal_ae_ctx *ctx; 167 STAILQ_ENTRY(versal_context_node) link; 168 }; 169 170 enum engine_state { 171 READY = 1, INIT = 2, FINALIZED = 3, 172 }; 173 174 static struct mutex engine_lock = MUTEX_INITIALIZER; 175 176 static struct versal_engine { 177 enum aes_key_src key_src; 178 enum engine_state state; 179 struct versal_init init; 180 struct refcount refc; 181 struct mutex *lock; /* protect the HW instance */ 182 STAILQ_HEAD(authenc_context_list, versal_context_node) context_list; 183 STAILQ_HEAD(authenc_replay_list, versal_node) replay_list; 184 } engine = { 185 .key_src = XSECURE_AES_USER_KEY_0, 186 .lock = &engine_lock, 187 }; 188 189 static struct versal_ae_ctx *to_versal_ctx(struct crypto_authenc_ctx *ctx) 190 { 191 assert(ctx); 192 193 return container_of(ctx, struct versal_ae_ctx, a_ctx); 194 } 195 196 static TEE_Result replay_init(void) 197 { 198 struct versal_cmd_args arg = { }; 199 uint32_t err = 0; 200 201 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) { 202 EMSG("AES_INIT error"); 203 return TEE_ERROR_GENERIC; 204 } 205 206 arg.data[arg.dlen++] = engine.init.key_len; 207 arg.data[arg.dlen++] = engine.key_src; 208 arg.ibuf[0].mem = engine.init.key; 209 210 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) { 211 EMSG("AES_WRITE_KEY error"); 212 return TEE_ERROR_GENERIC; 213 } 214 215 memset(&arg, 0, sizeof(arg)); 216 217 arg.ibuf[0].mem = engine.init.init_buf; 218 arg.ibuf[1].mem = engine.init.nonce; 219 arg.ibuf[1].only_cache = true; 220 221 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) { 222 EMSG("AES_OP_INIT error"); 223 return TEE_ERROR_GENERIC; 224 } 225 226 return TEE_SUCCESS; 227 } 228 229 static TEE_Result replay_aad(struct versal_aad *p) 230 { 231 struct versal_cmd_args arg = { }; 232 uint32_t err = 0; 233 234 arg.data[arg.dlen++] = p->mem.len % 16 ? p->mem.alloc_len : p->mem.len; 235 arg.ibuf[0].mem = p->mem; 236 237 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) { 238 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err)); 239 return TEE_ERROR_GENERIC; 240 } 241 242 return TEE_SUCCESS; 243 } 244 245 static TEE_Result replay_payload(struct versal_payload *p) 246 { 247 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE; 248 struct versal_cmd_args arg = { }; 249 uint32_t err = 0; 250 251 arg.ibuf[0].mem = p->input_cmd; 252 arg.ibuf[1].mem = p->dst; 253 arg.ibuf[2].mem = p->src; 254 255 if (p->encrypt) 256 id = VERSAL_AES_ENCRYPT_UPDATE; 257 258 if (versal_crypto_request(id, &arg, &err)) { 259 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err)); 260 return TEE_ERROR_GENERIC; 261 } 262 263 return TEE_SUCCESS; 264 } 265 266 static TEE_Result do_replay(void) 267 { 268 struct versal_node *node = NULL; 269 TEE_Result ret = TEE_SUCCESS; 270 271 ret = replay_init(); 272 if (ret) 273 return ret; 274 275 STAILQ_FOREACH(node, &engine.replay_list, link) { 276 if (node->is_aad) { 277 ret = replay_aad(&node->aad); 278 if (ret) 279 return ret; 280 } else { 281 ret = replay_payload(&node->payload); 282 if (ret) 283 return ret; 284 } 285 } 286 287 /* Engine has been init */ 288 engine.state = INIT; 289 290 return TEE_SUCCESS; 291 } 292 293 static bool engine_in_use(void) 294 { 295 if (STAILQ_EMPTY(&engine.context_list)) 296 return false; 297 298 return true; 299 } 300 301 static bool context_allowed(struct crypto_authenc_ctx *ctx) 302 { 303 struct versal_context_node *node = NULL; 304 305 STAILQ_FOREACH(node, &engine.context_list, link) { 306 if (node->ctx == to_versal_ctx(ctx)) 307 return true; 308 } 309 310 return false; 311 } 312 313 static TEE_Result do_init(struct drvcrypt_authenc_init *dinit) 314 { 315 uint32_t key_len = XSECURE_AES_KEY_SIZE_128; 316 struct versal_context_node *node = NULL; 317 struct versal_aes_init *init = NULL; 318 struct versal_mbox_mem init_buf = { }; 319 struct versal_mbox_mem key = { }; 320 struct versal_mbox_mem nonce = { }; 321 struct versal_cmd_args arg = { }; 322 TEE_Result ret = TEE_SUCCESS; 323 uint32_t err = 0; 324 325 if (engine_in_use()) { 326 EMSG("Versal: AES-GCM Engine busy"); 327 return TEE_ERROR_BUSY; 328 } 329 330 if (dinit->key.length != 32 && dinit->key.length != 16) 331 return TEE_ERROR_BAD_PARAMETERS; 332 333 if (dinit->key.length == 32) 334 key_len = XSECURE_AES_KEY_SIZE_256; 335 336 if (engine.state != READY) 337 return TEE_ERROR_BAD_STATE; 338 339 /* Initialize the AES engine */ 340 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) { 341 EMSG("AES_INIT error: %s", versal_aes_error(err)); 342 return TEE_ERROR_GENERIC; 343 } 344 345 /* Write the key */ 346 versal_mbox_alloc(dinit->key.length, dinit->key.data, &key); 347 348 arg.data[arg.dlen++] = key_len; 349 arg.data[arg.dlen++] = engine.key_src; 350 arg.ibuf[0].mem = key; 351 352 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) { 353 EMSG("AES_WRITE_KEY error: %s", versal_aes_error(err)); 354 ret = TEE_ERROR_GENERIC; 355 goto error; 356 } 357 358 memset(&arg, 0, sizeof(arg)); 359 360 /* Send the initialization structure */ 361 versal_mbox_alloc(sizeof(*init), NULL, &init_buf); 362 versal_mbox_alloc(dinit->nonce.length, dinit->nonce.data, &nonce); 363 364 init = init_buf.buf; 365 init->iv_addr = virt_to_phys(nonce.buf); 366 init->operation = dinit->encrypt ? XSECURE_ENCRYPT : XSECURE_DECRYPT; 367 init->key_src = engine.key_src; 368 init->key_len = key_len; 369 370 arg.ibuf[0].mem = init_buf; 371 arg.ibuf[1].mem = nonce; 372 arg.ibuf[1].only_cache = true; 373 374 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) { 375 EMSG("AES_OP_INIT error: %s", versal_aes_error(err)); 376 ret = TEE_ERROR_GENERIC; 377 goto error; 378 } 379 380 node = calloc(1, sizeof(*node)); 381 if (!node) { 382 ret = TEE_ERROR_OUT_OF_MEMORY; 383 goto error; 384 } 385 386 /* Save key context */ 387 engine.init.operation = dinit->encrypt ? 388 XSECURE_ENCRYPT : XSECURE_DECRYPT; 389 engine.init.key_len = key_len; 390 engine.init.init_buf = init_buf; 391 engine.init.nonce = nonce; 392 engine.init.key = key; 393 394 /* Active context */ 395 node->ctx = to_versal_ctx(dinit->ctx); 396 STAILQ_INSERT_TAIL(&engine.context_list, node, link); 397 398 /* Engine has been init*/ 399 engine.state = INIT; 400 401 return TEE_SUCCESS; 402 error: 403 free(key.buf); 404 free(init_buf.buf); 405 free(nonce.buf); 406 407 return ret; 408 } 409 410 static TEE_Result do_update_aad(struct drvcrypt_authenc_update_aad *dupdate) 411 { 412 struct versal_cmd_args arg = { }; 413 struct versal_mbox_mem p = { }; 414 TEE_Result ret = TEE_SUCCESS; 415 struct versal_node *node = NULL; 416 uint32_t err = 0; 417 418 /* This context has not been inited */ 419 if (!context_allowed(dupdate->ctx)) 420 return TEE_ERROR_BUSY; 421 422 /* There is a copy of the context: don't allow updates, only finalize */ 423 if (refcount_val(&engine.refc) > 1) 424 return TEE_ERROR_BUSY; 425 426 /* There was a copy of the context and it was finalized, then replay */ 427 if (engine.state == FINALIZED) 428 do_replay(); 429 430 versal_mbox_alloc(dupdate->aad.length, dupdate->aad.data, &p); 431 432 arg.data[arg.dlen++] = p.len % 16 ? p.alloc_len : p.len; 433 arg.ibuf[0].mem = p; 434 435 #if DEBUG_VERSAL_AES 436 IMSG("versal: aad length - requested: %zu, sent to plm: %"PRIu32, 437 dupdate->aad.length, arg.data[0]); 438 #endif 439 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) { 440 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err)); 441 ret = TEE_ERROR_GENERIC; 442 goto error; 443 } 444 445 node = calloc(1, sizeof(*node)); 446 if (!node) { 447 ret = TEE_ERROR_OUT_OF_MEMORY; 448 goto error; 449 } 450 451 /* Save the context */ 452 node->aad.mem = p; 453 node->is_aad = true; 454 STAILQ_INSERT_TAIL(&engine.replay_list, node, link); 455 456 return TEE_SUCCESS; 457 error: 458 free(p.buf); 459 return ret; 460 } 461 462 static TEE_Result 463 update_payload(struct drvcrypt_authenc_update_payload *dupdate, bool is_last) 464 { 465 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE; 466 struct versal_aes_input_param *input = NULL; 467 struct versal_mbox_mem input_cmd = { }; 468 struct versal_mbox_mem p = { }; 469 struct versal_mbox_mem q = { }; 470 TEE_Result ret = TEE_SUCCESS; 471 struct versal_cmd_args arg = { }; 472 struct versal_node *node = NULL; 473 uint32_t err = 0; 474 475 if (!context_allowed(dupdate->ctx)) 476 return TEE_ERROR_BUSY; 477 478 if (!dupdate->src.length || dupdate->src.length % 4) { 479 EMSG("Versal AES payload length not word aligned (len = %zu)", 480 dupdate->src.length); 481 return TEE_ERROR_BAD_PARAMETERS; 482 } 483 484 versal_mbox_alloc(dupdate->src.length, dupdate->src.data, &p); 485 versal_mbox_alloc(dupdate->dst.length, NULL, &q); 486 versal_mbox_alloc(sizeof(*input), NULL, &input_cmd); 487 488 input = input_cmd.buf; 489 input->input_addr = virt_to_phys(p.buf); 490 input->input_len = p.len % 16 ? p.alloc_len : p.len; 491 input->is_last = is_last; 492 493 arg.ibuf[0].mem = input_cmd; 494 arg.ibuf[1].mem = q; 495 arg.ibuf[2].mem = p; 496 497 if (dupdate->encrypt) 498 id = VERSAL_AES_ENCRYPT_UPDATE; 499 500 #if DEBUG_VERSAL_AES 501 IMSG("versal: payload length - requested %zu, sent to plm: %"PRIu32, 502 dupdate->src.length, input->input_len); 503 IMSG("versal: destination length - %zu ", dupdate->dst.length); 504 #endif 505 if (versal_crypto_request(id, &arg, &err)) { 506 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err)); 507 ret = TEE_ERROR_GENERIC; 508 goto out; 509 } 510 511 if (dupdate->dst.data) 512 memcpy(dupdate->dst.data, q.buf, dupdate->dst.length); 513 514 if (!is_last) { 515 node = calloc(1, sizeof(*node)); 516 if (!node) { 517 ret = TEE_ERROR_OUT_OF_MEMORY; 518 goto out; 519 } 520 521 node->is_aad = false; 522 node->payload.dst = q; 523 node->payload.src = p; 524 node->payload.input_cmd = input_cmd; 525 node->payload.encrypt = dupdate->encrypt; 526 STAILQ_INSERT_TAIL(&engine.replay_list, node, link); 527 528 return TEE_SUCCESS; 529 } 530 out: 531 free(p.buf); 532 free(q.buf); 533 free(input_cmd.buf); 534 535 return ret; 536 } 537 538 static TEE_Result do_update_payload(struct drvcrypt_authenc_update_payload *p) 539 { 540 TEE_Result ret = TEE_SUCCESS; 541 542 if (!context_allowed(p->ctx)) 543 return TEE_ERROR_BUSY; 544 545 /* 546 * If there is a copy, we don't allow updates until one of the copies 547 * has been deleted 548 */ 549 if (refcount_val(&engine.refc) > 1) 550 return TEE_ERROR_BUSY; 551 552 /* 553 * If there was a copy and it was finalized, we need to replay before 554 * we can update; do not clear the list so the state can be copied 555 */ 556 if (engine.state == FINALIZED) { 557 ret = do_replay(); 558 if (ret) 559 return ret; 560 } 561 562 return update_payload(p, false); 563 } 564 565 static TEE_Result do_enc_final(struct drvcrypt_authenc_final *dfinal) 566 { 567 struct drvcrypt_authenc_update_payload last = { }; 568 struct versal_cmd_args arg = { }; 569 struct versal_mbox_mem p = { }; 570 TEE_Result ret = TEE_SUCCESS; 571 uint32_t err = 0; 572 573 if (!context_allowed(dfinal->ctx)) 574 return TEE_ERROR_BUSY; 575 576 if (engine.state == FINALIZED) { 577 DMSG("Operation was already finalized"); 578 ret = do_replay(); 579 if (ret) 580 return ret; 581 } 582 583 if (engine.state != INIT) 584 panic(); 585 586 last.ctx = dfinal->ctx; 587 last.dst = dfinal->dst; 588 last.encrypt = true; 589 last.src = dfinal->src; 590 591 ret = update_payload(&last, true); 592 if (ret) 593 return ret; 594 595 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length); 596 597 versal_mbox_alloc(GCM_TAG_LEN, NULL, &p); 598 599 arg.ibuf[0].mem = p; 600 if (versal_crypto_request(VERSAL_AES_ENCRYPT_FINAL, &arg, &err)) { 601 EMSG("AES_ENCRYPT_FINAL error: %s", versal_aes_error(err)); 602 ret = TEE_ERROR_GENERIC; 603 goto out; 604 } 605 606 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN); 607 dfinal->tag.length = GCM_TAG_LEN; 608 out: 609 free(p.buf); 610 611 if (refcount_val(&engine.refc) > 1) 612 engine.state = FINALIZED; 613 else 614 engine.state = READY; 615 616 return ret; 617 } 618 619 static TEE_Result do_dec_final(struct drvcrypt_authenc_final *dfinal) 620 { 621 struct drvcrypt_authenc_update_payload last = { }; 622 struct versal_cmd_args arg = { }; 623 struct versal_mbox_mem p = { }; 624 TEE_Result ret = TEE_SUCCESS; 625 uint32_t err = 0; 626 627 if (!context_allowed(dfinal->ctx)) 628 return TEE_ERROR_BUSY; 629 630 if (engine.state == FINALIZED) { 631 DMSG("Operation was already finalized"); 632 ret = do_replay(); 633 if (ret) 634 return ret; 635 } 636 637 if (engine.state != INIT) 638 panic(); 639 640 last.encrypt = false; 641 last.ctx = dfinal->ctx; 642 last.dst = dfinal->dst; 643 last.src = dfinal->src; 644 645 ret = update_payload(&last, true); 646 if (ret) 647 return ret; 648 649 versal_mbox_alloc(dfinal->tag.length, dfinal->tag.data, &p); 650 arg.ibuf[0].mem = p; 651 652 if (versal_crypto_request(VERSAL_AES_DECRYPT_FINAL, &arg, &err)) { 653 EMSG("AES_DECRYPT_FINAL error: %s", versal_aes_error(err)); 654 ret = TEE_ERROR_GENERIC; 655 goto out; 656 } 657 658 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length); 659 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN); 660 dfinal->tag.length = GCM_TAG_LEN; 661 out: 662 free(p.buf); 663 664 if (refcount_val(&engine.refc) > 1) 665 engine.state = FINALIZED; 666 else 667 engine.state = READY; 668 669 return ret; 670 } 671 672 static void do_final(void *ctx __unused) 673 { 674 } 675 676 static void do_free(void *ctx) 677 { 678 struct versal_ae_ctx *c = to_versal_ctx(ctx); 679 struct versal_node *next = NULL; 680 struct versal_node *node = NULL; 681 struct versal_context_node *ctx_next = NULL; 682 struct versal_context_node *ctx_node = NULL; 683 bool release = false; 684 685 if (refcount_dec(&engine.refc)) { 686 /* this is a final release */ 687 release = true; 688 refcount_set(&engine.refc, 1); 689 engine.state = READY; 690 free(engine.init.init_buf.buf); 691 free(engine.init.nonce.buf); 692 free(engine.init.key.buf); 693 memset(&engine.init, 0, sizeof(engine.init)); 694 STAILQ_FOREACH_SAFE(node, &engine.replay_list, link, next) { 695 STAILQ_REMOVE(&engine.replay_list, node, 696 versal_node, link); 697 if (node->is_aad) { 698 free(node->aad.mem.buf); 699 } else { 700 free(node->payload.dst.buf); 701 free(node->payload.src.buf); 702 free(node->payload.input_cmd.buf); 703 } 704 free(node); 705 } 706 } 707 708 STAILQ_FOREACH_SAFE(ctx_node, &engine.context_list, link, ctx_next) { 709 if (c == ctx_node->ctx) { 710 STAILQ_REMOVE(&engine.context_list, ctx_node, 711 versal_context_node, link); 712 free(ctx_node); 713 } 714 } 715 716 if (release && engine_in_use()) 717 panic(); 718 719 free(c); 720 } 721 722 static void do_copy_state(void *dst_ctx, void *src_ctx) 723 { 724 struct versal_context_node *node = NULL; 725 726 STAILQ_FOREACH(node, &engine.context_list, link) { 727 if (node->ctx != to_versal_ctx(src_ctx)) 728 continue; 729 /* 730 * The running context has been copied: from now on we can only 731 * finalize any of the contexts until there is only one active 732 * again. 733 */ 734 node = calloc(1, sizeof(*node)); 735 if (!node) 736 panic(); 737 738 node->ctx = to_versal_ctx(dst_ctx); 739 STAILQ_INSERT_TAIL(&engine.context_list, node, link); 740 741 /* number of active contexts */ 742 refcount_inc(&engine.refc); 743 744 return; 745 } 746 747 panic(); 748 } 749 750 static TEE_Result do_allocate(void **ctx, uint32_t algo) 751 { 752 struct versal_ae_ctx *c = NULL; 753 754 if (algo != TEE_ALG_AES_GCM) 755 return TEE_ERROR_NOT_IMPLEMENTED; 756 757 c = calloc(1, sizeof(*c)); 758 if (!c) 759 return TEE_ERROR_OUT_OF_MEMORY; 760 761 *ctx = &c->a_ctx; 762 763 return TEE_SUCCESS; 764 } 765 766 static TEE_Result 767 do_update_payload_locked(struct drvcrypt_authenc_update_payload *p) 768 { 769 TEE_Result ret = TEE_SUCCESS; 770 771 mutex_lock(engine.lock); 772 ret = do_update_payload(p); 773 mutex_unlock(engine.lock); 774 return ret; 775 } 776 777 static TEE_Result 778 do_update_aad_locked(struct drvcrypt_authenc_update_aad *p) 779 { 780 TEE_Result ret = TEE_SUCCESS; 781 782 mutex_lock(engine.lock); 783 ret = do_update_aad(p); 784 mutex_unlock(engine.lock); 785 return ret; 786 } 787 788 static void do_copy_state_locked(void *dst, void *src) 789 { 790 mutex_lock(engine.lock); 791 do_copy_state(dst, src); 792 mutex_unlock(engine.lock); 793 } 794 795 static TEE_Result do_enc_final_locked(struct drvcrypt_authenc_final *p) 796 { 797 TEE_Result ret = TEE_SUCCESS; 798 799 mutex_lock(engine.lock); 800 ret = do_enc_final(p); 801 mutex_unlock(engine.lock); 802 return ret; 803 } 804 805 static TEE_Result do_dec_final_locked(struct drvcrypt_authenc_final *p) 806 { 807 TEE_Result ret = TEE_SUCCESS; 808 809 mutex_lock(engine.lock); 810 ret = do_dec_final(p); 811 mutex_unlock(engine.lock); 812 return ret; 813 } 814 815 static void do_free_locked(void *p) 816 { 817 mutex_lock(engine.lock); 818 do_free(p); 819 mutex_unlock(engine.lock); 820 } 821 822 static TEE_Result do_init_locked(struct drvcrypt_authenc_init *p) 823 { 824 TEE_Result ret = TEE_SUCCESS; 825 826 mutex_lock(engine.lock); 827 ret = do_init(p); 828 mutex_unlock(engine.lock); 829 return ret; 830 } 831 832 static struct drvcrypt_authenc versal_authenc = { 833 .update_payload = do_update_payload_locked, 834 .update_aad = do_update_aad_locked, 835 .copy_state = do_copy_state_locked, 836 .enc_final = do_enc_final_locked, 837 .dec_final = do_dec_final_locked, 838 .free_ctx = do_free_locked, 839 .alloc_ctx = do_allocate, 840 .init = do_init_locked, 841 .final = do_final, 842 }; 843 844 static TEE_Result enable_secure_status(void) 845 { 846 /* Once Linux has support, we need to reserve the device */ 847 return TEE_SUCCESS; 848 } 849 850 static TEE_Result versal_register_authenc(void) 851 { 852 TEE_Result ret = TEE_SUCCESS; 853 854 ret = drvcrypt_register_authenc(&versal_authenc); 855 if (ret) 856 return ret; 857 858 if (engine.key_src < XSECURE_AES_USER_KEY_0 || 859 engine.key_src > XSECURE_AES_USER_KEY_7) 860 return TEE_ERROR_GENERIC; 861 862 engine.state = READY; 863 STAILQ_INIT(&engine.replay_list); 864 STAILQ_INIT(&engine.context_list); 865 refcount_set(&engine.refc, 1); 866 867 return enable_secure_status(); 868 } 869 870 driver_init_late(versal_register_authenc); 871