1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (C) Foundries Ltd. 2022. 4 * Author: Jorge Ramirez <jorge@foundries.io> 5 */ 6 7 #include <assert.h> 8 #include <crypto/crypto.h> 9 #include <crypto/crypto_impl.h> 10 #include <crypto/internal_aes-gcm.h> 11 #include <drvcrypt.h> 12 #include <drvcrypt_authenc.h> 13 #include <initcall.h> 14 #include <ipi.h> 15 #include <kernel/boot.h> 16 #include <kernel/dt.h> 17 #include <kernel/panic.h> 18 #include <kernel/spinlock.h> 19 #include <libfdt.h> 20 #include <mm/core_memprot.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <string_ext.h> 24 #include <tee_api_types.h> 25 #include <tee/cache.h> 26 #include <utee_defines.h> 27 #include <util.h> 28 29 /* 30 * This driver does not queue/pad non-aligned data. 31 * 32 * Allow debug information for future PLM work: if the PLM can not implement 33 * the required changes, we might be able to do it in OP-TEE. 34 */ 35 #define DEBUG_VERSAL_AES 0 36 37 #define GCM_TAG_LEN 16 38 39 #define XSECURE_AES_KEY_SIZE_128 0 /* Key Length = 32 bytes = 256 bits */ 40 #define XSECURE_AES_KEY_SIZE_256 2 /* Key Length = 16 bytes = 128 bits */ 41 42 #define XSECURE_ENCRYPT 0 43 #define XSECURE_DECRYPT 1 44 45 enum versal_aes_err { 46 AES_GCM_TAG_MISMATCH = 0x40, 47 AES_KEY_CLEAR_ERROR, 48 AES_DPA_CM_NOT_SUPPORTED, 49 AES_KAT_WRITE_KEY_FAILED_ERROR, 50 AES_KAT_DECRYPT_INIT_FAILED_ERROR, 51 AES_KAT_GCM_TAG_MISMATCH_ERROR, 52 AES_KAT_DATA_MISMATCH_ERROR, 53 AES_KAT_FAILED_ERROR, 54 AESDPACM_KAT_WRITE_KEY_FAILED_ERROR, 55 AESDPACM_KAT_KEYLOAD_FAILED_ERROR, 56 AESDPACM_SSS_CFG_FAILED_ERROR, 57 AESDPACM_KAT_FAILED_ERROR, 58 AESDPACM_KAT_CHECK1_FAILED_ERROR, 59 AESDPACM_KAT_CHECK2_FAILED_ERROR, 60 AESDPACM_KAT_CHECK3_FAILED_ERROR, 61 AESDPACM_KAT_CHECK4_FAILED_ERROR, 62 AESDPACM_KAT_CHECK5_FAILED_ERROR, 63 AES_INVALID_PARAM, 64 AESKAT_INVALID_PARAM, 65 AES_STATE_MISMATCH_ERROR, 66 AES_DEVICE_KEY_NOT_ALLOWED, 67 }; 68 69 #define VERSAL_AES_ERROR(m) { . error = (m), .name = TO_STR(m) } 70 71 static const char *versal_aes_error(uint8_t err) 72 { 73 const struct { 74 enum versal_aes_err error; 75 const char *name; 76 } elist[] = { 77 VERSAL_AES_ERROR(AES_GCM_TAG_MISMATCH), 78 VERSAL_AES_ERROR(AES_KEY_CLEAR_ERROR), 79 VERSAL_AES_ERROR(AES_DPA_CM_NOT_SUPPORTED), 80 VERSAL_AES_ERROR(AES_KAT_WRITE_KEY_FAILED_ERROR), 81 VERSAL_AES_ERROR(AES_KAT_DECRYPT_INIT_FAILED_ERROR), 82 VERSAL_AES_ERROR(AES_KAT_GCM_TAG_MISMATCH_ERROR), 83 VERSAL_AES_ERROR(AES_KAT_DATA_MISMATCH_ERROR), 84 VERSAL_AES_ERROR(AES_KAT_FAILED_ERROR), 85 VERSAL_AES_ERROR(AESDPACM_KAT_WRITE_KEY_FAILED_ERROR), 86 VERSAL_AES_ERROR(AESDPACM_KAT_KEYLOAD_FAILED_ERROR), 87 VERSAL_AES_ERROR(AESDPACM_SSS_CFG_FAILED_ERROR), 88 VERSAL_AES_ERROR(AESDPACM_KAT_FAILED_ERROR), 89 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK1_FAILED_ERROR), 90 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK2_FAILED_ERROR), 91 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK3_FAILED_ERROR), 92 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK4_FAILED_ERROR), 93 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK5_FAILED_ERROR), 94 VERSAL_AES_ERROR(AES_INVALID_PARAM), 95 VERSAL_AES_ERROR(AESKAT_INVALID_PARAM), 96 VERSAL_AES_ERROR(AES_STATE_MISMATCH_ERROR), 97 VERSAL_AES_ERROR(AES_DEVICE_KEY_NOT_ALLOWED), 98 }; 99 100 if (err >= AES_GCM_TAG_MISMATCH && err <= AES_DEVICE_KEY_NOT_ALLOWED) { 101 if (elist[err - AES_GCM_TAG_MISMATCH].name) 102 return elist[err - AES_GCM_TAG_MISMATCH].name; 103 104 return "Invalid"; 105 } 106 107 return "Unknown"; 108 } 109 110 enum aes_key_src { 111 XSECURE_AES_BBRAM_KEY = 0, /* BBRAM Key */ 112 XSECURE_AES_BBRAM_RED_KEY, /* BBRAM Red Key */ 113 XSECURE_AES_BH_KEY, /* BH Key */ 114 XSECURE_AES_BH_RED_KEY, /* BH Red Key */ 115 XSECURE_AES_EFUSE_KEY, /* eFUSE Key */ 116 XSECURE_AES_EFUSE_RED_KEY, /* eFUSE Red Key */ 117 XSECURE_AES_EFUSE_USER_KEY_0, /* eFUSE User Key 0 */ 118 XSECURE_AES_EFUSE_USER_KEY_1, /* eFUSE User Key 1 */ 119 XSECURE_AES_EFUSE_USER_RED_KEY_0, /* eFUSE User Red Key 0 */ 120 XSECURE_AES_EFUSE_USER_RED_KEY_1, /* eFUSE User Red Key 1 */ 121 XSECURE_AES_KUP_KEY, /* KUP key */ 122 XSECURE_AES_PUF_KEY, /* PUF key */ 123 XSECURE_AES_USER_KEY_0, /* User Key 0 */ 124 XSECURE_AES_USER_KEY_1, /* User Key 1 */ 125 XSECURE_AES_USER_KEY_2, /* User Key 2 */ 126 XSECURE_AES_USER_KEY_3, /* User Key 3 */ 127 XSECURE_AES_USER_KEY_4, /* User Key 4 */ 128 XSECURE_AES_USER_KEY_5, /* User Key 5 */ 129 XSECURE_AES_USER_KEY_6, /* User Key 6 */ 130 XSECURE_AES_USER_KEY_7, /* User Key 7 */ 131 XSECURE_AES_EXPANDED_KEYS, /* Expanded keys */ 132 XSECURE_AES_ALL_KEYS, /* AES All keys */ 133 }; 134 135 struct versal_payload { 136 struct versal_mbox_mem input_cmd; 137 struct versal_mbox_mem src; 138 struct versal_mbox_mem dst; 139 bool encrypt; 140 }; 141 142 struct versal_aad { 143 struct versal_mbox_mem mem; 144 }; 145 146 struct versal_node { 147 struct versal_payload payload; 148 struct versal_aad aad; 149 bool is_aad; 150 STAILQ_ENTRY(versal_node) link; 151 }; 152 153 struct versal_init { 154 uint32_t key_len; 155 uint32_t operation; 156 struct versal_mbox_mem key; 157 struct versal_mbox_mem nonce; 158 struct versal_mbox_mem init_buf; 159 }; 160 161 struct versal_ae_ctx { 162 struct crypto_authenc_ctx a_ctx; 163 }; 164 165 struct versal_context_node { 166 struct versal_ae_ctx *ctx; 167 STAILQ_ENTRY(versal_context_node) link; 168 }; 169 170 enum engine_state { 171 READY = 1, INIT = 2, FINALIZED = 3, 172 }; 173 174 static struct mutex engine_lock = MUTEX_INITIALIZER; 175 176 static struct versal_engine { 177 enum aes_key_src key_src; 178 enum engine_state state; 179 struct versal_init init; 180 struct refcount refc; 181 struct mutex *lock; /* protect the HW instance */ 182 STAILQ_HEAD(authenc_context_list, versal_context_node) context_list; 183 STAILQ_HEAD(authenc_replay_list, versal_node) replay_list; 184 } engine = { 185 .key_src = XSECURE_AES_USER_KEY_0, 186 .lock = &engine_lock, 187 }; 188 189 static struct versal_ae_ctx *to_versal_ctx(struct crypto_authenc_ctx *ctx) 190 { 191 assert(ctx); 192 193 return container_of(ctx, struct versal_ae_ctx, a_ctx); 194 } 195 196 static TEE_Result replay_init(void) 197 { 198 struct versal_cmd_args arg = { }; 199 uint32_t err = 0; 200 201 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) { 202 EMSG("AES_INIT error"); 203 return TEE_ERROR_GENERIC; 204 } 205 206 arg.data[arg.dlen++] = engine.init.key_len; 207 arg.data[arg.dlen++] = engine.key_src; 208 arg.ibuf[0].mem = engine.init.key; 209 210 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) { 211 EMSG("AES_WRITE_KEY error"); 212 return TEE_ERROR_GENERIC; 213 } 214 215 memset(&arg, 0, sizeof(arg)); 216 217 arg.ibuf[0].mem = engine.init.init_buf; 218 arg.ibuf[1].mem = engine.init.nonce; 219 arg.ibuf[1].only_cache = true; 220 221 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) { 222 EMSG("AES_OP_INIT error"); 223 return TEE_ERROR_GENERIC; 224 } 225 226 return TEE_SUCCESS; 227 } 228 229 static TEE_Result replay_aad(struct versal_aad *p) 230 { 231 struct versal_cmd_args arg = { }; 232 uint32_t err = 0; 233 234 arg.data[arg.dlen++] = p->mem.len % 16 ? p->mem.alloc_len : p->mem.len; 235 arg.ibuf[0].mem = p->mem; 236 237 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) { 238 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err)); 239 return TEE_ERROR_GENERIC; 240 } 241 242 return TEE_SUCCESS; 243 } 244 245 static TEE_Result replay_payload(struct versal_payload *p) 246 { 247 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE; 248 struct versal_cmd_args arg = { }; 249 uint32_t err = 0; 250 251 arg.ibuf[0].mem = p->input_cmd; 252 arg.ibuf[1].mem = p->dst; 253 arg.ibuf[2].mem = p->src; 254 255 if (p->encrypt) 256 id = VERSAL_AES_ENCRYPT_UPDATE; 257 258 if (versal_crypto_request(id, &arg, &err)) { 259 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err)); 260 return TEE_ERROR_GENERIC; 261 } 262 263 return TEE_SUCCESS; 264 } 265 266 static TEE_Result do_replay(void) 267 { 268 struct versal_node *node = NULL; 269 TEE_Result ret = TEE_SUCCESS; 270 271 ret = replay_init(); 272 if (ret) 273 return ret; 274 275 STAILQ_FOREACH(node, &engine.replay_list, link) { 276 if (node->is_aad) { 277 ret = replay_aad(&node->aad); 278 if (ret) 279 return ret; 280 } else { 281 ret = replay_payload(&node->payload); 282 if (ret) 283 return ret; 284 } 285 } 286 287 /* Engine has been init */ 288 engine.state = INIT; 289 290 return TEE_SUCCESS; 291 } 292 293 static bool engine_in_use(void) 294 { 295 if (STAILQ_EMPTY(&engine.context_list)) 296 return false; 297 298 return true; 299 } 300 301 static bool context_allowed(struct crypto_authenc_ctx *ctx) 302 { 303 struct versal_context_node *node = NULL; 304 305 STAILQ_FOREACH(node, &engine.context_list, link) { 306 if (node->ctx == to_versal_ctx(ctx)) 307 return true; 308 } 309 310 return false; 311 } 312 313 static TEE_Result do_init(struct drvcrypt_authenc_init *dinit) 314 { 315 uint32_t key_len = XSECURE_AES_KEY_SIZE_128; 316 struct versal_context_node *node = NULL; 317 struct versal_aes_init *init = NULL; 318 struct versal_mbox_mem init_buf = { }; 319 struct versal_mbox_mem key = { }; 320 struct versal_mbox_mem nonce = { }; 321 struct versal_cmd_args arg = { }; 322 TEE_Result ret = TEE_SUCCESS; 323 uint32_t err = 0; 324 325 if (engine_in_use()) { 326 EMSG("Versal: AES-GCM Engine busy"); 327 return TEE_ERROR_BUSY; 328 } 329 330 if (dinit->key.length != 32 && dinit->key.length != 16) 331 return TEE_ERROR_BAD_PARAMETERS; 332 333 if (dinit->key.length == 32) 334 key_len = XSECURE_AES_KEY_SIZE_256; 335 336 if (engine.state != READY) 337 return TEE_ERROR_BAD_STATE; 338 339 /* Initialize the AES engine */ 340 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) { 341 EMSG("AES_INIT error: %s", versal_aes_error(err)); 342 return TEE_ERROR_GENERIC; 343 } 344 345 /* Write the key */ 346 ret = versal_mbox_alloc(dinit->key.length, dinit->key.data, &key); 347 if (ret) 348 return ret; 349 350 arg.data[arg.dlen++] = key_len; 351 arg.data[arg.dlen++] = engine.key_src; 352 arg.ibuf[0].mem = key; 353 354 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) { 355 EMSG("AES_WRITE_KEY error: %s", versal_aes_error(err)); 356 ret = TEE_ERROR_GENERIC; 357 goto error; 358 } 359 360 memset(&arg, 0, sizeof(arg)); 361 362 /* Send the initialization structure */ 363 ret = versal_mbox_alloc(sizeof(*init), NULL, &init_buf); 364 if (ret) 365 goto error; 366 ret = versal_mbox_alloc(dinit->nonce.length, dinit->nonce.data, &nonce); 367 if (ret) 368 goto error; 369 370 init = init_buf.buf; 371 init->iv_addr = virt_to_phys(nonce.buf); 372 init->operation = dinit->encrypt ? XSECURE_ENCRYPT : XSECURE_DECRYPT; 373 init->key_src = engine.key_src; 374 init->key_len = key_len; 375 376 arg.ibuf[0].mem = init_buf; 377 arg.ibuf[1].mem = nonce; 378 arg.ibuf[1].only_cache = true; 379 380 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) { 381 EMSG("AES_OP_INIT error: %s", versal_aes_error(err)); 382 ret = TEE_ERROR_GENERIC; 383 goto error; 384 } 385 386 node = calloc(1, sizeof(*node)); 387 if (!node) { 388 ret = TEE_ERROR_OUT_OF_MEMORY; 389 goto error; 390 } 391 392 /* Save key context */ 393 engine.init.operation = dinit->encrypt ? 394 XSECURE_ENCRYPT : XSECURE_DECRYPT; 395 engine.init.key_len = key_len; 396 engine.init.init_buf = init_buf; 397 engine.init.nonce = nonce; 398 engine.init.key = key; 399 400 /* Active context */ 401 node->ctx = to_versal_ctx(dinit->ctx); 402 STAILQ_INSERT_TAIL(&engine.context_list, node, link); 403 404 /* Engine has been init*/ 405 engine.state = INIT; 406 407 return TEE_SUCCESS; 408 error: 409 versal_mbox_free(&nonce); 410 versal_mbox_free(&init_buf); 411 versal_mbox_free(&key); 412 413 return ret; 414 } 415 416 static TEE_Result do_update_aad(struct drvcrypt_authenc_update_aad *dupdate) 417 { 418 struct versal_cmd_args arg = { }; 419 struct versal_mbox_mem p = { }; 420 TEE_Result ret = TEE_SUCCESS; 421 struct versal_node *node = NULL; 422 uint32_t err = 0; 423 424 /* This context has not been inited */ 425 if (!context_allowed(dupdate->ctx)) 426 return TEE_ERROR_BUSY; 427 428 /* There is a copy of the context: don't allow updates, only finalize */ 429 if (refcount_val(&engine.refc) > 1) 430 return TEE_ERROR_BUSY; 431 432 /* There was a copy of the context and it was finalized, then replay */ 433 if (engine.state == FINALIZED) 434 do_replay(); 435 436 ret = versal_mbox_alloc(dupdate->aad.length, dupdate->aad.data, &p); 437 if (ret) 438 return ret; 439 440 arg.data[arg.dlen++] = p.len % 16 ? p.alloc_len : p.len; 441 arg.ibuf[0].mem = p; 442 443 #if DEBUG_VERSAL_AES 444 IMSG("versal: aad length - requested: %zu, sent to plm: %"PRIu32, 445 dupdate->aad.length, arg.data[0]); 446 #endif 447 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) { 448 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err)); 449 ret = TEE_ERROR_GENERIC; 450 goto error; 451 } 452 453 node = calloc(1, sizeof(*node)); 454 if (!node) { 455 ret = TEE_ERROR_OUT_OF_MEMORY; 456 goto error; 457 } 458 459 /* Save the context */ 460 node->aad.mem = p; 461 node->is_aad = true; 462 STAILQ_INSERT_TAIL(&engine.replay_list, node, link); 463 464 return TEE_SUCCESS; 465 error: 466 versal_mbox_free(&p); 467 return ret; 468 } 469 470 static TEE_Result 471 update_payload(struct drvcrypt_authenc_update_payload *dupdate, bool is_last) 472 { 473 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE; 474 struct versal_aes_input_param *input = NULL; 475 struct versal_mbox_mem input_cmd = { }; 476 struct versal_mbox_mem p = { }; 477 struct versal_mbox_mem q = { }; 478 TEE_Result ret = TEE_SUCCESS; 479 struct versal_cmd_args arg = { }; 480 struct versal_node *node = NULL; 481 uint32_t err = 0; 482 483 if (!context_allowed(dupdate->ctx)) 484 return TEE_ERROR_BUSY; 485 486 if (!dupdate->src.length) { 487 EMSG("Versal AES payload length must be non-zero"); 488 return TEE_ERROR_BAD_PARAMETERS; 489 } 490 #if !defined(PLATFORM_FLAVOR_net) 491 if ((!is_last && dupdate->src.length % 4) || 492 (is_last && dupdate->src.length % 16)) { 493 EMSG("Versal AES payload length not properly aligned (len = %zu)", 494 dupdate->src.length); 495 return TEE_ERROR_BAD_PARAMETERS; 496 } 497 #endif 498 499 ret = versal_mbox_alloc(dupdate->src.length, dupdate->src.data, &p); 500 if (ret) 501 return ret; 502 ret = versal_mbox_alloc(dupdate->dst.length, NULL, &q); 503 if (ret) 504 goto out; 505 ret = versal_mbox_alloc(sizeof(*input), NULL, &input_cmd); 506 if (ret) 507 goto out; 508 509 input = input_cmd.buf; 510 input->input_addr = virt_to_phys(p.buf); 511 input->input_len = p.len % 16 ? p.alloc_len : p.len; 512 input->is_last = is_last; 513 514 arg.ibuf[0].mem = input_cmd; 515 arg.ibuf[1].mem = q; 516 arg.ibuf[2].mem = p; 517 518 if (dupdate->encrypt) 519 id = VERSAL_AES_ENCRYPT_UPDATE; 520 521 #if DEBUG_VERSAL_AES 522 IMSG("versal: payload length - requested %zu, sent to plm: %"PRIu32, 523 dupdate->src.length, input->input_len); 524 IMSG("versal: destination length - %zu ", dupdate->dst.length); 525 #endif 526 if (versal_crypto_request(id, &arg, &err)) { 527 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err)); 528 ret = TEE_ERROR_GENERIC; 529 goto out; 530 } 531 532 if (dupdate->dst.data) 533 memcpy(dupdate->dst.data, q.buf, dupdate->dst.length); 534 535 if (!is_last) { 536 node = calloc(1, sizeof(*node)); 537 if (!node) { 538 ret = TEE_ERROR_OUT_OF_MEMORY; 539 goto out; 540 } 541 542 node->is_aad = false; 543 node->payload.dst = q; 544 node->payload.src = p; 545 node->payload.input_cmd = input_cmd; 546 node->payload.encrypt = dupdate->encrypt; 547 STAILQ_INSERT_TAIL(&engine.replay_list, node, link); 548 549 return TEE_SUCCESS; 550 } 551 out: 552 versal_mbox_free(&input_cmd); 553 versal_mbox_free(&q); 554 versal_mbox_free(&p); 555 556 return ret; 557 } 558 559 static TEE_Result do_update_payload(struct drvcrypt_authenc_update_payload *p) 560 { 561 TEE_Result ret = TEE_SUCCESS; 562 563 if (!context_allowed(p->ctx)) 564 return TEE_ERROR_BUSY; 565 566 /* 567 * If there is a copy, we don't allow updates until one of the copies 568 * has been deleted 569 */ 570 if (refcount_val(&engine.refc) > 1) 571 return TEE_ERROR_BUSY; 572 573 /* 574 * If there was a copy and it was finalized, we need to replay before 575 * we can update; do not clear the list so the state can be copied 576 */ 577 if (engine.state == FINALIZED) { 578 ret = do_replay(); 579 if (ret) 580 return ret; 581 } 582 583 return update_payload(p, false); 584 } 585 586 static TEE_Result do_enc_final(struct drvcrypt_authenc_final *dfinal) 587 { 588 struct drvcrypt_authenc_update_payload last = { }; 589 struct versal_cmd_args arg = { }; 590 struct versal_mbox_mem p = { }; 591 TEE_Result ret = TEE_SUCCESS; 592 uint32_t err = 0; 593 594 if (!context_allowed(dfinal->ctx)) 595 return TEE_ERROR_BUSY; 596 597 if (engine.state == FINALIZED) { 598 DMSG("Operation was already finalized"); 599 ret = do_replay(); 600 if (ret) 601 return ret; 602 } 603 604 if (engine.state != INIT) 605 panic(); 606 607 last.ctx = dfinal->ctx; 608 last.dst = dfinal->dst; 609 last.encrypt = true; 610 last.src = dfinal->src; 611 612 ret = update_payload(&last, true); 613 if (ret) 614 return ret; 615 616 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length); 617 618 ret = versal_mbox_alloc(GCM_TAG_LEN, NULL, &p); 619 if (ret) 620 return ret; 621 622 arg.ibuf[0].mem = p; 623 if (versal_crypto_request(VERSAL_AES_ENCRYPT_FINAL, &arg, &err)) { 624 EMSG("AES_ENCRYPT_FINAL error: %s", versal_aes_error(err)); 625 ret = TEE_ERROR_GENERIC; 626 goto out; 627 } 628 629 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN); 630 dfinal->tag.length = GCM_TAG_LEN; 631 out: 632 versal_mbox_free(&p); 633 634 if (refcount_val(&engine.refc) > 1) 635 engine.state = FINALIZED; 636 else 637 engine.state = READY; 638 639 return ret; 640 } 641 642 static TEE_Result do_dec_final(struct drvcrypt_authenc_final *dfinal) 643 { 644 struct drvcrypt_authenc_update_payload last = { }; 645 struct versal_cmd_args arg = { }; 646 struct versal_mbox_mem p = { }; 647 TEE_Result ret = TEE_SUCCESS; 648 uint32_t err = 0; 649 650 if (!context_allowed(dfinal->ctx)) 651 return TEE_ERROR_BUSY; 652 653 if (engine.state == FINALIZED) { 654 DMSG("Operation was already finalized"); 655 ret = do_replay(); 656 if (ret) 657 return ret; 658 } 659 660 if (engine.state != INIT) 661 panic(); 662 663 last.encrypt = false; 664 last.ctx = dfinal->ctx; 665 last.dst = dfinal->dst; 666 last.src = dfinal->src; 667 668 ret = update_payload(&last, true); 669 if (ret) 670 return ret; 671 672 ret = versal_mbox_alloc(dfinal->tag.length, dfinal->tag.data, &p); 673 if (ret) 674 return ret; 675 676 arg.ibuf[0].mem = p; 677 678 if (versal_crypto_request(VERSAL_AES_DECRYPT_FINAL, &arg, &err)) { 679 EMSG("AES_DECRYPT_FINAL error: %s", versal_aes_error(err)); 680 ret = TEE_ERROR_GENERIC; 681 goto out; 682 } 683 684 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length); 685 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN); 686 dfinal->tag.length = GCM_TAG_LEN; 687 out: 688 versal_mbox_free(&p); 689 690 if (refcount_val(&engine.refc) > 1) 691 engine.state = FINALIZED; 692 else 693 engine.state = READY; 694 695 return ret; 696 } 697 698 static void do_final(void *ctx __unused) 699 { 700 } 701 702 static void do_free(void *ctx) 703 { 704 struct versal_ae_ctx *c = to_versal_ctx(ctx); 705 struct versal_node *next = NULL; 706 struct versal_node *node = NULL; 707 struct versal_context_node *ctx_next = NULL; 708 struct versal_context_node *ctx_node = NULL; 709 bool release = false; 710 711 if (refcount_dec(&engine.refc)) { 712 /* this is a final release */ 713 release = true; 714 refcount_set(&engine.refc, 1); 715 engine.state = READY; 716 versal_mbox_free(&engine.init.init_buf); 717 versal_mbox_free(&engine.init.nonce); 718 versal_mbox_free(&engine.init.key); 719 memset(&engine.init, 0, sizeof(engine.init)); 720 STAILQ_FOREACH_SAFE(node, &engine.replay_list, link, next) { 721 STAILQ_REMOVE(&engine.replay_list, node, 722 versal_node, link); 723 if (node->is_aad) { 724 versal_mbox_free(&node->aad.mem); 725 } else { 726 versal_mbox_free(&node->payload.dst); 727 versal_mbox_free(&node->payload.src); 728 versal_mbox_free(&node->payload.input_cmd); 729 } 730 free(node); 731 } 732 } 733 734 STAILQ_FOREACH_SAFE(ctx_node, &engine.context_list, link, ctx_next) { 735 if (c == ctx_node->ctx) { 736 STAILQ_REMOVE(&engine.context_list, ctx_node, 737 versal_context_node, link); 738 free(ctx_node); 739 } 740 } 741 742 if (release && engine_in_use()) 743 panic(); 744 745 free(c); 746 } 747 748 static void do_copy_state(void *dst_ctx, void *src_ctx) 749 { 750 struct versal_context_node *node = NULL; 751 752 STAILQ_FOREACH(node, &engine.context_list, link) { 753 if (node->ctx != to_versal_ctx(src_ctx)) 754 continue; 755 /* 756 * The running context has been copied: from now on we can only 757 * finalize any of the contexts until there is only one active 758 * again. 759 */ 760 node = calloc(1, sizeof(*node)); 761 if (!node) 762 panic(); 763 764 node->ctx = to_versal_ctx(dst_ctx); 765 STAILQ_INSERT_TAIL(&engine.context_list, node, link); 766 767 /* number of active contexts */ 768 refcount_inc(&engine.refc); 769 770 return; 771 } 772 773 panic(); 774 } 775 776 static TEE_Result do_allocate(void **ctx, uint32_t algo) 777 { 778 struct versal_ae_ctx *c = NULL; 779 780 if (algo != TEE_ALG_AES_GCM) 781 return TEE_ERROR_NOT_IMPLEMENTED; 782 783 c = calloc(1, sizeof(*c)); 784 if (!c) 785 return TEE_ERROR_OUT_OF_MEMORY; 786 787 *ctx = &c->a_ctx; 788 789 return TEE_SUCCESS; 790 } 791 792 static TEE_Result 793 do_update_payload_locked(struct drvcrypt_authenc_update_payload *p) 794 { 795 TEE_Result ret = TEE_SUCCESS; 796 797 mutex_lock(engine.lock); 798 ret = do_update_payload(p); 799 mutex_unlock(engine.lock); 800 return ret; 801 } 802 803 static TEE_Result 804 do_update_aad_locked(struct drvcrypt_authenc_update_aad *p) 805 { 806 TEE_Result ret = TEE_SUCCESS; 807 808 mutex_lock(engine.lock); 809 ret = do_update_aad(p); 810 mutex_unlock(engine.lock); 811 return ret; 812 } 813 814 static void do_copy_state_locked(void *dst, void *src) 815 { 816 mutex_lock(engine.lock); 817 do_copy_state(dst, src); 818 mutex_unlock(engine.lock); 819 } 820 821 static TEE_Result do_enc_final_locked(struct drvcrypt_authenc_final *p) 822 { 823 TEE_Result ret = TEE_SUCCESS; 824 825 mutex_lock(engine.lock); 826 ret = do_enc_final(p); 827 mutex_unlock(engine.lock); 828 return ret; 829 } 830 831 static TEE_Result do_dec_final_locked(struct drvcrypt_authenc_final *p) 832 { 833 TEE_Result ret = TEE_SUCCESS; 834 835 mutex_lock(engine.lock); 836 ret = do_dec_final(p); 837 mutex_unlock(engine.lock); 838 return ret; 839 } 840 841 static void do_free_locked(void *p) 842 { 843 mutex_lock(engine.lock); 844 do_free(p); 845 mutex_unlock(engine.lock); 846 } 847 848 static TEE_Result do_init_locked(struct drvcrypt_authenc_init *p) 849 { 850 TEE_Result ret = TEE_SUCCESS; 851 852 mutex_lock(engine.lock); 853 ret = do_init(p); 854 mutex_unlock(engine.lock); 855 return ret; 856 } 857 858 static struct drvcrypt_authenc versal_authenc = { 859 .update_payload = do_update_payload_locked, 860 .update_aad = do_update_aad_locked, 861 .copy_state = do_copy_state_locked, 862 .enc_final = do_enc_final_locked, 863 .dec_final = do_dec_final_locked, 864 .free_ctx = do_free_locked, 865 .alloc_ctx = do_allocate, 866 .init = do_init_locked, 867 .final = do_final, 868 }; 869 870 static TEE_Result enable_secure_status(void) 871 { 872 /* Once Linux has support, we need to reserve the device */ 873 return TEE_SUCCESS; 874 } 875 876 static TEE_Result versal_register_authenc(void) 877 { 878 TEE_Result ret = TEE_SUCCESS; 879 880 ret = drvcrypt_register_authenc(&versal_authenc); 881 if (ret) 882 return ret; 883 884 if (engine.key_src < XSECURE_AES_USER_KEY_0 || 885 engine.key_src > XSECURE_AES_USER_KEY_7) 886 return TEE_ERROR_GENERIC; 887 888 engine.state = READY; 889 STAILQ_INIT(&engine.replay_list); 890 STAILQ_INIT(&engine.context_list); 891 refcount_set(&engine.refc, 1); 892 893 return enable_secure_status(); 894 } 895 896 driver_init_late(versal_register_authenc); 897