1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright 2022-2024 HiSilicon Limited. 4 * Kunpeng hardware accelerator sec cipher algorithm implementation. 5 */ 6 7 #include <drvcrypt.h> 8 #include <drvcrypt_cipher.h> 9 #include <initcall.h> 10 #include <trace.h> 11 #include <utee_defines.h> 12 13 #include "sec_cipher.h" 14 #include "sec_main.h" 15 16 static TEE_Result sec_do_cipher_task(struct hisi_qp *qp, void *msg) 17 { 18 enum hisi_drv_status ret = HISI_QM_DRVCRYPT_NO_ERR; 19 20 ret = hisi_qp_send(qp, msg); 21 if (ret) { 22 EMSG("Fail to send task, ret=%d", ret); 23 return TEE_ERROR_BAD_STATE; 24 } 25 26 ret = hisi_qp_recv_sync(qp, msg); 27 if (ret) { 28 EMSG("Recv task error, ret=%d", ret); 29 return TEE_ERROR_BAD_STATE; 30 } 31 32 return TEE_SUCCESS; 33 } 34 35 static TEE_Result sec_cipher_des_get_c_key_len(size_t key_len, 36 uint8_t *c_key_len) 37 { 38 if (key_len == DES_KEY_SIZE) { 39 *c_key_len = CKEY_LEN_DES; 40 } else { 41 EMSG("Invalid DES key size"); 42 return TEE_ERROR_BAD_PARAMETERS; 43 } 44 45 return TEE_SUCCESS; 46 } 47 48 static TEE_Result sec_cipher_3des_get_c_key_len(size_t key_len, 49 uint8_t *c_key_len) 50 { 51 if (key_len == SEC_3DES_2KEY_SIZE) { 52 *c_key_len = CKEY_LEN_3DES_2KEY; 53 } else if (key_len == SEC_3DES_3KEY_SIZE) { 54 *c_key_len = CKEY_LEN_3DES_3KEY; 55 } else { 56 EMSG("Invalid 3DES key size"); 57 return TEE_ERROR_BAD_PARAMETERS; 58 } 59 60 return TEE_SUCCESS; 61 } 62 63 static TEE_Result sec_cipher_aes_get_c_key_len(size_t key_len, 64 enum sec_c_mode mode, 65 uint8_t *c_key_len) 66 { 67 switch (mode) { 68 case C_MODE_ECB: 69 case C_MODE_CBC: 70 case C_MODE_CTR: 71 switch (key_len) { 72 case AES_KEYSIZE_128: 73 *c_key_len = CKEY_LEN_128_BIT; 74 break; 75 case AES_KEYSIZE_192: 76 *c_key_len = CKEY_LEN_192_BIT; 77 break; 78 case AES_KEYSIZE_256: 79 *c_key_len = CKEY_LEN_256_BIT; 80 break; 81 default: 82 EMSG("Invalid AES key size"); 83 return TEE_ERROR_BAD_PARAMETERS; 84 } 85 break; 86 case C_MODE_XTS: 87 switch (key_len) { 88 case XTS_KEYSIZE_128: 89 *c_key_len = CKEY_LEN_128_BIT; 90 break; 91 case XTS_KEYSIZE_256: 92 *c_key_len = CKEY_LEN_256_BIT; 93 break; 94 default: 95 EMSG("Invalid AES-XTS key size"); 96 return TEE_ERROR_BAD_PARAMETERS; 97 } 98 break; 99 default: 100 EMSG("Unsupported AES mode"); 101 return TEE_ERROR_BAD_PARAMETERS; 102 } 103 104 return TEE_SUCCESS; 105 } 106 107 static TEE_Result sec_cipher_sm4_get_c_key_len(size_t key_len, 108 enum sec_c_mode mode, 109 uint8_t *c_key_len) 110 { 111 switch (mode) { 112 case C_MODE_ECB: 113 case C_MODE_CBC: 114 case C_MODE_CTR: 115 if (key_len != AES_KEYSIZE_128) { 116 EMSG("Invalid SM4 key size"); 117 return TEE_ERROR_BAD_PARAMETERS; 118 } 119 *c_key_len = CKEY_LEN_128_BIT; 120 break; 121 case C_MODE_XTS: 122 if (key_len != XTS_KEYSIZE_128) { 123 EMSG("Invalid SM4-XTS key size"); 124 return TEE_ERROR_BAD_PARAMETERS; 125 } 126 *c_key_len = CKEY_LEN_128_BIT; 127 break; 128 default: 129 EMSG("Unsupported SM4 mode"); 130 return TEE_ERROR_BAD_PARAMETERS; 131 } 132 133 return TEE_SUCCESS; 134 } 135 136 static TEE_Result sec_cipher_set_key(struct sec_cipher_ctx *c_ctx, 137 uint8_t *key1, size_t key1_len, 138 uint8_t *key2, size_t key2_len) 139 { 140 size_t key_len = key1_len + key2_len; 141 TEE_Result ret = TEE_SUCCESS; 142 uint8_t c_key_len = 0; 143 144 switch (c_ctx->alg) { 145 case C_ALG_DES: 146 ret = sec_cipher_des_get_c_key_len(key_len, &c_key_len); 147 break; 148 case C_ALG_3DES: 149 ret = sec_cipher_3des_get_c_key_len(key_len, &c_key_len); 150 break; 151 case C_ALG_AES: 152 ret = sec_cipher_aes_get_c_key_len(key_len, c_ctx->mode, 153 &c_key_len); 154 break; 155 case C_ALG_SM4: 156 ret = sec_cipher_sm4_get_c_key_len(key_len, c_ctx->mode, 157 &c_key_len); 158 break; 159 default: 160 EMSG("Invalid cipher type %#"PRIx8, c_ctx->alg); 161 ret = TEE_ERROR_NOT_IMPLEMENTED; 162 break; 163 } 164 165 if (ret) 166 return ret; 167 168 c_ctx->key_dma = virt_to_phys(c_ctx->key); 169 if (!c_ctx->key_dma) { 170 EMSG("c_key_dma is NULL"); 171 return TEE_ERROR_GENERIC; 172 } 173 174 c_ctx->key_len = key_len; 175 c_ctx->c_key_len = c_key_len; 176 177 memcpy(c_ctx->key, key1, key1_len); 178 memcpy(c_ctx->key + key1_len, key2, key2_len); 179 180 return TEE_SUCCESS; 181 } 182 183 static void ctr_iv_inc(uint64_t *ctr, uint32_t inc) 184 { 185 uint64_t v0 = TEE_U64_FROM_BIG_ENDIAN(ctr[0]); 186 uint64_t v1 = TEE_U64_FROM_BIG_ENDIAN(ctr[1]); 187 188 /* increment counter (128-bit int) by inc */ 189 if (ADD_OVERFLOW(v1, inc, &v1)) 190 v0++; 191 192 ctr[0] = TEE_U64_TO_BIG_ENDIAN(v0); 193 ctr[1] = TEE_U64_TO_BIG_ENDIAN(v1); 194 } 195 196 static void xts_multi_galois(unsigned char *data) 197 { 198 int i = 0; 199 uint8_t t = 0; 200 uint8_t tt = 0; 201 202 for (i = 0; i < AES_SM4_IV_SIZE; i++) { 203 tt = data[i] >> LEFT_MOST_BIT; 204 data[i] = ((data[i] << 1) | t) & 0xFF; 205 t = tt; 206 } 207 if (tt) 208 data[0] ^= 0x87; 209 } 210 211 /* 212 * When the IV is delivered by segment, 213 * the AES/SM4-ECB is used to update the IV to be used next time. 214 */ 215 static TEE_Result xts_iv_update(struct sec_cipher_ctx *c_ctx) 216 { 217 size_t xts_key_len = c_ctx->key_len / 2; 218 struct sec_cipher_ctx ecb_ctx = { }; 219 TEE_Result ret = TEE_SUCCESS; 220 size_t i = 0; 221 222 ecb_ctx.alg = c_ctx->alg; 223 ecb_ctx.mode = C_MODE_ECB; 224 ret = sec_cipher_set_key(&ecb_ctx, c_ctx->key + xts_key_len, 225 xts_key_len, NULL, 0); 226 if (ret) 227 return ret; 228 229 ecb_ctx.encrypt = true; 230 ecb_ctx.in = (uint8_t *)c_ctx->iv; 231 ecb_ctx.out = (uint8_t *)c_ctx->iv; 232 ecb_ctx.in_dma = c_ctx->iv_dma; 233 ecb_ctx.out_dma = c_ctx->iv_dma; 234 ecb_ctx.len = c_ctx->iv_len; 235 236 ret = sec_do_cipher_task(c_ctx->qp, &ecb_ctx); 237 if (ret) { 238 EMSG("Fail to encrypt xts iv, ret=%#"PRIx32, ret); 239 return ret; 240 } 241 242 for (i = 0; i < DIV_ROUND_UP(c_ctx->len, AES_SM4_BLOCK_SIZE); i++) 243 xts_multi_galois((uint8_t *)c_ctx->iv); 244 245 ecb_ctx.encrypt = false; 246 ret = sec_do_cipher_task(c_ctx->qp, &ecb_ctx); 247 if (ret) 248 EMSG("Fail to decrypt xts iv, ret=%#"PRIx32, ret); 249 250 return ret; 251 } 252 253 static TEE_Result sec_update_iv(struct sec_cipher_ctx *c_ctx) 254 { 255 TEE_Result ret = TEE_SUCCESS; 256 size_t offset = 0; 257 258 switch (c_ctx->mode) { 259 case C_MODE_CBC: 260 offset = c_ctx->len - c_ctx->iv_len; 261 if (c_ctx->encrypt && c_ctx->len >= c_ctx->iv_len) 262 memcpy(c_ctx->iv, c_ctx->out + offset, c_ctx->iv_len); 263 if (!c_ctx->encrypt && c_ctx->len >= c_ctx->iv_len) 264 memcpy(c_ctx->iv, c_ctx->in + offset, c_ctx->iv_len); 265 break; 266 case C_MODE_CTR: 267 /* 268 * Increase the iv counter with the number of processed blocks. 269 */ 270 ctr_iv_inc(c_ctx->iv, c_ctx->len >> CTR_MODE_LEN_SHIFT); 271 break; 272 case C_MODE_XTS: 273 ret = xts_iv_update(c_ctx); 274 break; 275 default: 276 break; 277 } 278 279 return ret; 280 } 281 282 static TEE_Result sec_cipher_iv_check(struct sec_cipher_ctx *c_ctx, 283 size_t iv_size) 284 { 285 TEE_Result ret = TEE_ERROR_BAD_PARAMETERS; 286 287 switch (c_ctx->mode) { 288 case C_MODE_ECB: 289 if (!iv_size) 290 ret = TEE_SUCCESS; 291 break; 292 case C_MODE_CBC: 293 if (c_ctx->alg == C_ALG_DES || c_ctx->alg == C_ALG_3DES) { 294 if (iv_size == DES_CBC_IV_SIZE) 295 ret = TEE_SUCCESS; 296 break; 297 } 298 fallthrough; 299 case C_MODE_XTS: 300 case C_MODE_CTR: 301 if (c_ctx->alg == C_ALG_AES || c_ctx->alg == C_ALG_SM4) { 302 if (iv_size == AES_SM4_IV_SIZE) 303 ret = TEE_SUCCESS; 304 } 305 break; 306 default: 307 break; 308 } 309 310 if (ret) 311 EMSG("Fail to check iv_size"); 312 313 return ret; 314 } 315 316 static TEE_Result sec_cipher_set_iv(struct sec_cipher_ctx *c_ctx, 317 uint8_t *iv, size_t iv_len) 318 { 319 TEE_Result ret = TEE_SUCCESS; 320 321 if (!iv && iv_len) { 322 EMSG("iv is NULL"); 323 return TEE_ERROR_BAD_PARAMETERS; 324 } 325 326 ret = sec_cipher_iv_check(c_ctx, iv_len); 327 if (ret) 328 return ret; 329 330 c_ctx->iv_len = iv_len; 331 c_ctx->iv_dma = virt_to_phys(c_ctx->iv); 332 if (!c_ctx->iv_dma) { 333 EMSG("c_iv_dma is NULL"); 334 return TEE_ERROR_BAD_PARAMETERS; 335 } 336 337 memcpy(c_ctx->iv, iv, c_ctx->iv_len); 338 339 return TEE_SUCCESS; 340 } 341 342 static enum hisi_drv_status sec_cipher_bd_fill(void *bd, void *msg) 343 { 344 struct sec_cipher_ctx *c_ctx = msg; 345 struct hisi_sec_sqe *sqe = bd; 346 uint8_t cipher = 0; 347 uint8_t scene = 0; 348 uint8_t de = 0; 349 350 sqe->type_auth_cipher = BD_TYPE2; 351 scene = SHIFT_U32(SCENE_NOTHING, SEC_SCENE_OFFSET); 352 de = SHIFT_U32(DATA_DST_ADDR_ENABLE, SEC_DE_OFFSET); 353 sqe->sds_sa_type = de | scene; 354 sqe->type2.clen_ivhlen = c_ctx->len; 355 356 sqe->type2.c_alg = c_ctx->alg; 357 sqe->type2.icvw_kmode = SHIFT_U32(c_ctx->mode, SEC_CMODE_OFFSET) | 358 SHIFT_U32(c_ctx->c_key_len, SEC_CKEY_OFFSET); 359 360 if (c_ctx->encrypt) 361 cipher = SHIFT_U32(CIPHER_ENCRYPT, SEC_CIPHER_OFFSET); 362 else 363 cipher = SHIFT_U32(CIPHER_DECRYPT, SEC_CIPHER_OFFSET); 364 365 sqe->type_auth_cipher |= cipher; 366 367 sqe->type2.data_dst_addr = c_ctx->out_dma; 368 sqe->type2.data_src_addr = c_ctx->in_dma; 369 sqe->type2.c_key_addr = c_ctx->key_dma; 370 sqe->type2.c_ivin_addr = c_ctx->iv_dma; 371 372 return HISI_QM_DRVCRYPT_NO_ERR; 373 } 374 375 static enum hisi_drv_status sec_cipher_bd_parse(void *bd, void *msg __unused) 376 { 377 struct hisi_sec_sqe *sqe = bd; 378 uint16_t done = 0; 379 380 done = SEC_GET_FIELD(sqe->type2.done_flag, SEC_DONE_MASK, 0); 381 if (done != SEC_HW_TASK_DONE || sqe->type2.error_type) { 382 EMSG("SEC BD2 fail! done=%#"PRIx16", etype=%#"PRIx8, 383 done, sqe->type2.error_type); 384 return HISI_QM_DRVCRYPT_IN_EPARA; 385 } 386 387 return HISI_QM_DRVCRYPT_NO_ERR; 388 } 389 390 static enum hisi_drv_status sec_cipher_bd3_fill(void *bd, void *msg) 391 { 392 struct hisi_sec_bd3_sqe *sqe = bd; 393 struct sec_cipher_ctx *c_ctx = msg; 394 395 sqe->bd_param = BD_TYPE3 | SHIFT_U32(SCENE_NOTHING, 396 SEC_SCENE_OFFSET_V3) | 397 SHIFT_U32(DATA_DST_ADDR_ENABLE, SEC_DE_OFFSET_V3); 398 sqe->c_len_ivin = c_ctx->len; 399 sqe->c_mode_alg = c_ctx->mode | 400 SHIFT_U32(c_ctx->alg, SEC_CALG_OFFSET_V3); 401 sqe->c_icv_key = SHIFT_U32(c_ctx->c_key_len, SEC_CKEY_OFFSET_V3); 402 403 if (c_ctx->encrypt) 404 sqe->c_icv_key |= CIPHER_ENCRYPT; 405 else 406 sqe->c_icv_key |= CIPHER_DECRYPT; 407 408 sqe->data_dst_addr = c_ctx->out_dma; 409 sqe->data_src_addr = c_ctx->in_dma; 410 sqe->c_key_addr = c_ctx->key_dma; 411 sqe->no_scene.c_ivin_addr = c_ctx->iv_dma; 412 413 return HISI_QM_DRVCRYPT_NO_ERR; 414 } 415 416 static enum hisi_drv_status sec_cipher_bd3_parse(void *bd, void *msg __unused) 417 { 418 struct hisi_sec_bd3_sqe *sqe = bd; 419 uint16_t done = 0; 420 421 done = SEC_GET_FIELD(sqe->done_flag, SEC_DONE_MASK, 0); 422 if (done != SEC_HW_TASK_DONE || sqe->error_type) { 423 EMSG("SEC BD3 fail! done=%#"PRIx16", etype=%#"PRIx8, 424 done, sqe->error_type); 425 return HISI_QM_DRVCRYPT_IN_EPARA; 426 } 427 428 return HISI_QM_DRVCRYPT_NO_ERR; 429 } 430 431 static TEE_Result cipher_algo_check(uint32_t algo) 432 { 433 switch (algo) { 434 case TEE_ALG_AES_ECB_NOPAD: 435 case TEE_ALG_AES_CBC_NOPAD: 436 case TEE_ALG_AES_CTR: 437 case TEE_ALG_AES_XTS: 438 case TEE_ALG_DES_ECB_NOPAD: 439 case TEE_ALG_DES3_ECB_NOPAD: 440 case TEE_ALG_DES_CBC_NOPAD: 441 case TEE_ALG_DES3_CBC_NOPAD: 442 case TEE_ALG_SM4_CBC_NOPAD: 443 case TEE_ALG_SM4_ECB_NOPAD: 444 case TEE_ALG_SM4_XTS: 445 case TEE_ALG_SM4_CTR: 446 break; 447 default: 448 return TEE_ERROR_NOT_IMPLEMENTED; 449 } 450 451 return TEE_SUCCESS; 452 } 453 454 static TEE_Result crypto_set_alg(struct sec_cipher_ctx *c_ctx, uint32_t alg) 455 { 456 TEE_Result ret = TEE_SUCCESS; 457 458 switch (alg) { 459 case TEE_MAIN_ALGO_DES: 460 c_ctx->alg = C_ALG_DES; 461 break; 462 case TEE_MAIN_ALGO_DES3: 463 c_ctx->alg = C_ALG_3DES; 464 break; 465 case TEE_MAIN_ALGO_AES: 466 c_ctx->alg = C_ALG_AES; 467 break; 468 case TEE_MAIN_ALGO_SM4: 469 c_ctx->alg = C_ALG_SM4; 470 break; 471 default: 472 EMSG("Invalid cipher type %#"PRIx32, alg); 473 ret = TEE_ERROR_NOT_IMPLEMENTED; 474 break; 475 } 476 477 return ret; 478 } 479 480 static TEE_Result crypto_set_mode(struct sec_cipher_ctx *c_ctx, uint32_t mode) 481 { 482 TEE_Result ret = TEE_SUCCESS; 483 484 switch (mode) { 485 case TEE_CHAIN_MODE_ECB_NOPAD: 486 c_ctx->mode = C_MODE_ECB; 487 break; 488 case TEE_CHAIN_MODE_CBC_NOPAD: 489 c_ctx->mode = C_MODE_CBC; 490 break; 491 case TEE_CHAIN_MODE_XTS: 492 c_ctx->mode = C_MODE_XTS; 493 break; 494 case TEE_CHAIN_MODE_CTR: 495 c_ctx->mode = C_MODE_CTR; 496 break; 497 default: 498 EMSG("Invalid cipher mode type %#"PRIx32, mode); 499 ret = TEE_ERROR_NOT_IMPLEMENTED; 500 break; 501 } 502 503 return ret; 504 } 505 506 static TEE_Result sec_cipher_ctx_allocate(void **ctx, uint32_t algo) 507 { 508 struct sec_cipher_ctx *c_ctx = NULL; 509 TEE_Result ret = TEE_SUCCESS; 510 511 if (!ctx) { 512 EMSG("ctx is NULL"); 513 return TEE_ERROR_BAD_PARAMETERS; 514 } 515 516 ret = cipher_algo_check(algo); 517 if (ret) 518 return ret; 519 520 c_ctx = calloc(1, sizeof(struct sec_cipher_ctx)); 521 if (!c_ctx) { 522 EMSG("c_ctx is NULL"); 523 return TEE_ERROR_OUT_OF_MEMORY; 524 } 525 526 ret = crypto_set_alg(c_ctx, TEE_ALG_GET_MAIN_ALG(algo)); 527 if (ret) 528 goto free_c_ctx; 529 530 ret = crypto_set_mode(c_ctx, TEE_ALG_GET_CHAIN_MODE(algo)); 531 if (ret) 532 goto free_c_ctx; 533 534 c_ctx->qp = sec_create_qp(HISI_QM_CHANNEL_TYPE0); 535 if (!c_ctx->qp) { 536 ret = TEE_ERROR_BUSY; 537 goto free_c_ctx; 538 } 539 540 if (c_ctx->qp->qm->version == HISI_QM_HW_V2) { 541 c_ctx->qp->fill_sqe = sec_cipher_bd_fill; 542 c_ctx->qp->parse_sqe = sec_cipher_bd_parse; 543 } else { 544 c_ctx->qp->fill_sqe = sec_cipher_bd3_fill; 545 c_ctx->qp->parse_sqe = sec_cipher_bd3_parse; 546 } 547 548 c_ctx->offs = 0; 549 *ctx = c_ctx; 550 551 return TEE_SUCCESS; 552 553 free_c_ctx: 554 free(c_ctx); 555 556 return ret; 557 } 558 559 static void sec_cipher_ctx_free(void *ctx) 560 { 561 struct sec_cipher_ctx *c_ctx = ctx; 562 563 if (!c_ctx) 564 return; 565 566 hisi_qm_release_qp(c_ctx->qp); 567 memzero_explicit(c_ctx->key, c_ctx->key_len); 568 free(c_ctx); 569 } 570 571 static TEE_Result sec_cipher_initialize(struct drvcrypt_cipher_init *dinit) 572 { 573 struct sec_cipher_ctx *c_ctx = NULL; 574 TEE_Result ret = TEE_SUCCESS; 575 576 if (!dinit || !dinit->ctx || !dinit->key1.data) { 577 EMSG("drvcrypt_cipher init param error"); 578 return TEE_ERROR_BAD_PARAMETERS; 579 } 580 581 c_ctx = dinit->ctx; 582 583 ret = sec_cipher_set_key(c_ctx, dinit->key1.data, dinit->key1.length, 584 dinit->key2.data, dinit->key2.length); 585 if (ret) 586 return ret; 587 588 ret = sec_cipher_set_iv(c_ctx, dinit->iv.data, dinit->iv.length); 589 if (ret) 590 return ret; 591 592 c_ctx->encrypt = dinit->encrypt; 593 594 return TEE_SUCCESS; 595 } 596 597 static TEE_Result sec_cipher_cryptlen_check(struct sec_cipher_ctx *c_ctx, 598 size_t length) 599 { 600 if (c_ctx->mode == C_MODE_XTS && length < AES_SM4_BLOCK_SIZE) { 601 EMSG("Invalid xts src len"); 602 return TEE_ERROR_BAD_PARAMETERS; 603 } 604 605 if ((c_ctx->mode == C_MODE_ECB || c_ctx->mode == C_MODE_CBC) && 606 (length & (AES_SM4_BLOCK_SIZE - 1))) { 607 EMSG("Invalid ecb cbc src len"); 608 return TEE_ERROR_BAD_PARAMETERS; 609 } 610 611 return TEE_SUCCESS; 612 } 613 614 static TEE_Result sec_cipher_param_check(struct drvcrypt_cipher_update *dupdate) 615 { 616 struct sec_cipher_ctx *c_ctx = NULL; 617 618 if (!dupdate || !dupdate->src.data || !dupdate->dst.data || 619 dupdate->src.length != dupdate->dst.length || 620 dupdate->src.length > MAX_CIPHER_LENGTH || !dupdate->src.length) { 621 EMSG("Dupdate input param error"); 622 return TEE_ERROR_BAD_PARAMETERS; 623 } 624 625 c_ctx = dupdate->ctx; 626 switch (c_ctx->alg) { 627 case C_ALG_SM4: 628 case C_ALG_AES: 629 if (sec_cipher_cryptlen_check(c_ctx, dupdate->src.length)) 630 return TEE_ERROR_BAD_PARAMETERS; 631 break; 632 case C_ALG_DES: 633 case C_ALG_3DES: 634 if (dupdate->src.length % TEE_DES_BLOCK_SIZE) { 635 EMSG("Invalid src len"); 636 return TEE_ERROR_BAD_PARAMETERS; 637 } 638 break; 639 default: 640 return TEE_ERROR_BAD_PARAMETERS; 641 } 642 643 return TEE_SUCCESS; 644 } 645 646 static TEE_Result sec_alloc_buffer(struct sec_cipher_ctx *c_ctx) 647 { 648 c_ctx->in = malloc(c_ctx->len); 649 if (!c_ctx->in) { 650 EMSG("Fail to alloc c_in buf"); 651 return TEE_ERROR_OUT_OF_MEMORY; 652 } 653 654 c_ctx->in_dma = virt_to_phys(c_ctx->in); 655 656 c_ctx->out = malloc(c_ctx->len); 657 if (!c_ctx->out) { 658 EMSG("Fail to alloc c_out buf"); 659 goto free_c_in; 660 } 661 662 c_ctx->out_dma = virt_to_phys(c_ctx->out); 663 664 return TEE_SUCCESS; 665 666 free_c_in: 667 free(c_ctx->in); 668 c_ctx->in = NULL; 669 return TEE_ERROR_OUT_OF_MEMORY; 670 } 671 672 static void sec_free_buffer(struct sec_cipher_ctx *c_ctx) 673 { 674 free(c_ctx->in); 675 free(c_ctx->out); 676 c_ctx->in = NULL; 677 c_ctx->out = NULL; 678 } 679 680 static TEE_Result sec_cipher_update(struct drvcrypt_cipher_update *dupdate) 681 { 682 struct sec_cipher_ctx *c_ctx = NULL; 683 TEE_Result ret = TEE_SUCCESS; 684 size_t padding_size = 0; 685 686 ret = sec_cipher_param_check(dupdate); 687 if (ret) 688 return ret; 689 690 c_ctx = dupdate->ctx; 691 if (c_ctx->mode == C_MODE_CTR && (c_ctx->offs & CTR_SRC_ALIGN_MASK)) 692 padding_size = c_ctx->offs % CTR_SRC_BLOCK_SIZE; 693 694 c_ctx->offs += dupdate->src.length; 695 c_ctx->len = dupdate->src.length + padding_size; 696 ret = sec_alloc_buffer(c_ctx); 697 if (ret) 698 return ret; 699 700 memset(c_ctx->in, 0, padding_size); 701 memcpy(c_ctx->in + padding_size, dupdate->src.data, 702 dupdate->src.length); 703 704 ret = sec_do_cipher_task(c_ctx->qp, c_ctx); 705 if (ret) 706 goto free_buffer; 707 708 ret = sec_update_iv(c_ctx); 709 if (ret) { 710 EMSG("Fail to update iv, ret=%#"PRIx32, ret); 711 goto free_buffer; 712 } 713 714 memcpy(dupdate->dst.data, c_ctx->out + padding_size, 715 dupdate->src.length); 716 717 free_buffer: 718 sec_free_buffer(c_ctx); 719 return ret; 720 } 721 722 static void sec_cipher_final(void *ctx __unused) 723 { 724 } 725 726 static void sec_cipher_copy_state(void *dst_ctx, void *src_ctx) 727 { 728 struct sec_cipher_ctx *dst_c_ctx = dst_ctx; 729 struct sec_cipher_ctx *src_c_ctx = src_ctx; 730 731 dst_c_ctx->alg = src_c_ctx->alg; 732 dst_c_ctx->mode = src_c_ctx->mode; 733 dst_c_ctx->encrypt = src_c_ctx->encrypt; 734 dst_c_ctx->offs = src_c_ctx->offs; 735 736 if (src_c_ctx->key_len) { 737 dst_c_ctx->key_len = src_c_ctx->key_len; 738 dst_c_ctx->c_key_len = src_c_ctx->c_key_len; 739 memcpy(dst_c_ctx->key, src_c_ctx->key, dst_c_ctx->key_len); 740 dst_c_ctx->key_dma = virt_to_phys(dst_c_ctx->key); 741 } else { 742 dst_c_ctx->key_len = 0; 743 dst_c_ctx->c_key_len = 0; 744 } 745 746 if (src_c_ctx->iv_len) { 747 dst_c_ctx->iv_len = src_c_ctx->iv_len; 748 memcpy(dst_c_ctx->iv, src_c_ctx->iv, dst_c_ctx->iv_len); 749 dst_c_ctx->iv_dma = virt_to_phys(dst_c_ctx->iv); 750 } else { 751 dst_c_ctx->iv_len = 0; 752 } 753 } 754 755 static struct drvcrypt_cipher driver_cipher = { 756 .alloc_ctx = sec_cipher_ctx_allocate, 757 .free_ctx = sec_cipher_ctx_free, 758 .init = sec_cipher_initialize, 759 .update = sec_cipher_update, 760 .final = sec_cipher_final, 761 .copy_state = sec_cipher_copy_state, 762 }; 763 764 static TEE_Result sec_cipher_init(void) 765 { 766 TEE_Result ret = TEE_SUCCESS; 767 768 ret = drvcrypt_register_cipher(&driver_cipher); 769 if (ret) 770 EMSG("Sec cipher register to crypto fail ret=%#"PRIx32, ret); 771 772 return ret; 773 } 774 driver_init(sec_cipher_init); 775