1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2017-2020, Linaro Limited 4 */ 5 6 #include <crypto/crypto_accel.h> 7 #include <crypto/ghash-ce-core.h> 8 #include <crypto/internal_aes-gcm.h> 9 #include <io.h> 10 #include <kernel/panic.h> 11 #include <kernel/thread.h> 12 #include <string.h> 13 #include <types_ext.h> 14 15 static void get_be_block(void *dst, const void *src) 16 { 17 uint64_t *d = dst; 18 19 d[1] = get_be64(src); 20 d[0] = get_be64((const uint8_t *)src + 8); 21 } 22 23 static void put_be_block(void *dst, const void *src) 24 { 25 const uint64_t *s = src; 26 27 put_be64(dst, s[1]); 28 put_be64((uint8_t *)dst + 8, s[0]); 29 } 30 31 void internal_aes_gcm_set_key(struct internal_aes_gcm_state *state, 32 const struct internal_aes_gcm_key *enc_key) 33 { 34 uint64_t k[2]; 35 uint64_t a; 36 uint64_t b; 37 38 internal_aes_gcm_encrypt_block(enc_key, state->ctr, k); 39 40 /* Store hash key in little endian and multiply by 'x' */ 41 b = get_be64(k); 42 a = get_be64(k + 1); 43 state->ghash_key.k[0] = (a << 1) | (b >> 63); 44 state->ghash_key.k[1] = (b << 1) | (a >> 63); 45 if (b >> 63) 46 state->ghash_key.k[1] ^= 0xc200000000000000UL; 47 } 48 49 void internal_aes_gcm_ghash_update(struct internal_aes_gcm_state *state, 50 const void *head, const void *data, 51 size_t num_blocks) 52 { 53 uint32_t vfp_state; 54 uint64_t dg[2]; 55 56 get_be_block(dg, state->hash_state); 57 58 vfp_state = thread_kernel_enable_vfp(); 59 60 #ifdef CFG_HWSUPP_PMULT_64 61 pmull_ghash_update_p64(num_blocks, dg, data, &state->ghash_key, head); 62 #else 63 pmull_ghash_update_p8(num_blocks, dg, data, &state->ghash_key, head); 64 #endif 65 thread_kernel_disable_vfp(vfp_state); 66 67 put_be_block(state->hash_state, dg); 68 } 69 70 TEE_Result internal_aes_gcm_expand_enc_key(const void *key, size_t key_len, 71 struct internal_aes_gcm_key *enc_key) 72 { 73 return crypto_accel_aes_expand_keys(key, key_len, enc_key->data, NULL, 74 sizeof(enc_key->data), 75 &enc_key->rounds); 76 } 77 78 void internal_aes_gcm_encrypt_block(const struct internal_aes_gcm_key *ek, 79 const void *src, void *dst) 80 { 81 crypto_accel_aes_ecb_enc(dst, src, ek->data, ek->rounds, 1); 82 } 83 84 #ifdef ARM64 85 void internal_aes_gcm_update_payload_block_aligned( 86 struct internal_aes_gcm_state *state, 87 const struct internal_aes_gcm_key *ek, 88 TEE_OperationMode mode, const void *src, 89 size_t num_blocks, void *dst) 90 { 91 uint32_t vfp_state; 92 uint64_t dg[2]; 93 uint64_t ctr[2]; 94 95 get_be_block(dg, state->hash_state); 96 get_be_block(ctr, state->ctr); 97 98 vfp_state = thread_kernel_enable_vfp(); 99 100 pmull_gcm_load_round_keys(ek->data, ek->rounds); 101 102 if (mode == TEE_MODE_ENCRYPT) 103 pmull_gcm_encrypt(num_blocks, dg, dst, src, &state->ghash_key, 104 ctr, ek->rounds, state->buf_cryp); 105 else 106 pmull_gcm_decrypt(num_blocks, dg, dst, src, &state->ghash_key, 107 ctr, ek->rounds); 108 109 thread_kernel_disable_vfp(vfp_state); 110 111 put_be_block(state->ctr, ctr); 112 put_be_block(state->hash_state, dg); 113 } 114 #endif /*ARM64*/ 115