1 // SPDX-License-Identifier: BSD-2-Clause 2 /* 3 * Copyright (c) 2017, Linaro Limited 4 */ 5 6 #include <crypto/internal_aes-gcm.h> 7 #include <crypto/ghash-ce-core.h> 8 #include <io.h> 9 #include <kernel/panic.h> 10 #include <kernel/thread.h> 11 #include <tomcrypt.h> 12 #include <types_ext.h> 13 14 static void get_be_block(void *dst, const void *src) 15 { 16 uint64_t *d = dst; 17 18 d[1] = get_be64(src); 19 d[0] = get_be64((const uint8_t *)src + 8); 20 } 21 22 static void put_be_block(void *dst, const void *src) 23 { 24 const uint64_t *s = src; 25 26 put_be64(dst, s[1]); 27 put_be64((uint8_t *)dst + 8, s[0]); 28 } 29 30 void internal_aes_gcm_set_key(struct internal_aes_gcm_state *state, 31 const struct internal_aes_gcm_key *enc_key) 32 { 33 uint64_t k[2]; 34 uint64_t a; 35 uint64_t b; 36 37 internal_aes_gcm_encrypt_block(enc_key, state->ctr, state->hash_subkey); 38 39 /* Store hash key in little endian and multiply by 'x' */ 40 b = get_be64(state->hash_subkey); 41 a = get_be64(state->hash_subkey + 8); 42 k[0] = (a << 1) | (b >> 63); 43 k[1] = (b << 1) | (a >> 63); 44 if (b >> 63) 45 k[1] ^= 0xc200000000000000UL; 46 47 memcpy(state->hash_subkey, k, TEE_AES_BLOCK_SIZE); 48 } 49 50 void internal_aes_gcm_ghash_update(struct internal_aes_gcm_state *state, 51 const void *head, const void *data, 52 size_t num_blocks) 53 { 54 uint32_t vfp_state; 55 uint64_t dg[2]; 56 uint64_t *k; 57 58 get_be_block(dg, state->hash_state); 59 60 k = (void *)state->hash_subkey; 61 62 vfp_state = thread_kernel_enable_vfp(); 63 64 #ifdef CFG_HWSUPP_PMULL 65 pmull_ghash_update_p64(num_blocks, dg, data, k, head); 66 #else 67 pmull_ghash_update_p8(num_blocks, dg, data, k, head); 68 #endif 69 thread_kernel_disable_vfp(vfp_state); 70 71 put_be_block(state->hash_state, dg); 72 } 73 74 #ifdef ARM64 75 static uint32_t ror32(uint32_t word, unsigned int shift) 76 { 77 return (word >> shift) | (word << (32 - shift)); 78 } 79 80 TEE_Result internal_aes_gcm_expand_enc_key(const void *key, size_t key_len, 81 struct internal_aes_gcm_key *enc_key) 82 { 83 /* The AES key schedule round constants */ 84 static uint8_t const rcon[] = { 85 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 86 }; 87 uint32_t vfp_state; 88 uint32_t kwords = key_len / sizeof(uint32_t); 89 void *p = enc_key->data; 90 uint32_t *k = p; 91 unsigned int i; 92 93 if (key_len != 16 && key_len != 24 && key_len != 32) 94 return TEE_ERROR_BAD_PARAMETERS; 95 96 memcpy(k, key, key_len); 97 /* 98 * # of rounds specified by AES: 99 * 128 bit key 10 rounds 100 * 192 bit key 12 rounds 101 * 256 bit key 14 rounds 102 * => n byte key => 6 + (n/4) rounds 103 */ 104 enc_key->rounds = 6 + key_len / 4; 105 106 vfp_state = thread_kernel_enable_vfp(); 107 for (i = 0; i < sizeof(rcon); i++) { 108 uint32_t *rki = k + (i * kwords); 109 uint32_t *rko = rki + kwords; 110 111 rko[0] = ror32(pmull_gcm_aes_sub(rki[kwords - 1]), 8) ^ 112 rcon[i] ^ rki[0]; 113 rko[1] = rko[0] ^ rki[1]; 114 rko[2] = rko[1] ^ rki[2]; 115 rko[3] = rko[2] ^ rki[3]; 116 117 if (key_len == 24) { 118 if (i >= 7) 119 break; 120 rko[4] = rko[3] ^ rki[4]; 121 rko[5] = rko[4] ^ rki[5]; 122 } else if (key_len == 32) { 123 if (i >= 6) 124 break; 125 rko[4] = pmull_gcm_aes_sub(rko[3]) ^ rki[4]; 126 rko[5] = rko[4] ^ rki[5]; 127 rko[6] = rko[5] ^ rki[6]; 128 rko[7] = rko[6] ^ rki[7]; 129 } 130 } 131 132 thread_kernel_disable_vfp(vfp_state); 133 return TEE_SUCCESS; 134 } 135 136 void internal_aes_gcm_encrypt_block(const struct internal_aes_gcm_key *ek, 137 const void *src, void *dst) 138 { 139 uint32_t vfp_state; 140 141 vfp_state = thread_kernel_enable_vfp(); 142 143 pmull_gcm_load_round_keys(ek->data, ek->rounds); 144 pmull_gcm_encrypt_block(dst, src, ek->rounds); 145 146 thread_kernel_disable_vfp(vfp_state); 147 } 148 149 void internal_aes_gcm_update_payload_block_aligned( 150 struct internal_aes_gcm_state *state, 151 const struct internal_aes_gcm_key *ek, 152 TEE_OperationMode mode, const void *src, 153 size_t num_blocks, void *dst) 154 { 155 uint32_t vfp_state; 156 uint64_t dg[2]; 157 uint64_t ctr[2]; 158 uint64_t *k; 159 160 get_be_block(dg, state->hash_state); 161 get_be_block(ctr, state->ctr); 162 163 k = (void *)state->hash_subkey; 164 165 vfp_state = thread_kernel_enable_vfp(); 166 167 pmull_gcm_load_round_keys(ek->data, ek->rounds); 168 169 if (mode == TEE_MODE_ENCRYPT) 170 pmull_gcm_encrypt(num_blocks, dg, dst, src, k, ctr, ek->rounds, 171 state->buf_cryp); 172 else 173 pmull_gcm_decrypt(num_blocks, dg, dst, src, k, ctr, ek->rounds); 174 175 thread_kernel_disable_vfp(vfp_state); 176 177 put_be_block(state->ctr, ctr); 178 put_be_block(state->hash_state, dg); 179 } 180 #endif /*ARM64*/ 181