1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0-only */ 2*4882a593Smuzhiyun/* 3*4882a593Smuzhiyun * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with 4*4882a593Smuzhiyun * Crypto Extensions 5*4882a593Smuzhiyun * 6*4882a593Smuzhiyun * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 7*4882a593Smuzhiyun */ 8*4882a593Smuzhiyun 9*4882a593Smuzhiyun#include <linux/linkage.h> 10*4882a593Smuzhiyun#include <asm/assembler.h> 11*4882a593Smuzhiyun 12*4882a593Smuzhiyun#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func) 13*4882a593Smuzhiyun#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func) 14*4882a593Smuzhiyun 15*4882a593Smuzhiyun .arch armv8-a+crypto 16*4882a593Smuzhiyun 17*4882a593Smuzhiyun xtsmask .req v16 18*4882a593Smuzhiyun cbciv .req v16 19*4882a593Smuzhiyun vctr .req v16 20*4882a593Smuzhiyun 21*4882a593Smuzhiyun .macro xts_reload_mask, tmp 22*4882a593Smuzhiyun .endm 23*4882a593Smuzhiyun 24*4882a593Smuzhiyun .macro xts_cts_skip_tw, reg, lbl 25*4882a593Smuzhiyun .endm 26*4882a593Smuzhiyun 27*4882a593Smuzhiyun /* preload all round keys */ 28*4882a593Smuzhiyun .macro load_round_keys, rounds, rk 29*4882a593Smuzhiyun cmp \rounds, #12 30*4882a593Smuzhiyun blo 2222f /* 128 bits */ 31*4882a593Smuzhiyun beq 1111f /* 192 bits */ 32*4882a593Smuzhiyun ld1 {v17.4s-v18.4s}, [\rk], #32 33*4882a593Smuzhiyun1111: ld1 {v19.4s-v20.4s}, [\rk], #32 34*4882a593Smuzhiyun2222: ld1 {v21.4s-v24.4s}, [\rk], #64 35*4882a593Smuzhiyun ld1 {v25.4s-v28.4s}, [\rk], #64 36*4882a593Smuzhiyun ld1 {v29.4s-v31.4s}, [\rk] 37*4882a593Smuzhiyun .endm 38*4882a593Smuzhiyun 39*4882a593Smuzhiyun /* prepare for encryption with key in rk[] */ 40*4882a593Smuzhiyun .macro enc_prepare, rounds, rk, temp 41*4882a593Smuzhiyun mov \temp, \rk 42*4882a593Smuzhiyun load_round_keys \rounds, \temp 43*4882a593Smuzhiyun .endm 44*4882a593Smuzhiyun 45*4882a593Smuzhiyun /* prepare for encryption (again) but with new key in rk[] */ 46*4882a593Smuzhiyun .macro enc_switch_key, rounds, rk, temp 47*4882a593Smuzhiyun mov \temp, \rk 48*4882a593Smuzhiyun load_round_keys \rounds, \temp 49*4882a593Smuzhiyun .endm 50*4882a593Smuzhiyun 51*4882a593Smuzhiyun /* prepare for decryption with key in rk[] */ 52*4882a593Smuzhiyun .macro dec_prepare, rounds, rk, temp 53*4882a593Smuzhiyun mov \temp, \rk 54*4882a593Smuzhiyun load_round_keys \rounds, \temp 55*4882a593Smuzhiyun .endm 56*4882a593Smuzhiyun 57*4882a593Smuzhiyun .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4 58*4882a593Smuzhiyun aes\de \i0\().16b, \k\().16b 59*4882a593Smuzhiyun aes\mc \i0\().16b, \i0\().16b 60*4882a593Smuzhiyun .ifnb \i1 61*4882a593Smuzhiyun aes\de \i1\().16b, \k\().16b 62*4882a593Smuzhiyun aes\mc \i1\().16b, \i1\().16b 63*4882a593Smuzhiyun .ifnb \i3 64*4882a593Smuzhiyun aes\de \i2\().16b, \k\().16b 65*4882a593Smuzhiyun aes\mc \i2\().16b, \i2\().16b 66*4882a593Smuzhiyun aes\de \i3\().16b, \k\().16b 67*4882a593Smuzhiyun aes\mc \i3\().16b, \i3\().16b 68*4882a593Smuzhiyun .ifnb \i4 69*4882a593Smuzhiyun aes\de \i4\().16b, \k\().16b 70*4882a593Smuzhiyun aes\mc \i4\().16b, \i4\().16b 71*4882a593Smuzhiyun .endif 72*4882a593Smuzhiyun .endif 73*4882a593Smuzhiyun .endif 74*4882a593Smuzhiyun .endm 75*4882a593Smuzhiyun 76*4882a593Smuzhiyun /* up to 5 interleaved encryption rounds with the same round key */ 77*4882a593Smuzhiyun .macro round_Nx, enc, k, i0, i1, i2, i3, i4 78*4882a593Smuzhiyun .ifc \enc, e 79*4882a593Smuzhiyun do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4 80*4882a593Smuzhiyun .else 81*4882a593Smuzhiyun do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4 82*4882a593Smuzhiyun .endif 83*4882a593Smuzhiyun .endm 84*4882a593Smuzhiyun 85*4882a593Smuzhiyun /* up to 5 interleaved final rounds */ 86*4882a593Smuzhiyun .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4 87*4882a593Smuzhiyun aes\de \i0\().16b, \k\().16b 88*4882a593Smuzhiyun .ifnb \i1 89*4882a593Smuzhiyun aes\de \i1\().16b, \k\().16b 90*4882a593Smuzhiyun .ifnb \i3 91*4882a593Smuzhiyun aes\de \i2\().16b, \k\().16b 92*4882a593Smuzhiyun aes\de \i3\().16b, \k\().16b 93*4882a593Smuzhiyun .ifnb \i4 94*4882a593Smuzhiyun aes\de \i4\().16b, \k\().16b 95*4882a593Smuzhiyun .endif 96*4882a593Smuzhiyun .endif 97*4882a593Smuzhiyun .endif 98*4882a593Smuzhiyun eor \i0\().16b, \i0\().16b, \k2\().16b 99*4882a593Smuzhiyun .ifnb \i1 100*4882a593Smuzhiyun eor \i1\().16b, \i1\().16b, \k2\().16b 101*4882a593Smuzhiyun .ifnb \i3 102*4882a593Smuzhiyun eor \i2\().16b, \i2\().16b, \k2\().16b 103*4882a593Smuzhiyun eor \i3\().16b, \i3\().16b, \k2\().16b 104*4882a593Smuzhiyun .ifnb \i4 105*4882a593Smuzhiyun eor \i4\().16b, \i4\().16b, \k2\().16b 106*4882a593Smuzhiyun .endif 107*4882a593Smuzhiyun .endif 108*4882a593Smuzhiyun .endif 109*4882a593Smuzhiyun .endm 110*4882a593Smuzhiyun 111*4882a593Smuzhiyun /* up to 5 interleaved blocks */ 112*4882a593Smuzhiyun .macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4 113*4882a593Smuzhiyun cmp \rounds, #12 114*4882a593Smuzhiyun blo 2222f /* 128 bits */ 115*4882a593Smuzhiyun beq 1111f /* 192 bits */ 116*4882a593Smuzhiyun round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4 117*4882a593Smuzhiyun round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4 118*4882a593Smuzhiyun1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4 119*4882a593Smuzhiyun round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4 120*4882a593Smuzhiyun2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 121*4882a593Smuzhiyun round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4 122*4882a593Smuzhiyun .endr 123*4882a593Smuzhiyun fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4 124*4882a593Smuzhiyun .endm 125*4882a593Smuzhiyun 126*4882a593Smuzhiyun .macro encrypt_block, in, rounds, t0, t1, t2 127*4882a593Smuzhiyun do_block_Nx e, \rounds, \in 128*4882a593Smuzhiyun .endm 129*4882a593Smuzhiyun 130*4882a593Smuzhiyun .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 131*4882a593Smuzhiyun do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 132*4882a593Smuzhiyun .endm 133*4882a593Smuzhiyun 134*4882a593Smuzhiyun .macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 135*4882a593Smuzhiyun do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4 136*4882a593Smuzhiyun .endm 137*4882a593Smuzhiyun 138*4882a593Smuzhiyun .macro decrypt_block, in, rounds, t0, t1, t2 139*4882a593Smuzhiyun do_block_Nx d, \rounds, \in 140*4882a593Smuzhiyun .endm 141*4882a593Smuzhiyun 142*4882a593Smuzhiyun .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 143*4882a593Smuzhiyun do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 144*4882a593Smuzhiyun .endm 145*4882a593Smuzhiyun 146*4882a593Smuzhiyun .macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 147*4882a593Smuzhiyun do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4 148*4882a593Smuzhiyun .endm 149*4882a593Smuzhiyun 150*4882a593Smuzhiyun#define MAX_STRIDE 5 151*4882a593Smuzhiyun 152*4882a593Smuzhiyun#include "aes-modes.S" 153