1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0-only */ 2*4882a593Smuzhiyun/* 3*4882a593Smuzhiyun * Scalar AES core transform 4*4882a593Smuzhiyun * 5*4882a593Smuzhiyun * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6*4882a593Smuzhiyun */ 7*4882a593Smuzhiyun 8*4882a593Smuzhiyun#include <linux/linkage.h> 9*4882a593Smuzhiyun#include <asm/assembler.h> 10*4882a593Smuzhiyun#include <asm/cache.h> 11*4882a593Smuzhiyun 12*4882a593Smuzhiyun .text 13*4882a593Smuzhiyun 14*4882a593Smuzhiyun rk .req x0 15*4882a593Smuzhiyun out .req x1 16*4882a593Smuzhiyun in .req x2 17*4882a593Smuzhiyun rounds .req x3 18*4882a593Smuzhiyun tt .req x2 19*4882a593Smuzhiyun 20*4882a593Smuzhiyun .macro __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift 21*4882a593Smuzhiyun .ifc \op\shift, b0 22*4882a593Smuzhiyun ubfiz \reg0, \in0, #2, #8 23*4882a593Smuzhiyun ubfiz \reg1, \in1e, #2, #8 24*4882a593Smuzhiyun .else 25*4882a593Smuzhiyun ubfx \reg0, \in0, #\shift, #8 26*4882a593Smuzhiyun ubfx \reg1, \in1e, #\shift, #8 27*4882a593Smuzhiyun .endif 28*4882a593Smuzhiyun 29*4882a593Smuzhiyun /* 30*4882a593Smuzhiyun * AArch64 cannot do byte size indexed loads from a table containing 31*4882a593Smuzhiyun * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a 32*4882a593Smuzhiyun * valid instruction. So perform the shift explicitly first for the 33*4882a593Smuzhiyun * high bytes (the low byte is shifted implicitly by using ubfiz rather 34*4882a593Smuzhiyun * than ubfx above) 35*4882a593Smuzhiyun */ 36*4882a593Smuzhiyun .ifnc \op, b 37*4882a593Smuzhiyun ldr \reg0, [tt, \reg0, uxtw #2] 38*4882a593Smuzhiyun ldr \reg1, [tt, \reg1, uxtw #2] 39*4882a593Smuzhiyun .else 40*4882a593Smuzhiyun .if \shift > 0 41*4882a593Smuzhiyun lsl \reg0, \reg0, #2 42*4882a593Smuzhiyun lsl \reg1, \reg1, #2 43*4882a593Smuzhiyun .endif 44*4882a593Smuzhiyun ldrb \reg0, [tt, \reg0, uxtw] 45*4882a593Smuzhiyun ldrb \reg1, [tt, \reg1, uxtw] 46*4882a593Smuzhiyun .endif 47*4882a593Smuzhiyun .endm 48*4882a593Smuzhiyun 49*4882a593Smuzhiyun .macro __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift 50*4882a593Smuzhiyun ubfx \reg0, \in0, #\shift, #8 51*4882a593Smuzhiyun ubfx \reg1, \in1d, #\shift, #8 52*4882a593Smuzhiyun ldr\op \reg0, [tt, \reg0, uxtw #\sz] 53*4882a593Smuzhiyun ldr\op \reg1, [tt, \reg1, uxtw #\sz] 54*4882a593Smuzhiyun .endm 55*4882a593Smuzhiyun 56*4882a593Smuzhiyun .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op 57*4882a593Smuzhiyun ldp \out0, \out1, [rk], #8 58*4882a593Smuzhiyun 59*4882a593Smuzhiyun __pair\enc \sz, \op, w12, w13, \in0, \in1, \in3, 0 60*4882a593Smuzhiyun __pair\enc \sz, \op, w14, w15, \in1, \in2, \in0, 8 61*4882a593Smuzhiyun __pair\enc \sz, \op, w16, w17, \in2, \in3, \in1, 16 62*4882a593Smuzhiyun __pair\enc \sz, \op, \t0, \t1, \in3, \in0, \in2, 24 63*4882a593Smuzhiyun 64*4882a593Smuzhiyun eor \out0, \out0, w12 65*4882a593Smuzhiyun eor \out1, \out1, w13 66*4882a593Smuzhiyun eor \out0, \out0, w14, ror #24 67*4882a593Smuzhiyun eor \out1, \out1, w15, ror #24 68*4882a593Smuzhiyun eor \out0, \out0, w16, ror #16 69*4882a593Smuzhiyun eor \out1, \out1, w17, ror #16 70*4882a593Smuzhiyun eor \out0, \out0, \t0, ror #8 71*4882a593Smuzhiyun eor \out1, \out1, \t1, ror #8 72*4882a593Smuzhiyun .endm 73*4882a593Smuzhiyun 74*4882a593Smuzhiyun .macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op 75*4882a593Smuzhiyun __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op 76*4882a593Smuzhiyun __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op 77*4882a593Smuzhiyun .endm 78*4882a593Smuzhiyun 79*4882a593Smuzhiyun .macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op 80*4882a593Smuzhiyun __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op 81*4882a593Smuzhiyun __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op 82*4882a593Smuzhiyun .endm 83*4882a593Smuzhiyun 84*4882a593Smuzhiyun .macro do_crypt, round, ttab, ltab, bsz 85*4882a593Smuzhiyun ldp w4, w5, [in] 86*4882a593Smuzhiyun ldp w6, w7, [in, #8] 87*4882a593Smuzhiyun ldp w8, w9, [rk], #16 88*4882a593Smuzhiyun ldp w10, w11, [rk, #-8] 89*4882a593Smuzhiyun 90*4882a593SmuzhiyunCPU_BE( rev w4, w4 ) 91*4882a593SmuzhiyunCPU_BE( rev w5, w5 ) 92*4882a593SmuzhiyunCPU_BE( rev w6, w6 ) 93*4882a593SmuzhiyunCPU_BE( rev w7, w7 ) 94*4882a593Smuzhiyun 95*4882a593Smuzhiyun eor w4, w4, w8 96*4882a593Smuzhiyun eor w5, w5, w9 97*4882a593Smuzhiyun eor w6, w6, w10 98*4882a593Smuzhiyun eor w7, w7, w11 99*4882a593Smuzhiyun 100*4882a593Smuzhiyun adr_l tt, \ttab 101*4882a593Smuzhiyun 102*4882a593Smuzhiyun tbnz rounds, #1, 1f 103*4882a593Smuzhiyun 104*4882a593Smuzhiyun0: \round w8, w9, w10, w11, w4, w5, w6, w7 105*4882a593Smuzhiyun \round w4, w5, w6, w7, w8, w9, w10, w11 106*4882a593Smuzhiyun 107*4882a593Smuzhiyun1: subs rounds, rounds, #4 108*4882a593Smuzhiyun \round w8, w9, w10, w11, w4, w5, w6, w7 109*4882a593Smuzhiyun b.ls 3f 110*4882a593Smuzhiyun2: \round w4, w5, w6, w7, w8, w9, w10, w11 111*4882a593Smuzhiyun b 0b 112*4882a593Smuzhiyun3: adr_l tt, \ltab 113*4882a593Smuzhiyun \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b 114*4882a593Smuzhiyun 115*4882a593SmuzhiyunCPU_BE( rev w4, w4 ) 116*4882a593SmuzhiyunCPU_BE( rev w5, w5 ) 117*4882a593SmuzhiyunCPU_BE( rev w6, w6 ) 118*4882a593SmuzhiyunCPU_BE( rev w7, w7 ) 119*4882a593Smuzhiyun 120*4882a593Smuzhiyun stp w4, w5, [out] 121*4882a593Smuzhiyun stp w6, w7, [out, #8] 122*4882a593Smuzhiyun ret 123*4882a593Smuzhiyun .endm 124*4882a593Smuzhiyun 125*4882a593SmuzhiyunSYM_FUNC_START(__aes_arm64_encrypt) 126*4882a593Smuzhiyun do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2 127*4882a593SmuzhiyunSYM_FUNC_END(__aes_arm64_encrypt) 128*4882a593Smuzhiyun 129*4882a593Smuzhiyun .align 5 130*4882a593SmuzhiyunSYM_FUNC_START(__aes_arm64_decrypt) 131*4882a593Smuzhiyun do_crypt iround, crypto_it_tab, crypto_aes_inv_sbox, 0 132*4882a593SmuzhiyunSYM_FUNC_END(__aes_arm64_decrypt) 133