1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0-only
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
6*4882a593Smuzhiyun */
7*4882a593Smuzhiyun
8*4882a593Smuzhiyun #include <linux/crc-t10dif.h>
9*4882a593Smuzhiyun #include <linux/init.h>
10*4882a593Smuzhiyun #include <linux/kernel.h>
11*4882a593Smuzhiyun #include <linux/module.h>
12*4882a593Smuzhiyun #include <linux/string.h>
13*4882a593Smuzhiyun
14*4882a593Smuzhiyun #include <crypto/internal/hash.h>
15*4882a593Smuzhiyun #include <crypto/internal/simd.h>
16*4882a593Smuzhiyun
17*4882a593Smuzhiyun #include <asm/neon.h>
18*4882a593Smuzhiyun #include <asm/simd.h>
19*4882a593Smuzhiyun
20*4882a593Smuzhiyun #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
21*4882a593Smuzhiyun
22*4882a593Smuzhiyun asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len);
23*4882a593Smuzhiyun
crct10dif_init(struct shash_desc * desc)24*4882a593Smuzhiyun static int crct10dif_init(struct shash_desc *desc)
25*4882a593Smuzhiyun {
26*4882a593Smuzhiyun u16 *crc = shash_desc_ctx(desc);
27*4882a593Smuzhiyun
28*4882a593Smuzhiyun *crc = 0;
29*4882a593Smuzhiyun return 0;
30*4882a593Smuzhiyun }
31*4882a593Smuzhiyun
crct10dif_update(struct shash_desc * desc,const u8 * data,unsigned int length)32*4882a593Smuzhiyun static int crct10dif_update(struct shash_desc *desc, const u8 *data,
33*4882a593Smuzhiyun unsigned int length)
34*4882a593Smuzhiyun {
35*4882a593Smuzhiyun u16 *crc = shash_desc_ctx(desc);
36*4882a593Smuzhiyun
37*4882a593Smuzhiyun if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
38*4882a593Smuzhiyun kernel_neon_begin();
39*4882a593Smuzhiyun *crc = crc_t10dif_pmull(*crc, data, length);
40*4882a593Smuzhiyun kernel_neon_end();
41*4882a593Smuzhiyun } else {
42*4882a593Smuzhiyun *crc = crc_t10dif_generic(*crc, data, length);
43*4882a593Smuzhiyun }
44*4882a593Smuzhiyun
45*4882a593Smuzhiyun return 0;
46*4882a593Smuzhiyun }
47*4882a593Smuzhiyun
crct10dif_final(struct shash_desc * desc,u8 * out)48*4882a593Smuzhiyun static int crct10dif_final(struct shash_desc *desc, u8 *out)
49*4882a593Smuzhiyun {
50*4882a593Smuzhiyun u16 *crc = shash_desc_ctx(desc);
51*4882a593Smuzhiyun
52*4882a593Smuzhiyun *(u16 *)out = *crc;
53*4882a593Smuzhiyun return 0;
54*4882a593Smuzhiyun }
55*4882a593Smuzhiyun
56*4882a593Smuzhiyun static struct shash_alg crc_t10dif_alg = {
57*4882a593Smuzhiyun .digestsize = CRC_T10DIF_DIGEST_SIZE,
58*4882a593Smuzhiyun .init = crct10dif_init,
59*4882a593Smuzhiyun .update = crct10dif_update,
60*4882a593Smuzhiyun .final = crct10dif_final,
61*4882a593Smuzhiyun .descsize = CRC_T10DIF_DIGEST_SIZE,
62*4882a593Smuzhiyun
63*4882a593Smuzhiyun .base.cra_name = "crct10dif",
64*4882a593Smuzhiyun .base.cra_driver_name = "crct10dif-arm-ce",
65*4882a593Smuzhiyun .base.cra_priority = 200,
66*4882a593Smuzhiyun .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
67*4882a593Smuzhiyun .base.cra_module = THIS_MODULE,
68*4882a593Smuzhiyun };
69*4882a593Smuzhiyun
crc_t10dif_mod_init(void)70*4882a593Smuzhiyun static int __init crc_t10dif_mod_init(void)
71*4882a593Smuzhiyun {
72*4882a593Smuzhiyun if (!(elf_hwcap2 & HWCAP2_PMULL))
73*4882a593Smuzhiyun return -ENODEV;
74*4882a593Smuzhiyun
75*4882a593Smuzhiyun return crypto_register_shash(&crc_t10dif_alg);
76*4882a593Smuzhiyun }
77*4882a593Smuzhiyun
crc_t10dif_mod_exit(void)78*4882a593Smuzhiyun static void __exit crc_t10dif_mod_exit(void)
79*4882a593Smuzhiyun {
80*4882a593Smuzhiyun crypto_unregister_shash(&crc_t10dif_alg);
81*4882a593Smuzhiyun }
82*4882a593Smuzhiyun
83*4882a593Smuzhiyun module_init(crc_t10dif_mod_init);
84*4882a593Smuzhiyun module_exit(crc_t10dif_mod_exit);
85*4882a593Smuzhiyun
86*4882a593Smuzhiyun MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
87*4882a593Smuzhiyun MODULE_LICENSE("GPL v2");
88*4882a593Smuzhiyun MODULE_ALIAS_CRYPTO("crct10dif");
89