1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Crypto acceleration support for Rockchip RK3288
4 *
5 * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
6 *
7 * Author: Zain Wang <zain.wang@rock-chips.com>
8 *
9 * Some ideas are from marvell/cesa.c and s5p-sss.c driver.
10 */
11 #include "rk_crypto_core.h"
12 #include "rk_crypto_v1.h"
13 #include "rk_crypto_v1_reg.h"
14
15 /*
16 * IC can not process zero message hash,
17 * so we put the fixed hash out when met zero message.
18 */
19
rk_alg_ctx_cast(struct rk_crypto_dev * rk_dev)20 static struct rk_alg_ctx *rk_alg_ctx_cast(
21 struct rk_crypto_dev *rk_dev)
22 {
23 struct ahash_request *req =
24 ahash_request_cast(rk_dev->async_req);
25
26 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
27 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
28
29 return &ctx->algs_ctx;
30 }
31
rk_crypto_irq_handle(int irq,void * dev_id)32 static int rk_crypto_irq_handle(int irq, void *dev_id)
33 {
34 struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id);
35 u32 interrupt_status;
36
37 interrupt_status = CRYPTO_READ(rk_dev, RK_CRYPTO_INTSTS);
38 CRYPTO_WRITE(rk_dev, RK_CRYPTO_INTSTS, interrupt_status);
39
40 if (interrupt_status & 0x0a) {
41 dev_warn(rk_dev->dev, "DMA Error\n");
42 rk_dev->err = -EFAULT;
43 }
44
45 return 0;
46 }
47
zero_message_process(struct ahash_request * req)48 static int zero_message_process(struct ahash_request *req)
49 {
50 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
51 int rk_digest_size = crypto_ahash_digestsize(tfm);
52
53 const u8 sha256_zero_msg_hash[SHA256_DIGEST_SIZE] = {
54 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
55 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
56 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
57 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55
58 };
59
60 const u8 sha1_zero_msg_hash[SHA1_DIGEST_SIZE] = {
61 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
62 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
63 0xaf, 0xd8, 0x07, 0x09
64 };
65
66 const u8 md5_zero_msg_hash[MD5_DIGEST_SIZE] = {
67 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
68 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
69 };
70
71 switch (rk_digest_size) {
72 case SHA1_DIGEST_SIZE:
73 memcpy(req->result, sha1_zero_msg_hash, rk_digest_size);
74 break;
75 case SHA256_DIGEST_SIZE:
76 memcpy(req->result, sha256_zero_msg_hash, rk_digest_size);
77 break;
78 case MD5_DIGEST_SIZE:
79 memcpy(req->result, md5_zero_msg_hash, rk_digest_size);
80 break;
81 default:
82 return -EINVAL;
83 }
84
85 return 0;
86 }
87
rk_ahash_crypto_complete(struct crypto_async_request * base,int err)88 static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
89 {
90 if (base->complete)
91 base->complete(base, err);
92 }
93
rk_ahash_reg_init(struct rk_crypto_dev * rk_dev)94 static void rk_ahash_reg_init(struct rk_crypto_dev *rk_dev)
95 {
96 struct ahash_request *req = ahash_request_cast(rk_dev->async_req);
97 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
98 struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
99 int reg_status = 0;
100
101 reg_status = CRYPTO_READ(rk_dev, RK_CRYPTO_CTRL) |
102 RK_CRYPTO_HASH_FLUSH | _SBF(0xffff, 16);
103 CRYPTO_WRITE(rk_dev, RK_CRYPTO_CTRL, reg_status);
104
105 reg_status = CRYPTO_READ(rk_dev, RK_CRYPTO_CTRL);
106 reg_status &= (~RK_CRYPTO_HASH_FLUSH);
107 reg_status |= _SBF(0xffff, 16);
108 CRYPTO_WRITE(rk_dev, RK_CRYPTO_CTRL, reg_status);
109
110 memset_io(rk_dev->reg + RK_CRYPTO_HASH_DOUT_0, 0, 32);
111
112 CRYPTO_WRITE(rk_dev, RK_CRYPTO_INTENA, RK_CRYPTO_HRDMA_ERR_ENA |
113 RK_CRYPTO_HRDMA_DONE_ENA);
114
115 CRYPTO_WRITE(rk_dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT |
116 RK_CRYPTO_HRDMA_DONE_INT);
117
118 CRYPTO_WRITE(rk_dev, RK_CRYPTO_HASH_CTRL, rctx->mode |
119 RK_CRYPTO_HASH_SWAP_DO);
120
121 CRYPTO_WRITE(rk_dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO |
122 RK_CRYPTO_BYTESWAP_BRFIFO |
123 RK_CRYPTO_BYTESWAP_BTFIFO);
124
125 CRYPTO_WRITE(rk_dev, RK_CRYPTO_HASH_MSG_LEN, alg_ctx->total);
126 }
127
rk_ahash_init(struct ahash_request * req)128 static int rk_ahash_init(struct ahash_request *req)
129 {
130 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
131 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
132 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
133
134 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
135 rctx->fallback_req.base.flags = req->base.flags &
136 CRYPTO_TFM_REQ_MAY_SLEEP;
137
138 return crypto_ahash_init(&rctx->fallback_req);
139 }
140
rk_ahash_update(struct ahash_request * req)141 static int rk_ahash_update(struct ahash_request *req)
142 {
143 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
144 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
145 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
146
147 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
148 rctx->fallback_req.base.flags = req->base.flags &
149 CRYPTO_TFM_REQ_MAY_SLEEP;
150 rctx->fallback_req.nbytes = req->nbytes;
151 rctx->fallback_req.src = req->src;
152
153 return crypto_ahash_update(&rctx->fallback_req);
154 }
155
rk_ahash_final(struct ahash_request * req)156 static int rk_ahash_final(struct ahash_request *req)
157 {
158 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
159 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
160 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
161
162 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
163 rctx->fallback_req.base.flags = req->base.flags &
164 CRYPTO_TFM_REQ_MAY_SLEEP;
165 rctx->fallback_req.result = req->result;
166
167 return crypto_ahash_final(&rctx->fallback_req);
168 }
169
rk_ahash_finup(struct ahash_request * req)170 static int rk_ahash_finup(struct ahash_request *req)
171 {
172 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
173 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
174 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
175
176 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
177 rctx->fallback_req.base.flags = req->base.flags &
178 CRYPTO_TFM_REQ_MAY_SLEEP;
179
180 rctx->fallback_req.nbytes = req->nbytes;
181 rctx->fallback_req.src = req->src;
182 rctx->fallback_req.result = req->result;
183
184 return crypto_ahash_finup(&rctx->fallback_req);
185 }
186
rk_ahash_import(struct ahash_request * req,const void * in)187 static int rk_ahash_import(struct ahash_request *req, const void *in)
188 {
189 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
190 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
191 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
192
193 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
194 rctx->fallback_req.base.flags = req->base.flags &
195 CRYPTO_TFM_REQ_MAY_SLEEP;
196
197 return crypto_ahash_import(&rctx->fallback_req, in);
198 }
199
rk_ahash_export(struct ahash_request * req,void * out)200 static int rk_ahash_export(struct ahash_request *req, void *out)
201 {
202 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
203 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
204 struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
205
206 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
207 rctx->fallback_req.base.flags = req->base.flags &
208 CRYPTO_TFM_REQ_MAY_SLEEP;
209
210 return crypto_ahash_export(&rctx->fallback_req, out);
211 }
212
rk_ahash_digest(struct ahash_request * req)213 static int rk_ahash_digest(struct ahash_request *req)
214 {
215 struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
216 struct rk_crypto_dev *rk_dev = tctx->rk_dev;
217
218 if (!req->nbytes)
219 return zero_message_process(req);
220 else
221 return rk_dev->enqueue(rk_dev, &req->base);
222 }
223
crypto_ahash_dma_start(struct rk_crypto_dev * rk_dev)224 static void crypto_ahash_dma_start(struct rk_crypto_dev *rk_dev)
225 {
226 struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
227
228 CRYPTO_WRITE(rk_dev, RK_CRYPTO_HRDMAS, alg_ctx->addr_in);
229 CRYPTO_WRITE(rk_dev, RK_CRYPTO_HRDMAL, (alg_ctx->count + 3) / 4);
230 CRYPTO_WRITE(rk_dev, RK_CRYPTO_CTRL, RK_CRYPTO_HASH_START |
231 (RK_CRYPTO_HASH_START << 16));
232 }
233
rk_ahash_set_data_start(struct rk_crypto_dev * rk_dev)234 static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev)
235 {
236 int err;
237 struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
238
239 err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, NULL);
240 if (!err)
241 crypto_ahash_dma_start(rk_dev);
242 return err;
243 }
244
rk_ahash_start(struct rk_crypto_dev * rk_dev)245 static int rk_ahash_start(struct rk_crypto_dev *rk_dev)
246 {
247 struct ahash_request *req = ahash_request_cast(rk_dev->async_req);
248 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
249 struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
250 struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
251
252 alg_ctx->total = req->nbytes;
253 alg_ctx->left_bytes = req->nbytes;
254 alg_ctx->sg_src = req->src;
255 alg_ctx->req_src = req->src;
256 alg_ctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
257
258 rctx->mode = 0;
259
260 switch (crypto_ahash_digestsize(tfm)) {
261 case SHA1_DIGEST_SIZE:
262 rctx->mode = RK_CRYPTO_HASH_SHA1;
263 break;
264 case SHA256_DIGEST_SIZE:
265 rctx->mode = RK_CRYPTO_HASH_SHA256;
266 break;
267 case MD5_DIGEST_SIZE:
268 rctx->mode = RK_CRYPTO_HASH_MD5;
269 break;
270 default:
271 return -EINVAL;
272 }
273
274 rk_ahash_reg_init(rk_dev);
275 return rk_ahash_set_data_start(rk_dev);
276 }
277
rk_ahash_crypto_rx(struct rk_crypto_dev * rk_dev)278 static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev)
279 {
280 int err = 0;
281 struct ahash_request *req = ahash_request_cast(rk_dev->async_req);
282 struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
283 struct crypto_ahash *tfm;
284
285 CRYPTO_TRACE("left_bytes = %u\n", alg_ctx->left_bytes);
286
287 err = rk_dev->unload_data(rk_dev);
288 if (err)
289 goto out_rx;
290
291 if (alg_ctx->left_bytes) {
292 if (alg_ctx->aligned) {
293 if (sg_is_last(alg_ctx->sg_src)) {
294 dev_warn(rk_dev->dev, "[%s:%d], Lack of data\n",
295 __func__, __LINE__);
296 err = -ENOMEM;
297 goto out_rx;
298 }
299 alg_ctx->sg_src = sg_next(alg_ctx->sg_src);
300 }
301 err = rk_ahash_set_data_start(rk_dev);
302 } else {
303 /*
304 * it will take some time to process date after last dma
305 * transmission.
306 *
307 * waiting time is relative with the last date len,
308 * so cannot set a fixed time here.
309 * 10us makes system not call here frequently wasting
310 * efficiency, and make it response quickly when dma
311 * complete.
312 */
313 while (!CRYPTO_READ(rk_dev, RK_CRYPTO_HASH_STS))
314 udelay(10);
315
316 tfm = crypto_ahash_reqtfm(req);
317 memcpy_fromio(req->result, rk_dev->reg + RK_CRYPTO_HASH_DOUT_0,
318 crypto_ahash_digestsize(tfm));
319 }
320
321 out_rx:
322 return err;
323 }
324
rk_cra_hash_init(struct crypto_tfm * tfm)325 static int rk_cra_hash_init(struct crypto_tfm *tfm)
326 {
327 struct rk_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
328 struct rk_crypto_algt *algt;
329 struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
330 const char *alg_name = crypto_tfm_alg_name(tfm);
331 struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;
332 struct rk_crypto_dev *rk_dev;
333
334 algt = container_of(alg, struct rk_crypto_algt, alg.hash);
335 rk_dev = algt->rk_dev;
336
337 memset(ctx, 0x00, sizeof(*ctx));
338
339 if (!rk_dev->request_crypto)
340 return -EFAULT;
341
342 rk_dev->request_crypto(rk_dev, crypto_tfm_alg_name(tfm));
343
344 alg_ctx->align_size = 4;
345
346 alg_ctx->ops.start = rk_ahash_start;
347 alg_ctx->ops.update = rk_ahash_crypto_rx;
348 alg_ctx->ops.complete = rk_ahash_crypto_complete;
349 alg_ctx->ops.irq_handle = rk_crypto_irq_handle;
350
351 ctx->rk_dev = rk_dev;
352
353 /* for fallback */
354 ctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0,
355 CRYPTO_ALG_NEED_FALLBACK);
356 if (IS_ERR(ctx->fallback_tfm)) {
357 dev_err(rk_dev->dev, "Could not load fallback driver.\n");
358 return PTR_ERR(ctx->fallback_tfm);
359 }
360 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
361 sizeof(struct rk_ahash_rctx) +
362 crypto_ahash_reqsize(ctx->fallback_tfm));
363
364 algt->alg.hash.halg.statesize = crypto_ahash_statesize(ctx->fallback_tfm);
365
366 return 0;
367 }
368
rk_cra_hash_exit(struct crypto_tfm * tfm)369 static void rk_cra_hash_exit(struct crypto_tfm *tfm)
370 {
371 struct rk_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
372
373 if (ctx->fallback_tfm)
374 crypto_free_ahash(ctx->fallback_tfm);
375
376 ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(tfm));
377 }
378
379 struct rk_crypto_algt rk_v1_ahash_sha1 = RK_HASH_ALGO_INIT(SHA1, sha1);
380 struct rk_crypto_algt rk_v1_ahash_sha256 = RK_HASH_ALGO_INIT(SHA256, sha256);
381 struct rk_crypto_algt rk_v1_ahash_md5 = RK_HASH_ALGO_INIT(MD5, md5);
382
383