69 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)  in safexcel_queued_len()  argument
71 	return req->len - req->processed;  in safexcel_queued_len()
108 				     struct safexcel_ahash_req *req,  in safexcel_context_control()  argument
122 	if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {  in safexcel_context_control()
123 		if (req->xcbcmac)  in safexcel_context_control()
126 			memcpy(ctx->base.ctxr->data, req->state, req->state_sz);  in safexcel_context_control()
128 		if (!req->finish && req->xcbcmac)  in safexcel_context_control()
133 				CONTEXT_CONTROL_SIZE(req->state_sz /  in safexcel_context_control()
139 				CONTEXT_CONTROL_SIZE(req->state_sz /  in safexcel_context_control()
142 	} else if (!req->processed) {  in safexcel_context_control()
144 		if (req->finish)  in safexcel_context_control()
145 			cdesc->control_data.control0 |= req->digest |  in safexcel_context_control()
151 			cdesc->control_data.control0 |= req->digest |  in safexcel_context_control()
161 	memcpy(ctx->base.ctxr->data, req->state, req->state_sz);  in safexcel_context_control()
163 	if (req->finish) {  in safexcel_context_control()
165 		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||  in safexcel_context_control()
166 		    req->hmac_zlen || (req->processed != req->block_sz)) {  in safexcel_context_control()
167 			count = req->processed / EIP197_COUNTER_BLOCK_SIZE;  in safexcel_context_control()
181 		if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||  in safexcel_context_control()
183 		    req->hmac_zlen ||  in safexcel_context_control()
185 		    (req->processed != req->block_sz)) {  in safexcel_context_control()
188 				CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |  in safexcel_context_control()
192 			if (req->hmac_zlen)  in safexcel_context_control()
197 			ctx->base.ctxr->data[req->state_sz >> 2] =  in safexcel_context_control()
199 			req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_context_control()
202 			req->hmac_zlen = false;  in safexcel_context_control()
205 			memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),  in safexcel_context_control()
206 			       &ctx->base.opad, req->state_sz);  in safexcel_context_control()
210 				CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |  in safexcel_context_control()
216 			CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |  in safexcel_context_control()
314 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_send_req()  local
324 	queued = safexcel_queued_len(req);  in safexcel_ahash_send_req()
330 	if (!req->finish && !req->last_req) {  in safexcel_ahash_send_req()
343 				   req->cache_next, extra,  in safexcel_ahash_send_req()
357 	if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {  in safexcel_ahash_send_req()
370 					req->cache + cache_len,  in safexcel_ahash_send_req()
374 			memset(req->cache + cache_len + skip, 0, extra);  in safexcel_ahash_send_req()
377 				req->cache[cache_len + skip] = 0x80;  in safexcel_ahash_send_req()
380 					u32 *cache = (void *)req->cache;  in safexcel_ahash_send_req()
393 		crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);  in safexcel_ahash_send_req()
399 		req->cache_dma = dma_map_single(priv->dev, req->cache,  in safexcel_ahash_send_req()
401 		if (dma_mapping_error(priv->dev, req->cache_dma))  in safexcel_ahash_send_req()
404 		req->cache_sz = cache_len;  in safexcel_ahash_send_req()
407 						 req->cache_dma, cache_len,  in safexcel_ahash_send_req()
422 	req->nents = dma_map_sg(priv->dev, areq->src,  in safexcel_ahash_send_req()
426 	if (!req->nents) {  in safexcel_ahash_send_req()
431 	for_each_sg(areq->src, sg, req->nents, i) {  in safexcel_ahash_send_req()
466 	safexcel_context_control(ctx, req, first_cdesc);  in safexcel_ahash_send_req()
469 	safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);  in safexcel_ahash_send_req()
471 	req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,  in safexcel_ahash_send_req()
473 	if (dma_mapping_error(priv->dev, req->result_dma)) {  in safexcel_ahash_send_req()
479 	rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,  in safexcel_ahash_send_req()
480 				   req->digest_sz);  in safexcel_ahash_send_req()
488 	req->processed += len - extra;  in safexcel_ahash_send_req()
495 	dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,  in safexcel_ahash_send_req()
498 	if (req->nents) {  in safexcel_ahash_send_req()
499 		dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);  in safexcel_ahash_send_req()
500 		req->nents = 0;  in safexcel_ahash_send_req()
506 	if (req->cache_dma) {  in safexcel_ahash_send_req()
507 		dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,  in safexcel_ahash_send_req()
509 		req->cache_dma = 0;  in safexcel_ahash_send_req()
510 		req->cache_sz = 0;  in safexcel_ahash_send_req()
571 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_handle_result()  local
574 	BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);  in safexcel_handle_result()
576 	if (req->needs_inv) {  in safexcel_handle_result()
577 		req->needs_inv = false;  in safexcel_handle_result()
610 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_send()  local
613 	if (req->needs_inv)  in safexcel_ahash_send()
625 	EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);  in safexcel_ahash_exit_inv()
626 	struct safexcel_ahash_req *rctx = ahash_request_ctx(req);  in safexcel_ahash_exit_inv()
630 	memset(req, 0, EIP197_AHASH_REQ_SIZE);  in safexcel_ahash_exit_inv()
634 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,  in safexcel_ahash_exit_inv()
637 	ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));  in safexcel_ahash_exit_inv()
638 	ctx = crypto_tfm_ctx(req->base.tfm);  in safexcel_ahash_exit_inv()
643 	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);  in safexcel_ahash_exit_inv()
665 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_cache()  local
669 	 * tot sz handled by update() - last req sz - tot sz handled by send()  in safexcel_ahash_cache()
671 	cache_len = safexcel_queued_len(req);  in safexcel_ahash_cache()
679 				   req->cache + cache_len,  in safexcel_ahash_cache()
691 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_enqueue()  local
695 	req->needs_inv = false;  in safexcel_ahash_enqueue()
700 		   ((req->not_first && !req->xcbcmac) ||  in safexcel_ahash_enqueue()
702 		     memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||  in safexcel_ahash_enqueue()
704 		     (req->finish && req->hmac &&  in safexcel_ahash_enqueue()
705 		      memcmp(ctx->base.ctxr->data + (req->state_sz>>2),  in safexcel_ahash_enqueue()
706 			     &ctx->base.opad, req->state_sz))))  in safexcel_ahash_enqueue()
717 			req->needs_inv = true;  in safexcel_ahash_enqueue()
727 	req->not_first = true;  in safexcel_ahash_enqueue()
743 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_update()  local
754 	req->len += areq->nbytes;  in safexcel_ahash_update()
760 	if ((ret && !req->finish) || req->last_req)  in safexcel_ahash_update()
768 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_final()  local
771 	req->finish = true;  in safexcel_ahash_final()
773 	if (unlikely(!req->len && !areq->nbytes)) {  in safexcel_ahash_final()
803 	} else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&  in safexcel_ahash_final()
805 			    req->len == sizeof(u32) && !areq->nbytes)) {  in safexcel_ahash_final()
809 	} else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&  in safexcel_ahash_final()
814 	} else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&  in safexcel_ahash_final()
828 	} else if (unlikely(req->hmac &&  in safexcel_ahash_final()
829 			    (req->len == req->block_sz) &&  in safexcel_ahash_final()
839 		memset(req->cache, 0, req->block_sz);  in safexcel_ahash_final()
841 		req->cache[0] = 0x80;  in safexcel_ahash_final()
843 		if (req->len_is_le) {  in safexcel_ahash_final()
845 			req->cache[req->block_sz-8] = (req->block_sz << 3) &  in safexcel_ahash_final()
847 			req->cache[req->block_sz-7] = (req->block_sz >> 5);  in safexcel_ahash_final()
850 			req->cache[req->block_sz-2] = (req->block_sz >> 5);  in safexcel_ahash_final()
851 			req->cache[req->block_sz-1] = (req->block_sz << 3) &  in safexcel_ahash_final()
855 		req->len += req->block_sz; /* plus 1 hash block */  in safexcel_ahash_final()
858 		req->hmac_zlen = true;  in safexcel_ahash_final()
861 		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_ahash_final()
862 	} else if (req->hmac) {  in safexcel_ahash_final()
864 		req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_ahash_final()
872 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_finup()  local
874 	req->finish = true;  in safexcel_ahash_finup()
882 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_export()  local
885 	export->len = req->len;  in safexcel_ahash_export()
886 	export->processed = req->processed;  in safexcel_ahash_export()
888 	export->digest = req->digest;  in safexcel_ahash_export()
890 	memcpy(export->state, req->state, req->state_sz);  in safexcel_ahash_export()
891 	memcpy(export->cache, req->cache, HASH_CACHE_SIZE);  in safexcel_ahash_export()
898 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_import()  local
906 	req->len = export->len;  in safexcel_ahash_import()
907 	req->processed = export->processed;  in safexcel_ahash_import()
909 	req->digest = export->digest;  in safexcel_ahash_import()
911 	memcpy(req->cache, export->cache, HASH_CACHE_SIZE);  in safexcel_ahash_import()
912 	memcpy(req->state, export->state, req->state_sz);  in safexcel_ahash_import()
937 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha1_init()  local
939 	memset(req, 0, sizeof(*req));  in safexcel_sha1_init()
942 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha1_init()
943 	req->state_sz = SHA1_DIGEST_SIZE;  in safexcel_sha1_init()
944 	req->digest_sz = SHA1_DIGEST_SIZE;  in safexcel_sha1_init()
945 	req->block_sz = SHA1_BLOCK_SIZE;  in safexcel_sha1_init()
1014 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha1_init()  local
1016 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha1_init()
1019 	memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);  in safexcel_hmac_sha1_init()
1021 	req->len	= SHA1_BLOCK_SIZE;  in safexcel_hmac_sha1_init()
1022 	req->processed	= SHA1_BLOCK_SIZE;  in safexcel_hmac_sha1_init()
1025 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha1_init()
1026 	req->state_sz = SHA1_DIGEST_SIZE;  in safexcel_hmac_sha1_init()
1027 	req->digest_sz = SHA1_DIGEST_SIZE;  in safexcel_hmac_sha1_init()
1028 	req->block_sz = SHA1_BLOCK_SIZE;  in safexcel_hmac_sha1_init()
1029 	req->hmac = true;  in safexcel_hmac_sha1_init()
1049 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)  in safexcel_ahash_complete()  argument
1051 	struct safexcel_ahash_result *result = req->data;  in safexcel_ahash_complete()
1112 	struct safexcel_ahash_req *req;  in safexcel_hmac_init_iv()  local
1126 	req = ahash_request_ctx(areq);  in safexcel_hmac_init_iv()
1127 	req->hmac = true;  in safexcel_hmac_init_iv()
1128 	req->last_req = true;  in safexcel_hmac_init_iv()
1266 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha256_init()  local
1268 	memset(req, 0, sizeof(*req));  in safexcel_sha256_init()
1271 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha256_init()
1272 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_sha256_init()
1273 	req->digest_sz = SHA256_DIGEST_SIZE;  in safexcel_sha256_init()
1274 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_sha256_init()
1323 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha224_init()  local
1325 	memset(req, 0, sizeof(*req));  in safexcel_sha224_init()
1328 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha224_init()
1329 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_sha224_init()
1330 	req->digest_sz = SHA256_DIGEST_SIZE;  in safexcel_sha224_init()
1331 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_sha224_init()
1387 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha224_init()  local
1389 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha224_init()
1392 	memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);  in safexcel_hmac_sha224_init()
1394 	req->len	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha224_init()
1395 	req->processed	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha224_init()
1398 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha224_init()
1399 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_hmac_sha224_init()
1400 	req->digest_sz = SHA256_DIGEST_SIZE;  in safexcel_hmac_sha224_init()
1401 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_hmac_sha224_init()
1402 	req->hmac = true;  in safexcel_hmac_sha224_init()
1459 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha256_init()  local
1461 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha256_init()
1464 	memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);  in safexcel_hmac_sha256_init()
1466 	req->len	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha256_init()
1467 	req->processed	= SHA256_BLOCK_SIZE;  in safexcel_hmac_sha256_init()
1470 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha256_init()
1471 	req->state_sz = SHA256_DIGEST_SIZE;  in safexcel_hmac_sha256_init()
1472 	req->digest_sz = SHA256_DIGEST_SIZE;  in safexcel_hmac_sha256_init()
1473 	req->block_sz = SHA256_BLOCK_SIZE;  in safexcel_hmac_sha256_init()
1474 	req->hmac = true;  in safexcel_hmac_sha256_init()
1524 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha512_init()  local
1526 	memset(req, 0, sizeof(*req));  in safexcel_sha512_init()
1529 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha512_init()
1530 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_sha512_init()
1531 	req->digest_sz = SHA512_DIGEST_SIZE;  in safexcel_sha512_init()
1532 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_sha512_init()
1581 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha384_init()  local
1583 	memset(req, 0, sizeof(*req));  in safexcel_sha384_init()
1586 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sha384_init()
1587 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_sha384_init()
1588 	req->digest_sz = SHA512_DIGEST_SIZE;  in safexcel_sha384_init()
1589 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_sha384_init()
1645 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha512_init()  local
1647 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha512_init()
1650 	memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);  in safexcel_hmac_sha512_init()
1652 	req->len	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha512_init()
1653 	req->processed	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha512_init()
1656 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha512_init()
1657 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_hmac_sha512_init()
1658 	req->digest_sz = SHA512_DIGEST_SIZE;  in safexcel_hmac_sha512_init()
1659 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_hmac_sha512_init()
1660 	req->hmac = true;  in safexcel_hmac_sha512_init()
1717 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha384_init()  local
1719 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha384_init()
1722 	memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);  in safexcel_hmac_sha384_init()
1724 	req->len	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha384_init()
1725 	req->processed	= SHA512_BLOCK_SIZE;  in safexcel_hmac_sha384_init()
1728 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sha384_init()
1729 	req->state_sz = SHA512_DIGEST_SIZE;  in safexcel_hmac_sha384_init()
1730 	req->digest_sz = SHA512_DIGEST_SIZE;  in safexcel_hmac_sha384_init()
1731 	req->block_sz = SHA512_BLOCK_SIZE;  in safexcel_hmac_sha384_init()
1732 	req->hmac = true;  in safexcel_hmac_sha384_init()
1782 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_md5_init()  local
1784 	memset(req, 0, sizeof(*req));  in safexcel_md5_init()
1787 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_md5_init()
1788 	req->state_sz = MD5_DIGEST_SIZE;  in safexcel_md5_init()
1789 	req->digest_sz = MD5_DIGEST_SIZE;  in safexcel_md5_init()
1790 	req->block_sz = MD5_HMAC_BLOCK_SIZE;  in safexcel_md5_init()
1839 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_md5_init()  local
1841 	memset(req, 0, sizeof(*req));  in safexcel_hmac_md5_init()
1844 	memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);  in safexcel_hmac_md5_init()
1846 	req->len	= MD5_HMAC_BLOCK_SIZE;  in safexcel_hmac_md5_init()
1847 	req->processed	= MD5_HMAC_BLOCK_SIZE;  in safexcel_hmac_md5_init()
1850 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_md5_init()
1851 	req->state_sz = MD5_DIGEST_SIZE;  in safexcel_hmac_md5_init()
1852 	req->digest_sz = MD5_DIGEST_SIZE;  in safexcel_hmac_md5_init()
1853 	req->block_sz = MD5_HMAC_BLOCK_SIZE;  in safexcel_hmac_md5_init()
1854 	req->len_is_le = true; /* MD5 is little endian! ... */  in safexcel_hmac_md5_init()
1855 	req->hmac = true;  in safexcel_hmac_md5_init()
1922 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_crc32_init()  local
1924 	memset(req, 0, sizeof(*req));  in safexcel_crc32_init()
1927 	req->state[0]	= cpu_to_le32(~ctx->base.ipad.word[0]);  in safexcel_crc32_init()
1929 	req->len	= sizeof(u32);  in safexcel_crc32_init()
1930 	req->processed	= sizeof(u32);  in safexcel_crc32_init()
1933 	req->digest = CONTEXT_CONTROL_DIGEST_XCM;  in safexcel_crc32_init()
1934 	req->state_sz = sizeof(u32);  in safexcel_crc32_init()
1935 	req->digest_sz = sizeof(u32);  in safexcel_crc32_init()
1936 	req->block_sz = sizeof(u32);  in safexcel_crc32_init()
1994 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_cbcmac_init()  local
1996 	memset(req, 0, sizeof(*req));  in safexcel_cbcmac_init()
1999 	memcpy(req->state, &ctx->base.ipad, ctx->key_sz);  in safexcel_cbcmac_init()
2001 	req->len	= AES_BLOCK_SIZE;  in safexcel_cbcmac_init()
2002 	req->processed	= AES_BLOCK_SIZE;  in safexcel_cbcmac_init()
2004 	req->digest   = CONTEXT_CONTROL_DIGEST_XCM;  in safexcel_cbcmac_init()
2005 	req->state_sz = ctx->key_sz;  in safexcel_cbcmac_init()
2006 	req->digest_sz = AES_BLOCK_SIZE;  in safexcel_cbcmac_init()
2007 	req->block_sz = AES_BLOCK_SIZE;  in safexcel_cbcmac_init()
2008 	req->xcbcmac  = true;  in safexcel_cbcmac_init()
2275 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sm3_init()  local
2277 	memset(req, 0, sizeof(*req));  in safexcel_sm3_init()
2280 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_sm3_init()
2281 	req->state_sz = SM3_DIGEST_SIZE;  in safexcel_sm3_init()
2282 	req->digest_sz = SM3_DIGEST_SIZE;  in safexcel_sm3_init()
2283 	req->block_sz = SM3_BLOCK_SIZE;  in safexcel_sm3_init()
2339 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sm3_init()  local
2341 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sm3_init()
2344 	memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);  in safexcel_hmac_sm3_init()
2346 	req->len	= SM3_BLOCK_SIZE;  in safexcel_hmac_sm3_init()
2347 	req->processed	= SM3_BLOCK_SIZE;  in safexcel_hmac_sm3_init()
2350 	req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;  in safexcel_hmac_sm3_init()
2351 	req->state_sz = SM3_DIGEST_SIZE;  in safexcel_hmac_sm3_init()
2352 	req->digest_sz = SM3_DIGEST_SIZE;  in safexcel_hmac_sm3_init()
2353 	req->block_sz = SM3_BLOCK_SIZE;  in safexcel_hmac_sm3_init()
2354 	req->hmac = true;  in safexcel_hmac_sm3_init()
2405 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_224_init()  local
2407 	memset(req, 0, sizeof(*req));  in safexcel_sha3_224_init()
2410 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;  in safexcel_sha3_224_init()
2411 	req->state_sz = SHA3_224_DIGEST_SIZE;  in safexcel_sha3_224_init()
2412 	req->digest_sz = SHA3_224_DIGEST_SIZE;  in safexcel_sha3_224_init()
2413 	req->block_sz = SHA3_224_BLOCK_SIZE;  in safexcel_sha3_224_init()
2419 static int safexcel_sha3_fbcheck(struct ahash_request *req)  in safexcel_sha3_fbcheck()  argument
2421 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_fbcheck()
2423 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_fbcheck()
2428 		ahash_request_set_callback(subreq, req->base.flags,  in safexcel_sha3_fbcheck()
2429 					   req->base.complete, req->base.data);  in safexcel_sha3_fbcheck()
2430 		ahash_request_set_crypt(subreq, req->src, req->result,  in safexcel_sha3_fbcheck()
2431 					req->nbytes);  in safexcel_sha3_fbcheck()
2456 static int safexcel_sha3_update(struct ahash_request *req)  in safexcel_sha3_update()  argument
2458 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_update()
2460 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_update()
2463 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);  in safexcel_sha3_update()
2466 static int safexcel_sha3_final(struct ahash_request *req)  in safexcel_sha3_final()  argument
2468 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_final()
2470 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_final()
2473 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);  in safexcel_sha3_final()
2476 static int safexcel_sha3_finup(struct ahash_request *req)  in safexcel_sha3_finup()  argument
2478 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_finup()
2480 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_finup()
2482 	ctx->do_fallback |= !req->nbytes;  in safexcel_sha3_finup()
2485 		return safexcel_sha3_fbcheck(req) ?:  in safexcel_sha3_finup()
2488 		return safexcel_ahash_finup(req);  in safexcel_sha3_finup()
2491 static int safexcel_sha3_digest_fallback(struct ahash_request *req)  in safexcel_sha3_digest_fallback()  argument
2493 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_digest_fallback()
2495 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_digest_fallback()
2499 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);  in safexcel_sha3_digest_fallback()
2502 static int safexcel_sha3_224_digest(struct ahash_request *req)  in safexcel_sha3_224_digest()  argument
2504 	if (req->nbytes)  in safexcel_sha3_224_digest()
2505 		return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);  in safexcel_sha3_224_digest()
2508 	return safexcel_sha3_digest_fallback(req);  in safexcel_sha3_224_digest()
2511 static int safexcel_sha3_export(struct ahash_request *req, void *out)  in safexcel_sha3_export()  argument
2513 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_export()
2515 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_export()
2518 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);  in safexcel_sha3_export()
2521 static int safexcel_sha3_import(struct ahash_request *req, const void *in)  in safexcel_sha3_import()  argument
2523 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);  in safexcel_sha3_import()
2525 	struct ahash_request *subreq = ahash_request_ctx(req);  in safexcel_sha3_import()
2528 	return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);  in safexcel_sha3_import()
2529 	// return safexcel_ahash_import(req, in);  in safexcel_sha3_import()
2598 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_256_init()  local
2600 	memset(req, 0, sizeof(*req));  in safexcel_sha3_256_init()
2603 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;  in safexcel_sha3_256_init()
2604 	req->state_sz = SHA3_256_DIGEST_SIZE;  in safexcel_sha3_256_init()
2605 	req->digest_sz = SHA3_256_DIGEST_SIZE;  in safexcel_sha3_256_init()
2606 	req->block_sz = SHA3_256_BLOCK_SIZE;  in safexcel_sha3_256_init()
2612 static int safexcel_sha3_256_digest(struct ahash_request *req)  in safexcel_sha3_256_digest()  argument
2614 	if (req->nbytes)  in safexcel_sha3_256_digest()
2615 		return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);  in safexcel_sha3_256_digest()
2618 	return safexcel_sha3_digest_fallback(req);  in safexcel_sha3_256_digest()
2656 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_384_init()  local
2658 	memset(req, 0, sizeof(*req));  in safexcel_sha3_384_init()
2661 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;  in safexcel_sha3_384_init()
2662 	req->state_sz = SHA3_384_DIGEST_SIZE;  in safexcel_sha3_384_init()
2663 	req->digest_sz = SHA3_384_DIGEST_SIZE;  in safexcel_sha3_384_init()
2664 	req->block_sz = SHA3_384_BLOCK_SIZE;  in safexcel_sha3_384_init()
2670 static int safexcel_sha3_384_digest(struct ahash_request *req)  in safexcel_sha3_384_digest()  argument
2672 	if (req->nbytes)  in safexcel_sha3_384_digest()
2673 		return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);  in safexcel_sha3_384_digest()
2676 	return safexcel_sha3_digest_fallback(req);  in safexcel_sha3_384_digest()
2714 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_512_init()  local
2716 	memset(req, 0, sizeof(*req));  in safexcel_sha3_512_init()
2719 	req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;  in safexcel_sha3_512_init()
2720 	req->state_sz = SHA3_512_DIGEST_SIZE;  in safexcel_sha3_512_init()
2721 	req->digest_sz = SHA3_512_DIGEST_SIZE;  in safexcel_sha3_512_init()
2722 	req->block_sz = SHA3_512_BLOCK_SIZE;  in safexcel_sha3_512_init()
2728 static int safexcel_sha3_512_digest(struct ahash_request *req)  in safexcel_sha3_512_digest()  argument
2730 	if (req->nbytes)  in safexcel_sha3_512_digest()
2731 		return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);  in safexcel_sha3_512_digest()
2734 	return safexcel_sha3_digest_fallback(req);  in safexcel_sha3_512_digest()
2864 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_224_init()  local
2866 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha3_224_init()
2869 	memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);  in safexcel_hmac_sha3_224_init()
2871 	req->len	= SHA3_224_BLOCK_SIZE;  in safexcel_hmac_sha3_224_init()
2872 	req->processed	= SHA3_224_BLOCK_SIZE;  in safexcel_hmac_sha3_224_init()
2874 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_hmac_sha3_224_init()
2875 	req->state_sz = SHA3_224_BLOCK_SIZE / 2;  in safexcel_hmac_sha3_224_init()
2876 	req->digest_sz = SHA3_224_DIGEST_SIZE;  in safexcel_hmac_sha3_224_init()
2877 	req->block_sz = SHA3_224_BLOCK_SIZE;  in safexcel_hmac_sha3_224_init()
2878 	req->hmac = true;  in safexcel_hmac_sha3_224_init()
2884 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)  in safexcel_hmac_sha3_224_digest()  argument
2886 	if (req->nbytes)  in safexcel_hmac_sha3_224_digest()
2887 		return safexcel_hmac_sha3_224_init(req) ?:  in safexcel_hmac_sha3_224_digest()
2888 		       safexcel_ahash_finup(req);  in safexcel_hmac_sha3_224_digest()
2891 	return safexcel_sha3_digest_fallback(req);  in safexcel_hmac_sha3_224_digest()
2935 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_256_init()  local
2937 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha3_256_init()
2940 	memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);  in safexcel_hmac_sha3_256_init()
2942 	req->len	= SHA3_256_BLOCK_SIZE;  in safexcel_hmac_sha3_256_init()
2943 	req->processed	= SHA3_256_BLOCK_SIZE;  in safexcel_hmac_sha3_256_init()
2945 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_hmac_sha3_256_init()
2946 	req->state_sz = SHA3_256_BLOCK_SIZE / 2;  in safexcel_hmac_sha3_256_init()
2947 	req->digest_sz = SHA3_256_DIGEST_SIZE;  in safexcel_hmac_sha3_256_init()
2948 	req->block_sz = SHA3_256_BLOCK_SIZE;  in safexcel_hmac_sha3_256_init()
2949 	req->hmac = true;  in safexcel_hmac_sha3_256_init()
2955 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)  in safexcel_hmac_sha3_256_digest()  argument
2957 	if (req->nbytes)  in safexcel_hmac_sha3_256_digest()
2958 		return safexcel_hmac_sha3_256_init(req) ?:  in safexcel_hmac_sha3_256_digest()
2959 		       safexcel_ahash_finup(req);  in safexcel_hmac_sha3_256_digest()
2962 	return safexcel_sha3_digest_fallback(req);  in safexcel_hmac_sha3_256_digest()
3006 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_384_init()  local
3008 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha3_384_init()
3011 	memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);  in safexcel_hmac_sha3_384_init()
3013 	req->len	= SHA3_384_BLOCK_SIZE;  in safexcel_hmac_sha3_384_init()
3014 	req->processed	= SHA3_384_BLOCK_SIZE;  in safexcel_hmac_sha3_384_init()
3016 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_hmac_sha3_384_init()
3017 	req->state_sz = SHA3_384_BLOCK_SIZE / 2;  in safexcel_hmac_sha3_384_init()
3018 	req->digest_sz = SHA3_384_DIGEST_SIZE;  in safexcel_hmac_sha3_384_init()
3019 	req->block_sz = SHA3_384_BLOCK_SIZE;  in safexcel_hmac_sha3_384_init()
3020 	req->hmac = true;  in safexcel_hmac_sha3_384_init()
3026 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)  in safexcel_hmac_sha3_384_digest()  argument
3028 	if (req->nbytes)  in safexcel_hmac_sha3_384_digest()
3029 		return safexcel_hmac_sha3_384_init(req) ?:  in safexcel_hmac_sha3_384_digest()
3030 		       safexcel_ahash_finup(req);  in safexcel_hmac_sha3_384_digest()
3033 	return safexcel_sha3_digest_fallback(req);  in safexcel_hmac_sha3_384_digest()
3077 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_512_init()  local
3079 	memset(req, 0, sizeof(*req));  in safexcel_hmac_sha3_512_init()
3082 	memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);  in safexcel_hmac_sha3_512_init()
3084 	req->len	= SHA3_512_BLOCK_SIZE;  in safexcel_hmac_sha3_512_init()
3085 	req->processed	= SHA3_512_BLOCK_SIZE;  in safexcel_hmac_sha3_512_init()
3087 	req->digest = CONTEXT_CONTROL_DIGEST_HMAC;  in safexcel_hmac_sha3_512_init()
3088 	req->state_sz = SHA3_512_BLOCK_SIZE / 2;  in safexcel_hmac_sha3_512_init()
3089 	req->digest_sz = SHA3_512_DIGEST_SIZE;  in safexcel_hmac_sha3_512_init()
3090 	req->block_sz = SHA3_512_BLOCK_SIZE;  in safexcel_hmac_sha3_512_init()
3091 	req->hmac = true;  in safexcel_hmac_sha3_512_init()
3097 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)  in safexcel_hmac_sha3_512_digest()  argument
3099 	if (req->nbytes)  in safexcel_hmac_sha3_512_digest()
3100 		return safexcel_hmac_sha3_512_init(req) ?:  in safexcel_hmac_sha3_512_digest()
3101 		       safexcel_ahash_finup(req);  in safexcel_hmac_sha3_512_digest()
3104 	return safexcel_sha3_digest_fallback(req);  in safexcel_hmac_sha3_512_digest()