223 static int safexcel_ahash_enqueue(struct ahash_request *areq);
231 	struct ahash_request *areq = ahash_request_cast(async);  in safexcel_handle_req_result()  local
232 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);  in safexcel_handle_req_result()
233 	struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);  in safexcel_handle_req_result()
251 		dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);  in safexcel_handle_req_result()
284 			areq->nbytes = 0;  in safexcel_handle_req_result()
285 			safexcel_ahash_enqueue(areq);  in safexcel_handle_req_result()
294 			*(__le32 *)areq->result = ~sreq->state[0];  in safexcel_handle_req_result()
296 			memcpy(areq->result, sreq->state,  in safexcel_handle_req_result()
313 	struct ahash_request *areq = ahash_request_cast(async);  in safexcel_ahash_send_req()  local
314 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_send_req()
315 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_ahash_send_req()
328 		cache_len = queued - areq->nbytes;  in safexcel_ahash_send_req()
342 		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),  in safexcel_ahash_send_req()
344 				   areq->nbytes - extra);  in safexcel_ahash_send_req()
368 				sg_pcopy_to_buffer(areq->src,  in safexcel_ahash_send_req()
369 					sg_nents(areq->src),  in safexcel_ahash_send_req()
422 	req->nents = dma_map_sg(priv->dev, areq->src,  in safexcel_ahash_send_req()
423 				sg_nents_for_len(areq->src,  in safexcel_ahash_send_req()
424 						 areq->nbytes),  in safexcel_ahash_send_req()
431 	for_each_sg(areq->src, sg, req->nents, i) {  in safexcel_ahash_send_req()
486 	safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);  in safexcel_ahash_send_req()
499 		dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);  in safexcel_ahash_send_req()
522 	struct ahash_request *areq = ahash_request_cast(async);  in safexcel_handle_inv_result()  local
523 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);  in safexcel_handle_inv_result()
570 	struct ahash_request *areq = ahash_request_cast(async);  in safexcel_handle_result()  local
571 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_handle_result()
591 	struct ahash_request *areq = ahash_request_cast(async);  in safexcel_ahash_send_inv()  local
592 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_ahash_send_inv()
609 	struct ahash_request *areq = ahash_request_cast(async);  in safexcel_ahash_send()  local
610 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_send()
663 static int safexcel_ahash_cache(struct ahash_request *areq)  in safexcel_ahash_cache()  argument
665 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_cache()
677 	if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {  in safexcel_ahash_cache()
678 		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),  in safexcel_ahash_cache()
680 				   areq->nbytes, 0);  in safexcel_ahash_cache()
688 static int safexcel_ahash_enqueue(struct ahash_request *areq)  in safexcel_ahash_enqueue()  argument
690 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_ahash_enqueue()
691 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_enqueue()
722 						 EIP197_GFP_FLAGS(areq->base),  in safexcel_ahash_enqueue()
732 	ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);  in safexcel_ahash_enqueue()
741 static int safexcel_ahash_update(struct ahash_request *areq)  in safexcel_ahash_update()  argument
743 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_update()
747 	if (!areq->nbytes)  in safexcel_ahash_update()
751 	ret = safexcel_ahash_cache(areq);  in safexcel_ahash_update()
754 	req->len += areq->nbytes;  in safexcel_ahash_update()
761 		return safexcel_ahash_enqueue(areq);  in safexcel_ahash_update()
766 static int safexcel_ahash_final(struct ahash_request *areq)  in safexcel_ahash_final()  argument
768 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_final()
769 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_ahash_final()
773 	if (unlikely(!req->len && !areq->nbytes)) {  in safexcel_ahash_final()
780 			memcpy(areq->result, md5_zero_message_hash,  in safexcel_ahash_final()
783 			memcpy(areq->result, sha1_zero_message_hash,  in safexcel_ahash_final()
786 			memcpy(areq->result, sha224_zero_message_hash,  in safexcel_ahash_final()
789 			memcpy(areq->result, sha256_zero_message_hash,  in safexcel_ahash_final()
792 			memcpy(areq->result, sha384_zero_message_hash,  in safexcel_ahash_final()
795 			memcpy(areq->result, sha512_zero_message_hash,  in safexcel_ahash_final()
798 			memcpy(areq->result,  in safexcel_ahash_final()
805 			    req->len == sizeof(u32) && !areq->nbytes)) {  in safexcel_ahash_final()
807 		memcpy(areq->result, &ctx->base.ipad, sizeof(u32));  in safexcel_ahash_final()
810 			    !areq->nbytes)) {  in safexcel_ahash_final()
812 		memset(areq->result, 0, AES_BLOCK_SIZE);  in safexcel_ahash_final()
815 			    !areq->nbytes)) {  in safexcel_ahash_final()
820 			u32 *result = (void *)areq->result;  in safexcel_ahash_final()
825 		areq->result[0] ^= 0x80;			// 10- padding  in safexcel_ahash_final()
826 		crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);  in safexcel_ahash_final()
830 			    !areq->nbytes)) {  in safexcel_ahash_final()
867 	return safexcel_ahash_enqueue(areq);  in safexcel_ahash_final()
870 static int safexcel_ahash_finup(struct ahash_request *areq)  in safexcel_ahash_finup()  argument
872 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_finup()
876 	safexcel_ahash_update(areq);  in safexcel_ahash_finup()
877 	return safexcel_ahash_final(areq);  in safexcel_ahash_finup()
880 static int safexcel_ahash_export(struct ahash_request *areq, void *out)  in safexcel_ahash_export()  argument
882 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_export()
896 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)  in safexcel_ahash_import()  argument
898 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_ahash_import()
902 	ret = crypto_ahash_init(areq);  in safexcel_ahash_import()
934 static int safexcel_sha1_init(struct ahash_request *areq)  in safexcel_sha1_init()  argument
936 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_sha1_init()
937 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha1_init()
950 static int safexcel_sha1_digest(struct ahash_request *areq)  in safexcel_sha1_digest()  argument
952 	int ret = safexcel_sha1_init(areq);  in safexcel_sha1_digest()
957 	return safexcel_ahash_finup(areq);  in safexcel_sha1_digest()
1011 static int safexcel_hmac_sha1_init(struct ahash_request *areq)  in safexcel_hmac_sha1_init()  argument
1013 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_sha1_init()
1014 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha1_init()
1034 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)  in safexcel_hmac_sha1_digest()  argument
1036 	int ret = safexcel_hmac_sha1_init(areq);  in safexcel_hmac_sha1_digest()
1041 	return safexcel_ahash_finup(areq);  in safexcel_hmac_sha1_digest()
1060 static int safexcel_hmac_init_pad(struct ahash_request *areq,  in safexcel_hmac_init_pad()  argument
1076 		ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,  in safexcel_hmac_init_pad()
1079 		ahash_request_set_crypt(areq, &sg, ipad, keylen);  in safexcel_hmac_init_pad()
1082 		ret = crypto_ahash_digest(areq);  in safexcel_hmac_init_pad()
1094 		keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));  in safexcel_hmac_init_pad()
1108 static int safexcel_hmac_init_iv(struct ahash_request *areq,  in safexcel_hmac_init_iv()  argument
1116 	ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,  in safexcel_hmac_init_iv()
1119 	ahash_request_set_crypt(areq, &sg, pad, blocksize);  in safexcel_hmac_init_iv()
1122 	ret = crypto_ahash_init(areq);  in safexcel_hmac_init_iv()
1126 	req = ahash_request_ctx(areq);  in safexcel_hmac_init_iv()
1130 	ret = crypto_ahash_update(areq);  in safexcel_hmac_init_iv()
1138 	return crypto_ahash_export(areq, state);  in safexcel_hmac_init_iv()
1145 	struct ahash_request *areq;  in __safexcel_hmac_setkey()  local
1155 	areq = ahash_request_alloc(tfm, GFP_KERNEL);  in __safexcel_hmac_setkey()
1156 	if (!areq) {  in __safexcel_hmac_setkey()
1172 	ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);  in __safexcel_hmac_setkey()
1176 	ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);  in __safexcel_hmac_setkey()
1180 	ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);  in __safexcel_hmac_setkey()
1185 	ahash_request_free(areq);  in __safexcel_hmac_setkey()
1263 static int safexcel_sha256_init(struct ahash_request *areq)  in safexcel_sha256_init()  argument
1265 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_sha256_init()
1266 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha256_init()
1279 static int safexcel_sha256_digest(struct ahash_request *areq)  in safexcel_sha256_digest()  argument
1281 	int ret = safexcel_sha256_init(areq);  in safexcel_sha256_digest()
1286 	return safexcel_ahash_finup(areq);  in safexcel_sha256_digest()
1320 static int safexcel_sha224_init(struct ahash_request *areq)  in safexcel_sha224_init()  argument
1322 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_sha224_init()
1323 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha224_init()
1336 static int safexcel_sha224_digest(struct ahash_request *areq)  in safexcel_sha224_digest()  argument
1338 	int ret = safexcel_sha224_init(areq);  in safexcel_sha224_digest()
1343 	return safexcel_ahash_finup(areq);  in safexcel_sha224_digest()
1384 static int safexcel_hmac_sha224_init(struct ahash_request *areq)  in safexcel_hmac_sha224_init()  argument
1386 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_sha224_init()
1387 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha224_init()
1407 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)  in safexcel_hmac_sha224_digest()  argument
1409 	int ret = safexcel_hmac_sha224_init(areq);  in safexcel_hmac_sha224_digest()
1414 	return safexcel_ahash_finup(areq);  in safexcel_hmac_sha224_digest()
1456 static int safexcel_hmac_sha256_init(struct ahash_request *areq)  in safexcel_hmac_sha256_init()  argument
1458 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_sha256_init()
1459 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha256_init()
1479 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)  in safexcel_hmac_sha256_digest()  argument
1481 	int ret = safexcel_hmac_sha256_init(areq);  in safexcel_hmac_sha256_digest()
1486 	return safexcel_ahash_finup(areq);  in safexcel_hmac_sha256_digest()
1521 static int safexcel_sha512_init(struct ahash_request *areq)  in safexcel_sha512_init()  argument
1523 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_sha512_init()
1524 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha512_init()
1537 static int safexcel_sha512_digest(struct ahash_request *areq)  in safexcel_sha512_digest()  argument
1539 	int ret = safexcel_sha512_init(areq);  in safexcel_sha512_digest()
1544 	return safexcel_ahash_finup(areq);  in safexcel_sha512_digest()
1578 static int safexcel_sha384_init(struct ahash_request *areq)  in safexcel_sha384_init()  argument
1580 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_sha384_init()
1581 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha384_init()
1594 static int safexcel_sha384_digest(struct ahash_request *areq)  in safexcel_sha384_digest()  argument
1596 	int ret = safexcel_sha384_init(areq);  in safexcel_sha384_digest()
1601 	return safexcel_ahash_finup(areq);  in safexcel_sha384_digest()
1642 static int safexcel_hmac_sha512_init(struct ahash_request *areq)  in safexcel_hmac_sha512_init()  argument
1644 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_sha512_init()
1645 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha512_init()
1665 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)  in safexcel_hmac_sha512_digest()  argument
1667 	int ret = safexcel_hmac_sha512_init(areq);  in safexcel_hmac_sha512_digest()
1672 	return safexcel_ahash_finup(areq);  in safexcel_hmac_sha512_digest()
1714 static int safexcel_hmac_sha384_init(struct ahash_request *areq)  in safexcel_hmac_sha384_init()  argument
1716 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_sha384_init()
1717 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha384_init()
1737 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)  in safexcel_hmac_sha384_digest()  argument
1739 	int ret = safexcel_hmac_sha384_init(areq);  in safexcel_hmac_sha384_digest()
1744 	return safexcel_ahash_finup(areq);  in safexcel_hmac_sha384_digest()
1779 static int safexcel_md5_init(struct ahash_request *areq)  in safexcel_md5_init()  argument
1781 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_md5_init()
1782 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_md5_init()
1795 static int safexcel_md5_digest(struct ahash_request *areq)  in safexcel_md5_digest()  argument
1797 	int ret = safexcel_md5_init(areq);  in safexcel_md5_digest()
1802 	return safexcel_ahash_finup(areq);  in safexcel_md5_digest()
1836 static int safexcel_hmac_md5_init(struct ahash_request *areq)  in safexcel_hmac_md5_init()  argument
1838 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_md5_init()
1839 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_md5_init()
1867 static int safexcel_hmac_md5_digest(struct ahash_request *areq)  in safexcel_hmac_md5_digest()  argument
1869 	int ret = safexcel_hmac_md5_init(areq);  in safexcel_hmac_md5_digest()
1874 	return safexcel_ahash_finup(areq);  in safexcel_hmac_md5_digest()
1919 static int safexcel_crc32_init(struct ahash_request *areq)  in safexcel_crc32_init()  argument
1921 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_crc32_init()
1922 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_crc32_init()
1953 static int safexcel_crc32_digest(struct ahash_request *areq)  in safexcel_crc32_digest()  argument
1955 	return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);  in safexcel_crc32_digest()
1991 static int safexcel_cbcmac_init(struct ahash_request *areq)  in safexcel_cbcmac_init()  argument
1993 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_cbcmac_init()
1994 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_cbcmac_init()
2044 static int safexcel_cbcmac_digest(struct ahash_request *areq)  in safexcel_cbcmac_digest()  argument
2046 	return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);  in safexcel_cbcmac_digest()
2272 static int safexcel_sm3_init(struct ahash_request *areq)  in safexcel_sm3_init()  argument
2274 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_sm3_init()
2275 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sm3_init()
2288 static int safexcel_sm3_digest(struct ahash_request *areq)  in safexcel_sm3_digest()  argument
2290 	int ret = safexcel_sm3_init(areq);  in safexcel_sm3_digest()
2295 	return safexcel_ahash_finup(areq);  in safexcel_sm3_digest()
2336 static int safexcel_hmac_sm3_init(struct ahash_request *areq)  in safexcel_hmac_sm3_init()  argument
2338 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));  in safexcel_hmac_sm3_init()
2339 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sm3_init()
2359 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)  in safexcel_hmac_sm3_digest()  argument
2361 	int ret = safexcel_hmac_sm3_init(areq);  in safexcel_hmac_sm3_digest()
2366 	return safexcel_ahash_finup(areq);  in safexcel_hmac_sm3_digest()
2401 static int safexcel_sha3_224_init(struct ahash_request *areq)  in safexcel_sha3_224_init()  argument
2403 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_sha3_224_init()
2405 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_224_init()
2594 static int safexcel_sha3_256_init(struct ahash_request *areq)  in safexcel_sha3_256_init()  argument
2596 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_sha3_256_init()
2598 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_256_init()
2652 static int safexcel_sha3_384_init(struct ahash_request *areq)  in safexcel_sha3_384_init()  argument
2654 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_sha3_384_init()
2656 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_384_init()
2710 static int safexcel_sha3_512_init(struct ahash_request *areq)  in safexcel_sha3_512_init()  argument
2712 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_sha3_512_init()
2714 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_sha3_512_init()
2860 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)  in safexcel_hmac_sha3_224_init()  argument
2862 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_hmac_sha3_224_init()
2864 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_224_init()
2931 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)  in safexcel_hmac_sha3_256_init()  argument
2933 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_hmac_sha3_256_init()
2935 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_256_init()
3002 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)  in safexcel_hmac_sha3_384_init()  argument
3004 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_hmac_sha3_384_init()
3006 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_384_init()
3073 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)  in safexcel_hmac_sha3_512_init()  argument
3075 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);  in safexcel_hmac_sha3_512_init()
3077 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);  in safexcel_hmac_sha3_512_init()