325 	struct talitos_edesc *edesc;  in get_request_hdr()  local
333 	edesc = container_of(request->desc, struct talitos_edesc, desc);  in get_request_hdr()
335 	return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;  in get_request_hdr()
490 		struct talitos_edesc *edesc;  local
492 		edesc = container_of(priv->chan[ch].fifo[iter].desc,
495 			(edesc->buf + edesc->dma_len))->hdr;
959 			     struct talitos_edesc *edesc,  in talitos_sg_unmap()  argument
966 	unsigned int src_nents = edesc->src_nents ? : 1;  in talitos_sg_unmap()
967 	unsigned int dst_nents = edesc->dst_nents ? : 1;  in talitos_sg_unmap()
970 		dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,  in talitos_sg_unmap()
972 		sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,  in talitos_sg_unmap()
987 			    struct talitos_edesc *edesc,  in ipsec_esp_unmap()  argument
995 	bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;  in ipsec_esp_unmap()
996 	struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];  in ipsec_esp_unmap()
999 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],  in ipsec_esp_unmap()
1003 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst,  in ipsec_esp_unmap()
1006 	if (edesc->dma_len)  in ipsec_esp_unmap()
1007 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in ipsec_esp_unmap()
1011 		unsigned int dst_nents = edesc->dst_nents ? : 1;  in ipsec_esp_unmap()
1028 	struct talitos_edesc *edesc;  in ipsec_esp_encrypt_done()  local
1030 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_encrypt_done()
1032 	ipsec_esp_unmap(dev, edesc, areq, true);  in ipsec_esp_encrypt_done()
1034 	dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);  in ipsec_esp_encrypt_done()
1036 	kfree(edesc);  in ipsec_esp_encrypt_done()
1048 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_swauth_done()  local
1051 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_swauth_done()
1053 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_swauth_done()
1057 		oicv = edesc->buf + edesc->dma_len;  in ipsec_esp_decrypt_swauth_done()
1063 	kfree(edesc);  in ipsec_esp_decrypt_swauth_done()
1073 	struct talitos_edesc *edesc;  in ipsec_esp_decrypt_hwauth_done()  local
1075 	edesc = container_of(desc, struct talitos_edesc, desc);  in ipsec_esp_decrypt_hwauth_done()
1077 	ipsec_esp_unmap(dev, edesc, req, false);  in ipsec_esp_decrypt_hwauth_done()
1084 	kfree(edesc);  in ipsec_esp_decrypt_hwauth_done()
1144 			      unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map_ext()  argument
1163 		to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1);  in talitos_sg_map_ext()
1167 					 &edesc->link_tbl[tbl_off], align);  in talitos_sg_map_ext()
1170 		copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);  in talitos_sg_map_ext()
1173 	to_talitos_ptr(ptr, edesc->dma_link_tbl +  in talitos_sg_map_ext()
1181 			  unsigned int len, struct talitos_edesc *edesc,  in talitos_sg_map()  argument
1185 	return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,  in talitos_sg_map()
1192 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,  in ipsec_esp()  argument
1202 	struct talitos_desc *desc = &edesc->desc;  in ipsec_esp()
1214 	dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;  in ipsec_esp()
1219 	sg_count = edesc->src_nents ?: 1;  in ipsec_esp()
1221 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in ipsec_esp()
1229 	ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,  in ipsec_esp()
1238 	to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);  in ipsec_esp()
1253 	ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],  in ipsec_esp()
1264 		sg_count = edesc->dst_nents ? : 1;  in ipsec_esp()
1273 	ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],  in ipsec_esp()
1279 		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];  in ipsec_esp()
1293 		talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],  in ipsec_esp()
1303 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in ipsec_esp()
1304 					   edesc->dma_len,  in ipsec_esp()
1309 		ipsec_esp_unmap(dev, edesc, areq, encrypt);  in ipsec_esp()
1310 		kfree(edesc);  in ipsec_esp()
1330 	struct talitos_edesc *edesc;  in talitos_edesc_alloc()  local
1395 	edesc = kmalloc(alloc_len, GFP_DMA | flags);  in talitos_edesc_alloc()
1396 	if (!edesc)  in talitos_edesc_alloc()
1399 		iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);  in talitos_edesc_alloc()
1402 	memset(&edesc->desc, 0, sizeof(edesc->desc));  in talitos_edesc_alloc()
1404 	edesc->src_nents = src_nents;  in talitos_edesc_alloc()
1405 	edesc->dst_nents = dst_nents;  in talitos_edesc_alloc()
1406 	edesc->iv_dma = iv_dma;  in talitos_edesc_alloc()
1407 	edesc->dma_len = dma_len;  in talitos_edesc_alloc()
1409 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],  in talitos_edesc_alloc()
1410 						     edesc->dma_len,  in talitos_edesc_alloc()
1413 	return edesc;  in talitos_edesc_alloc()
1435 	struct talitos_edesc *edesc;  in aead_encrypt()  local
1438 	edesc = aead_edesc_alloc(req, req->iv, 0, true);  in aead_encrypt()
1439 	if (IS_ERR(edesc))  in aead_encrypt()
1440 		return PTR_ERR(edesc);  in aead_encrypt()
1443 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in aead_encrypt()
1445 	return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);  in aead_encrypt()
1454 	struct talitos_edesc *edesc;  in aead_decrypt()  local
1458 	edesc = aead_edesc_alloc(req, req->iv, 1, false);  in aead_decrypt()
1459 	if (IS_ERR(edesc))  in aead_decrypt()
1460 		return PTR_ERR(edesc);  in aead_decrypt()
1462 	if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&  in aead_decrypt()
1464 	    ((!edesc->src_nents && !edesc->dst_nents) ||  in aead_decrypt()
1468 		edesc->desc.hdr = ctx->desc_hdr_template |  in aead_decrypt()
1474 		return ipsec_esp(edesc, req, false,  in aead_decrypt()
1479 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in aead_decrypt()
1482 	icvdata = edesc->buf + edesc->dma_len;  in aead_decrypt()
1484 	sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,  in aead_decrypt()
1487 	return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);  in aead_decrypt()
1532 				  struct talitos_edesc *edesc,  in common_nonsnoop_unmap()  argument
1535 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_unmap()
1537 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, 0);  in common_nonsnoop_unmap()
1538 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);  in common_nonsnoop_unmap()
1540 	if (edesc->dma_len)  in common_nonsnoop_unmap()
1541 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_unmap()
1553 	struct talitos_edesc *edesc;  in skcipher_done()  local
1555 	edesc = container_of(desc, struct talitos_edesc, desc);  in skcipher_done()
1557 	common_nonsnoop_unmap(dev, edesc, areq);  in skcipher_done()
1560 	kfree(edesc);  in skcipher_done()
1565 static int common_nonsnoop(struct talitos_edesc *edesc,  in common_nonsnoop()  argument
1574 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop()
1587 	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);  in common_nonsnoop()
1592 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop()
1594 		sg_copy_to_buffer(areq->src, sg_count, edesc->buf,  in common_nonsnoop()
1603 	sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3],  in common_nonsnoop()
1610 		sg_count = edesc->dst_nents ? : 1;  in common_nonsnoop()
1615 	ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],  in common_nonsnoop()
1616 			     sg_count, 0, (edesc->src_nents + 1));  in common_nonsnoop()
1627 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop()
1628 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop()
1632 		common_nonsnoop_unmap(dev, edesc, areq);  in common_nonsnoop()
1633 		kfree(edesc);  in common_nonsnoop()
1654 	struct talitos_edesc *edesc;  in skcipher_encrypt()  local
1665 	edesc = skcipher_edesc_alloc(areq, true);  in skcipher_encrypt()
1666 	if (IS_ERR(edesc))  in skcipher_encrypt()
1667 		return PTR_ERR(edesc);  in skcipher_encrypt()
1670 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;  in skcipher_encrypt()
1672 	return common_nonsnoop(edesc, areq, skcipher_done);  in skcipher_encrypt()
1679 	struct talitos_edesc *edesc;  in skcipher_decrypt()  local
1690 	edesc = skcipher_edesc_alloc(areq, false);  in skcipher_decrypt()
1691 	if (IS_ERR(edesc))  in skcipher_decrypt()
1692 		return PTR_ERR(edesc);  in skcipher_decrypt()
1694 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;  in skcipher_decrypt()
1696 	return common_nonsnoop(edesc, areq, skcipher_done);  in skcipher_decrypt()
1700 				       struct talitos_edesc *edesc,  in common_nonsnoop_hash_unmap()  argument
1707 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash_unmap()
1709 				     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash_unmap()
1711 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);  in common_nonsnoop_hash_unmap()
1720 		talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);  in common_nonsnoop_hash_unmap()
1723 	if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))  in common_nonsnoop_hash_unmap()
1724 		unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],  in common_nonsnoop_hash_unmap()
1734 	if (edesc->dma_len)  in common_nonsnoop_hash_unmap()
1735 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,  in common_nonsnoop_hash_unmap()
1738 	if (edesc->desc.next_desc)  in common_nonsnoop_hash_unmap()
1739 		dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),  in common_nonsnoop_hash_unmap()
1748 	struct talitos_edesc *edesc =  in ahash_done()  local
1757 	common_nonsnoop_hash_unmap(dev, edesc, areq);  in ahash_done()
1759 	kfree(edesc);  in ahash_done()
1769 			       struct talitos_edesc *edesc,  in talitos_handle_buggy_hash()  argument
1780 	edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;  in talitos_handle_buggy_hash()
1785 static int common_nonsnoop_hash(struct talitos_edesc *edesc,  in common_nonsnoop_hash()  argument
1795 	struct talitos_desc *desc = &edesc->desc;  in common_nonsnoop_hash()
1823 	sg_count = edesc->src_nents ?: 1;  in common_nonsnoop_hash()
1825 		sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);  in common_nonsnoop_hash()
1837 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1859 		talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);  in common_nonsnoop_hash()
1863 					     (edesc->buf + edesc->dma_len);  in common_nonsnoop_hash()
1883 		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,  in common_nonsnoop_hash()
1900 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,  in common_nonsnoop_hash()
1901 					   edesc->dma_len, DMA_BIDIRECTIONAL);  in common_nonsnoop_hash()
1905 		common_nonsnoop_hash_unmap(dev, edesc, areq);  in common_nonsnoop_hash()
1906 		kfree(edesc);  in common_nonsnoop_hash()
1985 	struct talitos_edesc *edesc;  in ahash_process_req()  local
2067 	edesc = ahash_edesc_alloc(areq, nbytes_to_hash);  in ahash_process_req()
2068 	if (IS_ERR(edesc))  in ahash_process_req()
2069 		return PTR_ERR(edesc);  in ahash_process_req()
2071 	edesc->desc.hdr = ctx->desc_hdr_template;  in ahash_process_req()
2075 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;  in ahash_process_req()
2077 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;  in ahash_process_req()
2081 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;  in ahash_process_req()
2087 		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;  in ahash_process_req()
2089 	return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);  in ahash_process_req()