57 	struct crypt_auth_op *caop = &kcaop->caop;  in get_userbuf_tls()  local
60 	if (caop->dst == NULL)  in get_userbuf_tls()
64 		if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask + 1))  in get_userbuf_tls()
66 					caop->dst, ses->alignmask + 1);  in get_userbuf_tls()
74 	pagecount = PAGECOUNT(caop->dst, kcaop->dst_len);  in get_userbuf_tls()
83 	rc = __cryptodev_get_userbuf(caop->dst, kcaop->dst_len, 1, pagecount,  in get_userbuf_tls()
109 	struct crypt_auth_op *caop = &kcaop->caop;  in get_userbuf_srtp()  local
112 	if (caop->dst == NULL && caop->auth_src == NULL) {  in get_userbuf_srtp()
118 		if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask + 1))  in get_userbuf_srtp()
120 					caop->dst, ses->alignmask + 1);  in get_userbuf_srtp()
121 		if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask + 1))  in get_userbuf_srtp()
123 					caop->auth_src, ses->alignmask + 1);  in get_userbuf_srtp()
126 	if (unlikely(kcaop->dst_len == 0 || caop->auth_len == 0)) {  in get_userbuf_srtp()
134 	auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);  in get_userbuf_srtp()
135 	diff = (int)(caop->src - caop->auth_src);  in get_userbuf_srtp()
149 	rc = __cryptodev_get_userbuf(caop->auth_src, caop->auth_len, 1, auth_pagecount,  in get_userbuf_srtp()
163 	cryptodev_sg_copy(ses->sg, (*dst_sg), caop->auth_len);  in get_userbuf_srtp()
192 static int cryptodev_get_dst_len(struct crypt_auth_op *caop, struct csession *ses_ptr)  in cryptodev_get_dst_len()  argument
194 	int dst_len = caop->len;  in cryptodev_get_dst_len()
195 	if (caop->op == COP_DECRYPT)  in cryptodev_get_dst_len()
198 	if (caop->flags & COP_FLAG_AEAD_RK_TYPE)  in cryptodev_get_dst_len()
201 	dst_len += caop->tag_len;  in cryptodev_get_dst_len()
205 	if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) {  in cryptodev_get_dst_len()
215 	struct crypt_auth_op *caop = &kcaop->caop;  in fill_kcaop_from_caop()  local
220 	ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);  in fill_kcaop_from_caop()
222 		derr(1, "invalid session ID=0x%08X", caop->ses);  in fill_kcaop_from_caop()
226 	if (caop->flags & COP_FLAG_AEAD_TLS_TYPE || caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {  in fill_kcaop_from_caop()
227 		if (caop->src != caop->dst) {  in fill_kcaop_from_caop()
234 	if (caop->tag_len == 0)  in fill_kcaop_from_caop()
235 		caop->tag_len = cryptodev_get_tag_len(ses_ptr);  in fill_kcaop_from_caop()
237 	kcaop->ivlen = caop->iv ? ses_ptr->cdata.ivsize : 0;  in fill_kcaop_from_caop()
238 	kcaop->dst_len = cryptodev_get_dst_len(caop, ses_ptr);  in fill_kcaop_from_caop()
242 	if (caop->iv) {  in fill_kcaop_from_caop()
243 		ret = copy_from_user(kcaop->iv, caop->iv, kcaop->ivlen);  in fill_kcaop_from_caop()
246 					kcaop->ivlen, ret, caop->iv);  in fill_kcaop_from_caop()
264 	kcaop->caop.len = kcaop->dst_len;  in fill_caop_from_kcaop()
266 	if (kcaop->ivlen && kcaop->caop.flags & COP_FLAG_WRITE_IV) {  in fill_caop_from_kcaop()
267 		ret = copy_to_user(kcaop->caop.iv,  in fill_caop_from_kcaop()
281 	if (unlikely(copy_from_user(&kcaop->caop, arg, sizeof(kcaop->caop)))) {  in cryptodev_kcaop_from_user()
300 	if (unlikely(copy_to_user(arg, &kcaop->caop, sizeof(kcaop->caop)))) {  in cryptodev_kcaop_to_user()
311 compat_to_crypt_auth_op(struct compat_crypt_auth_op *compat, struct crypt_auth_op *caop)  in compat_to_crypt_auth_op()  argument
313 	caop->ses      = compat->ses;  in compat_to_crypt_auth_op()
314 	caop->op       = compat->op;  in compat_to_crypt_auth_op()
315 	caop->flags    = compat->flags;  in compat_to_crypt_auth_op()
316 	caop->len      = compat->len;  in compat_to_crypt_auth_op()
317 	caop->auth_len = compat->auth_len;  in compat_to_crypt_auth_op()
318 	caop->tag_len  = compat->tag_len;  in compat_to_crypt_auth_op()
319 	caop->iv_len   = compat->iv_len;  in compat_to_crypt_auth_op()
321 	caop->auth_src = compat_ptr(compat->auth_src);  in compat_to_crypt_auth_op()
322 	caop->src      = compat_ptr(compat->src);  in compat_to_crypt_auth_op()
323 	caop->dst      = compat_ptr(compat->dst);  in compat_to_crypt_auth_op()
324 	caop->tag      = compat_ptr(compat->tag);  in compat_to_crypt_auth_op()
325 	caop->iv       = compat_ptr(compat->iv);  in compat_to_crypt_auth_op()
329 crypt_auth_op_to_compat(struct crypt_auth_op *caop, struct compat_crypt_auth_op *compat)  in crypt_auth_op_to_compat()  argument
331 	compat->ses      = caop->ses;  in crypt_auth_op_to_compat()
332 	compat->op       = caop->op;  in crypt_auth_op_to_compat()
333 	compat->flags    = caop->flags;  in crypt_auth_op_to_compat()
334 	compat->len      = caop->len;  in crypt_auth_op_to_compat()
335 	compat->auth_len = caop->auth_len;  in crypt_auth_op_to_compat()
336 	compat->tag_len  = caop->tag_len;  in crypt_auth_op_to_compat()
337 	compat->iv_len   = caop->iv_len;  in crypt_auth_op_to_compat()
339 	compat->auth_src = ptr_to_compat(caop->auth_src);  in crypt_auth_op_to_compat()
340 	compat->src      = ptr_to_compat(caop->src);  in crypt_auth_op_to_compat()
341 	compat->dst      = ptr_to_compat(caop->dst);  in crypt_auth_op_to_compat()
342 	compat->tag      = ptr_to_compat(caop->tag);  in crypt_auth_op_to_compat()
343 	compat->iv       = ptr_to_compat(caop->iv);  in crypt_auth_op_to_compat()
358 	compat_to_crypt_auth_op(&compat_auth_cop, &kcaop->caop);  in compat_kcaop_from_user()
375 	crypt_auth_op_to_compat(&kcaop->caop, &compat_auth_cop);  in compat_kcaop_to_user()
442 	struct crypt_auth_op *caop = &kcaop->caop;  in tls_auth_n_crypt()  local
448 	if (caop->op == COP_ENCRYPT) {  in tls_auth_n_crypt()
474 			copy_tls_hash(dst_sg, len, hash_output, caop->tag_len);  in tls_auth_n_crypt()
475 			len += caop->tag_len;  in tls_auth_n_crypt()
513 			if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {  in tls_auth_n_crypt()
518 			read_tls_hash(dst_sg, len, vhash, caop->tag_len);  in tls_auth_n_crypt()
519 			len -= caop->tag_len;  in tls_auth_n_crypt()
545 			if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {  in tls_auth_n_crypt()
546 				derr(2, "MAC verification failed (tag_len: %d)", caop->tag_len);  in tls_auth_n_crypt()
564 	struct crypt_auth_op *caop = &kcaop->caop;  in srtp_auth_n_crypt()  local
570 	if (caop->op == COP_ENCRYPT) {  in srtp_auth_n_crypt()
596 			if (unlikely(copy_to_user(caop->tag, hash_output, caop->tag_len)))  in srtp_auth_n_crypt()
602 			if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {  in srtp_auth_n_crypt()
607 			if (unlikely(copy_from_user(vhash, caop->tag, caop->tag_len)))  in srtp_auth_n_crypt()
623 			if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {  in srtp_auth_n_crypt()
650 	struct crypt_auth_op *caop = &kcaop->caop;  in rk_auth_n_crypt()  local
654 	if (unlikely(caop->tag_len > max_tag_len)) {  in rk_auth_n_crypt()
655 		derr(0, "Illegal tag length: %d", caop->tag_len);  in rk_auth_n_crypt()
659 	if (caop->tag_len)  in rk_auth_n_crypt()
660 		cryptodev_cipher_set_tag_size(&ses_ptr->cdata, caop->tag_len);  in rk_auth_n_crypt()
662 		caop->tag_len = max_tag_len;  in rk_auth_n_crypt()
666 	if (caop->op == COP_ENCRYPT) {  in rk_auth_n_crypt()
693 	struct crypt_auth_op *caop = &kcaop->caop;  in auth_n_crypt()  local
697 	if (unlikely(caop->tag_len > max_tag_len)) {  in auth_n_crypt()
698 		derr(0, "Illegal tag length: %d", caop->tag_len);  in auth_n_crypt()
702 	if (caop->tag_len)  in auth_n_crypt()
703 		cryptodev_cipher_set_tag_size(&ses_ptr->cdata, caop->tag_len);  in auth_n_crypt()
705 		caop->tag_len = max_tag_len;  in auth_n_crypt()
709 	if (caop->op == COP_ENCRYPT) {  in auth_n_crypt()
716 		kcaop->dst_len = len + caop->tag_len;  in auth_n_crypt()
717 		caop->tag = caop->dst + len;  in auth_n_crypt()
726 		kcaop->dst_len = len - caop->tag_len;  in auth_n_crypt()
727 		caop->tag = caop->dst + len - caop->tag_len;  in auth_n_crypt()
736 	struct crypt_auth_op *caop = &kcaop->caop;  in crypto_auth_zc_srtp()  local
751 	ret = srtp_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,  in crypto_auth_zc_srtp()
752 			dst_sg, caop->len);  in crypto_auth_zc_srtp()
761 	struct crypt_auth_op *caop = &kcaop->caop;  in crypto_auth_zc_tls()  local
767 	if (unlikely(caop->auth_len > PAGE_SIZE)) {  in crypto_auth_zc_tls()
778 	if (caop->auth_src && caop->auth_len > 0) {  in crypto_auth_zc_tls()
779 		if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {  in crypto_auth_zc_tls()
785 		sg_init_one(&tmp, auth_buf, caop->auth_len);  in crypto_auth_zc_tls()
797 	ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,  in crypto_auth_zc_tls()
798 			dst_sg, caop->len);  in crypto_auth_zc_tls()
810 	struct crypt_auth_op *caop = &kcaop->caop;  in crypto_auth_zc_aead()  local
828 	if (unlikely(caop->auth_len > PAGE_SIZE)) {  in crypto_auth_zc_aead()
839 	ret = cryptodev_get_userbuf(ses_ptr, caop->src, caop->len, caop->dst, kcaop->dst_len,  in crypto_auth_zc_aead()
847 	if (caop->auth_src && caop->auth_len > 0) {  in crypto_auth_zc_aead()
848 		if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {  in crypto_auth_zc_aead()
854 		sg_init_one(&tmp, auth_buf, caop->auth_len);  in crypto_auth_zc_aead()
860 	ret = auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,  in crypto_auth_zc_aead()
861 			src_sg, dst_sg, caop->len);  in crypto_auth_zc_aead()
863 	if (caop->auth_src && caop->auth_len > 0) {  in crypto_auth_zc_aead()
864 		if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {  in crypto_auth_zc_aead()
871 		sg_set_buf(auth1, auth_buf, caop->auth_len);  in crypto_auth_zc_aead()
879 			sg_set_buf(auth2, auth_buf, caop->auth_len);  in crypto_auth_zc_aead()
886 	ret = auth_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len,  in crypto_auth_zc_aead()
887 			src_sg, dst_sg, caop->len);  in crypto_auth_zc_aead()
924 	struct crypt_auth_op *caop = &kcaop->caop;  in crypto_auth_zc_rk()  local
935 	if (unlikely(caop->auth_len > PAGE_SIZE)) {  in crypto_auth_zc_rk()
940 	ret = cryptodev_get_userbuf(ses_ptr, caop->src, caop->len,  in crypto_auth_zc_rk()
941 				    caop->dst, kcaop->dst_len,  in crypto_auth_zc_rk()
953 	if (caop->tag && caop->tag_len > 0) {  in crypto_auth_zc_rk()
954 		tag_buf = kcalloc(caop->tag_len, sizeof(*tag_buf), GFP_KERNEL);  in crypto_auth_zc_rk()
956 			derr(1, "unable to kcalloc %d.", caop->tag_len);  in crypto_auth_zc_rk()
961 		if (unlikely(copy_from_user(tag_buf, caop->tag, caop->tag_len))) {  in crypto_auth_zc_rk()
968 		sg_set_buf(&tag[1], tag_buf, caop->tag_len);  in crypto_auth_zc_rk()
973 		if (caop->op == COP_ENCRYPT)  in crypto_auth_zc_rk()
987 	if (caop->auth_src && caop->auth_len > 0) {  in crypto_auth_zc_rk()
988 		if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {  in crypto_auth_zc_rk()
995 		sg_set_buf(auth_src, auth_buf, caop->auth_len);  in crypto_auth_zc_rk()
997 		sg_set_buf(auth_dst, auth_buf, caop->auth_len);  in crypto_auth_zc_rk()
1005 	if (caop->op == COP_ENCRYPT)  in crypto_auth_zc_rk()
1006 		ret = rk_auth_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len,  in crypto_auth_zc_rk()
1007 				      src, dst, caop->len);  in crypto_auth_zc_rk()
1009 		ret = rk_auth_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len,  in crypto_auth_zc_rk()
1010 				      src, dst, caop->len + caop->tag_len);  in crypto_auth_zc_rk()
1012 	if (!ret && caop->op == COP_ENCRYPT) {  in crypto_auth_zc_rk()
1013 		if (unlikely(copy_to_user(kcaop->caop.tag, tag_buf, caop->tag_len))) {  in crypto_auth_zc_rk()
1035 	struct crypt_auth_op *caop = &kcaop->caop;  in __crypto_auth_run_zc()  local
1038 	if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {  in __crypto_auth_run_zc()
1040 	} else if (caop->flags & COP_FLAG_AEAD_TLS_TYPE &&  in __crypto_auth_run_zc()
1043 	} else if (caop->flags & COP_FLAG_AEAD_RK_TYPE &&  in __crypto_auth_run_zc()
1059 	struct crypt_auth_op *caop = &kcaop->caop;  in crypto_auth_run()  local
1062 	if (unlikely(caop->op != COP_ENCRYPT && caop->op != COP_DECRYPT)) {  in crypto_auth_run()
1063 		ddebug(1, "invalid operation op=%u", caop->op);  in crypto_auth_run()
1068 	ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);  in crypto_auth_run()
1070 		derr(1, "invalid session ID=0x%08X", caop->ses);  in crypto_auth_run()