Lines Matching refs:rctx
134 struct iproc_reqctx_s *rctx, in spu_skcipher_rx_sg_create() argument
140 struct iproc_ctx_s *ctx = rctx->ctx; in spu_skcipher_rx_sg_create()
144 rctx->gfp); in spu_skcipher_rx_sg_create()
151 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_skcipher_rx_sg_create()
156 sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, in spu_skcipher_rx_sg_create()
160 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, in spu_skcipher_rx_sg_create()
161 rctx->dst_nents, chunksize); in spu_skcipher_rx_sg_create()
169 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_skcipher_rx_sg_create()
171 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_skcipher_rx_sg_create()
172 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_skcipher_rx_sg_create()
198 struct iproc_reqctx_s *rctx, in spu_skcipher_tx_sg_create() argument
203 struct iproc_ctx_s *ctx = rctx->ctx; in spu_skcipher_tx_sg_create()
208 rctx->gfp); in spu_skcipher_tx_sg_create()
215 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_skcipher_tx_sg_create()
221 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE); in spu_skcipher_tx_sg_create()
224 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_skcipher_tx_sg_create()
225 rctx->src_nents, chunksize); in spu_skcipher_tx_sg_create()
233 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_skcipher_tx_sg_create()
237 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_skcipher_tx_sg_create()
238 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_skcipher_tx_sg_create()
299 static int handle_skcipher_req(struct iproc_reqctx_s *rctx) in handle_skcipher_req() argument
302 struct crypto_async_request *areq = rctx->parent; in handle_skcipher_req()
305 struct iproc_ctx_s *ctx = rctx->ctx; in handle_skcipher_req()
330 cipher_parms.iv_len = rctx->iv_ctr_len; in handle_skcipher_req()
332 mssg = &rctx->mb_mssg; in handle_skcipher_req()
333 chunk_start = rctx->src_sent; in handle_skcipher_req()
334 remaining = rctx->total_todo - chunk_start; in handle_skcipher_req()
343 rctx->src_sent += chunksize; in handle_skcipher_req()
344 rctx->total_sent = rctx->src_sent; in handle_skcipher_req()
347 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); in handle_skcipher_req()
348 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); in handle_skcipher_req()
351 rctx->is_encrypt && chunk_start) in handle_skcipher_req()
356 sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr, in handle_skcipher_req()
357 rctx->iv_ctr_len, in handle_skcipher_req()
358 chunk_start - rctx->iv_ctr_len); in handle_skcipher_req()
360 if (rctx->iv_ctr_len) { in handle_skcipher_req()
362 __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr, in handle_skcipher_req()
363 rctx->iv_ctr_len); in handle_skcipher_req()
367 !rctx->is_encrypt) { in handle_skcipher_req()
372 sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr, in handle_skcipher_req()
373 rctx->iv_ctr_len, in handle_skcipher_req()
374 rctx->src_sent - rctx->iv_ctr_len); in handle_skcipher_req()
385 add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4); in handle_skcipher_req()
395 rctx->src_sent, chunk_start, remaining, chunksize); in handle_skcipher_req()
398 memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr, in handle_skcipher_req()
399 sizeof(rctx->msg_buf.bcm_spu_req_hdr)); in handle_skcipher_req()
401 spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_skcipher_req()
402 ctx->spu_req_hdr_len, !(rctx->is_encrypt), in handle_skcipher_req()
413 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0, in handle_skcipher_req()
415 rctx->total_sent, stat_pad_len); in handle_skcipher_req()
418 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_skcipher_req()
421 dump_sg(rctx->src_sg, rctx->src_skip, chunksize); in handle_skcipher_req()
422 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_skcipher_req()
430 mssg->ctx = rctx; /* Will be returned in response */ in handle_skcipher_req()
433 rx_frag_num += rctx->dst_nents; in handle_skcipher_req()
439 err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize, in handle_skcipher_req()
445 tx_frag_num += rctx->src_nents; in handle_skcipher_req()
453 err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize, in handle_skcipher_req()
458 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_skcipher_req()
470 static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) in handle_skcipher_resp() argument
474 struct crypto_async_request *areq = rctx->parent; in handle_skcipher_resp()
477 struct iproc_ctx_s *ctx = rctx->ctx; in handle_skcipher_resp()
481 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); in handle_skcipher_resp()
495 __func__, rctx->total_received, payload_len); in handle_skcipher_resp()
497 dump_sg(req->dst, rctx->total_received, payload_len); in handle_skcipher_resp()
499 rctx->total_received += payload_len; in handle_skcipher_resp()
500 if (rctx->total_received == rctx->total_todo) { in handle_skcipher_resp()
528 struct iproc_reqctx_s *rctx, in spu_ahash_rx_sg_create() argument
534 struct iproc_ctx_s *ctx = rctx->ctx; in spu_ahash_rx_sg_create()
537 rctx->gfp); in spu_ahash_rx_sg_create()
544 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_ahash_rx_sg_create()
547 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); in spu_ahash_rx_sg_create()
550 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_ahash_rx_sg_create()
552 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_ahash_rx_sg_create()
553 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_ahash_rx_sg_create()
580 struct iproc_reqctx_s *rctx, in spu_ahash_tx_sg_create() argument
592 rctx->gfp); in spu_ahash_tx_sg_create()
599 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_ahash_tx_sg_create()
603 sg_set_buf(sg++, rctx->hash_carry, hash_carry_len); in spu_ahash_tx_sg_create()
607 datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_ahash_tx_sg_create()
608 rctx->src_nents, new_data_len); in spu_ahash_tx_sg_create()
617 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_ahash_tx_sg_create()
621 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_ahash_tx_sg_create()
622 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_ahash_tx_sg_create()
654 static int handle_ahash_req(struct iproc_reqctx_s *rctx) in handle_ahash_req() argument
657 struct crypto_async_request *areq = rctx->parent; in handle_ahash_req()
662 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ahash_req()
697 rctx->total_todo, rctx->total_sent); in handle_ahash_req()
720 mssg = &rctx->mb_mssg; in handle_ahash_req()
721 chunk_start = rctx->src_sent; in handle_ahash_req()
727 nbytes_to_hash = rctx->total_todo - rctx->total_sent; in handle_ahash_req()
738 if (!rctx->is_final) { in handle_ahash_req()
739 u8 *dest = rctx->hash_carry + rctx->hash_carry_len; in handle_ahash_req()
748 new_len = rem - rctx->hash_carry_len; in handle_ahash_req()
750 rctx->src_sent); in handle_ahash_req()
751 rctx->hash_carry_len = rem; in handle_ahash_req()
753 rctx->hash_carry_len); in handle_ahash_req()
755 rctx->hash_carry, in handle_ahash_req()
756 rctx->hash_carry_len); in handle_ahash_req()
763 local_nbuf = rctx->hash_carry_len; in handle_ahash_req()
764 rctx->hash_carry_len = 0; in handle_ahash_req()
770 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, in handle_ahash_req()
777 hash_parms.type = spu->spu_hash_type(rctx->total_sent); in handle_ahash_req()
784 rctx->total_sent += chunksize; in handle_ahash_req()
786 rctx->src_sent += new_data_len; in handle_ahash_req()
788 if ((rctx->total_sent == rctx->total_todo) && rctx->is_final) in handle_ahash_req()
800 hash_parms.key_buf = rctx->incr_hash; in handle_ahash_req()
807 __func__, rctx->is_final, local_nbuf); in handle_ahash_req()
817 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); in handle_ahash_req()
820 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + in handle_ahash_req()
845 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len, in handle_ahash_req()
847 ctx->auth.mode, rctx->total_sent, in handle_ahash_req()
851 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_ahash_req()
853 packet_dump(" prebuf: ", rctx->hash_carry, local_nbuf); in handle_ahash_req()
855 dump_sg(rctx->src_sg, rctx->src_skip, new_data_len); in handle_ahash_req()
856 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_ahash_req()
864 mssg->ctx = rctx; /* Will be returned in response */ in handle_ahash_req()
867 err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize, in handle_ahash_req()
873 tx_frag_num += rctx->src_nents; in handle_ahash_req()
876 err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, in handle_ahash_req()
881 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_ahash_req()
945 static int ahash_req_done(struct iproc_reqctx_s *rctx) in ahash_req_done() argument
948 struct crypto_async_request *areq = rctx->parent; in ahash_req_done()
950 struct iproc_ctx_s *ctx = rctx->ctx; in ahash_req_done()
953 memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize); in ahash_req_done()
971 if (rctx->is_sw_hmac) { in ahash_req_done()
978 if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) { in ahash_req_done()
995 static void handle_ahash_resp(struct iproc_reqctx_s *rctx) in handle_ahash_resp() argument
997 struct iproc_ctx_s *ctx = rctx->ctx; in handle_ahash_resp()
999 struct crypto_async_request *areq = rctx->parent; in handle_ahash_resp()
1009 memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE); in handle_ahash_resp()
1016 if (rctx->is_final && (rctx->total_sent == rctx->total_todo)) in handle_ahash_resp()
1017 ahash_req_done(rctx); in handle_ahash_resp()
1046 struct iproc_reqctx_s *rctx, in spu_aead_rx_sg_create() argument
1054 struct iproc_ctx_s *ctx = rctx->ctx; in spu_aead_rx_sg_create()
1069 rctx->is_encrypt); in spu_aead_rx_sg_create()
1083 rctx->gfp); in spu_aead_rx_sg_create()
1091 sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len); in spu_aead_rx_sg_create()
1098 memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len); in spu_aead_rx_sg_create()
1099 sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len); in spu_aead_rx_sg_create()
1107 datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip, in spu_aead_rx_sg_create()
1108 rctx->dst_nents, resp_len); in spu_aead_rx_sg_create()
1118 memset(rctx->msg_buf.a.gcmpad, 0, data_padlen); in spu_aead_rx_sg_create()
1119 sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen); in spu_aead_rx_sg_create()
1123 sg_set_buf(sg++, rctx->msg_buf.digest, digestsize); in spu_aead_rx_sg_create()
1127 memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len); in spu_aead_rx_sg_create()
1128 sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len); in spu_aead_rx_sg_create()
1131 memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN); in spu_aead_rx_sg_create()
1132 sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len()); in spu_aead_rx_sg_create()
1165 struct iproc_reqctx_s *rctx, in spu_aead_tx_sg_create() argument
1178 struct iproc_ctx_s *ctx = rctx->ctx; in spu_aead_tx_sg_create()
1185 rctx->gfp); in spu_aead_tx_sg_create()
1192 sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr, in spu_aead_tx_sg_create()
1207 sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len); in spu_aead_tx_sg_create()
1210 memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len); in spu_aead_tx_sg_create()
1211 sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len); in spu_aead_tx_sg_create()
1219 written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip, in spu_aead_tx_sg_create()
1220 rctx->src_nents, datalen); in spu_aead_tx_sg_create()
1229 memset(rctx->msg_buf.spu_req_pad, 0, pad_len); in spu_aead_tx_sg_create()
1230 sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len); in spu_aead_tx_sg_create()
1234 sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize); in spu_aead_tx_sg_create()
1238 memset(rctx->msg_buf.tx_stat, 0, stat_len); in spu_aead_tx_sg_create()
1239 sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len); in spu_aead_tx_sg_create()
1261 static int handle_aead_req(struct iproc_reqctx_s *rctx) in handle_aead_req() argument
1264 struct crypto_async_request *areq = rctx->parent; in handle_aead_req()
1267 struct iproc_ctx_s *ctx = rctx->ctx; in handle_aead_req()
1290 chunksize = rctx->total_todo; in handle_aead_req()
1298 req_opts.is_inbound = !(rctx->is_encrypt); in handle_aead_req()
1308 cipher_parms.iv_buf = rctx->msg_buf.iv_ctr; in handle_aead_req()
1309 cipher_parms.iv_len = rctx->iv_ctr_len; in handle_aead_req()
1331 if (rctx->is_encrypt) { in handle_aead_req()
1346 rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize); in handle_aead_req()
1347 rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize); in handle_aead_req()
1349 assoc_nents = spu_sg_count(rctx->assoc, 0, in handle_aead_req()
1352 mssg = &rctx->mb_mssg; in handle_aead_req()
1354 rctx->total_sent = chunksize; in handle_aead_req()
1355 rctx->src_sent = chunksize; in handle_aead_req()
1359 rctx->is_encrypt)) in handle_aead_req()
1363 rctx->iv_ctr_len); in handle_aead_req()
1389 if (!rctx->is_encrypt) in handle_aead_req()
1396 chunksize, rctx->is_encrypt, in handle_aead_req()
1406 if (!rctx->is_encrypt) in handle_aead_req()
1419 if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) { in handle_aead_req()
1423 sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize, in handle_aead_req()
1424 req->assoclen + rctx->total_sent - in handle_aead_req()
1433 memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN); in handle_aead_req()
1435 spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr + in handle_aead_req()
1452 spu->spu_request_pad(rctx->msg_buf.spu_req_pad, in handle_aead_req()
1455 rctx->total_sent, stat_pad_len); in handle_aead_req()
1458 spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN, in handle_aead_req()
1460 dump_sg(rctx->assoc, 0, aead_parms.assoc_size); in handle_aead_req()
1461 packet_dump(" aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len); in handle_aead_req()
1463 dump_sg(rctx->src_sg, rctx->src_skip, chunksize); in handle_aead_req()
1464 packet_dump(" pad: ", rctx->msg_buf.spu_req_pad, pad_len); in handle_aead_req()
1472 mssg->ctx = rctx; /* Will be returned in response */ in handle_aead_req()
1475 rx_frag_num += rctx->dst_nents; in handle_aead_req()
1486 (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) { in handle_aead_req()
1494 rx_frag_num -= rctx->dst_nents; in handle_aead_req()
1497 err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num, in handle_aead_req()
1505 tx_frag_num += rctx->src_nents; in handle_aead_req()
1513 err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len, in handle_aead_req()
1514 rctx->assoc, aead_parms.assoc_size, in handle_aead_req()
1520 err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); in handle_aead_req()
1531 static void handle_aead_resp(struct iproc_reqctx_s *rctx) in handle_aead_resp() argument
1534 struct crypto_async_request *areq = rctx->parent; in handle_aead_resp()
1537 struct iproc_ctx_s *ctx = rctx->ctx; in handle_aead_resp()
1543 payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr); in handle_aead_resp()
1550 packet_dump(" assoc_data ", rctx->msg_buf.a.resp_aad, in handle_aead_resp()
1559 if (rctx->is_encrypt) { in handle_aead_resp()
1560 icv_offset = req->assoclen + rctx->total_sent; in handle_aead_resp()
1561 packet_dump(" ICV: ", rctx->msg_buf.digest, ctx->digestsize); in handle_aead_resp()
1563 sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest, in handle_aead_resp()
1591 static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx) in spu_chunk_cleanup() argument
1594 struct brcm_message *mssg = &rctx->mb_mssg; in spu_chunk_cleanup()
1609 static void finish_req(struct iproc_reqctx_s *rctx, int err) in finish_req() argument
1611 struct crypto_async_request *areq = rctx->parent; in finish_req()
1616 spu_chunk_cleanup(rctx); in finish_req()
1631 struct iproc_reqctx_s *rctx; in spu_rx_callback() local
1634 rctx = mssg->ctx; in spu_rx_callback()
1635 if (unlikely(!rctx)) { in spu_rx_callback()
1643 err = spu->spu_status_process(rctx->msg_buf.rx_stat); in spu_rx_callback()
1652 switch (rctx->ctx->alg->type) { in spu_rx_callback()
1654 handle_skcipher_resp(rctx); in spu_rx_callback()
1657 handle_ahash_resp(rctx); in spu_rx_callback()
1660 handle_aead_resp(rctx); in spu_rx_callback()
1671 if (rctx->total_sent < rctx->total_todo) { in spu_rx_callback()
1673 spu_chunk_cleanup(rctx); in spu_rx_callback()
1675 switch (rctx->ctx->alg->type) { in spu_rx_callback()
1677 err = handle_skcipher_req(rctx); in spu_rx_callback()
1680 err = handle_ahash_req(rctx); in spu_rx_callback()
1689 err = handle_aead_req(rctx); in spu_rx_callback()
1701 finish_req(rctx, err); in spu_rx_callback()
1717 struct iproc_reqctx_s *rctx = skcipher_request_ctx(req); in skcipher_enqueue() local
1724 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in skcipher_enqueue()
1726 rctx->parent = &req->base; in skcipher_enqueue()
1727 rctx->is_encrypt = encrypt; in skcipher_enqueue()
1728 rctx->bd_suppress = false; in skcipher_enqueue()
1729 rctx->total_todo = req->cryptlen; in skcipher_enqueue()
1730 rctx->src_sent = 0; in skcipher_enqueue()
1731 rctx->total_sent = 0; in skcipher_enqueue()
1732 rctx->total_received = 0; in skcipher_enqueue()
1733 rctx->ctx = ctx; in skcipher_enqueue()
1736 rctx->src_sg = req->src; in skcipher_enqueue()
1737 rctx->src_nents = 0; in skcipher_enqueue()
1738 rctx->src_skip = 0; in skcipher_enqueue()
1739 rctx->dst_sg = req->dst; in skcipher_enqueue()
1740 rctx->dst_nents = 0; in skcipher_enqueue()
1741 rctx->dst_skip = 0; in skcipher_enqueue()
1749 rctx->iv_ctr_len = in skcipher_enqueue()
1751 memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len); in skcipher_enqueue()
1753 rctx->iv_ctr_len = 0; in skcipher_enqueue()
1757 rctx->chan_idx = select_channel(); in skcipher_enqueue()
1758 err = handle_skcipher_req(rctx); in skcipher_enqueue()
1761 spu_chunk_cleanup(rctx); in skcipher_enqueue()
1907 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_enqueue() local
1915 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in ahash_enqueue()
1917 rctx->parent = &req->base; in ahash_enqueue()
1918 rctx->ctx = ctx; in ahash_enqueue()
1919 rctx->bd_suppress = true; in ahash_enqueue()
1920 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); in ahash_enqueue()
1923 rctx->src_sg = req->src; in ahash_enqueue()
1924 rctx->src_skip = 0; in ahash_enqueue()
1925 rctx->src_nents = 0; in ahash_enqueue()
1926 rctx->dst_sg = NULL; in ahash_enqueue()
1927 rctx->dst_skip = 0; in ahash_enqueue()
1928 rctx->dst_nents = 0; in ahash_enqueue()
1931 if ((rctx->is_final == 1) && (rctx->total_todo == 0) && in ahash_enqueue()
1935 rctx->is_final ? "" : "non-", alg_name); in ahash_enqueue()
1944 rctx->chan_idx = select_channel(); in ahash_enqueue()
1946 err = handle_ahash_req(rctx); in ahash_enqueue()
1949 spu_chunk_cleanup(rctx); in ahash_enqueue()
1964 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_init() local
1971 rctx->hash_carry_len = 0; in __ahash_init()
1972 rctx->is_final = 0; in __ahash_init()
1974 rctx->total_todo = 0; in __ahash_init()
1975 rctx->src_sent = 0; in __ahash_init()
1976 rctx->total_sent = 0; in __ahash_init()
1977 rctx->total_received = 0; in __ahash_init()
1983 rctx->is_sw_hmac = false; in __ahash_init()
2080 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_update() local
2086 rctx->total_todo += req->nbytes; in __ahash_update()
2087 rctx->src_sent = 0; in __ahash_update()
2138 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_final() local
2142 rctx->is_final = 1; in __ahash_final()
2175 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in __ahash_finup() local
2179 rctx->total_todo += req->nbytes; in __ahash_finup()
2180 rctx->src_sent = 0; in __ahash_finup()
2181 rctx->is_final = 1; in __ahash_finup()
2290 const struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_export() local
2293 spu_exp->total_todo = rctx->total_todo; in ahash_export()
2294 spu_exp->total_sent = rctx->total_sent; in ahash_export()
2295 spu_exp->is_sw_hmac = rctx->is_sw_hmac; in ahash_export()
2296 memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry)); in ahash_export()
2297 spu_exp->hash_carry_len = rctx->hash_carry_len; in ahash_export()
2298 memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash)); in ahash_export()
2305 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_import() local
2308 rctx->total_todo = spu_exp->total_todo; in ahash_import()
2309 rctx->total_sent = spu_exp->total_sent; in ahash_import()
2310 rctx->is_sw_hmac = spu_exp->is_sw_hmac; in ahash_import()
2311 memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry)); in ahash_import()
2312 rctx->hash_carry_len = spu_exp->hash_carry_len; in ahash_import()
2313 memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash)); in ahash_import()
2420 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_hmac_init() local
2433 rctx->is_sw_hmac = true; in ahash_hmac_init()
2436 memcpy(rctx->hash_carry, ctx->ipad, blocksize); in ahash_hmac_init()
2437 rctx->hash_carry_len = blocksize; in ahash_hmac_init()
2438 rctx->total_todo += blocksize; in ahash_hmac_init()
2470 struct iproc_reqctx_s *rctx = ahash_request_ctx(req); in ahash_hmac_digest() local
2490 rctx->is_sw_hmac = false; in ahash_hmac_digest()
2493 rctx->is_sw_hmac = true; in ahash_hmac_digest()
2496 memcpy(rctx->hash_carry, ctx->ipad, blocksize); in ahash_hmac_digest()
2497 rctx->hash_carry_len = blocksize; in ahash_hmac_digest()
2498 rctx->total_todo += blocksize; in ahash_hmac_digest()
2508 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_need_fallback() local
2521 if ((rctx->is_encrypt && (req->cryptlen == 0)) || in aead_need_fallback()
2522 (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) { in aead_need_fallback()
2556 rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE && in aead_need_fallback()
2579 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_complete() local
2586 areq->complete = rctx->old_complete; in aead_complete()
2587 areq->data = rctx->old_data; in aead_complete()
2596 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_do_fallback() local
2605 rctx->old_tfm = tfm; in aead_do_fallback()
2611 rctx->old_complete = req->base.complete; in aead_do_fallback()
2612 rctx->old_data = req->base.data; in aead_do_fallback()
2624 rctx->old_complete, req); in aead_do_fallback()
2625 req->base.data = rctx->old_data; in aead_do_fallback()
2639 struct iproc_reqctx_s *rctx = aead_request_ctx(req); in aead_enqueue() local
2653 rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | in aead_enqueue()
2655 rctx->parent = &req->base; in aead_enqueue()
2656 rctx->is_encrypt = is_encrypt; in aead_enqueue()
2657 rctx->bd_suppress = false; in aead_enqueue()
2658 rctx->total_todo = req->cryptlen; in aead_enqueue()
2659 rctx->src_sent = 0; in aead_enqueue()
2660 rctx->total_sent = 0; in aead_enqueue()
2661 rctx->total_received = 0; in aead_enqueue()
2662 rctx->is_sw_hmac = false; in aead_enqueue()
2663 rctx->ctx = ctx; in aead_enqueue()
2664 memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message)); in aead_enqueue()
2667 rctx->assoc = req->src; in aead_enqueue()
2674 if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg, in aead_enqueue()
2675 &rctx->src_skip) < 0) { in aead_enqueue()
2681 rctx->src_nents = 0; in aead_enqueue()
2682 rctx->dst_nents = 0; in aead_enqueue()
2684 rctx->dst_sg = rctx->src_sg; in aead_enqueue()
2685 rctx->dst_skip = rctx->src_skip; in aead_enqueue()
2692 if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg, in aead_enqueue()
2693 &rctx->dst_skip) < 0) { in aead_enqueue()
2705 rctx->iv_ctr_len = in aead_enqueue()
2709 rctx->iv_ctr_len = CCM_AES_IV_SIZE; in aead_enqueue()
2711 rctx->iv_ctr_len = 0; in aead_enqueue()
2714 rctx->hash_carry_len = 0; in aead_enqueue()
2718 rctx->src_sg, rctx->src_skip); in aead_enqueue()
2719 flow_log(" assoc: %p, assoclen %u\n", rctx->assoc, req->assoclen); in aead_enqueue()
2722 rctx->dst_sg, rctx->dst_skip); in aead_enqueue()
2723 flow_log(" iv_ctr_len:%u\n", rctx->iv_ctr_len); in aead_enqueue()
2724 flow_dump(" iv: ", req->iv, rctx->iv_ctr_len); in aead_enqueue()
2740 if (rctx->iv_ctr_len) { in aead_enqueue()
2742 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset, in aead_enqueue()
2744 memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len, in aead_enqueue()
2746 rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset); in aead_enqueue()
2749 rctx->chan_idx = select_channel(); in aead_enqueue()
2750 err = handle_aead_req(rctx); in aead_enqueue()
2753 spu_chunk_cleanup(rctx); in aead_enqueue()