25 static void crypto_finalize_request(struct crypto_engine *engine,  in crypto_finalize_request()  argument
38 	if (!engine->retry_support) {  in crypto_finalize_request()
39 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_finalize_request()
40 		if (engine->cur_req == req) {  in crypto_finalize_request()
42 			engine->cur_req = NULL;  in crypto_finalize_request()
44 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_finalize_request()
47 	if (finalize_req || engine->retry_support) {  in crypto_finalize_request()
51 			ret = enginectx->op.unprepare_request(engine, req);  in crypto_finalize_request()
53 				dev_err(engine->dev, "failed to unprepare request\n");  in crypto_finalize_request()
58 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_finalize_request()
70 static void crypto_pump_requests(struct crypto_engine *engine,  in crypto_pump_requests()  argument
79 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
82 	if (!engine->retry_support && engine->cur_req)  in crypto_pump_requests()
86 	if (engine->idling) {  in crypto_pump_requests()
87 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
92 	if (!crypto_queue_len(&engine->queue) || !engine->running) {  in crypto_pump_requests()
93 		if (!engine->busy)  in crypto_pump_requests()
98 			kthread_queue_work(engine->kworker,  in crypto_pump_requests()
99 					   &engine->pump_requests);  in crypto_pump_requests()
103 		engine->busy = false;  in crypto_pump_requests()
104 		engine->idling = true;  in crypto_pump_requests()
105 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
107 		if (engine->unprepare_crypt_hardware &&  in crypto_pump_requests()
108 		    engine->unprepare_crypt_hardware(engine))  in crypto_pump_requests()
109 			dev_err(engine->dev, "failed to unprepare crypt hardware\n");  in crypto_pump_requests()
111 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
112 		engine->idling = false;  in crypto_pump_requests()
118 	backlog = crypto_get_backlog(&engine->queue);  in crypto_pump_requests()
119 	async_req = crypto_dequeue_request(&engine->queue);  in crypto_pump_requests()
128 	if (!engine->retry_support)  in crypto_pump_requests()
129 		engine->cur_req = async_req;  in crypto_pump_requests()
134 	if (engine->busy)  in crypto_pump_requests()
137 		engine->busy = true;  in crypto_pump_requests()
139 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
142 	if (!was_busy && engine->prepare_crypt_hardware) {  in crypto_pump_requests()
143 		ret = engine->prepare_crypt_hardware(engine);  in crypto_pump_requests()
145 			dev_err(engine->dev, "failed to prepare crypt hardware\n");  in crypto_pump_requests()
153 		ret = enginectx->op.prepare_request(engine, async_req);  in crypto_pump_requests()
155 			dev_err(engine->dev, "failed to prepare request: %d\n",  in crypto_pump_requests()
161 		dev_err(engine->dev, "failed to do request\n");  in crypto_pump_requests()
166 	ret = enginectx->op.do_one_request(engine, async_req);  in crypto_pump_requests()
175 		if (!engine->retry_support ||  in crypto_pump_requests()
177 			dev_err(engine->dev,  in crypto_pump_requests()
188 			ret = enginectx->op.unprepare_request(engine,  in crypto_pump_requests()
191 				dev_err(engine->dev,  in crypto_pump_requests()
194 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
200 		crypto_enqueue_request_head(&engine->queue, async_req);  in crypto_pump_requests()
202 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_pump_requests()
210 		ret = enginectx->op.unprepare_request(engine, async_req);  in crypto_pump_requests()
212 			dev_err(engine->dev, "failed to unprepare request\n");  in crypto_pump_requests()
220 	if (engine->retry_support) {  in crypto_pump_requests()
221 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_pump_requests()
227 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_pump_requests()
233 	if (engine->do_batch_requests) {  in crypto_pump_requests()
234 		ret = engine->do_batch_requests(engine);  in crypto_pump_requests()
236 			dev_err(engine->dev, "failed to do batch requests: %d\n",  in crypto_pump_requests()
245 	struct crypto_engine *engine =  in crypto_pump_work()  local
248 	crypto_pump_requests(engine, true);  in crypto_pump_work()
256 static int crypto_transfer_request(struct crypto_engine *engine,  in crypto_transfer_request()  argument
263 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_transfer_request()
265 	if (!engine->running) {  in crypto_transfer_request()
266 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
270 	ret = crypto_enqueue_request(&engine->queue, req);  in crypto_transfer_request()
272 	if (!engine->busy && need_pump)  in crypto_transfer_request()
273 		kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_transfer_request()
275 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_transfer_request()
285 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_request_to_engine()  argument
288 	return crypto_transfer_request(engine, req, true);  in crypto_transfer_request_to_engine()
297 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_aead_request_to_engine()  argument
300 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_aead_request_to_engine()
310 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_akcipher_request_to_engine()  argument
313 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_akcipher_request_to_engine()
323 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_hash_request_to_engine()  argument
326 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_hash_request_to_engine()
336 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,  in crypto_transfer_skcipher_request_to_engine()  argument
339 	return crypto_transfer_request_to_engine(engine, &req->base);  in crypto_transfer_skcipher_request_to_engine()
350 void crypto_finalize_aead_request(struct crypto_engine *engine,  in crypto_finalize_aead_request()  argument
353 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_aead_request()
364 void crypto_finalize_akcipher_request(struct crypto_engine *engine,  in crypto_finalize_akcipher_request()  argument
367 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_akcipher_request()
378 void crypto_finalize_hash_request(struct crypto_engine *engine,  in crypto_finalize_hash_request()  argument
381 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_hash_request()
392 void crypto_finalize_skcipher_request(struct crypto_engine *engine,  in crypto_finalize_skcipher_request()  argument
395 	return crypto_finalize_request(engine, &req->base, err);  in crypto_finalize_skcipher_request()
405 int crypto_engine_start(struct crypto_engine *engine)  in crypto_engine_start()  argument
409 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_start()
411 	if (engine->running || engine->busy) {  in crypto_engine_start()
412 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
416 	engine->running = true;  in crypto_engine_start()
417 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_start()
419 	kthread_queue_work(engine->kworker, &engine->pump_requests);  in crypto_engine_start()
431 int crypto_engine_stop(struct crypto_engine *engine)  in crypto_engine_stop()  argument
437 	spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
443 	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {  in crypto_engine_stop()
444 		spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
446 		spin_lock_irqsave(&engine->queue_lock, flags);  in crypto_engine_stop()
449 	if (crypto_queue_len(&engine->queue) || engine->busy)  in crypto_engine_stop()
452 		engine->running = false;  in crypto_engine_stop()
454 	spin_unlock_irqrestore(&engine->queue_lock, flags);  in crypto_engine_stop()
457 		dev_warn(engine->dev, "could not stop engine\n");  in crypto_engine_stop()
483 						       int (*cbk_do_batch)(struct crypto_engine *engine),  in crypto_engine_alloc_init_and_set()  argument
486 	struct crypto_engine *engine;  in crypto_engine_alloc_init_and_set()  local
491 	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);  in crypto_engine_alloc_init_and_set()
492 	if (!engine)  in crypto_engine_alloc_init_and_set()
495 	engine->dev = dev;  in crypto_engine_alloc_init_and_set()
496 	engine->rt = rt;  in crypto_engine_alloc_init_and_set()
497 	engine->running = false;  in crypto_engine_alloc_init_and_set()
498 	engine->busy = false;  in crypto_engine_alloc_init_and_set()
499 	engine->idling = false;  in crypto_engine_alloc_init_and_set()
500 	engine->retry_support = retry_support;  in crypto_engine_alloc_init_and_set()
501 	engine->priv_data = dev;  in crypto_engine_alloc_init_and_set()
506 	engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;  in crypto_engine_alloc_init_and_set()
508 	snprintf(engine->name, sizeof(engine->name),  in crypto_engine_alloc_init_and_set()
511 	crypto_init_queue(&engine->queue, qlen);  in crypto_engine_alloc_init_and_set()
512 	spin_lock_init(&engine->queue_lock);  in crypto_engine_alloc_init_and_set()
514 	engine->kworker = kthread_create_worker(0, "%s", engine->name);  in crypto_engine_alloc_init_and_set()
515 	if (IS_ERR(engine->kworker)) {  in crypto_engine_alloc_init_and_set()
519 	kthread_init_work(&engine->pump_requests, crypto_pump_work);  in crypto_engine_alloc_init_and_set()
521 	if (engine->rt) {  in crypto_engine_alloc_init_and_set()
523 		sched_set_fifo(engine->kworker->task);  in crypto_engine_alloc_init_and_set()
526 	return engine;  in crypto_engine_alloc_init_and_set()
552 int crypto_engine_exit(struct crypto_engine *engine)  in crypto_engine_exit()  argument
556 	ret = crypto_engine_stop(engine);  in crypto_engine_exit()
560 	kthread_destroy_worker(engine->kworker);  in crypto_engine_exit()