84 	struct img_hash_dev	*hdev;  member
111 	struct img_hash_dev	*hdev;  member
148 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)  in img_hash_read()  argument
150 	return readl_relaxed(hdev->io_base + offset);  in img_hash_read()
153 static inline void img_hash_write(struct img_hash_dev *hdev,  in img_hash_write()  argument
156 	writel_relaxed(value, hdev->io_base + offset);  in img_hash_write()
159 static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)  in img_hash_read_result_queue()  argument
161 	return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));  in img_hash_read_result_queue()
164 static void img_hash_start(struct img_hash_dev *hdev, bool dma)  in img_hash_start()  argument
166 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_start()
177 	dev_dbg(hdev->dev, "Starting hash process\n");  in img_hash_start()
178 	img_hash_write(hdev, CR_CONTROL, cr);  in img_hash_start()
187 		img_hash_read(hdev, CR_CONTROL);  in img_hash_start()
190 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,  in img_hash_xmit_cpu()  argument
196 	dev_dbg(hdev->dev, "xmit_cpu:  length: %zu bytes\n", length);  in img_hash_xmit_cpu()
199 		hdev->flags |= DRIVER_FLAGS_FINAL;  in img_hash_xmit_cpu()
204 		writel_relaxed(buffer[count], hdev->cpu_addr);  in img_hash_xmit_cpu()
211 	struct img_hash_dev *hdev = (struct img_hash_dev *)data;  in img_hash_dma_callback()  local
212 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_dma_callback()
215 		img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);  in img_hash_dma_callback()
219 		tasklet_schedule(&hdev->dma_task);  in img_hash_dma_callback()
222 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)  in img_hash_xmit_dma()  argument
225 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_xmit_dma()
227 	ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);  in img_hash_xmit_dma()
229 		dev_err(hdev->dev, "Invalid DMA sg\n");  in img_hash_xmit_dma()
230 		hdev->err = -EINVAL;  in img_hash_xmit_dma()
234 	desc = dmaengine_prep_slave_sg(hdev->dma_lch,  in img_hash_xmit_dma()
240 		dev_err(hdev->dev, "Null DMA descriptor\n");  in img_hash_xmit_dma()
241 		hdev->err = -EINVAL;  in img_hash_xmit_dma()
242 		dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);  in img_hash_xmit_dma()
246 	desc->callback_param = hdev;  in img_hash_xmit_dma()
248 	dma_async_issue_pending(hdev->dma_lch);  in img_hash_xmit_dma()
253 static int img_hash_write_via_cpu(struct img_hash_dev *hdev)  in img_hash_write_via_cpu()  argument
255 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_write_via_cpu()
257 	ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),  in img_hash_write_via_cpu()
258 					ctx->buffer, hdev->req->nbytes);  in img_hash_write_via_cpu()
260 	ctx->total = hdev->req->nbytes;  in img_hash_write_via_cpu()
263 	hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);  in img_hash_write_via_cpu()
265 	img_hash_start(hdev, false);  in img_hash_write_via_cpu()
267 	return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);  in img_hash_write_via_cpu()
289 		hash[i] = img_hash_read_result_queue(ctx->hdev);  in img_hash_copy_hash()
295 	struct img_hash_dev *hdev =  ctx->hdev;  in img_hash_finish_req()  local
299 		if (DRIVER_FLAGS_FINAL & hdev->flags)  in img_hash_finish_req()
302 		dev_warn(hdev->dev, "Hash failed with error %d\n", err);  in img_hash_finish_req()
306 	hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |  in img_hash_finish_req()
313 static int img_hash_write_via_dma(struct img_hash_dev *hdev)  in img_hash_write_via_dma()  argument
315 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_write_via_dma()
317 	img_hash_start(hdev, true);  in img_hash_write_via_dma()
319 	dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);  in img_hash_write_via_dma()
322 		hdev->flags |= DRIVER_FLAGS_FINAL;  in img_hash_write_via_dma()
324 	hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;  in img_hash_write_via_dma()
326 	tasklet_schedule(&hdev->dma_task);  in img_hash_write_via_dma()
331 static int img_hash_dma_init(struct img_hash_dev *hdev)  in img_hash_dma_init()  argument
336 	hdev->dma_lch = dma_request_chan(hdev->dev, "tx");  in img_hash_dma_init()
337 	if (IS_ERR(hdev->dma_lch)) {  in img_hash_dma_init()
338 		dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");  in img_hash_dma_init()
339 		return PTR_ERR(hdev->dma_lch);  in img_hash_dma_init()
342 	dma_conf.dst_addr = hdev->bus_addr;  in img_hash_dma_init()
347 	err = dmaengine_slave_config(hdev->dma_lch,  &dma_conf);  in img_hash_dma_init()
349 		dev_err(hdev->dev, "Couldn't configure DMA slave.\n");  in img_hash_dma_init()
350 		dma_release_channel(hdev->dma_lch);  in img_hash_dma_init()
359 	struct img_hash_dev *hdev = (struct img_hash_dev *)d;  in img_hash_dma_task()  local
360 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_dma_task()
365 	if (!hdev->req || !ctx->sg)  in img_hash_dma_task()
386 		if (img_hash_xmit_dma(hdev, &tsg)) {  in img_hash_dma_task()
387 			dev_err(hdev->dev, "DMA failed, falling back to CPU");  in img_hash_dma_task()
389 			hdev->err = 0;  in img_hash_dma_task()
390 			img_hash_xmit_cpu(hdev, addr + ctx->offset,  in img_hash_dma_task()
422 			img_hash_dma_callback(hdev);  in img_hash_dma_task()
429 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)  in img_hash_write_via_dma_stop()  argument
431 	struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);  in img_hash_write_via_dma_stop()
434 		dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);  in img_hash_write_via_dma_stop()
439 static int img_hash_process_data(struct img_hash_dev *hdev)  in img_hash_process_data()  argument
441 	struct ahash_request *req = hdev->req;  in img_hash_process_data()
448 		dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",  in img_hash_process_data()
450 		err = img_hash_write_via_dma(hdev);  in img_hash_process_data()
452 		dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",  in img_hash_process_data()
454 		err = img_hash_write_via_cpu(hdev);  in img_hash_process_data()
459 static int img_hash_hw_init(struct img_hash_dev *hdev)  in img_hash_hw_init()  argument
464 	img_hash_write(hdev, CR_RESET, CR_RESET_SET);  in img_hash_hw_init()
465 	img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);  in img_hash_hw_init()
466 	img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);  in img_hash_hw_init()
468 	nbits = (u64)hdev->req->nbytes << 3;  in img_hash_hw_init()
471 	img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);  in img_hash_hw_init()
472 	img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);  in img_hash_hw_init()
474 	if (!(DRIVER_FLAGS_INIT & hdev->flags)) {  in img_hash_hw_init()
475 		hdev->flags |= DRIVER_FLAGS_INIT;  in img_hash_hw_init()
476 		hdev->err = 0;  in img_hash_hw_init()
478 	dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);  in img_hash_hw_init()
495 static int img_hash_handle_queue(struct img_hash_dev *hdev,  in img_hash_handle_queue()  argument
503 	spin_lock_irqsave(&hdev->lock, flags);  in img_hash_handle_queue()
506 		res = ahash_enqueue_request(&hdev->queue, req);  in img_hash_handle_queue()
508 	if (DRIVER_FLAGS_BUSY & hdev->flags) {  in img_hash_handle_queue()
509 		spin_unlock_irqrestore(&hdev->lock, flags);  in img_hash_handle_queue()
513 	backlog = crypto_get_backlog(&hdev->queue);  in img_hash_handle_queue()
514 	async_req = crypto_dequeue_request(&hdev->queue);  in img_hash_handle_queue()
516 		hdev->flags |= DRIVER_FLAGS_BUSY;  in img_hash_handle_queue()
518 	spin_unlock_irqrestore(&hdev->lock, flags);  in img_hash_handle_queue()
527 	hdev->req = req;  in img_hash_handle_queue()
531 	dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",  in img_hash_handle_queue()
534 	err = img_hash_hw_init(hdev);  in img_hash_handle_queue()
537 		err = img_hash_process_data(hdev);  in img_hash_handle_queue()
622 	struct img_hash_dev *hdev = NULL;  in img_hash_digest()  local
627 	if (!tctx->hdev) {  in img_hash_digest()
629 			hdev = tmp;  in img_hash_digest()
632 		tctx->hdev = hdev;  in img_hash_digest()
635 		hdev = tctx->hdev;  in img_hash_digest()
639 	ctx->hdev = hdev;  in img_hash_digest()
668 	err = img_hash_handle_queue(tctx->hdev, req);  in img_hash_digest()
725 	struct img_hash_dev *hdev = dev_id;  in img_irq_handler()  local
728 	reg = img_hash_read(hdev, CR_INTSTAT);  in img_irq_handler()
729 	img_hash_write(hdev, CR_INTCLEAR, reg);  in img_irq_handler()
732 		dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");  in img_irq_handler()
733 		if (DRIVER_FLAGS_BUSY & hdev->flags) {  in img_irq_handler()
734 			hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;  in img_irq_handler()
735 			if (!(DRIVER_FLAGS_CPU & hdev->flags))  in img_irq_handler()
736 				hdev->flags |= DRIVER_FLAGS_DMA_READY;  in img_irq_handler()
737 			tasklet_schedule(&hdev->done_task);  in img_irq_handler()
739 			dev_warn(hdev->dev,  in img_irq_handler()
743 		dev_warn(hdev->dev,  in img_irq_handler()
746 		dev_warn(hdev->dev,  in img_irq_handler()
749 		dev_warn(hdev->dev,  in img_irq_handler()
862 static int img_register_algs(struct img_hash_dev *hdev)  in img_register_algs()  argument
880 static int img_unregister_algs(struct img_hash_dev *hdev)  in img_unregister_algs()  argument
891 	struct img_hash_dev *hdev = (struct img_hash_dev *)data;  in img_hash_done_task()  local
894 	if (hdev->err == -EINVAL) {  in img_hash_done_task()
895 		err = hdev->err;  in img_hash_done_task()
899 	if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {  in img_hash_done_task()
900 		img_hash_handle_queue(hdev, NULL);  in img_hash_done_task()
904 	if (DRIVER_FLAGS_CPU & hdev->flags) {  in img_hash_done_task()
905 		if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {  in img_hash_done_task()
906 			hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;  in img_hash_done_task()
909 	} else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {  in img_hash_done_task()
910 		if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {  in img_hash_done_task()
911 			hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;  in img_hash_done_task()
912 			img_hash_write_via_dma_stop(hdev);  in img_hash_done_task()
913 			if (hdev->err) {  in img_hash_done_task()
914 				err = hdev->err;  in img_hash_done_task()
918 		if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {  in img_hash_done_task()
919 			hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |  in img_hash_done_task()
927 	img_hash_finish_req(hdev->req, err);  in img_hash_done_task()
938 	struct img_hash_dev *hdev;  in img_hash_probe()  local
944 	hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);  in img_hash_probe()
945 	if (hdev == NULL)  in img_hash_probe()
948 	spin_lock_init(&hdev->lock);  in img_hash_probe()
950 	hdev->dev = dev;  in img_hash_probe()
952 	platform_set_drvdata(pdev, hdev);  in img_hash_probe()
954 	INIT_LIST_HEAD(&hdev->list);  in img_hash_probe()
956 	tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);  in img_hash_probe()
957 	tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);  in img_hash_probe()
959 	crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);  in img_hash_probe()
962 	hdev->io_base = devm_platform_ioremap_resource(pdev, 0);  in img_hash_probe()
963 	if (IS_ERR(hdev->io_base)) {  in img_hash_probe()
964 		err = PTR_ERR(hdev->io_base);  in img_hash_probe()
972 	hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);  in img_hash_probe()
973 	if (IS_ERR(hdev->cpu_addr)) {  in img_hash_probe()
975 		err = PTR_ERR(hdev->cpu_addr);  in img_hash_probe()
978 	hdev->bus_addr = hash_res->start;  in img_hash_probe()
987 			       dev_name(dev), hdev);  in img_hash_probe()
994 	hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");  in img_hash_probe()
995 	if (IS_ERR(hdev->hash_clk)) {  in img_hash_probe()
997 		err = PTR_ERR(hdev->hash_clk);  in img_hash_probe()
1001 	hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");  in img_hash_probe()
1002 	if (IS_ERR(hdev->sys_clk)) {  in img_hash_probe()
1004 		err = PTR_ERR(hdev->sys_clk);  in img_hash_probe()
1008 	err = clk_prepare_enable(hdev->hash_clk);  in img_hash_probe()
1012 	err = clk_prepare_enable(hdev->sys_clk);  in img_hash_probe()
1016 	err = img_hash_dma_init(hdev);  in img_hash_probe()
1021 		dma_chan_name(hdev->dma_lch));  in img_hash_probe()
1024 	list_add_tail(&hdev->list, &img_hash.dev_list);  in img_hash_probe()
1027 	err = img_register_algs(hdev);  in img_hash_probe()
1036 	list_del(&hdev->list);  in img_hash_probe()
1038 	dma_release_channel(hdev->dma_lch);  in img_hash_probe()
1040 	clk_disable_unprepare(hdev->sys_clk);  in img_hash_probe()
1042 	clk_disable_unprepare(hdev->hash_clk);  in img_hash_probe()
1044 	tasklet_kill(&hdev->done_task);  in img_hash_probe()
1045 	tasklet_kill(&hdev->dma_task);  in img_hash_probe()
1052 	struct img_hash_dev *hdev;  in img_hash_remove()  local
1054 	hdev = platform_get_drvdata(pdev);  in img_hash_remove()
1056 	list_del(&hdev->list);  in img_hash_remove()
1059 	img_unregister_algs(hdev);  in img_hash_remove()
1061 	tasklet_kill(&hdev->done_task);  in img_hash_remove()
1062 	tasklet_kill(&hdev->dma_task);  in img_hash_remove()
1064 	dma_release_channel(hdev->dma_lch);  in img_hash_remove()
1066 	clk_disable_unprepare(hdev->hash_clk);  in img_hash_remove()
1067 	clk_disable_unprepare(hdev->sys_clk);  in img_hash_remove()
1075 	struct img_hash_dev *hdev = dev_get_drvdata(dev);  in img_hash_suspend()  local
1077 	clk_disable_unprepare(hdev->hash_clk);  in img_hash_suspend()
1078 	clk_disable_unprepare(hdev->sys_clk);  in img_hash_suspend()
1085 	struct img_hash_dev *hdev = dev_get_drvdata(dev);  in img_hash_resume()  local
1088 	ret = clk_prepare_enable(hdev->hash_clk);  in img_hash_resume()
1092 	ret = clk_prepare_enable(hdev->sys_clk);  in img_hash_resume()
1094 		clk_disable_unprepare(hdev->hash_clk);  in img_hash_resume()