388 struct rga_job *rga_job_commit(struct rga_req *rga_command_base, struct rga_request *request)  in rga_job_commit()  argument
400 	job->use_batch_mode = request->use_batch_mode;  in rga_job_commit()
401 	job->request_id = request->id;  in rga_job_commit()
402 	job->session = request->session;  in rga_job_commit()
403 	job->mm = request->current_mm;  in rga_job_commit()
495 static int rga_request_get_current_mm(struct rga_request *request)  in rga_request_get_current_mm()  argument
499 	for (i = 0; i < request->task_count; i++) {  in rga_request_get_current_mm()
500 		if (rga_is_need_current_mm(&(request->task_list[i]))) {  in rga_request_get_current_mm()
503 			request->current_mm = current->mm;  in rga_request_get_current_mm()
512 static void rga_request_put_current_mm(struct rga_request *request)  in rga_request_put_current_mm()  argument
514 	if (request->current_mm == NULL)  in rga_request_put_current_mm()
517 	mmput(request->current_mm);  in rga_request_put_current_mm()
518 	mmdrop(request->current_mm);  in rga_request_put_current_mm()
519 	request->current_mm = NULL;  in rga_request_put_current_mm()
523 						  struct rga_request *request,  in rga_request_add_acquire_fence_callback()  argument
557 	 * Ensure that the request will not be free early when  in rga_request_add_acquire_fence_callback()
561 	rga_request_get(request);  in rga_request_add_acquire_fence_callback()
564 	ret = rga_dma_fence_add_callback(acquire_fence, cb_func, (void *)request);  in rga_request_add_acquire_fence_callback()
570 		rga_request_put(request);  in rga_request_add_acquire_fence_callback()
581 		pr_err("user request id[%d] is invalid", req->id);  in rga_request_check()
586 		pr_err("invalied user request!\n");  in rga_request_check()
606 	struct rga_request *request = NULL;  in rga_request_lookup()  local
610 	request = idr_find(&manager->request_idr, id);  in rga_request_lookup()
612 	return request;  in rga_request_lookup()
615 static int rga_request_scheduler_job_abort(struct rga_request *request)  in rga_request_scheduler_job_abort()  argument
630 			if (request->id == job->request_id) {  in rga_request_scheduler_job_abort()
640 			if (request->id == scheduler->running_job->request_id) {  in rga_request_scheduler_job_abort()
653 				pr_err("reset core[%d] by request[%d] abort",  in rga_request_scheduler_job_abort()
654 				       scheduler->core, request->id);  in rga_request_scheduler_job_abort()
677 	pr_err("request[%d] abort! finished %d failed %d running_abort %d todo_abort %d\n",  in rga_request_scheduler_job_abort()
678 	       request->id, request->finished_task_count, request->failed_task_count,  in rga_request_scheduler_job_abort()
684 static void rga_request_release_abort(struct rga_request *request, int err_code)  in rga_request_release_abort()  argument
689 	if (rga_request_scheduler_job_abort(request) > 0)  in rga_request_release_abort()
692 	spin_lock_irqsave(&request->lock, flags);  in rga_request_release_abort()
694 	if (request->is_done) {  in rga_request_release_abort()
695 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_release_abort()
699 	request->is_running = false;  in rga_request_release_abort()
700 	request->is_done = false;  in rga_request_release_abort()
702 	rga_request_put_current_mm(request);  in rga_request_release_abort()
704 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_release_abort()
706 	rga_dma_fence_signal(request->release_fence, err_code);  in rga_request_release_abort()
709 	/* current submit request put */  in rga_request_release_abort()
710 	rga_request_put(request);  in rga_request_release_abort()
717 	struct rga_request *request;  in rga_request_session_destroy_abort()  local
728 	idr_for_each_entry(&request_manager->request_idr, request, request_id) {  in rga_request_session_destroy_abort()
729 		if (session == request->session) {  in rga_request_session_destroy_abort()
730 			pr_err("[tgid:%d pid:%d] destroy request[%d] when the user exits",  in rga_request_session_destroy_abort()
731 			       session->tgid, current->pid, request->id);  in rga_request_session_destroy_abort()
732 			rga_request_put(request);  in rga_request_session_destroy_abort()
739 static int rga_request_timeout_query_state(struct rga_request *request)  in rga_request_timeout_query_state()  argument
753 			if (request->id == job->request_id) {  in rga_request_timeout_query_state()
757 					return request->ret;  in rga_request_timeout_query_state()
761 					pr_err("request[%d] hardware has finished, but the software has timeout!\n",  in rga_request_timeout_query_state()
762 					       request->id);  in rga_request_timeout_query_state()
767 					pr_err("request[%d] hardware has timeout.\n", request->id);  in rga_request_timeout_query_state()
776 	return request->ret;  in rga_request_timeout_query_state()
779 static int rga_request_wait(struct rga_request *request)  in rga_request_wait()  argument
784 	left_time = wait_event_timeout(request->finished_wq, request->is_done,  in rga_request_wait()
785 				       RGA_JOB_TIMEOUT_DELAY * request->task_count);  in rga_request_wait()
789 		ret = rga_request_timeout_query_state(request);  in rga_request_wait()
795 		ret = request->ret;  in rga_request_wait()
802 	rga_request_release_abort(request, ret);  in rga_request_wait()
807 int rga_request_commit(struct rga_request *request)  in rga_request_commit()  argument
813 	for (i = 0; i < request->task_count; i++) {  in rga_request_commit()
814 		job = rga_job_commit(&(request->task_list[i]), request);  in rga_request_commit()
816 			pr_err("request[%d] task[%d] job_commit failed.\n", request->id, i);  in rga_request_commit()
817 			rga_request_release_abort(request, PTR_ERR(job));  in rga_request_commit()
823 	if (request->sync_mode == RGA_BLIT_SYNC) {  in rga_request_commit()
824 		ret = rga_request_wait(request);  in rga_request_commit()
836 	struct rga_request *request = (struct rga_request *)waiter->private;  in rga_request_acquire_fence_signaled_cb()  local
839 	if (rga_request_commit(request))  in rga_request_acquire_fence_signaled_cb()
840 		pr_err("rga request[%d] commit failed!\n", request->id);  in rga_request_acquire_fence_signaled_cb()
843 	rga_request_put(request);  in rga_request_acquire_fence_signaled_cb()
852 	struct rga_request *request;  in rga_request_release_signal()  local
864 	request = rga_request_lookup(request_manager, job->request_id);  in rga_request_release_signal()
865 	if (IS_ERR_OR_NULL(request)) {  in rga_request_release_signal()
866 		pr_err("can not find internal request from id[%d]", job->request_id);  in rga_request_release_signal()
871 	rga_request_get(request);  in rga_request_release_signal()
874 	spin_lock_irqsave(&request->lock, flags);  in rga_request_release_signal()
877 		request->failed_task_count++;  in rga_request_release_signal()
878 		request->ret = job->ret;  in rga_request_release_signal()
880 		request->finished_task_count++;  in rga_request_release_signal()
883 	failed_count = request->failed_task_count;  in rga_request_release_signal()
884 	finished_count = request->finished_task_count;  in rga_request_release_signal()
886 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_release_signal()
890 	if ((failed_count + finished_count) >= request->task_count) {  in rga_request_release_signal()
891 		spin_lock_irqsave(&request->lock, flags);  in rga_request_release_signal()
893 		request->is_running = false;  in rga_request_release_signal()
894 		request->is_done = true;  in rga_request_release_signal()
896 		rga_request_put_current_mm(request);  in rga_request_release_signal()
898 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_release_signal()
900 		rga_dma_fence_signal(request->release_fence, request->ret);  in rga_request_release_signal()
902 		wake_up(&request->finished_wq);  in rga_request_release_signal()
905 			pr_info("request[%d] finished %d failed %d\n",  in rga_request_release_signal()
906 				request->id, finished_count, failed_count);  in rga_request_release_signal()
908 		/* current submit request put */  in rga_request_release_signal()
910 		rga_request_put(request);  in rga_request_release_signal()
915 	rga_request_put(request);  in rga_request_release_signal()
926 	struct rga_request *request;  in rga_request_config()  local
937 	request = rga_request_lookup(request_manager, user_request->id);  in rga_request_config()
938 	if (IS_ERR_OR_NULL(request)) {  in rga_request_config()
939 		pr_err("can not find request from id[%d]", user_request->id);  in rga_request_config()
944 	rga_request_get(request);  in rga_request_config()
961 	spin_lock_irqsave(&request->lock, flags);  in rga_request_config()
963 	request->use_batch_mode = true;  in rga_request_config()
964 	request->task_list = task_list;  in rga_request_config()
965 	request->task_count = user_request->task_num;  in rga_request_config()
966 	request->sync_mode = user_request->sync_mode;  in rga_request_config()
967 	request->mpi_config_flags = user_request->mpi_config_flags;  in rga_request_config()
968 	request->acquire_fence_fd = user_request->acquire_fence_fd;  in rga_request_config()
970 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_config()
972 	return request;  in rga_request_config()
978 	rga_request_put(request);  in rga_request_config()
989 	struct rga_request *request;  in rga_request_kernel_config()  local
1000 	request = rga_request_lookup(request_manager, user_request->id);  in rga_request_kernel_config()
1001 	if (IS_ERR_OR_NULL(request)) {  in rga_request_kernel_config()
1002 		pr_err("can not find request from id[%d]", user_request->id);  in rga_request_kernel_config()
1007 	rga_request_get(request);  in rga_request_kernel_config()
1020 	spin_lock_irqsave(&request->lock, flags);  in rga_request_kernel_config()
1022 	request->use_batch_mode = true;  in rga_request_kernel_config()
1023 	request->task_list = task_list;  in rga_request_kernel_config()
1024 	request->task_count = user_request->task_num;  in rga_request_kernel_config()
1025 	request->sync_mode = user_request->sync_mode;  in rga_request_kernel_config()
1026 	request->mpi_config_flags = user_request->mpi_config_flags;  in rga_request_kernel_config()
1027 	request->acquire_fence_fd = user_request->acquire_fence_fd;  in rga_request_kernel_config()
1029 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_kernel_config()
1031 	return request;  in rga_request_kernel_config()
1035 	rga_request_put(request);  in rga_request_kernel_config()
1041 int rga_request_submit(struct rga_request *request)  in rga_request_submit()  argument
1047 	spin_lock_irqsave(&request->lock, flags);  in rga_request_submit()
1049 	if (request->is_running) {  in rga_request_submit()
1050 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_submit()
1052 		pr_err("can not re-config when request is running\n");  in rga_request_submit()
1056 	if (request->task_list == NULL) {  in rga_request_submit()
1057 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_submit()
1059 		pr_err("can not find task list from id[%d]\n", request->id);  in rga_request_submit()
1064 	request->is_running = true;  in rga_request_submit()
1065 	request->is_done = false;  in rga_request_submit()
1066 	request->finished_task_count = 0;  in rga_request_submit()
1067 	request->failed_task_count = 0;  in rga_request_submit()
1069 	rga_request_get_current_mm(request);  in rga_request_submit()
1071 	/* Unlock after ensuring that the current request will not be resubmitted. */  in rga_request_submit()
1072 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_submit()
1074 	if (request->sync_mode == RGA_BLIT_ASYNC) {  in rga_request_submit()
1081 		request->release_fence = release_fence;  in rga_request_submit()
1083 		if (request->acquire_fence_fd > 0) {  in rga_request_submit()
1085 				request->acquire_fence_fd, request,  in rga_request_submit()
1095 				       request->acquire_fence_fd);  in rga_request_submit()
1102 	ret = rga_request_commit(request);  in rga_request_submit()
1104 		pr_err("rga request[%d] commit failed!\n", request->id);  in rga_request_submit()
1109 	if (request->release_fence != NULL) {  in rga_request_submit()
1110 		ret = rga_dma_fence_get_fd(request->release_fence);  in rga_request_submit()
1113 			rga_request_release_abort(request, ret);  in rga_request_submit()
1117 		request->release_fence_fd = ret;  in rga_request_submit()
1123 	if (request->release_fence != NULL) {  in rga_request_submit()
1124 		rga_dma_fence_put(request->release_fence);  in rga_request_submit()
1125 		request->release_fence = NULL;  in rga_request_submit()
1129 	spin_lock_irqsave(&request->lock, flags);  in rga_request_submit()
1131 	rga_request_put_current_mm(request);  in rga_request_submit()
1132 	request->is_running = false;  in rga_request_submit()
1134 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_submit()
1139 int rga_request_mpi_submit(struct rga_req *req, struct rga_request *request)  in rga_request_mpi_submit()  argument
1145 	if (request->sync_mode == RGA_BLIT_ASYNC) {  in rga_request_mpi_submit()
1150 	spin_lock_irqsave(&request->lock, flags);  in rga_request_mpi_submit()
1152 	if (request->is_running) {  in rga_request_mpi_submit()
1153 		pr_err("can not re-config when request is running");  in rga_request_mpi_submit()
1154 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_mpi_submit()
1158 	if (request->task_list == NULL) {  in rga_request_mpi_submit()
1159 		pr_err("can not find task list from id[%d]", request->id);  in rga_request_mpi_submit()
1160 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_mpi_submit()
1165 	request->is_running = true;  in rga_request_mpi_submit()
1166 	request->is_done = false;  in rga_request_mpi_submit()
1167 	request->finished_task_count = 0;  in rga_request_mpi_submit()
1168 	request->failed_task_count = 0;  in rga_request_mpi_submit()
1170 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_mpi_submit()
1172 	job = rga_job_commit(req, request);  in rga_request_mpi_submit()
1178 	ret = rga_request_wait(request);  in rga_request_mpi_submit()
1185 int rga_request_free(struct rga_request *request)  in rga_request_free()  argument
1199 	if (IS_ERR_OR_NULL(request)) {  in rga_request_free()
1200 		pr_err("request already freed");  in rga_request_free()
1205 	idr_remove(&request_manager->request_idr, request->id);  in rga_request_free()
1207 	spin_lock_irqsave(&request->lock, flags);  in rga_request_free()
1209 	task_list = request->task_list;  in rga_request_free()
1211 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_free()
1216 	kfree(request);  in rga_request_free()
1223 	struct rga_request *request;  in rga_request_kref_release()  local
1226 	request = container_of(ref, struct rga_request, refcount);  in rga_request_kref_release()
1228 	if (rga_dma_fence_get_status(request->release_fence) == 0)  in rga_request_kref_release()
1229 		rga_dma_fence_signal(request->release_fence, -EFAULT);  in rga_request_kref_release()
1231 	spin_lock_irqsave(&request->lock, flags);  in rga_request_kref_release()
1233 	rga_request_put_current_mm(request);  in rga_request_kref_release()
1234 	rga_dma_fence_put(request->release_fence);  in rga_request_kref_release()
1236 	if (!request->is_running || request->is_done) {  in rga_request_kref_release()
1237 		spin_unlock_irqrestore(&request->lock, flags);  in rga_request_kref_release()
1241 	spin_unlock_irqrestore(&request->lock, flags);  in rga_request_kref_release()
1243 	rga_request_scheduler_job_abort(request);  in rga_request_kref_release()
1246 	rga_request_free(request);  in rga_request_kref_release()
1250  * Called at driver close to release the request's id references.
1261 	struct rga_request *request;  in rga_request_alloc()  local
1269 	request = kzalloc(sizeof(*request), GFP_KERNEL);  in rga_request_alloc()
1270 	if (request == NULL) {  in rga_request_alloc()
1275 	spin_lock_init(&request->lock);  in rga_request_alloc()
1276 	init_waitqueue_head(&request->finished_wq);  in rga_request_alloc()
1278 	request->pid = current->pid;  in rga_request_alloc()
1279 	request->flags = flags;  in rga_request_alloc()
1280 	request->session = session;  in rga_request_alloc()
1281 	kref_init(&request->refcount);  in rga_request_alloc()
1290 	new_id = idr_alloc_cyclic(&request_manager->request_idr, request, 1, 0, GFP_NOWAIT);  in rga_request_alloc()
1293 		pr_err("request alloc id failed!\n");  in rga_request_alloc()
1296 		kfree(request);  in rga_request_alloc()
1300 	request->id = new_id;  in rga_request_alloc()
1305 	return request->id;  in rga_request_alloc()
1308 int rga_request_put(struct rga_request *request)  in rga_request_put()  argument
1310 	return kref_put(&request->refcount, rga_request_kref_release);  in rga_request_put()
1313 void rga_request_get(struct rga_request *request)  in rga_request_get()  argument
1315 	kref_get(&request->refcount);  in rga_request_get()