45 static void			 rpc_release_task(struct rpc_task *task);
61 rpc_task_timeout(const struct rpc_task *task)  in rpc_task_timeout()  argument
63 	unsigned long timeout = READ_ONCE(task->tk_timeout);  in rpc_task_timeout()
75  * Disable the timer for a given RPC task. Should be called with
80 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_disable_timer()  argument
82 	if (list_empty(&task->u.tk_wait.timer_list))  in __rpc_disable_timer()
84 	task->tk_timeout = 0;  in __rpc_disable_timer()
85 	list_del(&task->u.tk_wait.timer_list);  in __rpc_disable_timer()
103  * Set up a timer for the current task.
106 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task,  in __rpc_add_timer()  argument
109 	task->tk_timeout = timeout;  in __rpc_add_timer()
112 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);  in __rpc_add_timer()
132 __rpc_list_enqueue_task(struct list_head *q, struct rpc_task *task)  in __rpc_list_enqueue_task()  argument
137 		if (t->tk_owner == task->tk_owner) {  in __rpc_list_enqueue_task()
138 			list_add_tail(&task->u.tk_wait.links,  in __rpc_list_enqueue_task()
140 			/* Cache the queue head in task->u.tk_wait.list */  in __rpc_list_enqueue_task()
141 			task->u.tk_wait.list.next = q;  in __rpc_list_enqueue_task()
142 			task->u.tk_wait.list.prev = NULL;  in __rpc_list_enqueue_task()
146 	INIT_LIST_HEAD(&task->u.tk_wait.links);  in __rpc_list_enqueue_task()
147 	list_add_tail(&task->u.tk_wait.list, q);  in __rpc_list_enqueue_task()
154 __rpc_list_dequeue_task(struct rpc_task *task)  in __rpc_list_dequeue_task()  argument
159 	if (task->u.tk_wait.list.prev == NULL) {  in __rpc_list_dequeue_task()
160 		list_del(&task->u.tk_wait.links);  in __rpc_list_dequeue_task()
163 	if (!list_empty(&task->u.tk_wait.links)) {  in __rpc_list_dequeue_task()
164 		t = list_first_entry(&task->u.tk_wait.links,  in __rpc_list_dequeue_task()
170 		list_del(&task->u.tk_wait.links);  in __rpc_list_dequeue_task()
172 	list_del(&task->u.tk_wait.list);  in __rpc_list_dequeue_task()
179 		struct rpc_task *task,  in __rpc_add_wait_queue_priority()  argument
184 	__rpc_list_enqueue_task(&queue->tasks[queue_priority], task);  in __rpc_add_wait_queue_priority()
191 		struct rpc_task *task,  in __rpc_add_wait_queue()  argument
194 	INIT_LIST_HEAD(&task->u.tk_wait.timer_list);  in __rpc_add_wait_queue()
196 		__rpc_add_wait_queue_priority(queue, task, queue_priority);  in __rpc_add_wait_queue()
198 		list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);  in __rpc_add_wait_queue()
199 	task->tk_waitqueue = queue;  in __rpc_add_wait_queue()
203 	rpc_set_queued(task);  in __rpc_add_wait_queue()
209 static void __rpc_remove_wait_queue_priority(struct rpc_task *task)  in __rpc_remove_wait_queue_priority()  argument
211 	__rpc_list_dequeue_task(task);  in __rpc_remove_wait_queue_priority()
218 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)  in __rpc_remove_wait_queue()  argument
220 	__rpc_disable_timer(queue, task);  in __rpc_remove_wait_queue()
222 		__rpc_remove_wait_queue_priority(task);  in __rpc_remove_wait_queue()
224 		list_del(&task->u.tk_wait.list);  in __rpc_remove_wait_queue()
271 static void rpc_task_set_debuginfo(struct rpc_task *task)  in rpc_task_set_debuginfo()  argument
275 	task->tk_pid = atomic_inc_return(&rpc_pid);  in rpc_task_set_debuginfo()
278 static inline void rpc_task_set_debuginfo(struct rpc_task *task)  in rpc_task_set_debuginfo()  argument
283 static void rpc_set_active(struct rpc_task *task)  in rpc_set_active()  argument
285 	rpc_task_set_debuginfo(task);  in rpc_set_active()
286 	set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);  in rpc_set_active()
287 	trace_rpc_task_begin(task, NULL);  in rpc_set_active()
294 static int rpc_complete_task(struct rpc_task *task)  in rpc_complete_task()  argument
296 	void *m = &task->tk_runstate;  in rpc_complete_task()
302 	trace_rpc_task_complete(task, NULL);  in rpc_complete_task()
305 	clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);  in rpc_complete_task()
306 	ret = atomic_dec_and_test(&task->tk_count);  in rpc_complete_task()
320 int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action)  in __rpc_wait_for_completion_task()  argument
324 	return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,  in __rpc_wait_for_completion_task()
330  * Make an RPC task runnable.
332  * Note: If the task is ASYNC, and is being made runnable after sitting on an
341 		struct rpc_task *task)  in rpc_make_runnable()  argument
343 	bool need_wakeup = !rpc_test_and_set_running(task);  in rpc_make_runnable()
345 	rpc_clear_queued(task);  in rpc_make_runnable()
348 	if (RPC_IS_ASYNC(task)) {  in rpc_make_runnable()
349 		INIT_WORK(&task->u.tk_work, rpc_async_schedule);  in rpc_make_runnable()
350 		queue_work(wq, &task->u.tk_work);  in rpc_make_runnable()
352 		wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);  in rpc_make_runnable()
358  * NB: An RPC task will only receive interrupt-driven events as long
362 		struct rpc_task *task,  in __rpc_do_sleep_on_priority()  argument
365 	trace_rpc_task_sleep(task, q);  in __rpc_do_sleep_on_priority()
367 	__rpc_add_wait_queue(q, task, queue_priority);  in __rpc_do_sleep_on_priority()
371 		struct rpc_task *task,  in __rpc_sleep_on_priority()  argument
374 	if (WARN_ON_ONCE(RPC_IS_QUEUED(task)))  in __rpc_sleep_on_priority()
376 	__rpc_do_sleep_on_priority(q, task, queue_priority);  in __rpc_sleep_on_priority()
380 		struct rpc_task *task, unsigned long timeout,  in __rpc_sleep_on_priority_timeout()  argument
383 	if (WARN_ON_ONCE(RPC_IS_QUEUED(task)))  in __rpc_sleep_on_priority_timeout()
386 		__rpc_do_sleep_on_priority(q, task, queue_priority);  in __rpc_sleep_on_priority_timeout()
387 		__rpc_add_timer(q, task, timeout);  in __rpc_sleep_on_priority_timeout()
389 		task->tk_status = -ETIMEDOUT;  in __rpc_sleep_on_priority_timeout()
392 static void rpc_set_tk_callback(struct rpc_task *task, rpc_action action)  in rpc_set_tk_callback()  argument
394 	if (action && !WARN_ON_ONCE(task->tk_callback != NULL))  in rpc_set_tk_callback()
395 		task->tk_callback = action;  in rpc_set_tk_callback()
398 static bool rpc_sleep_check_activated(struct rpc_task *task)  in rpc_sleep_check_activated()  argument
400 	/* We shouldn't ever put an inactive task to sleep */  in rpc_sleep_check_activated()
401 	if (WARN_ON_ONCE(!RPC_IS_ACTIVATED(task))) {  in rpc_sleep_check_activated()
402 		task->tk_status = -EIO;  in rpc_sleep_check_activated()
403 		rpc_put_task_async(task);  in rpc_sleep_check_activated()
409 void rpc_sleep_on_timeout(struct rpc_wait_queue *q, struct rpc_task *task,  in rpc_sleep_on_timeout()  argument
412 	if (!rpc_sleep_check_activated(task))  in rpc_sleep_on_timeout()
415 	rpc_set_tk_callback(task, action);  in rpc_sleep_on_timeout()
421 	__rpc_sleep_on_priority_timeout(q, task, timeout, task->tk_priority);  in rpc_sleep_on_timeout()
426 void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,  in rpc_sleep_on()  argument
429 	if (!rpc_sleep_check_activated(task))  in rpc_sleep_on()
432 	rpc_set_tk_callback(task, action);  in rpc_sleep_on()
434 	WARN_ON_ONCE(task->tk_timeout != 0);  in rpc_sleep_on()
439 	__rpc_sleep_on_priority(q, task, task->tk_priority);  in rpc_sleep_on()
445 		struct rpc_task *task, unsigned long timeout, int priority)  in rpc_sleep_on_priority_timeout()  argument
447 	if (!rpc_sleep_check_activated(task))  in rpc_sleep_on_priority_timeout()
455 	__rpc_sleep_on_priority_timeout(q, task, timeout, priority);  in rpc_sleep_on_priority_timeout()
460 void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task,  in rpc_sleep_on_priority()  argument
463 	if (!rpc_sleep_check_activated(task))  in rpc_sleep_on_priority()
466 	WARN_ON_ONCE(task->tk_timeout != 0);  in rpc_sleep_on_priority()
472 	__rpc_sleep_on_priority(q, task, priority);  in rpc_sleep_on_priority()
479  * @wq: workqueue on which to run task
481  * @task: task to be woken up
483  * Caller must hold queue->lock, and have cleared the task queued flag.
487 		struct rpc_task *task)  in __rpc_do_wake_up_task_on_wq()  argument
489 	/* Has the task been executed yet? If not, we cannot wake it up! */  in __rpc_do_wake_up_task_on_wq()
490 	if (!RPC_IS_ACTIVATED(task)) {  in __rpc_do_wake_up_task_on_wq()
491 		printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);  in __rpc_do_wake_up_task_on_wq()
495 	trace_rpc_task_wakeup(task, queue);  in __rpc_do_wake_up_task_on_wq()
497 	__rpc_remove_wait_queue(queue, task);  in __rpc_do_wake_up_task_on_wq()
499 	rpc_make_runnable(wq, task);  in __rpc_do_wake_up_task_on_wq()
503  * Wake up a queued task while the queue lock is being held
507 		struct rpc_wait_queue *queue, struct rpc_task *task,  in rpc_wake_up_task_on_wq_queue_action_locked()  argument
510 	if (RPC_IS_QUEUED(task)) {  in rpc_wake_up_task_on_wq_queue_action_locked()
512 		if (task->tk_waitqueue == queue) {  in rpc_wake_up_task_on_wq_queue_action_locked()
513 			if (action == NULL || action(task, data)) {  in rpc_wake_up_task_on_wq_queue_action_locked()
514 				__rpc_do_wake_up_task_on_wq(wq, queue, task);  in rpc_wake_up_task_on_wq_queue_action_locked()
515 				return task;  in rpc_wake_up_task_on_wq_queue_action_locked()
523  * Wake up a queued task while the queue lock is being held
526 					  struct rpc_task *task)  in rpc_wake_up_task_queue_locked()  argument
529 						   task, NULL, NULL);  in rpc_wake_up_task_queue_locked()
533  * Wake up a task on a specific queue
535 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)  in rpc_wake_up_queued_task()  argument
537 	if (!RPC_IS_QUEUED(task))  in rpc_wake_up_queued_task()
540 	rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_queued_task()
545 static bool rpc_task_action_set_status(struct rpc_task *task, void *status)  in rpc_task_action_set_status()  argument
547 	task->tk_status = *(int *)status;  in rpc_task_action_set_status()
553 		struct rpc_task *task, int status)  in rpc_wake_up_task_queue_set_status_locked()  argument
556 			task, rpc_task_action_set_status, &status);  in rpc_wake_up_task_queue_set_status_locked()
560  * rpc_wake_up_queued_task_set_status - wake up a task and set task->tk_status
562  * @task: pointer to rpc_task
565  * If @task is queued on @queue, then it is woken up, and @task->tk_status is
570 		struct rpc_task *task, int status)  in rpc_wake_up_queued_task_set_status()  argument
572 	if (!RPC_IS_QUEUED(task))  in rpc_wake_up_queued_task_set_status()
575 	rpc_wake_up_task_queue_set_status_locked(queue, task, status);  in rpc_wake_up_queued_task_set_status()
580  * Wake up the next task on a priority queue.
585 	struct rpc_task *task;  in __rpc_find_next_queued_priority()  local
592 		task = list_first_entry(q, struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued_priority()
602 		task = list_first_entry(q, struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued_priority()
615 			task = list_first_entry(q, struct rpc_task, u.tk_wait.list);  in __rpc_find_next_queued_priority()
626 	return task;  in __rpc_find_next_queued_priority()
639  * Wake up the first task on the wait queue.
645 	struct rpc_task	*task = NULL;  in rpc_wake_up_first_on_wq()  local
648 	task = __rpc_find_next_queued(queue);  in rpc_wake_up_first_on_wq()
649 	if (task != NULL)  in rpc_wake_up_first_on_wq()
650 		task = rpc_wake_up_task_on_wq_queue_action_locked(wq, queue,  in rpc_wake_up_first_on_wq()
651 				task, func, data);  in rpc_wake_up_first_on_wq()
654 	return task;  in rpc_wake_up_first_on_wq()
658  * Wake up the first task on the wait queue.
667 static bool rpc_wake_up_next_func(struct rpc_task *task, void *data)  in rpc_wake_up_next_func()  argument
673  * Wake up the next task on the wait queue.
688 	struct rpc_task *task;  in rpc_wake_up_locked()  local
691 		task = __rpc_find_next_queued(queue);  in rpc_wake_up_locked()
692 		if (task == NULL)  in rpc_wake_up_locked()
694 		rpc_wake_up_task_queue_locked(queue, task);  in rpc_wake_up_locked()
719 	struct rpc_task *task;  in rpc_wake_up_status_locked()  local
722 		task = __rpc_find_next_queued(queue);  in rpc_wake_up_status_locked()
723 		if (task == NULL)  in rpc_wake_up_status_locked()
725 		rpc_wake_up_task_queue_set_status_locked(queue, task, status);  in rpc_wake_up_status_locked()
749 	struct rpc_task *task, *n;  in __rpc_queue_timer_fn()  local
754 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {  in __rpc_queue_timer_fn()
755 		timeo = task->tk_timeout;  in __rpc_queue_timer_fn()
757 			trace_rpc_task_timeout(task, task->tk_action);  in __rpc_queue_timer_fn()
758 			task->tk_status = -ETIMEDOUT;  in __rpc_queue_timer_fn()
759 			rpc_wake_up_task_queue_locked(queue, task);  in __rpc_queue_timer_fn()
770 static void __rpc_atrun(struct rpc_task *task)  in __rpc_atrun()  argument
772 	if (task->tk_status == -ETIMEDOUT)  in __rpc_atrun()
773 		task->tk_status = 0;  in __rpc_atrun()
777  * Run a task at a later time
779 void rpc_delay(struct rpc_task *task, unsigned long delay)  in rpc_delay()  argument
781 	rpc_sleep_on_timeout(&delay_queue, task, __rpc_atrun, jiffies + delay);  in rpc_delay()
786  * Helper to call task->tk_ops->rpc_call_prepare
788 void rpc_prepare_task(struct rpc_task *task)  in rpc_prepare_task()  argument
790 	task->tk_ops->rpc_call_prepare(task, task->tk_calldata);  in rpc_prepare_task()
794 rpc_init_task_statistics(struct rpc_task *task)  in rpc_init_task_statistics()  argument
797 	task->tk_garb_retry = 2;  in rpc_init_task_statistics()
798 	task->tk_cred_retry = 2;  in rpc_init_task_statistics()
799 	task->tk_rebind_retry = 2;  in rpc_init_task_statistics()
802 	task->tk_start = ktime_get();  in rpc_init_task_statistics()
806 rpc_reset_task_statistics(struct rpc_task *task)  in rpc_reset_task_statistics()  argument
808 	task->tk_timeouts = 0;  in rpc_reset_task_statistics()
809 	task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_SENT);  in rpc_reset_task_statistics()
810 	rpc_init_task_statistics(task);  in rpc_reset_task_statistics()
814  * Helper that calls task->tk_ops->rpc_call_done if it exists
816 void rpc_exit_task(struct rpc_task *task)  in rpc_exit_task()  argument
818 	trace_rpc_task_end(task, task->tk_action);  in rpc_exit_task()
819 	task->tk_action = NULL;  in rpc_exit_task()
820 	if (task->tk_ops->rpc_count_stats)  in rpc_exit_task()
821 		task->tk_ops->rpc_count_stats(task, task->tk_calldata);  in rpc_exit_task()
822 	else if (task->tk_client)  in rpc_exit_task()
823 		rpc_count_iostats(task, task->tk_client->cl_metrics);  in rpc_exit_task()
824 	if (task->tk_ops->rpc_call_done != NULL) {  in rpc_exit_task()
825 		task->tk_ops->rpc_call_done(task, task->tk_calldata);  in rpc_exit_task()
826 		if (task->tk_action != NULL) {  in rpc_exit_task()
828 			xprt_release(task);  in rpc_exit_task()
829 			rpc_reset_task_statistics(task);  in rpc_exit_task()
834 void rpc_signal_task(struct rpc_task *task)  in rpc_signal_task()  argument
838 	if (!RPC_IS_ACTIVATED(task))  in rpc_signal_task()
841 	trace_rpc_task_signalled(task, task->tk_action);  in rpc_signal_task()
842 	set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate);  in rpc_signal_task()
844 	queue = READ_ONCE(task->tk_waitqueue);  in rpc_signal_task()
846 		rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS);  in rpc_signal_task()
849 void rpc_exit(struct rpc_task *task, int status)  in rpc_exit()  argument
851 	task->tk_status = status;  in rpc_exit()
852 	task->tk_action = rpc_exit_task;  in rpc_exit()
853 	rpc_wake_up_queued_task(task->tk_waitqueue, task);  in rpc_exit()
866 static void __rpc_execute(struct rpc_task *task)  in __rpc_execute()  argument
869 	int task_is_async = RPC_IS_ASYNC(task);  in __rpc_execute()
872 	WARN_ON_ONCE(RPC_IS_QUEUED(task));  in __rpc_execute()
873 	if (RPC_IS_QUEUED(task))  in __rpc_execute()
882 		 * tk_action may be NULL if the task has been killed.  in __rpc_execute()
886 		do_action = task->tk_action;  in __rpc_execute()
887 		if (task->tk_callback) {  in __rpc_execute()
888 			do_action = task->tk_callback;  in __rpc_execute()
889 			task->tk_callback = NULL;  in __rpc_execute()
893 		trace_rpc_task_run_action(task, do_action);  in __rpc_execute()
894 		do_action(task);  in __rpc_execute()
897 		 * Lockless check for whether task is sleeping or not.  in __rpc_execute()
899 		if (!RPC_IS_QUEUED(task))  in __rpc_execute()
905 		if (RPC_SIGNALLED(task)) {  in __rpc_execute()
906 			task->tk_rpc_status = -ERESTARTSYS;  in __rpc_execute()
907 			rpc_exit(task, -ERESTARTSYS);  in __rpc_execute()
919 		queue = task->tk_waitqueue;  in __rpc_execute()
921 		if (!RPC_IS_QUEUED(task)) {  in __rpc_execute()
925 		rpc_clear_running(task);  in __rpc_execute()
930 		/* sync task: sleep here */  in __rpc_execute()
931 		trace_rpc_task_sync_sleep(task, task->tk_action);  in __rpc_execute()
932 		status = out_of_line_wait_on_bit(&task->tk_runstate,  in __rpc_execute()
937 			 * When a sync task receives a signal, it exits with  in __rpc_execute()
942 			trace_rpc_task_signalled(task, task->tk_action);  in __rpc_execute()
943 			set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate);  in __rpc_execute()
944 			task->tk_rpc_status = -ERESTARTSYS;  in __rpc_execute()
945 			rpc_exit(task, -ERESTARTSYS);  in __rpc_execute()
947 		trace_rpc_task_sync_wake(task, task->tk_action);  in __rpc_execute()
950 	/* Release all resources associated with the task */  in __rpc_execute()
951 	rpc_release_task(task);  in __rpc_execute()
957  * This may be called recursively if e.g. an async NFS task updates
959  * NOTE: Upon exit of this function the task is guaranteed to be
961  *	 been called, so your task memory may have been freed.
963 void rpc_execute(struct rpc_task *task)  in rpc_execute()  argument
965 	bool is_async = RPC_IS_ASYNC(task);  in rpc_execute()
967 	rpc_set_active(task);  in rpc_execute()
968 	rpc_make_runnable(rpciod_workqueue, task);  in rpc_execute()
971 		__rpc_execute(task);  in rpc_execute()
986  * @task: RPC task
989  * RPC call and RPC reply that this task is being used for. When
1001 int rpc_malloc(struct rpc_task *task)  in rpc_malloc()  argument
1003 	struct rpc_rqst *rqst = task->tk_rqstp;  in rpc_malloc()
1008 	if (RPC_IS_ASYNC(task))  in rpc_malloc()
1010 	if (RPC_IS_SWAPPER(task))  in rpc_malloc()
1031  * @task: RPC task
1034 void rpc_free(struct rpc_task *task)  in rpc_free()  argument
1036 	void *buffer = task->tk_rqstp->rq_buffer;  in rpc_free()
1051  * Creation and deletion of RPC task structures
1053 static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)  in rpc_init_task()  argument
1055 	memset(task, 0, sizeof(*task));  in rpc_init_task()
1056 	atomic_set(&task->tk_count, 1);  in rpc_init_task()
1057 	task->tk_flags  = task_setup_data->flags;  in rpc_init_task()
1058 	task->tk_ops = task_setup_data->callback_ops;  in rpc_init_task()
1059 	task->tk_calldata = task_setup_data->callback_data;  in rpc_init_task()
1060 	INIT_LIST_HEAD(&task->tk_task);  in rpc_init_task()
1062 	task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;  in rpc_init_task()
1063 	task->tk_owner = current->tgid;  in rpc_init_task()
1066 	task->tk_workqueue = task_setup_data->workqueue;  in rpc_init_task()
1068 	task->tk_xprt = rpc_task_get_xprt(task_setup_data->rpc_client,  in rpc_init_task()
1071 	task->tk_op_cred = get_rpccred(task_setup_data->rpc_op_cred);  in rpc_init_task()
1073 	if (task->tk_ops->rpc_call_prepare != NULL)  in rpc_init_task()
1074 		task->tk_action = rpc_prepare_task;  in rpc_init_task()
1076 	rpc_init_task_statistics(task);  in rpc_init_task()
1086  * Create a new task for the specified client.
1090 	struct rpc_task	*task = setup_data->task;  in rpc_new_task()  local
1093 	if (task == NULL) {  in rpc_new_task()
1094 		task = rpc_alloc_task();  in rpc_new_task()
1098 	rpc_init_task(task, setup_data);  in rpc_new_task()
1099 	task->tk_flags |= flags;  in rpc_new_task()
1100 	return task;  in rpc_new_task()
1104  * rpc_free_task - release rpc task and perform cleanups
1122 static void rpc_free_task(struct rpc_task *task)  in rpc_free_task()  argument
1124 	unsigned short tk_flags = task->tk_flags;  in rpc_free_task()
1126 	put_rpccred(task->tk_op_cred);  in rpc_free_task()
1127 	rpc_release_calldata(task->tk_ops, task->tk_calldata);  in rpc_free_task()
1130 		mempool_free(task, rpc_task_mempool);  in rpc_free_task()
1141 static void rpc_release_resources_task(struct rpc_task *task)  in rpc_release_resources_task()  argument
1143 	xprt_release(task);  in rpc_release_resources_task()
1144 	if (task->tk_msg.rpc_cred) {  in rpc_release_resources_task()
1145 		if (!(task->tk_flags & RPC_TASK_CRED_NOREF))  in rpc_release_resources_task()
1146 			put_cred(task->tk_msg.rpc_cred);  in rpc_release_resources_task()
1147 		task->tk_msg.rpc_cred = NULL;  in rpc_release_resources_task()
1149 	rpc_task_release_client(task);  in rpc_release_resources_task()
1152 static void rpc_final_put_task(struct rpc_task *task,  in rpc_final_put_task()  argument
1156 		INIT_WORK(&task->u.tk_work, rpc_async_release);  in rpc_final_put_task()
1157 		queue_work(q, &task->u.tk_work);  in rpc_final_put_task()
1159 		rpc_free_task(task);  in rpc_final_put_task()
1162 static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q)  in rpc_do_put_task()  argument
1164 	if (atomic_dec_and_test(&task->tk_count)) {  in rpc_do_put_task()
1165 		rpc_release_resources_task(task);  in rpc_do_put_task()
1166 		rpc_final_put_task(task, q);  in rpc_do_put_task()
1170 void rpc_put_task(struct rpc_task *task)  in rpc_put_task()  argument
1172 	rpc_do_put_task(task, NULL);  in rpc_put_task()
1176 void rpc_put_task_async(struct rpc_task *task)  in rpc_put_task_async()  argument
1178 	rpc_do_put_task(task, task->tk_workqueue);  in rpc_put_task_async()
1182 static void rpc_release_task(struct rpc_task *task)  in rpc_release_task()  argument
1184 	WARN_ON_ONCE(RPC_IS_QUEUED(task));  in rpc_release_task()
1186 	rpc_release_resources_task(task);  in rpc_release_task()
1190 	 * so it should be safe to use task->tk_count as a test for whether  in rpc_release_task()
1193 	if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) {  in rpc_release_task()
1194 		/* Wake up anyone who may be waiting for task completion */  in rpc_release_task()
1195 		if (!rpc_complete_task(task))  in rpc_release_task()
1198 		if (!atomic_dec_and_test(&task->tk_count))  in rpc_release_task()
1201 	rpc_final_put_task(task, task->tk_workqueue);  in rpc_release_task()