60 		struct work_struct work;  member
163 						   struct io_wq_work *work)  in io_work_get_acct()  argument
165 	return io_get_acct(wqe, !(work->flags & IO_WQ_WORK_UNBOUND));  in io_work_get_acct()
409 			     struct io_wq_work *work)  in __io_worker_busy()  argument
434 static inline unsigned int io_get_work_hash(struct io_wq_work *work)  in io_get_work_hash()  argument
436 	return work->flags >> IO_WQ_HASH_SHIFT;  in io_get_work_hash()
462 	struct io_wq_work *work, *tail;  in io_get_next_work()  local
469 		work = container_of(node, struct io_wq_work, list);  in io_get_next_work()
472 		if (!io_wq_is_hashed(work)) {  in io_get_next_work()
474 			return work;  in io_get_next_work()
477 		hash = io_get_work_hash(work);  in io_get_next_work()
485 			return work;  in io_get_next_work()
525 				   struct io_wq_work *work)  in io_assign_current_work()  argument
527 	if (work) {  in io_assign_current_work()
533 	worker->cur_work = work;  in io_assign_current_work()
537 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
548 		struct io_wq_work *work;  in io_worker_handle_work()  local
557 		work = io_get_next_work(acct, worker);  in io_worker_handle_work()
558 		if (work)  in io_worker_handle_work()
559 			__io_worker_busy(wqe, worker, work);  in io_worker_handle_work()
562 		if (!work)  in io_worker_handle_work()
564 		io_assign_current_work(worker, work);  in io_worker_handle_work()
570 			unsigned int hash = io_get_work_hash(work);  in io_worker_handle_work()
572 			next_hashed = wq_next_work(work);  in io_worker_handle_work()
574 			if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND))  in io_worker_handle_work()
575 				work->flags |= IO_WQ_WORK_CANCEL;  in io_worker_handle_work()
576 			wq->do_work(work);  in io_worker_handle_work()
579 			linked = wq->free_work(work);  in io_worker_handle_work()
580 			work = next_hashed;  in io_worker_handle_work()
581 			if (!work && linked && !io_wq_is_hashed(linked)) {  in io_worker_handle_work()
582 				work = linked;  in io_worker_handle_work()
585 			io_assign_current_work(worker, work);  in io_worker_handle_work()
599 				if (!work)  in io_worker_handle_work()
603 		} while (work);  in io_worker_handle_work()
720 static bool io_wq_work_match_all(struct io_wq_work *work, void *data)  in io_wq_work_match_all()  argument
782 	schedule_work(&worker->work);  in create_worker_cont()
785 static void io_workqueue_create(struct work_struct *work)  in io_workqueue_create()  argument
787 	struct io_worker *worker = container_of(work, struct io_worker, work);  in io_workqueue_create()
828 		INIT_WORK(&worker->work, io_workqueue_create);  in create_io_worker()
829 		schedule_work(&worker->work);  in create_io_worker()
867 static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)  in io_run_cancel()  argument
872 		work->flags |= IO_WQ_WORK_CANCEL;  in io_run_cancel()
873 		wq->do_work(work);  in io_run_cancel()
874 		work = wq->free_work(work);  in io_run_cancel()
875 	} while (work);  in io_run_cancel()
878 static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work)  in io_wqe_insert_work()  argument
880 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_insert_work()
884 	if (!io_wq_is_hashed(work)) {  in io_wqe_insert_work()
886 		wq_list_add_tail(&work->list, &acct->work_list);  in io_wqe_insert_work()
890 	hash = io_get_work_hash(work);  in io_wqe_insert_work()
892 	wqe->hash_tail[hash] = work;  in io_wqe_insert_work()
896 	wq_list_add_after(&work->list, &tail->list, &acct->work_list);  in io_wqe_insert_work()
899 static bool io_wq_work_match_item(struct io_wq_work *work, void *data)  in io_wq_work_match_item()  argument
901 	return work == data;  in io_wq_work_match_item()
904 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)  in io_wqe_enqueue()  argument
906 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_enqueue()
907 	unsigned work_flags = work->flags;  in io_wqe_enqueue()
915 	    (work->flags & IO_WQ_WORK_CANCEL)) {  in io_wqe_enqueue()
916 		io_run_cancel(work, wqe);  in io_wqe_enqueue()
921 	io_wqe_insert_work(wqe, work);  in io_wqe_enqueue()
943 				.data		= work,  in io_wqe_enqueue()
954 void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)  in io_wq_enqueue()  argument
958 	io_wqe_enqueue(wqe, work);  in io_wq_enqueue()
965 void io_wq_hash_work(struct io_wq_work *work, void *val)  in io_wq_hash_work()  argument
970 	work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));  in io_wq_hash_work()
993 					 struct io_wq_work *work,  in io_wqe_remove_pending()  argument
996 	struct io_wqe_acct *acct = io_work_get_acct(wqe, work);  in io_wqe_remove_pending()
997 	unsigned int hash = io_get_work_hash(work);  in io_wqe_remove_pending()
1000 	if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) {  in io_wqe_remove_pending()
1008 	wq_list_del(&acct->work_list, &work->list, prev);  in io_wqe_remove_pending()
1017 	struct io_wq_work *work;  in io_acct_cancel_pending_work()  local
1020 		work = container_of(node, struct io_wq_work, list);  in io_acct_cancel_pending_work()
1021 		if (!match->fn(work, match->data))  in io_acct_cancel_pending_work()
1023 		io_wqe_remove_pending(wqe, work, prev);  in io_acct_cancel_pending_work()
1025 		io_run_cancel(work, wqe);  in io_acct_cancel_pending_work()