78 	struct virtio_fs_vq *fsvq;  member
82 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
136 static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq)  in inc_in_flight_req()  argument
138 	fsvq->in_flight++;  in inc_in_flight_req()
142 static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq)  in dec_in_flight_req()  argument
144 	WARN_ON(fsvq->in_flight <= 0);  in dec_in_flight_req()
145 	fsvq->in_flight--;  in dec_in_flight_req()
146 	if (!fsvq->in_flight)  in dec_in_flight_req()
147 		complete(&fsvq->in_flight_zero);  in dec_in_flight_req()
173 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)  in virtio_fs_drain_queue()  argument
175 	WARN_ON(fsvq->in_flight < 0);  in virtio_fs_drain_queue()
178 	spin_lock(&fsvq->lock);  in virtio_fs_drain_queue()
179 	if (fsvq->in_flight) {  in virtio_fs_drain_queue()
183 		reinit_completion(&fsvq->in_flight_zero);  in virtio_fs_drain_queue()
184 		spin_unlock(&fsvq->lock);  in virtio_fs_drain_queue()
185 		wait_for_completion(&fsvq->in_flight_zero);  in virtio_fs_drain_queue()
187 		spin_unlock(&fsvq->lock);  in virtio_fs_drain_queue()
190 	flush_work(&fsvq->done_work);  in virtio_fs_drain_queue()
191 	flush_delayed_work(&fsvq->dispatch_work);  in virtio_fs_drain_queue()
196 	struct virtio_fs_vq *fsvq;  in virtio_fs_drain_all_queues_locked()  local
200 		fsvq = &fs->vqs[i];  in virtio_fs_drain_all_queues_locked()
201 		virtio_fs_drain_queue(fsvq);  in virtio_fs_drain_all_queues_locked()
220 	struct virtio_fs_vq *fsvq;  in virtio_fs_start_all_queues()  local
224 		fsvq = &fs->vqs[i];  in virtio_fs_start_all_queues()
225 		spin_lock(&fsvq->lock);  in virtio_fs_start_all_queues()
226 		fsvq->connected = true;  in virtio_fs_start_all_queues()
227 		spin_unlock(&fsvq->lock);  in virtio_fs_start_all_queues()
281 		struct virtio_fs_vq *fsvq = &fs->vqs[i];  in virtio_fs_free_devs()  local
283 		if (!fsvq->fud)  in virtio_fs_free_devs()
286 		fuse_dev_free(fsvq->fud);  in virtio_fs_free_devs()
287 		fsvq->fud = NULL;  in virtio_fs_free_devs()
318 	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,  in virtio_fs_hiprio_done_work()  local
320 	struct virtqueue *vq = fsvq->vq;  in virtio_fs_hiprio_done_work()
323 	spin_lock(&fsvq->lock);  in virtio_fs_hiprio_done_work()
332 			dec_in_flight_req(fsvq);  in virtio_fs_hiprio_done_work()
335 	spin_unlock(&fsvq->lock);  in virtio_fs_hiprio_done_work()
341 	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,  in virtio_fs_request_dispatch_work()  local
347 		spin_lock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
348 		req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,  in virtio_fs_request_dispatch_work()
351 			spin_unlock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
356 		spin_unlock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
362 		spin_lock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
363 		req = list_first_entry_or_null(&fsvq->queued_reqs,  in virtio_fs_request_dispatch_work()
366 			spin_unlock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
370 		spin_unlock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
372 		ret = virtio_fs_enqueue_req(fsvq, req, true);  in virtio_fs_request_dispatch_work()
375 				spin_lock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
376 				list_add_tail(&req->list, &fsvq->queued_reqs);  in virtio_fs_request_dispatch_work()
377 				schedule_delayed_work(&fsvq->dispatch_work,  in virtio_fs_request_dispatch_work()
379 				spin_unlock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
383 			spin_lock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
384 			dec_in_flight_req(fsvq);  in virtio_fs_request_dispatch_work()
385 			spin_unlock(&fsvq->lock);  in virtio_fs_request_dispatch_work()
397 static int send_forget_request(struct virtio_fs_vq *fsvq,  in send_forget_request()  argument
407 	spin_lock(&fsvq->lock);  in send_forget_request()
408 	if (!fsvq->connected) {  in send_forget_request()
410 			dec_in_flight_req(fsvq);  in send_forget_request()
416 	vq = fsvq->vq;  in send_forget_request()
424 			list_add_tail(&forget->list, &fsvq->queued_reqs);  in send_forget_request()
425 			schedule_delayed_work(&fsvq->dispatch_work,  in send_forget_request()
428 				inc_in_flight_req(fsvq);  in send_forget_request()
436 				dec_in_flight_req(fsvq);  in send_forget_request()
442 		inc_in_flight_req(fsvq);  in send_forget_request()
444 	spin_unlock(&fsvq->lock);  in send_forget_request()
450 	spin_unlock(&fsvq->lock);  in send_forget_request()
457 	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,  in virtio_fs_hiprio_dispatch_work()  local
461 		spin_lock(&fsvq->lock);  in virtio_fs_hiprio_dispatch_work()
462 		forget = list_first_entry_or_null(&fsvq->queued_reqs,  in virtio_fs_hiprio_dispatch_work()
465 			spin_unlock(&fsvq->lock);  in virtio_fs_hiprio_dispatch_work()
470 		spin_unlock(&fsvq->lock);  in virtio_fs_hiprio_dispatch_work()
471 		if (send_forget_request(fsvq, forget, true))  in virtio_fs_hiprio_dispatch_work()
545 				       struct virtio_fs_vq *fsvq)  in virtio_fs_request_complete()  argument
547 	struct fuse_pqueue *fpq = &fsvq->fud->pq;  in virtio_fs_request_complete()
581 	spin_lock(&fsvq->lock);  in virtio_fs_request_complete()
582 	dec_in_flight_req(fsvq);  in virtio_fs_request_complete()
583 	spin_unlock(&fsvq->lock);  in virtio_fs_request_complete()
591 	virtio_fs_request_complete(w->req, w->fsvq);  in virtio_fs_complete_req_work()
597 	struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,  in virtio_fs_requests_done_work()  local
599 	struct fuse_pqueue *fpq = &fsvq->fud->pq;  in virtio_fs_requests_done_work()
600 	struct virtqueue *vq = fsvq->vq;  in virtio_fs_requests_done_work()
607 	spin_lock(&fsvq->lock);  in virtio_fs_requests_done_work()
617 	spin_unlock(&fsvq->lock);  in virtio_fs_requests_done_work()
629 			w->fsvq = fsvq;  in virtio_fs_requests_done_work()
633 			virtio_fs_request_complete(req, fsvq);  in virtio_fs_requests_done_work()
641 	struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);  in virtio_fs_vq_done()  local
643 	dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);  in virtio_fs_vq_done()
645 	schedule_work(&fsvq->done_work);  in virtio_fs_vq_done()
648 static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name,  in virtio_fs_init_vq()  argument
651 	strncpy(fsvq->name, name, VQ_NAME_LEN);  in virtio_fs_init_vq()
652 	spin_lock_init(&fsvq->lock);  in virtio_fs_init_vq()
653 	INIT_LIST_HEAD(&fsvq->queued_reqs);  in virtio_fs_init_vq()
654 	INIT_LIST_HEAD(&fsvq->end_reqs);  in virtio_fs_init_vq()
655 	init_completion(&fsvq->in_flight_zero);  in virtio_fs_init_vq()
658 		INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work);  in virtio_fs_init_vq()
659 		INIT_DELAYED_WORK(&fsvq->dispatch_work,  in virtio_fs_init_vq()
662 		INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work);  in virtio_fs_init_vq()
663 		INIT_DELAYED_WORK(&fsvq->dispatch_work,  in virtio_fs_init_vq()
909 	struct virtio_fs_vq *fsvq;  in virtio_fs_stop_all_queues()  local
913 		fsvq = &fs->vqs[i];  in virtio_fs_stop_all_queues()
914 		spin_lock(&fsvq->lock);  in virtio_fs_stop_all_queues()
915 		fsvq->connected = false;  in virtio_fs_stop_all_queues()
916 		spin_unlock(&fsvq->lock);  in virtio_fs_stop_all_queues()
981 	struct virtio_fs_vq *fsvq;  in virtio_fs_wake_forget_and_unlock()  local
988 	fsvq = &fs->vqs[VQ_HIPRIO];  in virtio_fs_wake_forget_and_unlock()
1005 	send_forget_request(fsvq, forget, false);  in virtio_fs_wake_forget_and_unlock()
1122 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,  in virtio_fs_enqueue_req()  argument
1179 	spin_lock(&fsvq->lock);  in virtio_fs_enqueue_req()
1181 	if (!fsvq->connected) {  in virtio_fs_enqueue_req()
1182 		spin_unlock(&fsvq->lock);  in virtio_fs_enqueue_req()
1187 	vq = fsvq->vq;  in virtio_fs_enqueue_req()
1190 		spin_unlock(&fsvq->lock);  in virtio_fs_enqueue_req()
1195 	fpq = &fsvq->fud->pq;  in virtio_fs_enqueue_req()
1204 		inc_in_flight_req(fsvq);  in virtio_fs_enqueue_req()
1207 	spin_unlock(&fsvq->lock);  in virtio_fs_enqueue_req()
1232 	struct virtio_fs_vq *fsvq;  in virtio_fs_wake_pending_and_unlock()  local
1249 	fsvq = &fs->vqs[queue_id];  in virtio_fs_wake_pending_and_unlock()
1250 	ret = virtio_fs_enqueue_req(fsvq, req, false);  in virtio_fs_wake_pending_and_unlock()
1257 			spin_lock(&fsvq->lock);  in virtio_fs_wake_pending_and_unlock()
1258 			list_add_tail(&req->list, &fsvq->queued_reqs);  in virtio_fs_wake_pending_and_unlock()
1259 			inc_in_flight_req(fsvq);  in virtio_fs_wake_pending_and_unlock()
1260 			schedule_delayed_work(&fsvq->dispatch_work,  in virtio_fs_wake_pending_and_unlock()
1262 			spin_unlock(&fsvq->lock);  in virtio_fs_wake_pending_and_unlock()
1269 		spin_lock(&fsvq->lock);  in virtio_fs_wake_pending_and_unlock()
1270 		list_add_tail(&req->list, &fsvq->end_reqs);  in virtio_fs_wake_pending_and_unlock()
1271 		schedule_delayed_work(&fsvq->dispatch_work, 0);  in virtio_fs_wake_pending_and_unlock()
1272 		spin_unlock(&fsvq->lock);  in virtio_fs_wake_pending_and_unlock()
1321 		struct virtio_fs_vq *fsvq = &fs->vqs[i];  in virtio_fs_fill_super()  local
1323 		fsvq->fud = fuse_dev_alloc();  in virtio_fs_fill_super()
1324 		if (!fsvq->fud)  in virtio_fs_fill_super()
1344 		struct virtio_fs_vq *fsvq = &fs->vqs[i];  in virtio_fs_fill_super()  local
1346 		fuse_dev_install(fsvq->fud, fc);  in virtio_fs_fill_super()
1366 	struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];  in virtio_fs_conn_destroy()  local
1375 	spin_lock(&fsvq->lock);  in virtio_fs_conn_destroy()
1376 	fsvq->connected = false;  in virtio_fs_conn_destroy()
1377 	spin_unlock(&fsvq->lock);  in virtio_fs_conn_destroy()