69 	struct virtio_vsock *vsock;  in virtio_transport_get_local_cid()  local
73 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_get_local_cid()
74 	if (!vsock) {  in virtio_transport_get_local_cid()
79 	ret = vsock->guest_cid;  in virtio_transport_get_local_cid()
88 	struct virtio_vsock *vsock =  in virtio_transport_send_pkt_work()  local
94 	mutex_lock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
96 	if (!vsock->tx_run)  in virtio_transport_send_pkt_work()
99 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_pkt_work()
107 		spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
108 		if (list_empty(&vsock->send_pkt_list)) {  in virtio_transport_send_pkt_work()
109 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
113 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_transport_send_pkt_work()
116 		spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
134 			spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
135 			list_add(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt_work()
136 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
141 			struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_send_pkt_work()
144 			val = atomic_dec_return(&vsock->queued_replies);  in virtio_transport_send_pkt_work()
158 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
161 		queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_send_pkt_work()
167 	struct virtio_vsock *vsock;  in virtio_transport_send_pkt()  local
171 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_send_pkt()
172 	if (!vsock) {  in virtio_transport_send_pkt()
178 	if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid) {  in virtio_transport_send_pkt()
185 		atomic_inc(&vsock->queued_replies);  in virtio_transport_send_pkt()
187 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
188 	list_add_tail(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt()
189 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
191 	queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_send_pkt()
201 	struct virtio_vsock *vsock;  in virtio_transport_cancel_pkt()  local
207 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_cancel_pkt()
208 	if (!vsock) {  in virtio_transport_cancel_pkt()
213 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
214 	list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {  in virtio_transport_cancel_pkt()
219 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
229 		struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_cancel_pkt()
232 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in virtio_transport_cancel_pkt()
235 			queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_cancel_pkt()
245 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)  in virtio_vsock_rx_fill()  argument
253 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_vsock_rx_fill()
279 		vsock->rx_buf_nr++;  in virtio_vsock_rx_fill()
281 	if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)  in virtio_vsock_rx_fill()
282 		vsock->rx_buf_max_nr = vsock->rx_buf_nr;  in virtio_vsock_rx_fill()
288 	struct virtio_vsock *vsock =  in virtio_transport_tx_work()  local
293 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_tx_work()
294 	mutex_lock(&vsock->tx_lock);  in virtio_transport_tx_work()
296 	if (!vsock->tx_run)  in virtio_transport_tx_work()
311 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_tx_work()
314 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_tx_work()
318 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)  in virtio_transport_more_replies()  argument
320 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_more_replies()
324 	val = atomic_read(&vsock->queued_replies);  in virtio_transport_more_replies()
330 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,  in virtio_vsock_event_fill_one()  argument
336 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_vsock_event_fill_one()
344 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)  in virtio_vsock_event_fill()  argument
348 	for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {  in virtio_vsock_event_fill()
349 		struct virtio_vsock_event *event = &vsock->event_list[i];  in virtio_vsock_event_fill()
351 		virtio_vsock_event_fill_one(vsock, event);  in virtio_vsock_event_fill()
354 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_vsock_event_fill()
369 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)  in virtio_vsock_update_guest_cid()  argument
371 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_update_guest_cid()
376 	vsock->guest_cid = le64_to_cpu(guest_cid);  in virtio_vsock_update_guest_cid()
380 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,  in virtio_vsock_event_handle()  argument
385 		virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_event_handle()
393 	struct virtio_vsock *vsock =  in virtio_transport_event_work()  local
397 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_transport_event_work()
399 	mutex_lock(&vsock->event_lock);  in virtio_transport_event_work()
401 	if (!vsock->event_run)  in virtio_transport_event_work()
411 				virtio_vsock_event_handle(vsock, event);  in virtio_transport_event_work()
413 			virtio_vsock_event_fill_one(vsock, event);  in virtio_transport_event_work()
417 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_transport_event_work()
419 	mutex_unlock(&vsock->event_lock);  in virtio_transport_event_work()
424 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_event_done()  local
426 	if (!vsock)  in virtio_vsock_event_done()
428 	queue_work(virtio_vsock_workqueue, &vsock->event_work);  in virtio_vsock_event_done()
433 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_tx_done()  local
435 	if (!vsock)  in virtio_vsock_tx_done()
437 	queue_work(virtio_vsock_workqueue, &vsock->tx_work);  in virtio_vsock_tx_done()
442 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_rx_done()  local
444 	if (!vsock)  in virtio_vsock_rx_done()
446 	queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_vsock_rx_done()
493 	struct virtio_vsock *vsock =  in virtio_transport_rx_work()  local
497 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_rx_work()
499 	mutex_lock(&vsock->rx_lock);  in virtio_transport_rx_work()
501 	if (!vsock->rx_run)  in virtio_transport_rx_work()
510 			if (!virtio_transport_more_replies(vsock)) {  in virtio_transport_rx_work()
523 			vsock->rx_buf_nr--;  in virtio_transport_rx_work()
539 	if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)  in virtio_transport_rx_work()
540 		virtio_vsock_rx_fill(vsock);  in virtio_transport_rx_work()
541 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_rx_work()
556 	struct virtio_vsock *vsock = NULL;  in virtio_vsock_probe()  local
570 	vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);  in virtio_vsock_probe()
571 	if (!vsock) {  in virtio_vsock_probe()
576 	vsock->vdev = vdev;  in virtio_vsock_probe()
578 	ret = virtio_find_vqs(vsock->vdev, VSOCK_VQ_MAX,  in virtio_vsock_probe()
579 			      vsock->vqs, callbacks, names,  in virtio_vsock_probe()
584 	virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_probe()
586 	vsock->rx_buf_nr = 0;  in virtio_vsock_probe()
587 	vsock->rx_buf_max_nr = 0;  in virtio_vsock_probe()
588 	atomic_set(&vsock->queued_replies, 0);  in virtio_vsock_probe()
590 	mutex_init(&vsock->tx_lock);  in virtio_vsock_probe()
591 	mutex_init(&vsock->rx_lock);  in virtio_vsock_probe()
592 	mutex_init(&vsock->event_lock);  in virtio_vsock_probe()
593 	spin_lock_init(&vsock->send_pkt_list_lock);  in virtio_vsock_probe()
594 	INIT_LIST_HEAD(&vsock->send_pkt_list);  in virtio_vsock_probe()
595 	INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);  in virtio_vsock_probe()
596 	INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);  in virtio_vsock_probe()
597 	INIT_WORK(&vsock->event_work, virtio_transport_event_work);  in virtio_vsock_probe()
598 	INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);  in virtio_vsock_probe()
600 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_probe()
601 	vsock->tx_run = true;  in virtio_vsock_probe()
602 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_probe()
604 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_probe()
605 	virtio_vsock_rx_fill(vsock);  in virtio_vsock_probe()
606 	vsock->rx_run = true;  in virtio_vsock_probe()
607 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_probe()
609 	mutex_lock(&vsock->event_lock);  in virtio_vsock_probe()
610 	virtio_vsock_event_fill(vsock);  in virtio_vsock_probe()
611 	vsock->event_run = true;  in virtio_vsock_probe()
612 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_probe()
614 	vdev->priv = vsock;  in virtio_vsock_probe()
615 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_probe()
621 	kfree(vsock);  in virtio_vsock_probe()
628 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_remove()  local
643 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
644 	vsock->rx_run = false;  in virtio_vsock_remove()
645 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
647 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
648 	vsock->tx_run = false;  in virtio_vsock_remove()
649 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
651 	mutex_lock(&vsock->event_lock);  in virtio_vsock_remove()
652 	vsock->event_run = false;  in virtio_vsock_remove()
653 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_remove()
660 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
661 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))  in virtio_vsock_remove()
663 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
665 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
666 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))  in virtio_vsock_remove()
668 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
670 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
671 	while (!list_empty(&vsock->send_pkt_list)) {  in virtio_vsock_remove()
672 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_vsock_remove()
677 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
685 	flush_work(&vsock->rx_work);  in virtio_vsock_remove()
686 	flush_work(&vsock->tx_work);  in virtio_vsock_remove()
687 	flush_work(&vsock->event_work);  in virtio_vsock_remove()
688 	flush_work(&vsock->send_pkt_work);  in virtio_vsock_remove()
692 	kfree(vsock);  in virtio_vsock_remove()