85 			    struct vhost_virtqueue *vq)  in vhost_transport_do_send_pkt()  argument
92 	mutex_lock(&vq->mutex);  in vhost_transport_do_send_pkt()
94 	if (!vhost_vq_get_backend(vq))  in vhost_transport_do_send_pkt()
98 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
111 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
120 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_transport_do_send_pkt()
129 		if (head == vq->num) {  in vhost_transport_do_send_pkt()
137 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
138 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
146 			vq_err(vq, "Expected 0 output buffers, got %u\n", out);  in vhost_transport_do_send_pkt()
150 		iov_len = iov_length(&vq->iov[out], in);  in vhost_transport_do_send_pkt()
153 			vq_err(vq, "Buffer len [%zu] too small\n", iov_len);  in vhost_transport_do_send_pkt()
157 		iov_iter_init(&iov_iter, READ, &vq->iov[out], in, iov_len);  in vhost_transport_do_send_pkt()
172 			vq_err(vq, "Faulted on copying pkt hdr\n");  in vhost_transport_do_send_pkt()
180 			vq_err(vq, "Faulted on copying pkt buf\n");  in vhost_transport_do_send_pkt()
189 		vhost_add_used(vq, head, sizeof(pkt->hdr) + payload_len);  in vhost_transport_do_send_pkt()
223 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_transport_do_send_pkt()
225 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
228 	mutex_unlock(&vq->mutex);  in vhost_transport_do_send_pkt()
236 	struct vhost_virtqueue *vq;  in vhost_transport_send_pkt_work()  local
240 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
242 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
321 vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,  in vhost_vsock_alloc_pkt()  argument
330 		vq_err(vq, "Expected 0 input buffers, got %u\n", in);  in vhost_vsock_alloc_pkt()
338 	len = iov_length(vq->iov, out);  in vhost_vsock_alloc_pkt()
339 	iov_iter_init(&iov_iter, WRITE, vq->iov, out, len);  in vhost_vsock_alloc_pkt()
343 		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
372 		vq_err(vq, "Expected %u byte payload, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
384 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()  local
390 	return val < vq->num;  in vhost_vsock_more_replies()
438 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_tx_kick()  local
440 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()
447 	mutex_lock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
449 	if (!vhost_vq_get_backend(vq))  in vhost_vsock_handle_tx_kick()
452 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
464 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_vsock_handle_tx_kick()
469 		if (head == vq->num) {  in vhost_vsock_handle_tx_kick()
470 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
471 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
477 		pkt = vhost_vsock_alloc_pkt(vq, out, in);  in vhost_vsock_handle_tx_kick()
479 			vq_err(vq, "Faulted on pkt\n");  in vhost_vsock_handle_tx_kick()
497 		vhost_add_used(vq, head, 0);  in vhost_vsock_handle_tx_kick()
500 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_vsock_handle_tx_kick()
504 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
507 	mutex_unlock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
512 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_rx_kick()  local
514 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()
517 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
522 	struct vhost_virtqueue *vq;  in vhost_vsock_start()  local
533 		vq = &vsock->vqs[i];  in vhost_vsock_start()
535 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
537 		if (!vhost_vq_access_ok(vq)) {  in vhost_vsock_start()
542 		if (!vhost_vq_get_backend(vq)) {  in vhost_vsock_start()
543 			vhost_vq_set_backend(vq, vsock);  in vhost_vsock_start()
544 			ret = vhost_vq_init_access(vq);  in vhost_vsock_start()
549 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
561 	vhost_vq_set_backend(vq, NULL);  in vhost_vsock_start()
562 	mutex_unlock(&vq->mutex);  in vhost_vsock_start()
565 		vq = &vsock->vqs[i];  in vhost_vsock_start()
567 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
568 		vhost_vq_set_backend(vq, NULL);  in vhost_vsock_start()
569 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
590 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()  local
592 		mutex_lock(&vq->mutex);  in vhost_vsock_stop()
593 		vhost_vq_set_backend(vq, NULL);  in vhost_vsock_stop()
594 		mutex_unlock(&vq->mutex);  in vhost_vsock_stop()
767 	struct vhost_virtqueue *vq;  in vhost_vsock_set_features()  local
781 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
782 		mutex_lock(&vq->mutex);  in vhost_vsock_set_features()
783 		vq->acked_features = features;  in vhost_vsock_set_features()
784 		mutex_unlock(&vq->mutex);  in vhost_vsock_set_features()