125 static inline int get_request(struct pvcalls_bedata *bedata, int *req_id)  in get_request()  argument
127 	*req_id = bedata->ring.req_prod_pvt & (RING_SIZE(&bedata->ring) - 1);  in get_request()
128 	if (RING_FULL(&bedata->ring) ||  in get_request()
129 	    bedata->rsp[*req_id].req_id != PVCALLS_INVALID_ID)  in get_request()
168 	struct pvcalls_bedata *bedata;  in pvcalls_front_event_handler()  local
177 	bedata = dev_get_drvdata(&dev->dev);  in pvcalls_front_event_handler()
178 	if (bedata == NULL) {  in pvcalls_front_event_handler()
184 	while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) {  in pvcalls_front_event_handler()
185 		rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons);  in pvcalls_front_event_handler()
203 			dst = (uint8_t *)&bedata->rsp[req_id] +  in pvcalls_front_event_handler()
212 			bedata->rsp[req_id].req_id = req_id;  in pvcalls_front_event_handler()
216 		bedata->ring.rsp_cons++;  in pvcalls_front_event_handler()
219 	RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more);  in pvcalls_front_event_handler()
223 		wake_up(&bedata->inflight_req);  in pvcalls_front_event_handler()
228 static void pvcalls_front_free_map(struct pvcalls_bedata *bedata,  in pvcalls_front_free_map()  argument
235 	spin_lock(&bedata->socket_lock);  in pvcalls_front_free_map()
238 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_free_map()
262 	struct pvcalls_bedata *bedata;  in pvcalls_front_socket()  local
282 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_socket()
290 	spin_lock(&bedata->socket_lock);  in pvcalls_front_socket()
292 	ret = get_request(bedata, &req_id);  in pvcalls_front_socket()
295 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_socket()
307 	list_add_tail(&map->list, &bedata->socket_mappings);  in pvcalls_front_socket()
309 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_socket()
317 	bedata->ring.req_prod_pvt++;  in pvcalls_front_socket()
318 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_socket()
319 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_socket()
321 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_socket()
323 	wait_event(bedata->inflight_req,  in pvcalls_front_socket()
324 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_socket()
328 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_socket()
329 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_socket()
415 	struct pvcalls_bedata *bedata;  in pvcalls_front_connect()  local
428 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_connect()
435 	spin_lock(&bedata->socket_lock);  in pvcalls_front_connect()
436 	ret = get_request(bedata, &req_id);  in pvcalls_front_connect()
438 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_connect()
445 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_connect()
451 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_connect()
463 	bedata->ring.req_prod_pvt++;  in pvcalls_front_connect()
464 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_connect()
465 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_connect()
468 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_connect()
470 	wait_event(bedata->inflight_req,  in pvcalls_front_connect()
471 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_connect()
475 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_connect()
476 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_connect()
663 	struct pvcalls_bedata *bedata;  in pvcalls_front_bind()  local
674 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_bind()
676 	spin_lock(&bedata->socket_lock);  in pvcalls_front_bind()
677 	ret = get_request(bedata, &req_id);  in pvcalls_front_bind()
679 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_bind()
683 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_bind()
695 	bedata->ring.req_prod_pvt++;  in pvcalls_front_bind()
696 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_bind()
697 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_bind()
699 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_bind()
701 	wait_event(bedata->inflight_req,  in pvcalls_front_bind()
702 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_bind()
706 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_bind()
707 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_bind()
716 	struct pvcalls_bedata *bedata;  in pvcalls_front_listen()  local
724 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_listen()
731 	spin_lock(&bedata->socket_lock);  in pvcalls_front_listen()
732 	ret = get_request(bedata, &req_id);  in pvcalls_front_listen()
734 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_listen()
738 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_listen()
744 	bedata->ring.req_prod_pvt++;  in pvcalls_front_listen()
745 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_listen()
746 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_listen()
748 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_listen()
750 	wait_event(bedata->inflight_req,  in pvcalls_front_listen()
751 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_listen()
755 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_listen()
756 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_listen()
765 	struct pvcalls_bedata *bedata;  in pvcalls_front_accept()  local
775 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_accept()
791 		    READ_ONCE(bedata->rsp[req_id].req_id) == req_id) {  in pvcalls_front_accept()
822 	spin_lock(&bedata->socket_lock);  in pvcalls_front_accept()
823 	ret = get_request(bedata, &req_id);  in pvcalls_front_accept()
827 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_accept()
840 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_accept()
844 	list_add_tail(&map2->list, &bedata->socket_mappings);  in pvcalls_front_accept()
846 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_accept()
855 	bedata->ring.req_prod_pvt++;  in pvcalls_front_accept()
856 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_accept()
857 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_accept()
859 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_accept()
867 	if (wait_event_interruptible(bedata->inflight_req,  in pvcalls_front_accept()
868 		READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) {  in pvcalls_front_accept()
879 		bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_accept()
883 		pvcalls_front_free_map(bedata, map2);  in pvcalls_front_accept()
889 	ret = bedata->rsp[req_id].ret;  in pvcalls_front_accept()
890 	bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_accept()
901 					       struct pvcalls_bedata *bedata,  in pvcalls_front_poll_passive()  argument
913 		    READ_ONCE(bedata->rsp[req_id].req_id) == req_id)  in pvcalls_front_poll_passive()
932 		poll_wait(file, &bedata->inflight_req, wait);  in pvcalls_front_poll_passive()
936 	spin_lock(&bedata->socket_lock);  in pvcalls_front_poll_passive()
937 	ret = get_request(bedata, &req_id);  in pvcalls_front_poll_passive()
939 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_poll_passive()
942 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_poll_passive()
947 	bedata->ring.req_prod_pvt++;  in pvcalls_front_poll_passive()
948 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_poll_passive()
949 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_poll_passive()
951 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_poll_passive()
953 	poll_wait(file, &bedata->inflight_req, wait);  in pvcalls_front_poll_passive()
958 					      struct pvcalls_bedata *bedata,  in pvcalls_front_poll_active()  argument
983 	struct pvcalls_bedata *bedata;  in pvcalls_front_poll()  local
990 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_poll()
993 		ret = pvcalls_front_poll_active(file, bedata, map, wait);  in pvcalls_front_poll()
995 		ret = pvcalls_front_poll_passive(file, bedata, map, wait);  in pvcalls_front_poll()
1002 	struct pvcalls_bedata *bedata;  in pvcalls_front_release()  local
1017 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_release()
1019 	spin_lock(&bedata->socket_lock);  in pvcalls_front_release()
1020 	ret = get_request(bedata, &req_id);  in pvcalls_front_release()
1022 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_release()
1028 	req = RING_GET_REQUEST(&bedata->ring, req_id);  in pvcalls_front_release()
1033 	bedata->ring.req_prod_pvt++;  in pvcalls_front_release()
1034 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);  in pvcalls_front_release()
1035 	spin_unlock(&bedata->socket_lock);  in pvcalls_front_release()
1037 		notify_remote_via_irq(bedata->irq);  in pvcalls_front_release()
1039 	wait_event(bedata->inflight_req,  in pvcalls_front_release()
1040 		   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);  in pvcalls_front_release()
1059 		pvcalls_front_free_map(bedata, map);  in pvcalls_front_release()
1061 		wake_up(&bedata->inflight_req);  in pvcalls_front_release()
1067 		spin_lock(&bedata->socket_lock);  in pvcalls_front_release()
1069 		spin_unlock(&bedata->socket_lock);  in pvcalls_front_release()
1072 			pvcalls_front_free_map(bedata,  in pvcalls_front_release()
1077 	WRITE_ONCE(bedata->rsp[req_id].req_id, PVCALLS_INVALID_ID);  in pvcalls_front_release()
1090 	struct pvcalls_bedata *bedata;  in pvcalls_front_remove()  local
1093 	bedata = dev_get_drvdata(&pvcalls_front_dev->dev);  in pvcalls_front_remove()
1096 	if (bedata->irq >= 0)  in pvcalls_front_remove()
1097 		unbind_from_irqhandler(bedata->irq, dev);  in pvcalls_front_remove()
1099 	list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {  in pvcalls_front_remove()
1110 	list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {  in pvcalls_front_remove()
1113 			pvcalls_front_free_map(bedata, map);  in pvcalls_front_remove()
1119 	if (bedata->ref != -1)  in pvcalls_front_remove()
1120 		gnttab_end_foreign_access(bedata->ref, 0, 0);  in pvcalls_front_remove()
1121 	kfree(bedata->ring.sring);  in pvcalls_front_remove()
1122 	kfree(bedata);  in pvcalls_front_remove()
1136 	struct pvcalls_bedata *bedata = NULL;  in pvcalls_front_probe()  local
1165 	bedata = kzalloc(sizeof(struct pvcalls_bedata), GFP_KERNEL);  in pvcalls_front_probe()
1166 	if (!bedata)  in pvcalls_front_probe()
1169 	dev_set_drvdata(&dev->dev, bedata);  in pvcalls_front_probe()
1171 	init_waitqueue_head(&bedata->inflight_req);  in pvcalls_front_probe()
1172 	INIT_LIST_HEAD(&bedata->socket_mappings);  in pvcalls_front_probe()
1173 	spin_lock_init(&bedata->socket_lock);  in pvcalls_front_probe()
1174 	bedata->irq = -1;  in pvcalls_front_probe()
1175 	bedata->ref = -1;  in pvcalls_front_probe()
1178 		bedata->rsp[i].req_id = PVCALLS_INVALID_ID;  in pvcalls_front_probe()
1185 	FRONT_RING_INIT(&bedata->ring, sring, XEN_PAGE_SIZE);  in pvcalls_front_probe()
1191 	bedata->irq = bind_evtchn_to_irqhandler(evtchn,  in pvcalls_front_probe()
1194 	if (bedata->irq < 0) {  in pvcalls_front_probe()
1195 		ret = bedata->irq;  in pvcalls_front_probe()
1205 	bedata->ref = ret;  in pvcalls_front_probe()
1206 	gnttab_grant_foreign_access_ref(bedata->ref, dev->otherend_id,  in pvcalls_front_probe()
1218 	ret = xenbus_printf(xbt, dev->nodename, "ring-ref", "%d", bedata->ref);  in pvcalls_front_probe()