147 	struct xsk_socket *xsk;  member
193 static int xsk_get_xdp_stats(int fd, struct xsk_socket_info *xsk)  in xsk_get_xdp_stats()  argument
205 		xsk->ring_stats.rx_dropped_npkts = stats.rx_dropped;  in xsk_get_xdp_stats()
206 		xsk->ring_stats.rx_invalid_npkts = stats.rx_invalid_descs;  in xsk_get_xdp_stats()
207 		xsk->ring_stats.tx_invalid_npkts = stats.tx_invalid_descs;  in xsk_get_xdp_stats()
208 		xsk->ring_stats.rx_full_npkts = stats.rx_ring_full;  in xsk_get_xdp_stats()
209 		xsk->ring_stats.rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs;  in xsk_get_xdp_stats()
210 		xsk->ring_stats.tx_empty_npkts = stats.tx_ring_empty_descs;  in xsk_get_xdp_stats()
381 			if (!xsk_get_xdp_stats(xsk_socket__fd(xsks[i]->xsk), xsks[i])) {  in dump_stats()
488 		xsk_socket__delete(xsks[i]->xsk);  in xdpsock_cleanup()
844 	struct xsk_socket_info *xsk;  in xsk_configure_socket()  local
849 	xsk = calloc(1, sizeof(*xsk));  in xsk_configure_socket()
850 	if (!xsk)  in xsk_configure_socket()
853 	xsk->umem = umem;  in xsk_configure_socket()
863 	rxr = rx ? &xsk->rx : NULL;  in xsk_configure_socket()
864 	txr = tx ? &xsk->tx : NULL;  in xsk_configure_socket()
865 	ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem,  in xsk_configure_socket()
874 	xsk->app_stats.rx_empty_polls = 0;  in xsk_configure_socket()
875 	xsk->app_stats.fill_fail_polls = 0;  in xsk_configure_socket()
876 	xsk->app_stats.copy_tx_sendtos = 0;  in xsk_configure_socket()
877 	xsk->app_stats.tx_wakeup_sendtos = 0;  in xsk_configure_socket()
878 	xsk->app_stats.opt_polls = 0;  in xsk_configure_socket()
879 	xsk->app_stats.prev_rx_empty_polls = 0;  in xsk_configure_socket()
880 	xsk->app_stats.prev_fill_fail_polls = 0;  in xsk_configure_socket()
881 	xsk->app_stats.prev_copy_tx_sendtos = 0;  in xsk_configure_socket()
882 	xsk->app_stats.prev_tx_wakeup_sendtos = 0;  in xsk_configure_socket()
883 	xsk->app_stats.prev_opt_polls = 0;  in xsk_configure_socket()
885 	return xsk;  in xsk_configure_socket()
1090 static void kick_tx(struct xsk_socket_info *xsk)  in kick_tx()  argument
1094 	ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);  in kick_tx()
1101 static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk,  in complete_tx_l2fwd()  argument
1104 	struct xsk_umem_info *umem = xsk->umem;  in complete_tx_l2fwd()
1109 	if (!xsk->outstanding_tx)  in complete_tx_l2fwd()
1118 		xsk->app_stats.copy_tx_sendtos++;  in complete_tx_l2fwd()
1119 		kick_tx(xsk);  in complete_tx_l2fwd()
1122 	ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size :  in complete_tx_l2fwd()
1123 		xsk->outstanding_tx;  in complete_tx_l2fwd()
1136 				xsk->app_stats.fill_fail_polls++;  in complete_tx_l2fwd()
1146 		xsk_ring_prod__submit(&xsk->umem->fq, rcvd);  in complete_tx_l2fwd()
1147 		xsk_ring_cons__release(&xsk->umem->cq, rcvd);  in complete_tx_l2fwd()
1148 		xsk->outstanding_tx -= rcvd;  in complete_tx_l2fwd()
1149 		xsk->ring_stats.tx_npkts += rcvd;  in complete_tx_l2fwd()
1153 static inline void complete_tx_only(struct xsk_socket_info *xsk,  in complete_tx_only()  argument
1159 	if (!xsk->outstanding_tx)  in complete_tx_only()
1162 	if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) {  in complete_tx_only()
1163 		xsk->app_stats.tx_wakeup_sendtos++;  in complete_tx_only()
1164 		kick_tx(xsk);  in complete_tx_only()
1167 	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);  in complete_tx_only()
1169 		xsk_ring_cons__release(&xsk->umem->cq, rcvd);  in complete_tx_only()
1170 		xsk->outstanding_tx -= rcvd;  in complete_tx_only()
1171 		xsk->ring_stats.tx_npkts += rcvd;  in complete_tx_only()
1175 static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds)  in rx_drop()  argument
1181 	rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);  in rx_drop()
1183 		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {  in rx_drop()
1184 			xsk->app_stats.rx_empty_polls++;  in rx_drop()
1190 	ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);  in rx_drop()
1194 		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {  in rx_drop()
1195 			xsk->app_stats.fill_fail_polls++;  in rx_drop()
1198 		ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);  in rx_drop()
1202 		u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;  in rx_drop()
1203 		u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;  in rx_drop()
1207 		char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);  in rx_drop()
1210 		*xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;  in rx_drop()
1213 	xsk_ring_prod__submit(&xsk->umem->fq, rcvd);  in rx_drop()
1214 	xsk_ring_cons__release(&xsk->rx, rcvd);  in rx_drop()
1215 	xsk->ring_stats.rx_npkts += rcvd;  in rx_drop()
1224 		fds[i].fd = xsk_socket__fd(xsks[i]->xsk);  in rx_drop_all()
1245 static void tx_only(struct xsk_socket_info *xsk, u32 *frame_nb, int batch_size)  in tx_only()  argument
1250 	while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) <  in tx_only()
1252 		complete_tx_only(xsk, batch_size);  in tx_only()
1258 		struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx,  in tx_only()
1264 	xsk_ring_prod__submit(&xsk->tx, batch_size);  in tx_only()
1265 	xsk->outstanding_tx += batch_size;  in tx_only()
1268 	complete_tx_only(xsk, batch_size);  in tx_only()
1306 		fds[0].fd = xsk_socket__fd(xsks[i]->xsk);  in tx_only_all()
1337 static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds)  in l2fwd()  argument
1343 	complete_tx_l2fwd(xsk, fds);  in l2fwd()
1345 	rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);  in l2fwd()
1347 		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {  in l2fwd()
1348 			xsk->app_stats.rx_empty_polls++;  in l2fwd()
1354 	ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);  in l2fwd()
1358 		complete_tx_l2fwd(xsk, fds);  in l2fwd()
1359 		if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {  in l2fwd()
1360 			xsk->app_stats.tx_wakeup_sendtos++;  in l2fwd()
1361 			kick_tx(xsk);  in l2fwd()
1363 		ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);  in l2fwd()
1367 		u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;  in l2fwd()
1368 		u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;  in l2fwd()
1372 		char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);  in l2fwd()
1377 		xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig;  in l2fwd()
1378 		xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;  in l2fwd()
1381 	xsk_ring_prod__submit(&xsk->tx, rcvd);  in l2fwd()
1382 	xsk_ring_cons__release(&xsk->rx, rcvd);  in l2fwd()
1384 	xsk->ring_stats.rx_npkts += rcvd;  in l2fwd()
1385 	xsk->outstanding_tx += rcvd;  in l2fwd()
1394 		fds[i].fd = xsk_socket__fd(xsks[i]->xsk);  in l2fwd_all()
1454 		int fd = xsk_socket__fd(xsks[i]->xsk);  in enter_xsks_into_map()