143 	u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start;  in xp_get_handle()
151 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)  in __xsk_rcv_zc()  argument
153 	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);  in __xsk_rcv_zc()
186 static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len,  in __xsk_rcv()  argument
203 	xsk_copy_xdp(xsk_xdp, xdp, len);  in __xsk_rcv()
210 		xdp_return_buff(xdp);  in __xsk_rcv()
232 static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp,  in xsk_rcv()  argument
240 	if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index)  in xsk_rcv()
243 	len = xdp->data_end - xdp->data;  in xsk_rcv()
245 	return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?  in xsk_rcv()
246 		__xsk_rcv_zc(xs, xdp, len) :  in xsk_rcv()
247 		__xsk_rcv(xs, xdp, len, explicit_free);  in xsk_rcv()
257 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)  in xsk_generic_rcv()  argument
262 	err = xsk_rcv(xs, xdp, false);  in xsk_generic_rcv()
268 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)  in __xsk_map_redirect()  argument
273 	err = xsk_rcv(xs, xdp, true);  in __xsk_map_redirect()
615 	mutex_lock(&net->xdp.lock);  in xsk_release()
617 	mutex_unlock(&net->xdp.lock);  in xsk_release()
1115 		mutex_lock(&net->xdp.lock);  in xsk_notifier()
1116 		sk_for_each(sk, &net->xdp.list) {  in xsk_notifier()
1132 		mutex_unlock(&net->xdp.lock);  in xsk_notifier()
1217 	mutex_lock(&net->xdp.lock);  in xsk_create()
1218 	sk_add_node_rcu(sk, &net->xdp.list);  in xsk_create()
1219 	mutex_unlock(&net->xdp.lock);  in xsk_create()
1240 	mutex_init(&net->xdp.lock);  in xsk_net_init()
1241 	INIT_HLIST_HEAD(&net->xdp.list);  in xsk_net_init()
1247 	WARN_ON_ONCE(!hlist_empty(&net->xdp.list));  in xsk_net_exit()