Lines Matching +full:broken +full:- +full:prefetch +full:- +full:cmd
1 // SPDX-License-Identifier: GPL-2.0-or-later
33 * Fred Van Kempen : Net2e support for sk->broadcast.
63 * bound-to-device socket
141 sk_for_each(sk2, &hslot->head) { in udp_lib_lport_inuse()
144 (bitmap || udp_sk(sk2)->udp_port_hash == num) && in udp_lib_lport_inuse()
145 (!sk2->sk_reuse || !sk->sk_reuse) && in udp_lib_lport_inuse()
146 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || in udp_lib_lport_inuse()
147 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && in udp_lib_lport_inuse()
149 if (sk2->sk_reuseport && sk->sk_reuseport && in udp_lib_lport_inuse()
150 !rcu_access_pointer(sk->sk_reuseport_cb) && in udp_lib_lport_inuse()
157 __set_bit(udp_sk(sk2)->udp_port_hash >> log, in udp_lib_lport_inuse()
177 spin_lock(&hslot2->lock); in udp_lib_lport_inuse2()
178 udp_portaddr_for_each_entry(sk2, &hslot2->head) { in udp_lib_lport_inuse2()
181 (udp_sk(sk2)->udp_port_hash == num) && in udp_lib_lport_inuse2()
182 (!sk2->sk_reuse || !sk->sk_reuse) && in udp_lib_lport_inuse2()
183 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || in udp_lib_lport_inuse2()
184 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && in udp_lib_lport_inuse2()
186 if (sk2->sk_reuseport && sk->sk_reuseport && in udp_lib_lport_inuse2()
187 !rcu_access_pointer(sk->sk_reuseport_cb) && in udp_lib_lport_inuse2()
196 spin_unlock(&hslot2->lock); in udp_lib_lport_inuse2()
206 sk_for_each(sk2, &hslot->head) { in udp_reuseport_add_sock()
209 sk2->sk_family == sk->sk_family && in udp_reuseport_add_sock()
211 (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) && in udp_reuseport_add_sock()
212 (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && in udp_reuseport_add_sock()
213 sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && in udp_reuseport_add_sock()
224 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
228 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
235 struct udp_table *udptable = sk->sk_prot->h.udp_table; in udp_lib_get_port()
246 remaining = (high - low) + 1; in udp_lib_get_port()
253 rand = (rand | 1) * (udptable->mask + 1); in udp_lib_get_port()
254 last = first + udptable->mask + 1; in udp_lib_get_port()
258 spin_lock_bh(&hslot->lock); in udp_lib_get_port()
260 udptable->log); in udp_lib_get_port()
270 !test_bit(snum >> udptable->log, bitmap) && in udp_lib_get_port()
275 spin_unlock_bh(&hslot->lock); in udp_lib_get_port()
281 spin_lock_bh(&hslot->lock); in udp_lib_get_port()
282 if (hslot->count > 10) { in udp_lib_get_port()
284 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; in udp_lib_get_port()
286 slot2 &= udptable->mask; in udp_lib_get_port()
287 hash2_nulladdr &= udptable->mask; in udp_lib_get_port()
290 if (hslot->count < hslot2->count) in udp_lib_get_port()
309 inet_sk(sk)->inet_num = snum; in udp_lib_get_port()
310 udp_sk(sk)->udp_port_hash = snum; in udp_lib_get_port()
311 udp_sk(sk)->udp_portaddr_hash ^= snum; in udp_lib_get_port()
313 if (sk->sk_reuseport && in udp_lib_get_port()
315 inet_sk(sk)->inet_num = 0; in udp_lib_get_port()
316 udp_sk(sk)->udp_port_hash = 0; in udp_lib_get_port()
317 udp_sk(sk)->udp_portaddr_hash ^= snum; in udp_lib_get_port()
321 sk_add_node_rcu(sk, &hslot->head); in udp_lib_get_port()
322 hslot->count++; in udp_lib_get_port()
323 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in udp_lib_get_port()
325 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); in udp_lib_get_port()
326 spin_lock(&hslot2->lock); in udp_lib_get_port()
327 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && in udp_lib_get_port()
328 sk->sk_family == AF_INET6) in udp_lib_get_port()
329 hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node, in udp_lib_get_port()
330 &hslot2->head); in udp_lib_get_port()
332 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, in udp_lib_get_port()
333 &hslot2->head); in udp_lib_get_port()
334 hslot2->count++; in udp_lib_get_port()
335 spin_unlock(&hslot2->lock); in udp_lib_get_port()
340 spin_unlock_bh(&hslot->lock); in udp_lib_get_port()
351 ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); in udp_v4_get_port()
354 udp_sk(sk)->udp_portaddr_hash = hash2_partial; in udp_v4_get_port()
368 udp_sk(sk)->udp_port_hash != hnum || in compute_score()
370 return -1; in compute_score()
372 if (sk->sk_rcv_saddr != daddr) in compute_score()
373 return -1; in compute_score()
375 score = (sk->sk_family == PF_INET) ? 2 : 1; in compute_score()
378 if (inet->inet_daddr) { in compute_score()
379 if (inet->inet_daddr != saddr) in compute_score()
380 return -1; in compute_score()
384 if (inet->inet_dport) { in compute_score()
385 if (inet->inet_dport != sport) in compute_score()
386 return -1; in compute_score()
390 dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, in compute_score()
393 return -1; in compute_score()
394 if (sk->sk_bound_dev_if) in compute_score()
397 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) in compute_score()
422 if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) { in lookup_reuseport()
443 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { in udp4_lib_lookup2()
484 * harder than this. -DaveM
496 slot2 = hash2 & udptable->mask; in __udp4_lib_lookup()
497 hslot2 = &udptable->hash2[slot2]; in __udp4_lib_lookup()
499 /* Lookup connected or non-wildcard socket */ in __udp4_lib_lookup()
503 if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED) in __udp4_lib_lookup()
516 /* Got non-wildcard socket or error on first lookup */ in __udp4_lib_lookup()
522 slot2 = hash2 & udptable->mask; in __udp4_lib_lookup()
523 hslot2 = &udptable->hash2[slot2]; in __udp4_lib_lookup()
541 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, in __udp4_lib_lookup_skb()
542 iph->daddr, dport, inet_iif(skb), in __udp4_lib_lookup_skb()
551 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, in udp4_lib_lookup_skb()
552 iph->daddr, dport, inet_iif(skb), in udp4_lib_lookup_skb()
568 if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) in udp4_lib_lookup()
583 udp_sk(sk)->udp_port_hash != hnum || in __udp_is_mcast_sock()
584 (inet->inet_daddr && inet->inet_daddr != rmt_addr) || in __udp_is_mcast_sock()
585 (inet->inet_dport != rmt_port && inet->inet_dport) || in __udp_is_mcast_sock()
586 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) || in __udp_is_mcast_sock()
588 !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) in __udp_is_mcast_sock()
622 handler = encap->err_handler; in __udp4_lib_err_encap_no_sk()
627 return -ENOENT; in __udp4_lib_err_encap_no_sk()
663 skb_set_transport_header(skb, iph->ihl << 2); in __udp4_lib_err_encap()
665 sk = __udp4_lib_lookup(net, iph->daddr, uh->source, in __udp4_lib_err_encap()
666 iph->saddr, uh->dest, skb->dev->ifindex, 0, in __udp4_lib_err_encap()
672 lookup = READ_ONCE(up->encap_err_lookup); in __udp4_lib_err_encap()
700 const struct iphdr *iph = (const struct iphdr *)skb->data; in __udp4_lib_err()
701 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2)); in __udp4_lib_err()
702 const int type = icmp_hdr(skb)->type; in __udp4_lib_err()
703 const int code = icmp_hdr(skb)->code; in __udp4_lib_err()
708 struct net *net = dev_net(skb->dev); in __udp4_lib_err()
710 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest, in __udp4_lib_err()
711 iph->saddr, uh->source, skb->dev->ifindex, in __udp4_lib_err()
715 sk = ERR_PTR(-ENOENT); in __udp4_lib_err()
749 if (inet->pmtudisc != IP_PMTUDISC_DONT) { in __udp4_lib_err()
775 if (!inet->recverr) { in __udp4_lib_err()
776 if (!harderr || sk->sk_state != TCP_ESTABLISHED) in __udp4_lib_err()
779 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1)); in __udp4_lib_err()
781 sk->sk_err = err; in __udp4_lib_err()
782 sk->sk_error_report(sk); in __udp4_lib_err()
799 if (up->pending) { in udp_flush_pending_frames()
800 up->len = 0; in udp_flush_pending_frames()
801 up->pending = 0; in udp_flush_pending_frames()
808 * udp4_hwcsum - handle outgoing HW checksumming
809 * @skb: sk_buff containing the filled-in UDP header
818 int len = skb->len - offset; in udp4_hwcsum()
826 skb->csum_start = skb_transport_header(skb) - skb->head; in udp4_hwcsum()
827 skb->csum_offset = offsetof(struct udphdr, check); in udp4_hwcsum()
828 uh->check = ~csum_tcpudp_magic(src, dst, len, in udp4_hwcsum()
834 * HW-checksum won't work as there are two or more in udp4_hwcsum()
839 csum = csum_add(csum, frags->csum); in udp4_hwcsum()
840 hlen -= frags->len; in udp4_hwcsum()
844 skb->ip_summed = CHECKSUM_NONE; in udp4_hwcsum()
846 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum); in udp4_hwcsum()
847 if (uh->check == 0) in udp4_hwcsum()
848 uh->check = CSUM_MANGLED_0; in udp4_hwcsum()
862 uh->check = 0; in udp_set_csum()
864 uh->check = ~udp_v4_check(len, saddr, daddr, 0); in udp_set_csum()
865 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { in udp_set_csum()
866 uh->check = 0; in udp_set_csum()
867 uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb)); in udp_set_csum()
868 if (uh->check == 0) in udp_set_csum()
869 uh->check = CSUM_MANGLED_0; in udp_set_csum()
871 skb->ip_summed = CHECKSUM_PARTIAL; in udp_set_csum()
872 skb->csum_start = skb_transport_header(skb) - skb->head; in udp_set_csum()
873 skb->csum_offset = offsetof(struct udphdr, check); in udp_set_csum()
874 uh->check = ~udp_v4_check(len, saddr, daddr, 0); in udp_set_csum()
882 struct sock *sk = skb->sk; in udp_send_skb()
888 int len = skb->len - offset; in udp_send_skb()
889 int datalen = len - sizeof(*uh); in udp_send_skb()
896 uh->source = inet->inet_sport; in udp_send_skb()
897 uh->dest = fl4->fl4_dport; in udp_send_skb()
898 uh->len = htons(len); in udp_send_skb()
899 uh->check = 0; in udp_send_skb()
901 if (cork->gso_size) { in udp_send_skb()
905 if (hlen + cork->gso_size > cork->fragsize) { in udp_send_skb()
907 return -EINVAL; in udp_send_skb()
909 if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) { in udp_send_skb()
911 return -EINVAL; in udp_send_skb()
913 if (sk->sk_no_check_tx) { in udp_send_skb()
915 return -EINVAL; in udp_send_skb()
917 if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite || in udp_send_skb()
920 return -EIO; in udp_send_skb()
923 if (datalen > cork->gso_size) { in udp_send_skb()
924 skb_shinfo(skb)->gso_size = cork->gso_size; in udp_send_skb()
925 skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4; in udp_send_skb()
926 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen, in udp_send_skb()
927 cork->gso_size); in udp_send_skb()
932 if (is_udplite) /* UDP-Lite */ in udp_send_skb()
935 else if (sk->sk_no_check_tx) { /* UDP csum off */ in udp_send_skb()
937 skb->ip_summed = CHECKSUM_NONE; in udp_send_skb()
940 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */ in udp_send_skb()
943 udp4_hwcsum(skb, fl4->saddr, fl4->daddr); in udp_send_skb()
949 /* add protocol-dependent pseudo-header */ in udp_send_skb()
950 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len, in udp_send_skb()
951 sk->sk_protocol, csum); in udp_send_skb()
952 if (uh->check == 0) in udp_send_skb()
953 uh->check = CSUM_MANGLED_0; in udp_send_skb()
958 if (err == -ENOBUFS && !inet->recverr) { in udp_send_skb()
976 struct flowi4 *fl4 = &inet->cork.fl.u.ip4; in udp_push_pending_frames()
984 err = udp_send_skb(skb, fl4, &inet->cork.base); in udp_push_pending_frames()
987 up->len = 0; in udp_push_pending_frames()
988 up->pending = 0; in udp_push_pending_frames()
995 switch (cmsg->cmsg_type) { in __udp_cmsg_send()
997 if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16))) in __udp_cmsg_send()
998 return -EINVAL; in __udp_cmsg_send()
1002 return -EINVAL; in __udp_cmsg_send()
1014 return -EINVAL; in udp_cmsg_send()
1016 if (cmsg->cmsg_level != SOL_UDP) { in udp_cmsg_send()
1034 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); in udp_sendmsg()
1046 int corkreq = READ_ONCE(up->corkflag) || msg->msg_flags&MSG_MORE; in udp_sendmsg()
1052 return -EMSGSIZE; in udp_sendmsg()
1058 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */ in udp_sendmsg()
1059 return -EOPNOTSUPP; in udp_sendmsg()
1064 fl4 = &inet->cork.fl.u.ip4; in udp_sendmsg()
1065 if (up->pending) { in udp_sendmsg()
1071 if (likely(up->pending)) { in udp_sendmsg()
1072 if (unlikely(up->pending != AF_INET)) { in udp_sendmsg()
1074 return -EINVAL; in udp_sendmsg()
1086 if (msg->msg_namelen < sizeof(*usin)) in udp_sendmsg()
1087 return -EINVAL; in udp_sendmsg()
1088 if (usin->sin_family != AF_INET) { in udp_sendmsg()
1089 if (usin->sin_family != AF_UNSPEC) in udp_sendmsg()
1090 return -EAFNOSUPPORT; in udp_sendmsg()
1093 daddr = usin->sin_addr.s_addr; in udp_sendmsg()
1094 dport = usin->sin_port; in udp_sendmsg()
1096 return -EINVAL; in udp_sendmsg()
1098 if (sk->sk_state != TCP_ESTABLISHED) in udp_sendmsg()
1099 return -EDESTADDRREQ; in udp_sendmsg()
1100 daddr = inet->inet_daddr; in udp_sendmsg()
1101 dport = inet->inet_dport; in udp_sendmsg()
1109 ipc.gso_size = READ_ONCE(up->gso_size); in udp_sendmsg()
1111 if (msg->msg_controllen) { in udp_sendmsg()
1115 sk->sk_family == AF_INET6); in udp_sendmsg()
1128 inet_opt = rcu_dereference(inet->inet_opt); in udp_sendmsg()
1131 sizeof(*inet_opt) + inet_opt->opt.optlen); in udp_sendmsg()
1143 if (usin->sin_port == 0) { in udp_sendmsg()
1145 err = -EINVAL; in udp_sendmsg()
1148 daddr = usin->sin_addr.s_addr; in udp_sendmsg()
1149 dport = usin->sin_port; in udp_sendmsg()
1156 if (ipc.opt && ipc.opt->opt.srr) { in udp_sendmsg()
1158 err = -EINVAL; in udp_sendmsg()
1161 faddr = ipc.opt->opt.faddr; in udp_sendmsg()
1166 (msg->msg_flags & MSG_DONTROUTE) || in udp_sendmsg()
1167 (ipc.opt && ipc.opt->opt.is_strictroute)) { in udp_sendmsg()
1174 ipc.oif = inet->mc_index; in udp_sendmsg()
1176 saddr = inet->mc_addr; in udp_sendmsg()
1179 ipc.oif = inet->uc_index; in udp_sendmsg()
1180 } else if (ipv4_is_lbcast(daddr) && inet->uc_index) { in udp_sendmsg()
1187 if (ipc.oif != inet->uc_index && in udp_sendmsg()
1189 inet->uc_index)) { in udp_sendmsg()
1190 ipc.oif = inet->uc_index; in udp_sendmsg()
1204 RT_SCOPE_UNIVERSE, sk->sk_protocol, in udp_sendmsg()
1206 faddr, saddr, dport, inet->inet_sport, in udp_sendmsg()
1207 sk->sk_uid); in udp_sendmsg()
1214 if (err == -ENETUNREACH) in udp_sendmsg()
1219 err = -EACCES; in udp_sendmsg()
1220 if ((rt->rt_flags & RTCF_BROADCAST) && in udp_sendmsg()
1224 sk_dst_set(sk, dst_clone(&rt->dst)); in udp_sendmsg()
1227 if (msg->msg_flags&MSG_CONFIRM) in udp_sendmsg()
1231 saddr = fl4->saddr; in udp_sendmsg()
1233 daddr = ipc.addr = fl4->daddr; in udp_sendmsg()
1235 /* Lockless fast path for the non-corking case. */ in udp_sendmsg()
1241 &cork, msg->msg_flags); in udp_sendmsg()
1249 if (unlikely(up->pending)) { in udp_sendmsg()
1251 /* ... which is an evident application bug. --ANK */ in udp_sendmsg()
1255 err = -EINVAL; in udp_sendmsg()
1261 fl4 = &inet->cork.fl.u.ip4; in udp_sendmsg()
1262 fl4->daddr = daddr; in udp_sendmsg()
1263 fl4->saddr = saddr; in udp_sendmsg()
1264 fl4->fl4_dport = dport; in udp_sendmsg()
1265 fl4->fl4_sport = inet->inet_sport; in udp_sendmsg()
1266 up->pending = AF_INET; in udp_sendmsg()
1269 up->len += ulen; in udp_sendmsg()
1272 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); in udp_sendmsg()
1277 else if (unlikely(skb_queue_empty(&sk->sk_write_queue))) in udp_sendmsg()
1278 up->pending = 0; in udp_sendmsg()
1295 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { in udp_sendmsg()
1302 if (msg->msg_flags & MSG_PROBE) in udp_sendmsg()
1303 dst_confirm_neigh(&rt->dst, &fl4->daddr); in udp_sendmsg()
1304 if (!(msg->msg_flags&MSG_PROBE) || len) in udp_sendmsg()
1321 if (!up->pending) { in udp_sendpage()
1335 if (unlikely(!up->pending)) { in udp_sendpage()
1339 return -EINVAL; in udp_sendpage()
1342 ret = ip_append_page(sk, &inet->cork.fl.u.ip4, in udp_sendpage()
1344 if (ret == -EOPNOTSUPP) { in udp_sendpage()
1346 return sock_no_sendpage(sk->sk_socket, page, offset, in udp_sendpage()
1354 up->len += size; in udp_sendpage()
1355 if (!(READ_ONCE(up->corkflag) || (flags&MSG_MORE))) in udp_sendpage()
1392 scratch->_tsize_state = skb->truesize; in udp_set_dev_scratch()
1394 scratch->len = skb->len; in udp_set_dev_scratch()
1395 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb); in udp_set_dev_scratch()
1396 scratch->is_linear = !skb_is_nonlinear(skb); in udp_set_dev_scratch()
1399 scratch->_tsize_state |= UDP_SKB_IS_STATELESS; in udp_set_dev_scratch()
1406 * set skb->csum_valid to 1. in udp_skb_csum_unnecessary_set()
1412 udp_skb_scratch(skb)->csum_unnecessary = true; in udp_skb_csum_unnecessary_set()
1418 return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS; in udp_skb_truesize()
1423 return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS); in udp_skb_has_head_state()
1435 up->forward_deficit += size; in udp_rmem_release()
1436 size = up->forward_deficit; in udp_rmem_release()
1437 if (size < (sk->sk_rcvbuf >> 2) && in udp_rmem_release()
1438 !skb_queue_empty(&up->reader_queue)) in udp_rmem_release()
1441 size += up->forward_deficit; in udp_rmem_release()
1443 up->forward_deficit = 0; in udp_rmem_release()
1448 sk_queue = &sk->sk_receive_queue; in udp_rmem_release()
1450 spin_lock(&sk_queue->lock); in udp_rmem_release()
1453 sk->sk_forward_alloc += size; in udp_rmem_release()
1454 amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1); in udp_rmem_release()
1455 sk->sk_forward_alloc -= amt; in udp_rmem_release()
1460 atomic_sub(size, &sk->sk_rmem_alloc); in udp_rmem_release()
1463 skb_queue_splice_tail_init(sk_queue, &up->reader_queue); in udp_rmem_release()
1466 spin_unlock(&sk_queue->lock); in udp_rmem_release()
1470 * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1476 prefetch(&skb->data); in udp_skb_destructor()
1484 prefetch(&skb->data); in udp_skb_dtor_locked()
1515 struct sk_buff_head *list = &sk->sk_receive_queue; in __udp_enqueue_schedule_skb()
1516 int rmem, delta, amt, err = -ENOMEM; in __udp_enqueue_schedule_skb()
1523 rmem = atomic_read(&sk->sk_rmem_alloc); in __udp_enqueue_schedule_skb()
1524 if (rmem > sk->sk_rcvbuf) in __udp_enqueue_schedule_skb()
1529 * - Reduce memory overhead and thus increase receive queue capacity in __udp_enqueue_schedule_skb()
1530 * - Less cache line misses at copyout() time in __udp_enqueue_schedule_skb()
1531 * - Less work at consume_skb() (less alien page frag freeing) in __udp_enqueue_schedule_skb()
1533 if (rmem > (sk->sk_rcvbuf >> 1)) { in __udp_enqueue_schedule_skb()
1538 size = skb->truesize; in __udp_enqueue_schedule_skb()
1544 rmem = atomic_add_return(size, &sk->sk_rmem_alloc); in __udp_enqueue_schedule_skb()
1545 if (rmem > (size + (unsigned int)sk->sk_rcvbuf)) in __udp_enqueue_schedule_skb()
1548 spin_lock(&list->lock); in __udp_enqueue_schedule_skb()
1549 if (size >= sk->sk_forward_alloc) { in __udp_enqueue_schedule_skb()
1553 err = -ENOBUFS; in __udp_enqueue_schedule_skb()
1554 spin_unlock(&list->lock); in __udp_enqueue_schedule_skb()
1558 sk->sk_forward_alloc += delta; in __udp_enqueue_schedule_skb()
1561 sk->sk_forward_alloc -= size; in __udp_enqueue_schedule_skb()
1569 spin_unlock(&list->lock); in __udp_enqueue_schedule_skb()
1572 sk->sk_data_ready(sk); in __udp_enqueue_schedule_skb()
1578 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); in __udp_enqueue_schedule_skb()
1581 atomic_inc(&sk->sk_drops); in __udp_enqueue_schedule_skb()
1594 skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue); in udp_destruct_sock()
1595 while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) { in udp_destruct_sock()
1596 total += skb->truesize; in udp_destruct_sock()
1607 skb_queue_head_init(&udp_sk(sk)->reader_queue); in udp_init_sock()
1608 sk->sk_destruct = udp_destruct_sock; in udp_init_sock()
1615 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { in skb_consume_udp()
1646 atomic_inc(&sk->sk_drops); in __first_packet_length()
1648 *total += skb->truesize; in __first_packet_length()
1659 * first_packet_length - return length of first packet in receive queue
1663 * Returns the length of found skb, or -1 if none is found.
1667 struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue; in first_packet_length()
1668 struct sk_buff_head *sk_queue = &sk->sk_receive_queue; in first_packet_length()
1673 spin_lock_bh(&rcvq->lock); in first_packet_length()
1676 spin_lock(&sk_queue->lock); in first_packet_length()
1678 spin_unlock(&sk_queue->lock); in first_packet_length()
1682 res = skb ? skb->len : -1; in first_packet_length()
1685 spin_unlock_bh(&rcvq->lock); in first_packet_length()
1693 int udp_ioctl(struct sock *sk, int cmd, unsigned long arg) in udp_ioctl() argument
1695 switch (cmd) { in udp_ioctl()
1711 return -ENOIOCTLCMD; in udp_ioctl()
1721 struct sk_buff_head *sk_queue = &sk->sk_receive_queue; in __skb_recv_udp()
1727 queue = &udp_sk(sk)->reader_queue; in __skb_recv_udp()
1737 error = -EAGAIN; in __skb_recv_udp()
1739 spin_lock_bh(&queue->lock); in __skb_recv_udp()
1745 spin_unlock_bh(&queue->lock); in __skb_recv_udp()
1750 spin_unlock_bh(&queue->lock); in __skb_recv_udp()
1755 * keep both queues locked to avoid re-acquiring in __skb_recv_udp()
1759 spin_lock(&sk_queue->lock); in __skb_recv_udp()
1766 spin_unlock(&sk_queue->lock); in __skb_recv_udp()
1767 spin_unlock_bh(&queue->lock); in __skb_recv_udp()
1780 !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue, in __skb_recv_udp()
1798 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); in udp_recvmsg()
1817 if (copied > ulen - off) in udp_recvmsg()
1818 copied = ulen - off; in udp_recvmsg()
1820 msg->msg_flags |= MSG_TRUNC; in udp_recvmsg()
1825 * coverage checksum (UDP-Lite), do it before the copy. in udp_recvmsg()
1829 (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { in udp_recvmsg()
1838 err = copy_linear_skb(skb, copied, off, &msg->msg_iter); in udp_recvmsg()
1844 if (err == -EINVAL) in udp_recvmsg()
1850 atomic_inc(&sk->sk_drops); in udp_recvmsg()
1866 sin->sin_family = AF_INET; in udp_recvmsg()
1867 sin->sin_port = udp_hdr(skb)->source; in udp_recvmsg()
1868 sin->sin_addr.s_addr = ip_hdr(skb)->saddr; in udp_recvmsg()
1869 memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); in udp_recvmsg()
1877 if (udp_sk(sk)->gro_enabled) in udp_recvmsg()
1880 if (inet->cmsg_flags) in udp_recvmsg()
1887 skb_consume_udp(sk, skb, peeking ? -err : err); in udp_recvmsg()
1891 if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags, in udp_recvmsg()
1900 msg->msg_flags &= ~MSG_TRUNC; in udp_recvmsg()
1911 return -EINVAL; in udp_pre_connect()
1921 * 1003.1g - break association. in __udp_disconnect()
1924 sk->sk_state = TCP_CLOSE; in __udp_disconnect()
1925 inet->inet_daddr = 0; in __udp_disconnect()
1926 inet->inet_dport = 0; in __udp_disconnect()
1928 sk->sk_bound_dev_if = 0; in __udp_disconnect()
1929 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) { in __udp_disconnect()
1931 if (sk->sk_prot->rehash && in __udp_disconnect()
1932 (sk->sk_userlocks & SOCK_BINDPORT_LOCK)) in __udp_disconnect()
1933 sk->sk_prot->rehash(sk); in __udp_disconnect()
1936 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) { in __udp_disconnect()
1937 sk->sk_prot->unhash(sk); in __udp_disconnect()
1938 inet->inet_sport = 0; in __udp_disconnect()
1957 struct udp_table *udptable = sk->sk_prot->h.udp_table; in udp_lib_unhash()
1961 udp_sk(sk)->udp_port_hash); in udp_lib_unhash()
1962 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); in udp_lib_unhash()
1964 spin_lock_bh(&hslot->lock); in udp_lib_unhash()
1965 if (rcu_access_pointer(sk->sk_reuseport_cb)) in udp_lib_unhash()
1968 hslot->count--; in udp_lib_unhash()
1969 inet_sk(sk)->inet_num = 0; in udp_lib_unhash()
1970 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in udp_lib_unhash()
1972 spin_lock(&hslot2->lock); in udp_lib_unhash()
1973 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); in udp_lib_unhash()
1974 hslot2->count--; in udp_lib_unhash()
1975 spin_unlock(&hslot2->lock); in udp_lib_unhash()
1977 spin_unlock_bh(&hslot->lock); in udp_lib_unhash()
1988 struct udp_table *udptable = sk->sk_prot->h.udp_table; in udp_lib_rehash()
1991 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); in udp_lib_rehash()
1993 udp_sk(sk)->udp_portaddr_hash = newhash; in udp_lib_rehash()
1996 rcu_access_pointer(sk->sk_reuseport_cb)) { in udp_lib_rehash()
1998 udp_sk(sk)->udp_port_hash); in udp_lib_rehash()
2000 spin_lock_bh(&hslot->lock); in udp_lib_rehash()
2001 if (rcu_access_pointer(sk->sk_reuseport_cb)) in udp_lib_rehash()
2005 spin_lock(&hslot2->lock); in udp_lib_rehash()
2006 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); in udp_lib_rehash()
2007 hslot2->count--; in udp_lib_rehash()
2008 spin_unlock(&hslot2->lock); in udp_lib_rehash()
2010 spin_lock(&nhslot2->lock); in udp_lib_rehash()
2011 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, in udp_lib_rehash()
2012 &nhslot2->head); in udp_lib_rehash()
2013 nhslot2->count++; in udp_lib_rehash()
2014 spin_unlock(&nhslot2->lock); in udp_lib_rehash()
2017 spin_unlock_bh(&hslot->lock); in udp_lib_rehash()
2026 inet_sk(sk)->inet_rcv_saddr, in udp_v4_rehash()
2027 inet_sk(sk)->inet_num); in udp_v4_rehash()
2035 if (inet_sk(sk)->inet_daddr) { in __udp_queue_rcv_skb()
2048 if (rc == -ENOMEM) in __udp_queue_rcv_skb()
2054 return -1; in __udp_queue_rcv_skb()
2061 * -1: error
2080 if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) { in udp_queue_rcv_one_skb()
2087 * up->encap_rcv() returns the following value: in udp_queue_rcv_one_skb()
2091 * <0 if skb should be resubmitted as proto -N in udp_queue_rcv_one_skb()
2095 encap_rcv = READ_ONCE(up->encap_rcv); in udp_queue_rcv_one_skb()
2108 return -ret; in udp_queue_rcv_one_skb()
2112 /* FALLTHROUGH -- it's a UDP Packet */ in udp_queue_rcv_one_skb()
2116 * UDP-Lite specific tests, ignored on UDP sockets in udp_queue_rcv_one_skb()
2118 if ((up->pcflag & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) { in udp_queue_rcv_one_skb()
2131 if (up->pcrlen == 0) { /* full coverage was set */ in udp_queue_rcv_one_skb()
2133 UDP_SKB_CB(skb)->cscov, skb->len); in udp_queue_rcv_one_skb()
2140 * Therefore the above ...()->partial_cov statement is essential. in udp_queue_rcv_one_skb()
2142 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) { in udp_queue_rcv_one_skb()
2144 UDP_SKB_CB(skb)->cscov, up->pcrlen); in udp_queue_rcv_one_skb()
2149 prefetch(&sk->sk_rmem_alloc); in udp_queue_rcv_one_skb()
2150 if (rcu_access_pointer(sk->sk_filter) && in udp_queue_rcv_one_skb()
2166 atomic_inc(&sk->sk_drops); in udp_queue_rcv_one_skb()
2168 return -1; in udp_queue_rcv_one_skb()
2180 __skb_push(skb, -skb_mac_offset(skb)); in udp_queue_rcv_skb()
2186 ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret); in udp_queue_rcv_skb()
2199 old = xchg((__force struct dst_entry **)&sk->sk_rx_dst, dst); in udp_sk_rx_dst_set()
2219 unsigned short hnum = ntohs(uh->dest); in __udp4_lib_mcast_deliver()
2221 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); in __udp4_lib_mcast_deliver()
2223 int dif = skb->dev->ifindex; in __udp4_lib_mcast_deliver()
2230 udptable->mask; in __udp4_lib_mcast_deliver()
2231 hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask; in __udp4_lib_mcast_deliver()
2233 hslot = &udptable->hash2[hash2]; in __udp4_lib_mcast_deliver()
2237 sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) { in __udp4_lib_mcast_deliver()
2238 if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr, in __udp4_lib_mcast_deliver()
2239 uh->source, saddr, dif, sdif, hnum)) in __udp4_lib_mcast_deliver()
2249 atomic_inc(&sk->sk_drops); in __udp4_lib_mcast_deliver()
2280 * including udp header and folding it to skb->csum.
2287 UDP_SKB_CB(skb)->partial_cov = 0; in udp4_csum_init()
2288 UDP_SKB_CB(skb)->cscov = skb->len; in udp4_csum_init()
2295 if (UDP_SKB_CB(skb)->partial_cov) { in udp4_csum_init()
2296 skb->csum = inet_compute_pseudo(skb, proto); in udp4_csum_init()
2304 err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check, in udp4_csum_init()
2309 if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) { in udp4_csum_init()
2311 if (skb->csum_complete_sw) in udp4_csum_init()
2315 * skb->csum is no longer the full packet checksum, in udp4_csum_init()
2332 if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk)) in udp_unicast_rcv_skb()
2338 * it wants the return to be -protocol, or 0 in udp_unicast_rcv_skb()
2341 return -ret; in udp_unicast_rcv_skb()
2357 struct net *net = dev_net(skb->dev); in __udp4_lib_rcv()
2367 ulen = ntohs(uh->len); in __udp4_lib_rcv()
2368 saddr = ip_hdr(skb)->saddr; in __udp4_lib_rcv()
2369 daddr = ip_hdr(skb)->daddr; in __udp4_lib_rcv()
2371 if (ulen > skb->len) in __udp4_lib_rcv()
2389 if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst)) in __udp4_lib_rcv()
2398 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST)) in __udp4_lib_rcv()
2402 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable); in __udp4_lib_rcv()
2427 &saddr, ntohs(uh->source), in __udp4_lib_rcv()
2428 ulen, skb->len, in __udp4_lib_rcv()
2429 &daddr, ntohs(uh->dest)); in __udp4_lib_rcv()
2439 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest), in __udp4_lib_rcv()
2462 if (hslot->count > 10) in __udp4_lib_mcast_demux_lookup()
2466 sk_for_each_rcu(sk, &hslot->head) { in __udp4_lib_mcast_demux_lookup()
2495 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { in __udp4_lib_demux_lookup()
2506 struct net *net = dev_net(skb->dev); in udp_v4_early_demux()
2512 int dif = skb->dev->ifindex; in udp_v4_early_demux()
2523 if (skb->pkt_type == PACKET_MULTICAST) { in udp_v4_early_demux()
2524 in_dev = __in_dev_get_rcu(skb->dev); in udp_v4_early_demux()
2529 ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr, in udp_v4_early_demux()
2530 iph->protocol); in udp_v4_early_demux()
2534 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr, in udp_v4_early_demux()
2535 uh->source, iph->saddr, in udp_v4_early_demux()
2537 } else if (skb->pkt_type == PACKET_HOST) { in udp_v4_early_demux()
2538 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr, in udp_v4_early_demux()
2539 uh->source, iph->saddr, dif, sdif); in udp_v4_early_demux()
2542 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) in udp_v4_early_demux()
2545 skb->sk = sk; in udp_v4_early_demux()
2546 skb->destructor = sock_efree; in udp_v4_early_demux()
2547 dst = rcu_dereference(sk->sk_rx_dst); in udp_v4_early_demux()
2563 if (!inet_sk(sk)->inet_daddr && in_dev) in udp_v4_early_demux()
2564 return ip_mc_validate_source(skb, iph->daddr, in udp_v4_early_demux()
2565 iph->saddr, in udp_v4_early_demux()
2566 iph->tos & IPTOS_RT_MASK, in udp_v4_early_demux()
2567 skb->dev, in_dev, &itag); in udp_v4_early_demux()
2587 if (up->encap_type) { in udp_destroy_sock()
2589 encap_destroy = READ_ONCE(up->encap_destroy); in udp_destroy_sock()
2593 if (up->encap_enabled) in udp_destroy_sock()
2611 return -EINVAL; in udp_lib_setsockopt()
2614 return -EFAULT; in udp_lib_setsockopt()
2621 WRITE_ONCE(up->corkflag, 1); in udp_lib_setsockopt()
2623 WRITE_ONCE(up->corkflag, 0); in udp_lib_setsockopt()
2637 if (sk->sk_family == AF_INET6) in udp_lib_setsockopt()
2638 up->encap_rcv = ipv6_stub->xfrm6_udp_encap_rcv; in udp_lib_setsockopt()
2641 up->encap_rcv = xfrm4_udp_encap_rcv; in udp_lib_setsockopt()
2645 up->encap_type = val; in udp_lib_setsockopt()
2647 udp_tunnel_encap_enable(sk->sk_socket); in udp_lib_setsockopt()
2651 err = -ENOPROTOOPT; in udp_lib_setsockopt()
2657 up->no_check6_tx = valbool; in udp_lib_setsockopt()
2661 up->no_check6_rx = valbool; in udp_lib_setsockopt()
2666 return -EINVAL; in udp_lib_setsockopt()
2667 WRITE_ONCE(up->gso_size, val); in udp_lib_setsockopt()
2675 udp_tunnel_encap_enable(sk->sk_socket); in udp_lib_setsockopt()
2676 up->gro_enabled = valbool; in udp_lib_setsockopt()
2677 up->accept_udp_l4 = valbool; in udp_lib_setsockopt()
2682 * UDP-Lite's partial checksum coverage (RFC 3828). in udp_lib_setsockopt()
2688 return -ENOPROTOOPT; in udp_lib_setsockopt()
2693 up->pcslen = val; in udp_lib_setsockopt()
2694 up->pcflag |= UDPLITE_SEND_CC; in udp_lib_setsockopt()
2702 return -ENOPROTOOPT; in udp_lib_setsockopt()
2707 up->pcrlen = val; in udp_lib_setsockopt()
2708 up->pcflag |= UDPLITE_RECV_CC; in udp_lib_setsockopt()
2712 err = -ENOPROTOOPT; in udp_lib_setsockopt()
2737 return -EFAULT; in udp_lib_getsockopt()
2742 return -EINVAL; in udp_lib_getsockopt()
2746 val = READ_ONCE(up->corkflag); in udp_lib_getsockopt()
2750 val = up->encap_type; in udp_lib_getsockopt()
2754 val = up->no_check6_tx; in udp_lib_getsockopt()
2758 val = up->no_check6_rx; in udp_lib_getsockopt()
2762 val = READ_ONCE(up->gso_size); in udp_lib_getsockopt()
2766 val = up->gro_enabled; in udp_lib_getsockopt()
2772 val = up->pcslen; in udp_lib_getsockopt()
2776 val = up->pcrlen; in udp_lib_getsockopt()
2780 return -ENOPROTOOPT; in udp_lib_getsockopt()
2784 return -EFAULT; in udp_lib_getsockopt()
2786 return -EFAULT; in udp_lib_getsockopt()
2800 * udp_poll - wait for a UDP event.
2801 * @file: - file struct
2802 * @sock: - socket
2803 * @wait: - poll table
2810 * to work around these arguably broken applications.
2815 struct sock *sk = sock->sk; in udp_poll()
2817 if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) in udp_poll()
2821 if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) && in udp_poll()
2822 !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1) in udp_poll()
2840 sk->sk_err = err; in udp_abort()
2841 sk->sk_error_report(sk); in udp_abort()
2881 /* ------------------------------------------------------------------------ */
2888 struct udp_iter_state *state = seq->private; in udp_get_first()
2891 if (state->bpf_seq_afinfo) in udp_get_first()
2892 afinfo = state->bpf_seq_afinfo; in udp_get_first()
2894 afinfo = PDE_DATA(file_inode(seq->file)); in udp_get_first()
2896 for (state->bucket = start; state->bucket <= afinfo->udp_table->mask; in udp_get_first()
2897 ++state->bucket) { in udp_get_first()
2898 struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket]; in udp_get_first()
2900 if (hlist_empty(&hslot->head)) in udp_get_first()
2903 spin_lock_bh(&hslot->lock); in udp_get_first()
2904 sk_for_each(sk, &hslot->head) { in udp_get_first()
2907 if (afinfo->family == AF_UNSPEC || in udp_get_first()
2908 sk->sk_family == afinfo->family) in udp_get_first()
2911 spin_unlock_bh(&hslot->lock); in udp_get_first()
2921 struct udp_iter_state *state = seq->private; in udp_get_next()
2924 if (state->bpf_seq_afinfo) in udp_get_next()
2925 afinfo = state->bpf_seq_afinfo; in udp_get_next()
2927 afinfo = PDE_DATA(file_inode(seq->file)); in udp_get_next()
2932 (afinfo->family != AF_UNSPEC && in udp_get_next()
2933 sk->sk_family != afinfo->family))); in udp_get_next()
2936 if (state->bucket <= afinfo->udp_table->mask) in udp_get_next()
2937 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); in udp_get_next()
2938 return udp_get_first(seq, state->bucket + 1); in udp_get_next()
2949 --pos; in udp_get_idx()
2955 struct udp_iter_state *state = seq->private; in udp_seq_start()
2956 state->bucket = MAX_UDP_PORTS; in udp_seq_start()
2958 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; in udp_seq_start()
2979 struct udp_iter_state *state = seq->private; in udp_seq_stop()
2981 if (state->bpf_seq_afinfo) in udp_seq_stop()
2982 afinfo = state->bpf_seq_afinfo; in udp_seq_stop()
2984 afinfo = PDE_DATA(file_inode(seq->file)); in udp_seq_stop()
2986 if (state->bucket <= afinfo->udp_table->mask) in udp_seq_stop()
2987 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); in udp_seq_stop()
2991 /* ------------------------------------------------------------------------ */
2996 __be32 dest = inet->inet_daddr; in udp4_format_sock()
2997 __be32 src = inet->inet_rcv_saddr; in udp4_format_sock()
2998 __u16 destp = ntohs(inet->inet_dport); in udp4_format_sock()
2999 __u16 srcp = ntohs(inet->inet_sport); in udp4_format_sock()
3003 bucket, src, srcp, dest, destp, sp->sk_state, in udp4_format_sock()
3009 refcount_read(&sp->sk_refcnt), sp, in udp4_format_sock()
3010 atomic_read(&sp->sk_drops)); in udp4_format_sock()
3018 "rx_queue tr tm->when retrnsmt uid timeout " in udp4_seq_show()
3021 struct udp_iter_state *state = seq->private; in udp4_seq_show()
3023 udp4_format_sock(v, seq, state->bucket); in udp4_seq_show()
3042 meta->seq_num--; /* skip SEQ_START_TOKEN */ in udp_prog_seq_show()
3052 struct udp_iter_state *state = seq->private; in bpf_iter_udp_seq_show()
3064 return udp_prog_seq_show(prog, &meta, v, uid, state->bucket); in bpf_iter_udp_seq_show()
3105 if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops, in udp4_proc_init_net()
3107 return -ENOMEM; in udp4_proc_init_net()
3113 remove_proc_entry("udp", net->proc_net); in udp4_proc_exit_net()
3154 table->hash = alloc_large_system_hash(name, in udp_table_init()
3159 &table->log, in udp_table_init()
3160 &table->mask, in udp_table_init()
3164 table->hash2 = table->hash + (table->mask + 1); in udp_table_init()
3165 for (i = 0; i <= table->mask; i++) { in udp_table_init()
3166 INIT_HLIST_HEAD(&table->hash[i].head); in udp_table_init()
3167 table->hash[i].count = 0; in udp_table_init()
3168 spin_lock_init(&table->hash[i].lock); in udp_table_init()
3170 for (i = 0; i <= table->mask; i++) { in udp_table_init()
3171 INIT_HLIST_HEAD(&table->hash2[i].head); in udp_table_init()
3172 table->hash2[i].count = 0; in udp_table_init()
3173 spin_lock_init(&table->hash2[i].lock); in udp_table_init()
3189 net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM; in __udp_sysctl_init()
3190 net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM; in __udp_sysctl_init()
3193 net->ipv4.sysctl_udp_l3mdev_accept = 0; in __udp_sysctl_init()
3219 return -ENOMEM; in DEFINE_BPF_ITER_FUNC()
3221 afinfo->family = AF_UNSPEC; in DEFINE_BPF_ITER_FUNC()
3222 afinfo->udp_table = &udp_table; in DEFINE_BPF_ITER_FUNC()
3223 st->bpf_seq_afinfo = afinfo; in DEFINE_BPF_ITER_FUNC()
3234 kfree(st->bpf_seq_afinfo); in bpf_iter_fini_udp()