14 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
16 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
20 	int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);  in __skb_udp_tunnel_segment()
23 	struct udphdr *uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
24 	u16 mac_offset = skb->mac_header;  in __skb_udp_tunnel_segment()
25 	__be16 protocol = skb->protocol;  in __skb_udp_tunnel_segment()
26 	u16 mac_len = skb->mac_len;  in __skb_udp_tunnel_segment()
31 	if (unlikely(!pskb_may_pull(skb, tnl_hlen)))  in __skb_udp_tunnel_segment()
40 	if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)  in __skb_udp_tunnel_segment()
43 		partial = (__force __wsum)htonl(skb->len);  in __skb_udp_tunnel_segment()
47 	skb->encapsulation = 0;  in __skb_udp_tunnel_segment()
48 	SKB_GSO_CB(skb)->encap_level = 0;  in __skb_udp_tunnel_segment()
49 	__skb_pull(skb, tnl_hlen);  in __skb_udp_tunnel_segment()
50 	skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
51 	skb_set_network_header(skb, skb_inner_network_offset(skb));  in __skb_udp_tunnel_segment()
52 	skb->mac_len = skb_inner_network_offset(skb);  in __skb_udp_tunnel_segment()
53 	skb->protocol = new_protocol;  in __skb_udp_tunnel_segment()
55 	need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);  in __skb_udp_tunnel_segment()
56 	skb->encap_hdr_csum = need_csum;  in __skb_udp_tunnel_segment()
58 	remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);  in __skb_udp_tunnel_segment()
59 	skb->remcsum_offload = remcsum;  in __skb_udp_tunnel_segment()
61 	need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));  in __skb_udp_tunnel_segment()
65 			  (skb->dev->features &  in __skb_udp_tunnel_segment()
69 	features &= skb->dev->hw_enc_features;  in __skb_udp_tunnel_segment()
82 	segs = gso_inner_segment(skb, features);  in __skb_udp_tunnel_segment()
84 		skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,  in __skb_udp_tunnel_segment()
91 	outer_hlen = skb_tnl_header_len(skb);  in __skb_udp_tunnel_segment()
93 	skb = segs;  in __skb_udp_tunnel_segment()
98 			skb->ip_summed = CHECKSUM_NONE;  in __skb_udp_tunnel_segment()
101 		if (skb->ip_summed == CHECKSUM_PARTIAL) {  in __skb_udp_tunnel_segment()
102 			skb_reset_inner_headers(skb);  in __skb_udp_tunnel_segment()
103 			skb->encapsulation = 1;  in __skb_udp_tunnel_segment()
106 		skb->mac_len = mac_len;  in __skb_udp_tunnel_segment()
107 		skb->protocol = protocol;  in __skb_udp_tunnel_segment()
109 		__skb_push(skb, outer_hlen);  in __skb_udp_tunnel_segment()
110 		skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
111 		skb_set_network_header(skb, mac_len);  in __skb_udp_tunnel_segment()
112 		skb_set_transport_header(skb, udp_offset);  in __skb_udp_tunnel_segment()
113 		len = skb->len - udp_offset;  in __skb_udp_tunnel_segment()
114 		uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
120 		if (gso_partial && skb_is_gso(skb)) {  in __skb_udp_tunnel_segment()
121 			uh->len = htons(skb_shinfo(skb)->gso_size +  in __skb_udp_tunnel_segment()
122 					SKB_GSO_CB(skb)->data_offset +  in __skb_udp_tunnel_segment()
123 					skb->head - (unsigned char *)uh);  in __skb_udp_tunnel_segment()
134 		if (skb->encapsulation || !offload_csum) {  in __skb_udp_tunnel_segment()
135 			uh->check = gso_make_checksum(skb, ~uh->check);  in __skb_udp_tunnel_segment()
139 			skb->ip_summed = CHECKSUM_PARTIAL;  in __skb_udp_tunnel_segment()
140 			skb->csum_start = skb_transport_header(skb) - skb->head;  in __skb_udp_tunnel_segment()
141 			skb->csum_offset = offsetof(struct udphdr, check);  in __skb_udp_tunnel_segment()
143 	} while ((skb = skb->next));  in __skb_udp_tunnel_segment()
148 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,  in skb_udp_tunnel_segment()  argument
152 	__be16 protocol = skb->protocol;  in skb_udp_tunnel_segment()
156 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in skb_udp_tunnel_segment()
161 	switch (skb->inner_protocol_type) {  in skb_udp_tunnel_segment()
163 		protocol = skb->inner_protocol;  in skb_udp_tunnel_segment()
168 		ops = rcu_dereference(offloads[skb->inner_ipproto]);  in skb_udp_tunnel_segment()
177 	segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,  in skb_udp_tunnel_segment()
245 static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,  in __udp_gso_segment_list()  argument
249 	unsigned int mss = skb_shinfo(skb)->gso_size;  in __udp_gso_segment_list()
251 	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));  in __udp_gso_segment_list()
252 	if (IS_ERR(skb))  in __udp_gso_segment_list()
253 		return skb;  in __udp_gso_segment_list()
255 	udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);  in __udp_gso_segment_list()
257 	return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);  in __udp_gso_segment_list()
364 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,  in udp4_ufo_fragment()  argument
373 	if (skb->encapsulation &&  in udp4_ufo_fragment()
374 	    (skb_shinfo(skb)->gso_type &  in udp4_ufo_fragment()
376 		segs = skb_udp_tunnel_segment(skb, features, false);  in udp4_ufo_fragment()
380 	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))  in udp4_ufo_fragment()
383 	if (!pskb_may_pull(skb, sizeof(struct udphdr)))  in udp4_ufo_fragment()
386 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)  in udp4_ufo_fragment()
387 		return __udp_gso_segment(skb, features, false);  in udp4_ufo_fragment()
389 	mss = skb_shinfo(skb)->gso_size;  in udp4_ufo_fragment()
390 	if (unlikely(skb->len <= mss))  in udp4_ufo_fragment()
398 	uh = udp_hdr(skb);  in udp4_ufo_fragment()
399 	iph = ip_hdr(skb);  in udp4_ufo_fragment()
402 	csum = skb_checksum(skb, 0, skb->len, 0);  in udp4_ufo_fragment()
403 	uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);  in udp4_ufo_fragment()
407 	skb->ip_summed = CHECKSUM_UNNECESSARY;  in udp4_ufo_fragment()
413 	if (!skb->encap_hdr_csum)  in udp4_ufo_fragment()
419 	segs = skb_segment(skb, features);  in udp4_ufo_fragment()
426 					       struct sk_buff *skb)  in udp_gro_receive_segment()  argument
428 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp_gro_receive_segment()
437 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
443 	if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {  in udp_gro_receive_segment()
444 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
448 	skb_gro_pull(skb, sizeof(struct udphdr));  in udp_gro_receive_segment()
462 		if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {  in udp_gro_receive_segment()
463 			NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
476 			if (NAPI_GRO_CB(skb)->is_flist) {  in udp_gro_receive_segment()
477 				if (!pskb_may_pull(skb, skb_gro_offset(skb))) {  in udp_gro_receive_segment()
478 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
481 				if ((skb->ip_summed != p->ip_summed) ||  in udp_gro_receive_segment()
482 				    (skb->csum_level != p->csum_level)) {  in udp_gro_receive_segment()
483 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
486 				ret = skb_gro_receive_list(p, skb);  in udp_gro_receive_segment()
488 				skb_gro_postpull_rcsum(skb, uh,  in udp_gro_receive_segment()
491 				ret = skb_gro_receive(p, skb);  in udp_gro_receive_segment()
506 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,  in udp_gro_receive()  argument
512 	unsigned int off = skb_gro_offset(skb);  in udp_gro_receive()
515 	NAPI_GRO_CB(skb)->is_flist = 0;  in udp_gro_receive()
516 	if (skb->dev->features & NETIF_F_GRO_FRAGLIST)  in udp_gro_receive()
517 		NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled: 1;  in udp_gro_receive()
519 	if ((sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist) {  in udp_gro_receive()
520 		pp = call_gro_receive(udp_gro_receive_segment, head, skb);  in udp_gro_receive()
524 	if (!sk || NAPI_GRO_CB(skb)->encap_mark ||  in udp_gro_receive()
525 	    (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&  in udp_gro_receive()
526 	     NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in udp_gro_receive()
527 	     !NAPI_GRO_CB(skb)->csum_valid) ||  in udp_gro_receive()
532 	NAPI_GRO_CB(skb)->encap_mark = 1;  in udp_gro_receive()
552 	skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */  in udp_gro_receive()
553 	skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));  in udp_gro_receive()
554 	pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);  in udp_gro_receive()
557 	skb_gro_flush_final(skb, pp, flush);  in udp_gro_receive()
562 static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,  in udp4_gro_lookup_skb()  argument
565 	const struct iphdr *iph = skb_gro_network_header(skb);  in udp4_gro_lookup_skb()
567 	return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,  in udp4_gro_lookup_skb()
568 				 iph->daddr, dport, inet_iif(skb),  in udp4_gro_lookup_skb()
569 				 inet_sdif(skb), &udp_table, NULL);  in udp4_gro_lookup_skb()
573 struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)  in udp4_gro_receive()  argument
575 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp4_gro_receive()
583 	if (NAPI_GRO_CB(skb)->flush)  in udp4_gro_receive()
586 	if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,  in udp4_gro_receive()
590 		skb_gro_checksum_try_convert(skb, IPPROTO_UDP,  in udp4_gro_receive()
593 	NAPI_GRO_CB(skb)->is_ipv6 = 0;  in udp4_gro_receive()
597 		sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);  in udp4_gro_receive()
599 	pp = udp_gro_receive(head, skb, uh, sk);  in udp4_gro_receive()
604 	NAPI_GRO_CB(skb)->flush = 1;  in udp4_gro_receive()
608 static int udp_gro_complete_segment(struct sk_buff *skb)  in udp_gro_complete_segment()  argument
610 	struct udphdr *uh = udp_hdr(skb);  in udp_gro_complete_segment()
612 	skb->csum_start = (unsigned char *)uh - skb->head;  in udp_gro_complete_segment()
613 	skb->csum_offset = offsetof(struct udphdr, check);  in udp_gro_complete_segment()
614 	skb->ip_summed = CHECKSUM_PARTIAL;  in udp_gro_complete_segment()
616 	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp_gro_complete_segment()
617 	skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;  in udp_gro_complete_segment()
619 	if (skb->encapsulation)  in udp_gro_complete_segment()
620 		skb->inner_transport_header = skb->transport_header;  in udp_gro_complete_segment()
625 int udp_gro_complete(struct sk_buff *skb, int nhoff,  in udp_gro_complete()  argument
628 	__be16 newlen = htons(skb->len - nhoff);  in udp_gro_complete()
629 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp_gro_complete()
637 				udp4_lib_lookup_skb, skb, uh->source, uh->dest);  in udp_gro_complete()
639 		skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM  in udp_gro_complete()
645 		skb->encapsulation = 1;  in udp_gro_complete()
646 		err = udp_sk(sk)->gro_complete(sk, skb,  in udp_gro_complete()
649 		err = udp_gro_complete_segment(skb);  in udp_gro_complete()
653 	if (skb->remcsum_offload)  in udp_gro_complete()
654 		skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;  in udp_gro_complete()
660 INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)  in udp4_gro_complete()  argument
662 	const struct iphdr *iph = ip_hdr(skb);  in udp4_gro_complete()
663 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp4_gro_complete()
665 	if (NAPI_GRO_CB(skb)->is_flist) {  in udp4_gro_complete()
666 		uh->len = htons(skb->len - nhoff);  in udp4_gro_complete()
668 		skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);  in udp4_gro_complete()
669 		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp4_gro_complete()
671 		if (skb->ip_summed == CHECKSUM_UNNECESSARY) {  in udp4_gro_complete()
672 			if (skb->csum_level < SKB_MAX_CSUM_LEVEL)  in udp4_gro_complete()
673 				skb->csum_level++;  in udp4_gro_complete()
675 			skb->ip_summed = CHECKSUM_UNNECESSARY;  in udp4_gro_complete()
676 			skb->csum_level = 0;  in udp4_gro_complete()
683 		uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,  in udp4_gro_complete()
686 	return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);  in udp4_gro_complete()