31 #define indirect_call_gro_receive_l4(f2, f1, cb, head, skb)	\  argument
33 	unlikely(gro_recursion_inc_test(skb)) ?			\
34 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
35 		INDIRECT_CALL_L4(cb, f2, f1, head, skb);	\
38 static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)  in ipv6_gso_pull_exthdrs()  argument
56 		if (unlikely(!pskb_may_pull(skb, 8)))  in ipv6_gso_pull_exthdrs()
59 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
62 		if (unlikely(!pskb_may_pull(skb, len)))  in ipv6_gso_pull_exthdrs()
65 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
67 		__skb_pull(skb, len);  in ipv6_gso_pull_exthdrs()
73 static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,  in ipv6_gso_segment()  argument
88 	skb_reset_network_header(skb);  in ipv6_gso_segment()
89 	nhoff = skb_network_header(skb) - skb_mac_header(skb);  in ipv6_gso_segment()
90 	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))  in ipv6_gso_segment()
93 	encap = SKB_GSO_CB(skb)->encap_level > 0;  in ipv6_gso_segment()
95 		features &= skb->dev->hw_enc_features;  in ipv6_gso_segment()
96 	SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);  in ipv6_gso_segment()
98 	ipv6h = ipv6_hdr(skb);  in ipv6_gso_segment()
99 	__skb_pull(skb, sizeof(*ipv6h));  in ipv6_gso_segment()
102 	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);  in ipv6_gso_segment()
104 	if (skb->encapsulation &&  in ipv6_gso_segment()
105 	    skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))  in ipv6_gso_segment()
107 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
109 		udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&  in ipv6_gso_segment()
110 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
114 		skb_reset_transport_header(skb);  in ipv6_gso_segment()
115 		segs = ops->callbacks.gso_segment(skb, features);  in ipv6_gso_segment()
117 			skb->network_header = skb_mac_header(skb) + nhoff - skb->head;  in ipv6_gso_segment()
125 	for (skb = segs; skb; skb = skb->next) {  in ipv6_gso_segment()
126 		ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);  in ipv6_gso_segment()
127 		if (gso_partial && skb_is_gso(skb))  in ipv6_gso_segment()
128 			payload_len = skb_shinfo(skb)->gso_size +  in ipv6_gso_segment()
129 				      SKB_GSO_CB(skb)->data_offset +  in ipv6_gso_segment()
130 				      skb->head - (unsigned char *)(ipv6h + 1);  in ipv6_gso_segment()
132 			payload_len = skb->len - nhoff - sizeof(*ipv6h);  in ipv6_gso_segment()
134 		skb->network_header = (u8 *)ipv6h - skb->head;  in ipv6_gso_segment()
135 		skb_reset_mac_len(skb);  in ipv6_gso_segment()
138 			int err = ip6_find_1stfragopt(skb, &prevhdr);  in ipv6_gso_segment()
145 			if (skb->next)  in ipv6_gso_segment()
151 			skb_reset_inner_headers(skb);  in ipv6_gso_segment()
185 							 struct sk_buff *skb)  in ipv6_gro_receive()  argument
197 	off = skb_gro_offset(skb);  in ipv6_gro_receive()
199 	iph = skb_gro_header_fast(skb, off);  in ipv6_gro_receive()
200 	if (skb_gro_header_hard(skb, hlen)) {  in ipv6_gro_receive()
201 		iph = skb_gro_header_slow(skb, hlen, off);  in ipv6_gro_receive()
206 	skb_set_network_header(skb, off);  in ipv6_gro_receive()
207 	skb_gro_pull(skb, sizeof(*iph));  in ipv6_gro_receive()
208 	skb_set_transport_header(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
210 	flush += ntohs(iph->payload_len) != skb_gro_len(skb);  in ipv6_gro_receive()
216 		__pskb_pull(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
217 		skb_gro_frag0_invalidate(skb);  in ipv6_gro_receive()
218 		proto = ipv6_gso_pull_exthdrs(skb, proto);  in ipv6_gro_receive()
219 		skb_gro_pull(skb, -skb_transport_offset(skb));  in ipv6_gro_receive()
220 		skb_reset_transport_header(skb);  in ipv6_gro_receive()
221 		__skb_push(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
227 		iph = ipv6_hdr(skb);  in ipv6_gro_receive()
230 	NAPI_GRO_CB(skb)->proto = proto;  in ipv6_gro_receive()
233 	nlen = skb_network_header_len(skb);  in ipv6_gro_receive()
271 		if (NAPI_GRO_CB(skb)->is_atomic)  in ipv6_gro_receive()
275 	NAPI_GRO_CB(skb)->is_atomic = true;  in ipv6_gro_receive()
276 	NAPI_GRO_CB(skb)->flush |= flush;  in ipv6_gro_receive()
278 	skb_gro_postpull_rcsum(skb, iph, nlen);  in ipv6_gro_receive()
281 					 ops->callbacks.gro_receive, head, skb);  in ipv6_gro_receive()
287 	skb_gro_flush_final(skb, pp, flush);  in ipv6_gro_receive()
293 					      struct sk_buff *skb)  in sit_ip6ip6_gro_receive()  argument
297 	if (NAPI_GRO_CB(skb)->encap_mark) {  in sit_ip6ip6_gro_receive()
298 		NAPI_GRO_CB(skb)->flush = 1;  in sit_ip6ip6_gro_receive()
302 	NAPI_GRO_CB(skb)->encap_mark = 1;  in sit_ip6ip6_gro_receive()
304 	return ipv6_gro_receive(head, skb);  in sit_ip6ip6_gro_receive()
308 					  struct sk_buff *skb)  in ip4ip6_gro_receive()  argument
312 	if (NAPI_GRO_CB(skb)->encap_mark) {  in ip4ip6_gro_receive()
313 		NAPI_GRO_CB(skb)->flush = 1;  in ip4ip6_gro_receive()
317 	NAPI_GRO_CB(skb)->encap_mark = 1;  in ip4ip6_gro_receive()
319 	return inet_gro_receive(head, skb);  in ip4ip6_gro_receive()
322 INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)  in ipv6_gro_complete()  argument
325 	struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + nhoff);  in ipv6_gro_complete()
328 	if (skb->encapsulation) {  in ipv6_gro_complete()
329 		skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));  in ipv6_gro_complete()
330 		skb_set_inner_network_header(skb, nhoff);  in ipv6_gro_complete()
333 	iph->payload_len = htons(skb->len - nhoff - sizeof(*iph));  in ipv6_gro_complete()
342 			       udp6_gro_complete, skb, nhoff);  in ipv6_gro_complete()
350 static int sit_gro_complete(struct sk_buff *skb, int nhoff)  in sit_gro_complete()  argument
352 	skb->encapsulation = 1;  in sit_gro_complete()
353 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;  in sit_gro_complete()
354 	return ipv6_gro_complete(skb, nhoff);  in sit_gro_complete()
357 static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip6ip6_gro_complete()  argument
359 	skb->encapsulation = 1;  in ip6ip6_gro_complete()
360 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip6ip6_gro_complete()
361 	return ipv6_gro_complete(skb, nhoff);  in ip6ip6_gro_complete()
364 static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip4ip6_gro_complete()  argument
366 	skb->encapsulation = 1;  in ip4ip6_gro_complete()
367 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip4ip6_gro_complete()
368 	return inet_gro_complete(skb, nhoff);  in ip4ip6_gro_complete()
380 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,  in sit_gso_segment()  argument
383 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))  in sit_gso_segment()
386 	return ipv6_gso_segment(skb, features);  in sit_gso_segment()
389 static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,  in ip4ip6_gso_segment()  argument
392 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip4ip6_gso_segment()
395 	return inet_gso_segment(skb, features);  in ip4ip6_gso_segment()
398 static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,  in ip6ip6_gso_segment()  argument
401 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip6ip6_gso_segment()
404 	return ipv6_gso_segment(skb, features);  in ip6ip6_gso_segment()