13 int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,  in __tcp_bpf_recvmsg()  argument
21 	msg_rx = list_first_entry_or_null(&psock->ingress_msg,  in __tcp_bpf_recvmsg()
71 			if (msg_rx == list_last_entry(&psock->ingress_msg,  in __tcp_bpf_recvmsg()
85 		msg_rx = list_first_entry_or_null(&psock->ingress_msg,  in __tcp_bpf_recvmsg()
93 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,  in bpf_tcp_ingress()  argument
135 		sk_psock_queue_msg(psock, tmp);  in bpf_tcp_ingress()
136 		sk_psock_data_ready(sk, psock);  in bpf_tcp_ingress()
218 	struct sk_psock *psock = sk_psock_get(sk);  in tcp_bpf_sendmsg_redir()  local
221 	if (unlikely(!psock))  in tcp_bpf_sendmsg_redir()
224 	ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :  in tcp_bpf_sendmsg_redir()
226 	sk_psock_put(sk, psock);  in tcp_bpf_sendmsg_redir()
234 	struct sk_psock *psock;  in tcp_bpf_stream_read()  local
238 	psock = sk_psock(sk);  in tcp_bpf_stream_read()
239 	if (likely(psock))  in tcp_bpf_stream_read()
240 		empty = list_empty(&psock->ingress_msg);  in tcp_bpf_stream_read()
245 static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,  in tcp_bpf_wait_data()  argument
260 			    !list_empty(&psock->ingress_msg) ||  in tcp_bpf_wait_data()
270 	struct sk_psock *psock;  in tcp_bpf_recvmsg()  local
276 	psock = sk_psock_get(sk);  in tcp_bpf_recvmsg()
277 	if (unlikely(!psock))  in tcp_bpf_recvmsg()
280 	    sk_psock_queue_empty(psock)) {  in tcp_bpf_recvmsg()
281 		sk_psock_put(sk, psock);  in tcp_bpf_recvmsg()
286 	copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg()
292 		data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);  in tcp_bpf_recvmsg()
294 			if (!sk_psock_queue_empty(psock))  in tcp_bpf_recvmsg()
297 			sk_psock_put(sk, psock);  in tcp_bpf_recvmsg()
309 	sk_psock_put(sk, psock);  in tcp_bpf_recvmsg()
313 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,  in tcp_bpf_send_verdict()  argument
323 	if (psock->eval == __SK_NONE) {  in tcp_bpf_send_verdict()
330 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);  in tcp_bpf_send_verdict()
336 		psock->cork_bytes = msg->cork_bytes - msg->sg.size;  in tcp_bpf_send_verdict()
337 		if (!psock->cork) {  in tcp_bpf_send_verdict()
338 			psock->cork = kzalloc(sizeof(*psock->cork),  in tcp_bpf_send_verdict()
340 			if (!psock->cork)  in tcp_bpf_send_verdict()
343 		memcpy(psock->cork, msg, sizeof(*msg));  in tcp_bpf_send_verdict()
348 	if (psock->apply_bytes && psock->apply_bytes < tosend)  in tcp_bpf_send_verdict()
349 		tosend = psock->apply_bytes;  in tcp_bpf_send_verdict()
351 	switch (psock->eval) {  in tcp_bpf_send_verdict()
358 		sk_msg_apply_bytes(psock, tosend);  in tcp_bpf_send_verdict()
361 		sk_redir = psock->sk_redir;  in tcp_bpf_send_verdict()
362 		sk_msg_apply_bytes(psock, tosend);  in tcp_bpf_send_verdict()
363 		if (!psock->apply_bytes) {  in tcp_bpf_send_verdict()
365 			eval = psock->eval;  in tcp_bpf_send_verdict()
366 			psock->eval = __SK_NONE;  in tcp_bpf_send_verdict()
367 			psock->sk_redir = NULL;  in tcp_bpf_send_verdict()
369 		if (psock->cork) {  in tcp_bpf_send_verdict()
371 			psock->cork = NULL;  in tcp_bpf_send_verdict()
400 		sk_msg_apply_bytes(psock, tosend);  in tcp_bpf_send_verdict()
406 		if (!psock->apply_bytes) {  in tcp_bpf_send_verdict()
407 			psock->eval =  __SK_NONE;  in tcp_bpf_send_verdict()
408 			if (psock->sk_redir) {  in tcp_bpf_send_verdict()
409 				sock_put(psock->sk_redir);  in tcp_bpf_send_verdict()
410 				psock->sk_redir = NULL;  in tcp_bpf_send_verdict()
428 	struct sk_psock *psock;  in tcp_bpf_sendmsg()  local
436 	psock = sk_psock_get(sk);  in tcp_bpf_sendmsg()
437 	if (unlikely(!psock))  in tcp_bpf_sendmsg()
454 		if (psock->cork) {  in tcp_bpf_sendmsg()
455 			msg_tx = psock->cork;  in tcp_bpf_sendmsg()
478 		if (psock->cork_bytes) {  in tcp_bpf_sendmsg()
479 			if (size > psock->cork_bytes)  in tcp_bpf_sendmsg()
480 				psock->cork_bytes = 0;  in tcp_bpf_sendmsg()
482 				psock->cork_bytes -= size;  in tcp_bpf_sendmsg()
483 			if (psock->cork_bytes && !enospc)  in tcp_bpf_sendmsg()
486 			psock->eval = __SK_NONE;  in tcp_bpf_sendmsg()
487 			psock->cork_bytes = 0;  in tcp_bpf_sendmsg()
490 		err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);  in tcp_bpf_sendmsg()
499 			if (msg_tx && msg_tx != psock->cork)  in tcp_bpf_sendmsg()
508 	sk_psock_put(sk, psock);  in tcp_bpf_sendmsg()
517 	struct sk_psock *psock;  in tcp_bpf_sendpage()  local
520 	psock = sk_psock_get(sk);  in tcp_bpf_sendpage()
521 	if (unlikely(!psock))  in tcp_bpf_sendpage()
525 	if (psock->cork) {  in tcp_bpf_sendpage()
526 		msg = psock->cork;  in tcp_bpf_sendpage()
541 	if (psock->cork_bytes) {  in tcp_bpf_sendpage()
542 		if (size > psock->cork_bytes)  in tcp_bpf_sendpage()
543 			psock->cork_bytes = 0;  in tcp_bpf_sendpage()
545 			psock->cork_bytes -= size;  in tcp_bpf_sendpage()
546 		if (psock->cork_bytes && !enospc)  in tcp_bpf_sendpage()
549 		psock->eval = __SK_NONE;  in tcp_bpf_sendpage()
550 		psock->cork_bytes = 0;  in tcp_bpf_sendpage()
553 	err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);  in tcp_bpf_sendpage()
556 	sk_psock_put(sk, psock);  in tcp_bpf_sendpage()
619 struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)  in tcp_bpf_get_proto()  argument
622 	int config = psock->progs.msg_parser   ? TCP_BPF_TX   : TCP_BPF_BASE;  in tcp_bpf_get_proto()
625 		if (tcp_bpf_assert_proto_ops(psock->sk_proto))  in tcp_bpf_get_proto()
628 		tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);  in tcp_bpf_get_proto()