431 					struct sk_psock *psock,  in sk_psock_skb_ingress_enqueue()  argument
454 	sk_psock_queue_msg(psock, msg);  in sk_psock_skb_ingress_enqueue()
455 	sk_psock_data_ready(sk, psock);  in sk_psock_skb_ingress_enqueue()
459 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb);
461 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)  in sk_psock_skb_ingress()  argument
463 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress()
472 		return sk_psock_skb_ingress_self(psock, skb);  in sk_psock_skb_ingress()
484 	err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg);  in sk_psock_skb_ingress()
494 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb)  in sk_psock_skb_ingress_self()  argument
497 	struct sock *sk = psock->sk;  in sk_psock_skb_ingress_self()
504 	err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg);  in sk_psock_skb_ingress_self()
510 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,  in sk_psock_handle_skb()  argument
514 		if (!sock_writeable(psock->sk))  in sk_psock_handle_skb()
516 		return skb_send_sock_locked(psock->sk, skb, off, len);  in sk_psock_handle_skb()
518 	return sk_psock_skb_ingress(psock, skb);  in sk_psock_handle_skb()
523 	struct sk_psock *psock = container_of(work, struct sk_psock, work);  in sk_psock_backlog()  local
524 	struct sk_psock_work_state *state = &psock->work_state;  in sk_psock_backlog()
531 	lock_sock(psock->sk);  in sk_psock_backlog()
540 	while ((skb = skb_dequeue(&psock->ingress_skb))) {  in sk_psock_backlog()
547 			if (likely(psock->sk->sk_socket))  in sk_psock_backlog()
548 				ret = sk_psock_handle_skb(psock, skb, off,  in sk_psock_backlog()
558 				sk_psock_report_error(psock, ret ? -ret : EPIPE);  in sk_psock_backlog()
559 				sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_backlog()
571 	release_sock(psock->sk);  in sk_psock_backlog()
576 	struct sk_psock *psock;  in sk_psock_init()  local
582 		psock = ERR_PTR(-EINVAL);  in sk_psock_init()
587 		psock = ERR_PTR(-EBUSY);  in sk_psock_init()
591 	psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);  in sk_psock_init()
592 	if (!psock) {  in sk_psock_init()
593 		psock = ERR_PTR(-ENOMEM);  in sk_psock_init()
598 	psock->sk = sk;  in sk_psock_init()
599 	psock->eval = __SK_NONE;  in sk_psock_init()
600 	psock->sk_proto = prot;  in sk_psock_init()
601 	psock->saved_unhash = prot->unhash;  in sk_psock_init()
602 	psock->saved_close = prot->close;  in sk_psock_init()
603 	psock->saved_write_space = sk->sk_write_space;  in sk_psock_init()
605 	INIT_LIST_HEAD(&psock->link);  in sk_psock_init()
606 	spin_lock_init(&psock->link_lock);  in sk_psock_init()
608 	INIT_WORK(&psock->work, sk_psock_backlog);  in sk_psock_init()
609 	INIT_LIST_HEAD(&psock->ingress_msg);  in sk_psock_init()
610 	skb_queue_head_init(&psock->ingress_skb);  in sk_psock_init()
612 	sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_init()
613 	refcount_set(&psock->refcnt, 1);  in sk_psock_init()
615 	__rcu_assign_sk_user_data_with_flags(sk, psock,  in sk_psock_init()
622 	return psock;  in sk_psock_init()
626 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)  in sk_psock_link_pop()  argument
630 	spin_lock_bh(&psock->link_lock);  in sk_psock_link_pop()
631 	link = list_first_entry_or_null(&psock->link, struct sk_psock_link,  in sk_psock_link_pop()
635 	spin_unlock_bh(&psock->link_lock);  in sk_psock_link_pop()
639 void __sk_psock_purge_ingress_msg(struct sk_psock *psock)  in __sk_psock_purge_ingress_msg()  argument
643 	list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {  in __sk_psock_purge_ingress_msg()
645 		sk_msg_free(psock->sk, msg);  in __sk_psock_purge_ingress_msg()
650 static void sk_psock_zap_ingress(struct sk_psock *psock)  in sk_psock_zap_ingress()  argument
652 	__skb_queue_purge(&psock->ingress_skb);  in sk_psock_zap_ingress()
653 	__sk_psock_purge_ingress_msg(psock);  in sk_psock_zap_ingress()
656 static void sk_psock_link_destroy(struct sk_psock *psock)  in sk_psock_link_destroy()  argument
660 	list_for_each_entry_safe(link, tmp, &psock->link, list) {  in sk_psock_link_destroy()
668 	struct sk_psock *psock = container_of(gc, struct sk_psock, gc);  in sk_psock_destroy_deferred()  local
673 	if (psock->progs.skb_parser)  in sk_psock_destroy_deferred()
674 		strp_done(&psock->parser.strp);  in sk_psock_destroy_deferred()
676 	cancel_work_sync(&psock->work);  in sk_psock_destroy_deferred()
678 	psock_progs_drop(&psock->progs);  in sk_psock_destroy_deferred()
680 	sk_psock_link_destroy(psock);  in sk_psock_destroy_deferred()
681 	sk_psock_cork_free(psock);  in sk_psock_destroy_deferred()
682 	sk_psock_zap_ingress(psock);  in sk_psock_destroy_deferred()
684 	if (psock->sk_redir)  in sk_psock_destroy_deferred()
685 		sock_put(psock->sk_redir);  in sk_psock_destroy_deferred()
686 	sock_put(psock->sk);  in sk_psock_destroy_deferred()
687 	kfree(psock);  in sk_psock_destroy_deferred()
692 	struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);  in sk_psock_destroy()  local
694 	INIT_WORK(&psock->gc, sk_psock_destroy_deferred);  in sk_psock_destroy()
695 	schedule_work(&psock->gc);  in sk_psock_destroy()
698 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)  in sk_psock_drop()  argument
700 	sk_psock_cork_free(psock);  in sk_psock_drop()
701 	sk_psock_zap_ingress(psock);  in sk_psock_drop()
704 	sk_psock_restore_proto(sk, psock);  in sk_psock_drop()
706 	if (psock->progs.skb_parser)  in sk_psock_drop()
707 		sk_psock_stop_strp(sk, psock);  in sk_psock_drop()
708 	else if (psock->progs.skb_verdict)  in sk_psock_drop()
709 		sk_psock_stop_verdict(sk, psock);  in sk_psock_drop()
711 	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);  in sk_psock_drop()
713 	call_rcu(&psock->rcu, sk_psock_destroy);  in sk_psock_drop()
730 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,  in sk_psock_msg_verdict()  argument
737 	prog = READ_ONCE(psock->progs.msg_parser);  in sk_psock_msg_verdict()
747 	psock->apply_bytes = msg->apply_bytes;  in sk_psock_msg_verdict()
749 		if (psock->sk_redir)  in sk_psock_msg_verdict()
750 			sock_put(psock->sk_redir);  in sk_psock_msg_verdict()
751 		psock->sk_redir = msg->sk_redir;  in sk_psock_msg_verdict()
752 		if (!psock->sk_redir) {  in sk_psock_msg_verdict()
756 		sock_hold(psock->sk_redir);  in sk_psock_msg_verdict()
764 static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog,  in sk_psock_bpf_run()  argument
820 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)  in sk_psock_tls_strp_read()  argument
826 	prog = READ_ONCE(psock->progs.skb_verdict);  in sk_psock_tls_strp_read()
828 		skb->sk = psock->sk;  in sk_psock_tls_strp_read()
830 		ret = sk_psock_bpf_run(psock, prog, skb);  in sk_psock_tls_strp_read()
834 	sk_psock_tls_verdict_apply(skb, psock->sk, ret);  in sk_psock_tls_strp_read()
840 static void sk_psock_verdict_apply(struct sk_psock *psock,  in sk_psock_verdict_apply()  argument
849 		sk_other = psock->sk;  in sk_psock_verdict_apply()
851 		    !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {  in sk_psock_verdict_apply()
864 		if (skb_queue_empty(&psock->ingress_skb)) {  in sk_psock_verdict_apply()
865 			err = sk_psock_skb_ingress_self(psock, skb);  in sk_psock_verdict_apply()
868 			skb_queue_tail(&psock->ingress_skb, skb);  in sk_psock_verdict_apply()
869 			schedule_work(&psock->work);  in sk_psock_verdict_apply()
884 	struct sk_psock *psock;  in sk_psock_strp_read()  local
891 	psock = sk_psock(sk);  in sk_psock_strp_read()
892 	if (unlikely(!psock)) {  in sk_psock_strp_read()
896 	prog = READ_ONCE(psock->progs.skb_verdict);  in sk_psock_strp_read()
900 		ret = sk_psock_bpf_run(psock, prog, skb);  in sk_psock_strp_read()
904 	sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_strp_read()
916 	struct sk_psock *psock = sk_psock_from_strp(strp);  in sk_psock_strp_parse()  local
921 	prog = READ_ONCE(psock->progs.skb_parser);  in sk_psock_strp_parse()
923 		skb->sk = psock->sk;  in sk_psock_strp_parse()
924 		ret = sk_psock_bpf_run(psock, prog, skb);  in sk_psock_strp_parse()
934 	struct sk_psock *psock;  in sk_psock_strp_data_ready()  local
937 	psock = sk_psock(sk);  in sk_psock_strp_data_ready()
938 	if (likely(psock)) {  in sk_psock_strp_data_ready()
940 			psock->parser.saved_data_ready(sk);  in sk_psock_strp_data_ready()
943 			strp_data_ready(&psock->parser.strp);  in sk_psock_strp_data_ready()
954 	struct sk_psock *psock;  in sk_psock_verdict_recv()  local
967 	psock = sk_psock(sk);  in sk_psock_verdict_recv()
968 	if (unlikely(!psock)) {  in sk_psock_verdict_recv()
973 	prog = READ_ONCE(psock->progs.skb_verdict);  in sk_psock_verdict_recv()
977 		ret = sk_psock_bpf_run(psock, prog, skb);  in sk_psock_verdict_recv()
981 	sk_psock_verdict_apply(psock, skb, ret);  in sk_psock_verdict_recv()
1004 	struct sk_psock *psock;  in sk_psock_write_space()  local
1008 	psock = sk_psock(sk);  in sk_psock_write_space()
1009 	if (likely(psock)) {  in sk_psock_write_space()
1010 		if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))  in sk_psock_write_space()
1011 			schedule_work(&psock->work);  in sk_psock_write_space()
1012 		write_space = psock->saved_write_space;  in sk_psock_write_space()
1019 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_init_strp()  argument
1027 	psock->parser.enabled = false;  in sk_psock_init_strp()
1028 	return strp_init(&psock->parser.strp, sk, &cb);  in sk_psock_init_strp()
1031 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_verdict()  argument
1033 	struct sk_psock_parser *parser = &psock->parser;  in sk_psock_start_verdict()
1044 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_start_strp()  argument
1046 	struct sk_psock_parser *parser = &psock->parser;  in sk_psock_start_strp()
1057 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_strp()  argument
1059 	struct sk_psock_parser *parser = &psock->parser;  in sk_psock_stop_strp()
1070 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)  in sk_psock_stop_verdict()  argument
1072 	struct sk_psock_parser *parser = &psock->parser;  in sk_psock_stop_verdict()