Lines Matching refs:psock
53 static void kcm_abort_tx_psock(struct kcm_psock *psock, int err, in kcm_abort_tx_psock() argument
56 struct sock *csk = psock->sk; in kcm_abort_tx_psock()
57 struct kcm_mux *mux = psock->mux; in kcm_abort_tx_psock()
63 if (psock->tx_stopped) { in kcm_abort_tx_psock()
68 psock->tx_stopped = 1; in kcm_abort_tx_psock()
69 KCM_STATS_INCR(psock->stats.tx_aborts); in kcm_abort_tx_psock()
71 if (!psock->tx_kcm) { in kcm_abort_tx_psock()
73 list_del(&psock->psock_avail_list); in kcm_abort_tx_psock()
82 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_abort_tx_psock()
93 struct kcm_psock *psock) in kcm_update_rx_mux_stats() argument
96 psock->strp.stats.bytes - in kcm_update_rx_mux_stats()
97 psock->saved_rx_bytes); in kcm_update_rx_mux_stats()
99 psock->strp.stats.msgs - psock->saved_rx_msgs; in kcm_update_rx_mux_stats()
100 psock->saved_rx_msgs = psock->strp.stats.msgs; in kcm_update_rx_mux_stats()
101 psock->saved_rx_bytes = psock->strp.stats.bytes; in kcm_update_rx_mux_stats()
105 struct kcm_psock *psock) in kcm_update_tx_mux_stats() argument
108 psock->stats.tx_bytes - psock->saved_tx_bytes); in kcm_update_tx_mux_stats()
110 psock->stats.tx_msgs - psock->saved_tx_msgs; in kcm_update_tx_mux_stats()
111 psock->saved_tx_msgs = psock->stats.tx_msgs; in kcm_update_tx_mux_stats()
112 psock->saved_tx_bytes = psock->stats.tx_bytes; in kcm_update_tx_mux_stats()
124 struct kcm_psock *psock; in kcm_rcv_ready() local
140 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock, in kcm_rcv_ready()
143 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) { in kcm_rcv_ready()
152 list_del(&psock->psock_ready_list); in kcm_rcv_ready()
153 psock->ready_rx_msg = NULL; in kcm_rcv_ready()
157 strp_unpause(&psock->strp); in kcm_rcv_ready()
158 strp_check_rcv(&psock->strp); in kcm_rcv_ready()
252 static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, in reserve_rx_kcm() argument
255 struct kcm_mux *mux = psock->mux; in reserve_rx_kcm()
258 WARN_ON(psock->ready_rx_msg); in reserve_rx_kcm()
260 if (psock->rx_kcm) in reserve_rx_kcm()
261 return psock->rx_kcm; in reserve_rx_kcm()
265 if (psock->rx_kcm) { in reserve_rx_kcm()
267 return psock->rx_kcm; in reserve_rx_kcm()
270 kcm_update_rx_mux_stats(mux, psock); in reserve_rx_kcm()
273 psock->ready_rx_msg = head; in reserve_rx_kcm()
274 strp_pause(&psock->strp); in reserve_rx_kcm()
275 list_add_tail(&psock->psock_ready_list, in reserve_rx_kcm()
287 psock->rx_kcm = kcm; in reserve_rx_kcm()
289 WRITE_ONCE(kcm->rx_psock, psock); in reserve_rx_kcm()
304 static void unreserve_rx_kcm(struct kcm_psock *psock, in unreserve_rx_kcm() argument
307 struct kcm_sock *kcm = psock->rx_kcm; in unreserve_rx_kcm()
308 struct kcm_mux *mux = psock->mux; in unreserve_rx_kcm()
315 psock->rx_kcm = NULL; in unreserve_rx_kcm()
349 struct kcm_psock *psock; in psock_data_ready() local
353 psock = (struct kcm_psock *)sk->sk_user_data; in psock_data_ready()
354 if (likely(psock)) in psock_data_ready()
355 strp_data_ready(&psock->strp); in psock_data_ready()
363 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_rcv_strparser() local
367 kcm = reserve_rx_kcm(psock, skb); in kcm_rcv_strparser()
377 unreserve_rx_kcm(psock, false); in kcm_rcv_strparser()
384 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_parse_func_strparser() local
385 struct bpf_prog *prog = psock->bpf_prog; in kcm_parse_func_strparser()
394 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_read_sock_done() local
396 unreserve_rx_kcm(psock, true); in kcm_read_sock_done()
413 struct kcm_psock *psock; in psock_write_space() local
419 psock = (struct kcm_psock *)sk->sk_user_data; in psock_write_space()
420 if (unlikely(!psock)) in psock_write_space()
422 mux = psock->mux; in psock_write_space()
427 kcm = psock->tx_kcm; in psock_write_space()
442 struct kcm_psock *psock; in reserve_psock() local
444 psock = kcm->tx_psock; in reserve_psock()
448 if (psock) { in reserve_psock()
450 if (unlikely(psock->tx_stopped)) in reserve_psock()
461 psock = kcm->tx_psock; in reserve_psock()
462 if (unlikely(psock)) { in reserve_psock()
469 psock = list_first_entry(&mux->psocks_avail, in reserve_psock()
472 list_del(&psock->psock_avail_list); in reserve_psock()
477 kcm->tx_psock = psock; in reserve_psock()
478 psock->tx_kcm = kcm; in reserve_psock()
479 KCM_STATS_INCR(psock->stats.reserved); in reserve_psock()
488 return psock; in reserve_psock()
492 static void psock_now_avail(struct kcm_psock *psock) in psock_now_avail() argument
494 struct kcm_mux *mux = psock->mux; in psock_now_avail()
498 list_add_tail(&psock->psock_avail_list, in psock_now_avail()
506 psock->tx_kcm = kcm; in psock_now_avail()
513 kcm->tx_psock = psock; in psock_now_avail()
514 KCM_STATS_INCR(psock->stats.reserved); in psock_now_avail()
522 struct kcm_psock *psock; in unreserve_psock() local
527 psock = kcm->tx_psock; in unreserve_psock()
529 if (WARN_ON(!psock)) { in unreserve_psock()
536 kcm_update_tx_mux_stats(mux, psock); in unreserve_psock()
541 psock->tx_kcm = NULL; in unreserve_psock()
542 KCM_STATS_INCR(psock->stats.unreserved); in unreserve_psock()
544 if (unlikely(psock->tx_stopped)) { in unreserve_psock()
545 if (psock->done) { in unreserve_psock()
547 list_del(&psock->psock_list); in unreserve_psock()
549 sock_put(psock->sk); in unreserve_psock()
550 fput(psock->sk->sk_socket->file); in unreserve_psock()
551 kmem_cache_free(kcm_psockp, psock); in unreserve_psock()
561 psock_now_avail(psock); in unreserve_psock()
581 struct kcm_psock *psock; in kcm_write_msgs() local
589 psock = kcm->tx_psock; in kcm_write_msgs()
590 if (unlikely(psock && psock->tx_stopped)) { in kcm_write_msgs()
610 if (WARN_ON(!psock)) { in kcm_write_msgs()
623 psock = reserve_psock(kcm); in kcm_write_msgs()
624 if (!psock) in kcm_write_msgs()
650 ret = kernel_sendpage(psock->sk->sk_socket, in kcm_write_msgs()
674 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE, in kcm_write_msgs()
687 KCM_STATS_ADD(psock->stats.tx_bytes, ret); in kcm_write_msgs()
709 KCM_STATS_INCR(psock->stats.tx_msgs); in kcm_write_msgs()
1337 struct kcm_psock *psock = NULL, *tpsock; in kcm_attach() local
1366 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL); in kcm_attach()
1367 if (!psock) { in kcm_attach()
1372 psock->mux = mux; in kcm_attach()
1373 psock->sk = csk; in kcm_attach()
1374 psock->bpf_prog = prog; in kcm_attach()
1383 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1388 err = strp_init(&psock->strp, csk, &cb); in kcm_attach()
1391 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1395 psock->save_data_ready = csk->sk_data_ready; in kcm_attach()
1396 psock->save_write_space = csk->sk_write_space; in kcm_attach()
1397 psock->save_state_change = csk->sk_state_change; in kcm_attach()
1398 csk->sk_user_data = psock; in kcm_attach()
1417 list_add(&psock->psock_list, head); in kcm_attach()
1418 psock->index = index; in kcm_attach()
1422 psock_now_avail(psock); in kcm_attach()
1426 strp_check_rcv(&psock->strp); in kcm_attach()
1464 static void kcm_unattach(struct kcm_psock *psock) in kcm_unattach() argument
1466 struct sock *csk = psock->sk; in kcm_unattach()
1467 struct kcm_mux *mux = psock->mux; in kcm_unattach()
1476 csk->sk_data_ready = psock->save_data_ready; in kcm_unattach()
1477 csk->sk_write_space = psock->save_write_space; in kcm_unattach()
1478 csk->sk_state_change = psock->save_state_change; in kcm_unattach()
1479 strp_stop(&psock->strp); in kcm_unattach()
1481 if (WARN_ON(psock->rx_kcm)) { in kcm_unattach()
1492 if (psock->ready_rx_msg) { in kcm_unattach()
1493 list_del(&psock->psock_ready_list); in kcm_unattach()
1494 kfree_skb(psock->ready_rx_msg); in kcm_unattach()
1495 psock->ready_rx_msg = NULL; in kcm_unattach()
1505 strp_done(&psock->strp); in kcm_unattach()
1508 bpf_prog_put(psock->bpf_prog); in kcm_unattach()
1512 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats); in kcm_unattach()
1513 save_strp_stats(&psock->strp, &mux->aggregate_strp_stats); in kcm_unattach()
1517 if (psock->tx_kcm) { in kcm_unattach()
1529 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_unattach()
1532 if (!psock->tx_kcm) { in kcm_unattach()
1536 psock->done = 1; in kcm_unattach()
1542 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_unattach()
1546 if (!psock->tx_stopped) in kcm_unattach()
1547 list_del(&psock->psock_avail_list); in kcm_unattach()
1548 list_del(&psock->psock_list); in kcm_unattach()
1554 kmem_cache_free(kcm_psockp, psock); in kcm_unattach()
1564 struct kcm_psock *psock; in kcm_unattach_ioctl() local
1583 list_for_each_entry(psock, &mux->psocks, psock_list) { in kcm_unattach_ioctl()
1584 if (psock->sk != csk) in kcm_unattach_ioctl()
1589 if (psock->unattaching || WARN_ON(psock->done)) { in kcm_unattach_ioctl()
1594 psock->unattaching = 1; in kcm_unattach_ioctl()
1599 kcm_unattach(psock); in kcm_unattach_ioctl()
1712 struct kcm_psock *psock, *tmp_psock; in release_mux() local
1715 list_for_each_entry_safe(psock, tmp_psock, in release_mux()
1717 if (!WARN_ON(psock->unattaching)) in release_mux()
1718 kcm_unattach(psock); in release_mux()
1795 struct kcm_psock *psock; in kcm_release() local
1837 psock = kcm->tx_psock; in kcm_release()
1838 if (psock) { in kcm_release()
1843 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_release()