57 	struct kcm_mux *mux = psock->mux;  in kcm_abort_tx_psock()  local
61 	spin_lock_bh(&mux->lock);  in kcm_abort_tx_psock()
64 		spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
85 	spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
91 /* RX mux lock held. */
92 static void kcm_update_rx_mux_stats(struct kcm_mux *mux,  in kcm_update_rx_mux_stats()  argument
95 	STRP_STATS_ADD(mux->stats.rx_bytes,  in kcm_update_rx_mux_stats()
98 	mux->stats.rx_msgs +=  in kcm_update_rx_mux_stats()
104 static void kcm_update_tx_mux_stats(struct kcm_mux *mux,  in kcm_update_tx_mux_stats()  argument
107 	KCM_STATS_ADD(mux->stats.tx_bytes,  in kcm_update_tx_mux_stats()
109 	mux->stats.tx_msgs +=  in kcm_update_tx_mux_stats()
119  * pending ready messages on a psock. RX mux lock held.
123 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()  local
130 	while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {  in kcm_rcv_ready()
133 			skb_queue_head(&mux->rx_hold_queue, skb);  in kcm_rcv_ready()
139 	while (!list_empty(&mux->psocks_ready)) {  in kcm_rcv_ready()
140 		psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,  in kcm_rcv_ready()
163 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
172 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()  local
183 		spin_lock_bh(&mux->rx_lock);  in kcm_rfree()
185 		spin_unlock_bh(&mux->rx_lock);  in kcm_rfree()
217  * RX mux lock held.
219 static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)  in requeue_rx_msgs()  argument
229 		if (list_empty(&mux->kcm_rx_waiters)) {  in requeue_rx_msgs()
230 			skb_queue_tail(&mux->rx_hold_queue, skb);  in requeue_rx_msgs()
234 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
255 	struct kcm_mux *mux = psock->mux;  in reserve_rx_kcm()  local
263 	spin_lock_bh(&mux->rx_lock);  in reserve_rx_kcm()
266 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
270 	kcm_update_rx_mux_stats(mux, psock);  in reserve_rx_kcm()
272 	if (list_empty(&mux->kcm_rx_waiters)) {  in reserve_rx_kcm()
276 			      &mux->psocks_ready);  in reserve_rx_kcm()
277 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
281 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
291 	spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
308 	struct kcm_mux *mux = psock->mux;  in unreserve_rx_kcm()  local
313 	spin_lock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
325 		spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
336 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
343 	spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
414 	struct kcm_mux *mux;  in psock_write_space()  local
422 	mux = psock->mux;  in psock_write_space()
424 	spin_lock_bh(&mux->lock);  in psock_write_space()
431 	spin_unlock_bh(&mux->lock);  in psock_write_space()
441 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()  local
456 	spin_lock_bh(&mux->lock);  in reserve_psock()
464 		spin_unlock_bh(&mux->lock);  in reserve_psock()
468 	if (!list_empty(&mux->psocks_avail)) {  in reserve_psock()
469 		psock = list_first_entry(&mux->psocks_avail,  in reserve_psock()
482 			      &mux->kcm_tx_waiters);  in reserve_psock()
486 	spin_unlock_bh(&mux->lock);  in reserve_psock()
491 /* mux lock held */
494 	struct kcm_mux *mux = psock->mux;  in psock_now_avail()  local
497 	if (list_empty(&mux->kcm_tx_waiters)) {  in psock_now_avail()
499 			      &mux->psocks_avail);  in psock_now_avail()
501 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
523 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()  local
525 	spin_lock_bh(&mux->lock);  in unreserve_psock()
530 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
536 	kcm_update_tx_mux_stats(mux, psock);  in unreserve_psock()
548 			mux->psocks_cnt--;  in unreserve_psock()
556 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
563 	spin_unlock_bh(&mux->lock);  in unreserve_psock()
568 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()  local
570 	spin_lock_bh(&mux->lock);  in kcm_report_tx_retry()
571 	KCM_STATS_INCR(mux->stats.tx_retries);  in kcm_report_tx_retry()
572 	spin_unlock_bh(&mux->lock);  in kcm_report_tx_retry()
1190 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()  local
1195 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_disable()
1207 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1210 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_disable()
1216 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()  local
1221 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_enable()
1226 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_enable()
1294 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1306 	/* Add to mux's kcm sockets list */  in init_kcm_sock()
1307 	kcm->mux = mux;  in init_kcm_sock()
1308 	spin_lock_bh(&mux->lock);  in init_kcm_sock()
1310 	head = &mux->kcm_socks;  in init_kcm_sock()
1311 	list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {  in init_kcm_sock()
1321 	mux->kcm_socks_cnt++;  in init_kcm_sock()
1322 	spin_unlock_bh(&mux->lock);  in init_kcm_sock()
1326 	spin_lock_bh(&mux->rx_lock);  in init_kcm_sock()
1328 	spin_unlock_bh(&mux->rx_lock);  in init_kcm_sock()
1335 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()  local
1372 	psock->mux = mux;  in kcm_attach()
1407 	/* Finished initialization, now add the psock to the MUX. */  in kcm_attach()
1408 	spin_lock_bh(&mux->lock);  in kcm_attach()
1409 	head = &mux->psocks;  in kcm_attach()
1410 	list_for_each_entry(tpsock, &mux->psocks, psock_list) {  in kcm_attach()
1420 	KCM_STATS_INCR(mux->stats.psock_attach);  in kcm_attach()
1421 	mux->psocks_cnt++;  in kcm_attach()
1423 	spin_unlock_bh(&mux->lock);  in kcm_attach()
1467 	struct kcm_mux *mux = psock->mux;  in kcm_unattach()  local
1487 	spin_lock_bh(&mux->rx_lock);  in kcm_unattach()
1496 		KCM_STATS_INCR(mux->stats.rx_ready_drops);  in kcm_unattach()
1499 	spin_unlock_bh(&mux->rx_lock);  in kcm_unattach()
1510 	spin_lock_bh(&mux->lock);  in kcm_unattach()
1512 	aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);  in kcm_unattach()
1513 	save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);  in kcm_unattach()
1515 	KCM_STATS_INCR(mux->stats.psock_unattach);  in kcm_unattach()
1522 		KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);  in kcm_unattach()
1523 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1527 		 * to do this without the mux lock.  in kcm_unattach()
1531 		spin_lock_bh(&mux->lock);  in kcm_unattach()
1533 			/* psock now unreserved in window mux was unlocked */  in kcm_unattach()
1543 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1549 		mux->psocks_cnt--;  in kcm_unattach()
1550 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1563 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()  local
1581 	spin_lock_bh(&mux->lock);  in kcm_unattach_ioctl()
1583 	list_for_each_entry(psock, &mux->psocks, psock_list) {  in kcm_unattach_ioctl()
1596 		spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1605 	spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1640 	init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);  in kcm_clone()
1703 	struct kcm_mux *mux = container_of(rcu,  in free_mux()  local
1706 	kmem_cache_free(kcm_muxp, mux);  in free_mux()
1709 static void release_mux(struct kcm_mux *mux)  in release_mux()  argument
1711 	struct kcm_net *knet = mux->knet;  in release_mux()
1716 				 &mux->psocks, psock_list) {  in release_mux()
1721 	if (WARN_ON(mux->psocks_cnt))  in release_mux()
1724 	__skb_queue_purge(&mux->rx_hold_queue);  in release_mux()
1727 	aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);  in release_mux()
1728 	aggregate_psock_stats(&mux->aggregate_psock_stats,  in release_mux()
1730 	aggregate_strp_stats(&mux->aggregate_strp_stats,  in release_mux()
1732 	list_del_rcu(&mux->kcm_mux_list);  in release_mux()
1736 	call_rcu(&mux->rcu, free_mux);  in release_mux()
1741 	struct kcm_mux *mux = kcm->mux;  in kcm_done()  local
1745 	spin_lock_bh(&mux->rx_lock);  in kcm_done()
1751 		spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1761 	requeue_rx_msgs(mux, &sk->sk_receive_queue);  in kcm_done()
1763 	spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1768 	/* Detach from MUX */  in kcm_done()
1769 	spin_lock_bh(&mux->lock);  in kcm_done()
1772 	mux->kcm_socks_cnt--;  in kcm_done()
1773 	socks_cnt = mux->kcm_socks_cnt;  in kcm_done()
1775 	spin_unlock_bh(&mux->lock);  in kcm_done()
1778 		/* We are done with the mux now. */  in kcm_done()
1779 		release_mux(mux);  in kcm_done()
1788  * If this is the last KCM socket on the MUX, destroy the MUX.
1794 	struct kcm_mux *mux;  in kcm_release()  local
1801 	mux = kcm->mux;  in kcm_release()
1821 	spin_lock_bh(&mux->lock);  in kcm_release()
1829 	spin_unlock_bh(&mux->lock);  in kcm_release()
1907 	struct kcm_mux *mux;  in kcm_create()  local
1927 	/* Allocate a kcm mux, shared between KCM sockets */  in kcm_create()
1928 	mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);  in kcm_create()
1929 	if (!mux) {  in kcm_create()
1934 	spin_lock_init(&mux->lock);  in kcm_create()
1935 	spin_lock_init(&mux->rx_lock);  in kcm_create()
1936 	INIT_LIST_HEAD(&mux->kcm_socks);  in kcm_create()
1937 	INIT_LIST_HEAD(&mux->kcm_rx_waiters);  in kcm_create()
1938 	INIT_LIST_HEAD(&mux->kcm_tx_waiters);  in kcm_create()
1940 	INIT_LIST_HEAD(&mux->psocks);  in kcm_create()
1941 	INIT_LIST_HEAD(&mux->psocks_ready);  in kcm_create()
1942 	INIT_LIST_HEAD(&mux->psocks_avail);  in kcm_create()
1944 	mux->knet = knet;  in kcm_create()
1946 	/* Add new MUX to list */  in kcm_create()
1948 	list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);  in kcm_create()
1952 	skb_queue_head_init(&mux->rx_hold_queue);  in kcm_create()
1956 	init_kcm_sock(kcm_sk(sk), mux);  in kcm_create()