Lines Matching refs:msk
29 struct mptcp_sock msk; member
48 static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) in __mptcp_nmpc_socket() argument
50 if (!msk->subflow || READ_ONCE(msk->can_ack)) in __mptcp_nmpc_socket()
53 return msk->subflow; in __mptcp_nmpc_socket()
80 static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk) in __mptcp_tcp_fallback() argument
82 sock_owned_by_me((const struct sock *)msk); in __mptcp_tcp_fallback()
84 if (likely(!__mptcp_check_fallback(msk))) in __mptcp_tcp_fallback()
87 return msk->first; in __mptcp_tcp_fallback()
90 static int __mptcp_socket_create(struct mptcp_sock *msk) in __mptcp_socket_create() argument
93 struct sock *sk = (struct sock *)msk; in __mptcp_socket_create()
101 msk->first = ssock->sk; in __mptcp_socket_create()
102 msk->subflow = ssock; in __mptcp_socket_create()
104 list_add(&subflow->node, &msk->conn_list); in __mptcp_socket_create()
110 RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); in __mptcp_socket_create()
141 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, in mptcp_ooo_try_coalesce() argument
147 return mptcp_try_coalesce((struct sock *)msk, to, from); in mptcp_ooo_try_coalesce()
154 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) in mptcp_data_queue_ofo() argument
156 struct sock *sk = (struct sock *)msk; in mptcp_data_queue_ofo()
165 max_seq = space > 0 ? space + msk->ack_seq : msk->ack_seq; in mptcp_data_queue_ofo()
167 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, in mptcp_data_queue_ofo()
168 RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_data_queue_ofo()
176 p = &msk->out_of_order_queue.rb_node; in mptcp_data_queue_ofo()
178 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { in mptcp_data_queue_ofo()
180 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
181 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
188 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { in mptcp_data_queue_ofo()
195 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { in mptcp_data_queue_ofo()
197 parent = &msk->ooo_last_skb->rbnode; in mptcp_data_queue_ofo()
229 &msk->out_of_order_queue); in mptcp_data_queue_ofo()
234 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { in mptcp_data_queue_ofo()
244 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
251 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); in mptcp_data_queue_ofo()
257 msk->ooo_last_skb = skb; in mptcp_data_queue_ofo()
264 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, in __mptcp_move_skb() argument
269 struct sock *sk = (struct sock *)msk; in __mptcp_move_skb()
296 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { in __mptcp_move_skb()
298 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); in __mptcp_move_skb()
306 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { in __mptcp_move_skb()
307 mptcp_data_queue_ofo(msk, skb); in __mptcp_move_skb()
330 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin_ack() local
332 if (__mptcp_check_fallback(msk)) in mptcp_check_data_fin_ack()
338 msk->write_seq == atomic64_read(&msk->snd_una)) { in mptcp_check_data_fin_ack()
341 WRITE_ONCE(msk->snd_data_fin_enable, 0); in mptcp_check_data_fin_ack()
365 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin() local
367 if (READ_ONCE(msk->rcv_data_fin) && in mptcp_pending_data_fin()
370 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); in mptcp_pending_data_fin()
372 if (msk->ack_seq == rcv_data_fin_seq) { in mptcp_pending_data_fin()
395 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin() local
398 if (__mptcp_check_fallback(msk) || !msk->first) in mptcp_check_data_fin()
417 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); in mptcp_check_data_fin()
418 WRITE_ONCE(msk->rcv_data_fin, 0); in mptcp_check_data_fin()
422 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_check_data_fin()
442 mptcp_for_each_subflow(msk, subflow) { in mptcp_check_data_fin()
460 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, in __mptcp_move_skbs_from_subflow() argument
465 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs_from_subflow()
472 pr_debug("msk=%p ssk=%p", msk, ssk); in __mptcp_move_skbs_from_subflow()
496 if (__mptcp_check_fallback(msk)) { in __mptcp_move_skbs_from_subflow()
518 if (__mptcp_move_skb(msk, ssk, skb, offset, len)) in __mptcp_move_skbs_from_subflow()
546 static bool mptcp_ofo_queue(struct mptcp_sock *msk) in mptcp_ofo_queue() argument
548 struct sock *sk = (struct sock *)msk; in mptcp_ofo_queue()
554 p = rb_first(&msk->out_of_order_queue); in mptcp_ofo_queue()
555 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); in mptcp_ofo_queue()
558 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) in mptcp_ofo_queue()
562 rb_erase(&skb->rbnode, &msk->out_of_order_queue); in mptcp_ofo_queue()
565 msk->ack_seq))) { in mptcp_ofo_queue()
573 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { in mptcp_ofo_queue()
574 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; in mptcp_ofo_queue()
578 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, in mptcp_ofo_queue()
583 msk->ack_seq = end_seq; in mptcp_ofo_queue()
592 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) in move_skbs_to_msk() argument
594 struct sock *sk = (struct sock *)msk; in move_skbs_to_msk()
605 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in move_skbs_to_msk()
606 mptcp_ofo_queue(msk); in move_skbs_to_msk()
614 schedule_work(&msk->work)) in move_skbs_to_msk()
626 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_data_ready() local
635 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_data_ready()
638 move_skbs_to_msk(msk, ssk)) in mptcp_data_ready()
653 move_skbs_to_msk(msk, ssk); in mptcp_data_ready()
660 static void __mptcp_flush_join_list(struct mptcp_sock *msk) in __mptcp_flush_join_list() argument
662 if (likely(list_empty(&msk->join_list))) in __mptcp_flush_join_list()
665 spin_lock_bh(&msk->join_list_lock); in __mptcp_flush_join_list()
666 list_splice_tail_init(&msk->join_list, &msk->conn_list); in __mptcp_flush_join_list()
667 spin_unlock_bh(&msk->join_list_lock); in __mptcp_flush_join_list()
699 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_subflow_eof() local
701 if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && in mptcp_subflow_eof()
702 schedule_work(&msk->work)) in mptcp_subflow_eof()
706 static void mptcp_check_for_eof(struct mptcp_sock *msk) in mptcp_check_for_eof() argument
709 struct sock *sk = (struct sock *)msk; in mptcp_check_for_eof()
712 mptcp_for_each_subflow(msk, subflow) in mptcp_check_for_eof()
722 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_check_for_eof()
727 static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) in mptcp_ext_cache_refill() argument
729 const struct sock *sk = (const struct sock *)msk; in mptcp_ext_cache_refill()
731 if (!msk->cached_ext) in mptcp_ext_cache_refill()
732 msk->cached_ext = __skb_ext_alloc(sk->sk_allocation); in mptcp_ext_cache_refill()
734 return !!msk->cached_ext; in mptcp_ext_cache_refill()
737 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) in mptcp_subflow_recv_lookup() argument
740 struct sock *sk = (struct sock *)msk; in mptcp_subflow_recv_lookup()
744 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_recv_lookup()
768 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, in mptcp_frag_can_collapse_to() argument
774 df->data_seq + df->data_len == msk->write_seq; in mptcp_frag_can_collapse_to()
792 static bool mptcp_is_writeable(struct mptcp_sock *msk) in mptcp_is_writeable() argument
796 if (!sk_stream_is_writeable((struct sock *)msk)) in mptcp_is_writeable()
799 mptcp_for_each_subflow(msk, subflow) { in mptcp_is_writeable()
808 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_clean_una() local
816 if (__mptcp_check_fallback(msk)) in mptcp_clean_una()
817 atomic64_set(&msk->snd_una, msk->write_seq); in mptcp_clean_una()
818 snd_una = atomic64_read(&msk->snd_una); in mptcp_clean_una()
820 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { in mptcp_clean_una()
848 if (mptcp_is_writeable(msk)) { in mptcp_clean_una()
875 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, in mptcp_carve_data_frag() argument
883 dfrag->data_seq = msk->write_seq; in mptcp_carve_data_frag()
898 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_frag() local
914 write_seq = &msk->write_seq; in mptcp_sendmsg_frag()
950 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); in mptcp_sendmsg_frag()
952 dfrag = mptcp_carve_data_frag(msk, pfrag, offset); in mptcp_sendmsg_frag()
999 list_add_tail(&dfrag->list, &msk->rtx_queue); in mptcp_sendmsg_frag()
1024 mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); in mptcp_sendmsg_frag()
1025 msk->cached_ext = NULL; in mptcp_sendmsg_frag()
1047 static void mptcp_nospace(struct mptcp_sock *msk) in mptcp_nospace() argument
1051 clear_bit(MPTCP_SEND_SPACE, &msk->flags); in mptcp_nospace()
1054 mptcp_for_each_subflow(msk, subflow) { in mptcp_nospace()
1087 static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk, in mptcp_subflow_get_send() argument
1097 sock_owned_by_me((struct sock *)msk); in mptcp_subflow_get_send()
1100 if (!mptcp_ext_cache_refill(msk)) in mptcp_subflow_get_send()
1103 if (__mptcp_check_fallback(msk)) { in mptcp_subflow_get_send()
1104 if (!msk->first) in mptcp_subflow_get_send()
1106 *sndbuf = msk->first->sk_sndbuf; in mptcp_subflow_get_send()
1107 return sk_stream_memory_free(msk->first) ? msk->first : NULL; in mptcp_subflow_get_send()
1111 if (msk->last_snd && msk->snd_burst > 0 && in mptcp_subflow_get_send()
1112 sk_stream_memory_free(msk->last_snd) && in mptcp_subflow_get_send()
1113 mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) { in mptcp_subflow_get_send()
1114 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1118 return msk->last_snd; in mptcp_subflow_get_send()
1126 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_send()
1149 msk, nr_active, send_info[0].ssk, send_info[0].ratio, in mptcp_subflow_get_send()
1157 msk->last_snd = send_info[0].ssk; in mptcp_subflow_get_send()
1158 msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE, in mptcp_subflow_get_send()
1159 sk_stream_wspace(msk->last_snd)); in mptcp_subflow_get_send()
1160 return msk->last_snd; in mptcp_subflow_get_send()
1165 static void ssk_check_wmem(struct mptcp_sock *msk) in ssk_check_wmem() argument
1167 if (unlikely(!mptcp_is_writeable(msk))) in ssk_check_wmem()
1168 mptcp_nospace(msk); in ssk_check_wmem()
1174 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg() local
1204 __mptcp_flush_join_list(msk); in mptcp_sendmsg()
1205 ssk = mptcp_subflow_get_send(msk, &sndbuf); in mptcp_sendmsg()
1222 mptcp_nospace(msk); in mptcp_sendmsg()
1229 ssk = mptcp_subflow_get_send(msk, &sndbuf); in mptcp_sendmsg()
1230 if (list_empty(&msk->conn_list)) { in mptcp_sendmsg()
1260 msk->snd_burst -= ret; in mptcp_sendmsg()
1269 !mptcp_ext_cache_refill(msk)) { in mptcp_sendmsg()
1319 ssk_check_wmem(msk); in mptcp_sendmsg()
1327 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_wait_data() local
1333 test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); in mptcp_wait_data()
1339 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, in __mptcp_recvmsg_mskq() argument
1343 struct sock *sk = (struct sock *)msk; in __mptcp_recvmsg_mskq()
1381 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) in mptcp_rcv_space_adjust() argument
1384 struct sock *sk = (struct sock *)msk; in mptcp_rcv_space_adjust()
1393 msk->rcvq_space.copied += copied; in mptcp_rcv_space_adjust()
1396 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); in mptcp_rcv_space_adjust()
1398 rtt_us = msk->rcvq_space.rtt_us; in mptcp_rcv_space_adjust()
1403 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1417 msk->rcvq_space.rtt_us = rtt_us; in mptcp_rcv_space_adjust()
1421 if (msk->rcvq_space.copied <= msk->rcvq_space.space) in mptcp_rcv_space_adjust()
1429 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; in mptcp_rcv_space_adjust()
1431 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1433 do_div(grow, msk->rcvq_space.space); in mptcp_rcv_space_adjust()
1455 mptcp_for_each_subflow(msk, subflow) { in mptcp_rcv_space_adjust()
1469 msk->rcvq_space.space = msk->rcvq_space.copied; in mptcp_rcv_space_adjust()
1471 msk->rcvq_space.copied = 0; in mptcp_rcv_space_adjust()
1472 msk->rcvq_space.time = mstamp; in mptcp_rcv_space_adjust()
1475 static bool __mptcp_move_skbs(struct mptcp_sock *msk) in __mptcp_move_skbs() argument
1481 if (((struct sock *)msk)->sk_state == TCP_CLOSE) in __mptcp_move_skbs()
1484 __mptcp_flush_join_list(msk); in __mptcp_move_skbs()
1486 struct sock *ssk = mptcp_subflow_recv_lookup(msk); in __mptcp_move_skbs()
1492 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); in __mptcp_move_skbs()
1496 if (mptcp_ofo_queue(msk) || moved > 0) { in __mptcp_move_skbs()
1497 mptcp_check_data_fin((struct sock *)msk); in __mptcp_move_skbs()
1506 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recvmsg() local
1519 __mptcp_flush_join_list(msk); in mptcp_recvmsg()
1524 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); in mptcp_recvmsg()
1534 __mptcp_move_skbs(msk)) in mptcp_recvmsg()
1556 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_recvmsg()
1557 mptcp_check_for_eof(msk); in mptcp_recvmsg()
1584 clear_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_recvmsg()
1589 if (unlikely(__mptcp_move_skbs(msk))) in mptcp_recvmsg()
1590 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_recvmsg()
1591 } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { in mptcp_recvmsg()
1593 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_recvmsg()
1597 msk, test_bit(MPTCP_DATA_READY, &msk->flags), in mptcp_recvmsg()
1599 mptcp_rcv_space_adjust(msk, copied); in mptcp_recvmsg()
1607 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_retransmit_handler() local
1609 if (atomic64_read(&msk->snd_una) == READ_ONCE(msk->write_seq)) { in mptcp_retransmit_handler()
1612 set_bit(MPTCP_WORK_RTX, &msk->flags); in mptcp_retransmit_handler()
1613 if (schedule_work(&msk->work)) in mptcp_retransmit_handler()
1642 static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) in mptcp_subflow_get_retrans() argument
1647 sock_owned_by_me((const struct sock *)msk); in mptcp_subflow_get_retrans()
1649 if (__mptcp_check_fallback(msk)) in mptcp_subflow_get_retrans()
1650 return msk->first; in mptcp_subflow_get_retrans()
1652 mptcp_for_each_subflow(msk, subflow) { in mptcp_subflow_get_retrans()
1707 static void pm_work(struct mptcp_sock *msk) in pm_work() argument
1709 struct mptcp_pm_data *pm = &msk->pm; in pm_work()
1711 spin_lock_bh(&msk->pm.lock); in pm_work()
1713 pr_debug("msk=%p status=%x", msk, pm->status); in pm_work()
1716 mptcp_pm_nl_add_addr_received(msk); in pm_work()
1720 mptcp_pm_nl_rm_addr_received(msk); in pm_work()
1724 mptcp_pm_nl_fully_established(msk); in pm_work()
1728 mptcp_pm_nl_subflow_established(msk); in pm_work()
1731 spin_unlock_bh(&msk->pm.lock); in pm_work()
1734 static void __mptcp_close_subflow(struct mptcp_sock *msk) in __mptcp_close_subflow() argument
1738 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { in __mptcp_close_subflow()
1744 __mptcp_close_ssk((struct sock *)msk, ssk, subflow, 0); in __mptcp_close_subflow()
1750 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); in mptcp_worker() local
1751 struct sock *ssk, *sk = &msk->sk.icsk_inet.sk; in mptcp_worker()
1764 __mptcp_flush_join_list(msk); in mptcp_worker()
1765 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) in mptcp_worker()
1766 __mptcp_close_subflow(msk); in mptcp_worker()
1768 __mptcp_move_skbs(msk); in mptcp_worker()
1770 if (msk->pm.status) in mptcp_worker()
1771 pm_work(msk); in mptcp_worker()
1773 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) in mptcp_worker()
1774 mptcp_check_for_eof(msk); in mptcp_worker()
1778 if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) in mptcp_worker()
1785 if (!mptcp_ext_cache_refill(msk)) in mptcp_worker()
1788 ssk = mptcp_subflow_get_retrans(msk); in mptcp_worker()
1808 if (!mptcp_ext_cache_refill(msk)) in mptcp_worker()
1833 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_init_sock() local
1835 spin_lock_init(&msk->join_list_lock); in __mptcp_init_sock()
1837 INIT_LIST_HEAD(&msk->conn_list); in __mptcp_init_sock()
1838 INIT_LIST_HEAD(&msk->join_list); in __mptcp_init_sock()
1839 INIT_LIST_HEAD(&msk->rtx_queue); in __mptcp_init_sock()
1840 __set_bit(MPTCP_SEND_SPACE, &msk->flags); in __mptcp_init_sock()
1841 INIT_WORK(&msk->work, mptcp_worker); in __mptcp_init_sock()
1842 msk->out_of_order_queue = RB_ROOT; in __mptcp_init_sock()
1844 msk->first = NULL; in __mptcp_init_sock()
1847 mptcp_pm_data_init(msk); in __mptcp_init_sock()
1850 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); in __mptcp_init_sock()
1883 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clear_xmit() local
1886 sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); in __mptcp_clear_xmit()
1888 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) in __mptcp_clear_xmit()
1894 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_cancel_work() local
1896 if (cancel_work_sync(&msk->work)) in mptcp_cancel_work()
1958 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_close() local
1971 if (__mptcp_check_fallback(msk)) { in mptcp_close()
1975 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in mptcp_close()
1976 WRITE_ONCE(msk->snd_data_fin_enable, 1); in mptcp_close()
1978 mptcp_for_each_subflow(msk, subflow) { in mptcp_close()
1994 spin_lock_bh(&msk->join_list_lock); in mptcp_close()
1995 list_splice_tail_init(&msk->join_list, &msk->conn_list); in mptcp_close()
1996 spin_unlock_bh(&msk->join_list_lock); in mptcp_close()
1997 list_splice_init(&msk->conn_list, &conn_list); in mptcp_close()
2015 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) in mptcp_copy_inaddrs() argument
2019 struct ipv6_pinfo *msk6 = inet6_sk(msk); in mptcp_copy_inaddrs()
2021 msk->sk_v6_daddr = ssk->sk_v6_daddr; in mptcp_copy_inaddrs()
2022 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; in mptcp_copy_inaddrs()
2030 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; in mptcp_copy_inaddrs()
2031 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; in mptcp_copy_inaddrs()
2032 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; in mptcp_copy_inaddrs()
2033 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; in mptcp_copy_inaddrs()
2034 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; in mptcp_copy_inaddrs()
2035 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; in mptcp_copy_inaddrs()
2063 struct mptcp_sock *msk; in mptcp_sk_clone() local
2076 msk = mptcp_sk(nsk); in mptcp_sk_clone()
2077 msk->local_key = subflow_req->local_key; in mptcp_sk_clone()
2078 msk->token = subflow_req->token; in mptcp_sk_clone()
2079 msk->subflow = NULL; in mptcp_sk_clone()
2080 WRITE_ONCE(msk->fully_established, false); in mptcp_sk_clone()
2082 msk->write_seq = subflow_req->idsn + 1; in mptcp_sk_clone()
2083 atomic64_set(&msk->snd_una, msk->write_seq); in mptcp_sk_clone()
2085 msk->can_ack = true; in mptcp_sk_clone()
2086 msk->remote_key = mp_opt->sndr_key; in mptcp_sk_clone()
2087 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); in mptcp_sk_clone()
2089 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_sk_clone()
2104 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) in mptcp_rcv_space_init() argument
2108 msk->rcvq_space.copied = 0; in mptcp_rcv_space_init()
2109 msk->rcvq_space.rtt_us = 0; in mptcp_rcv_space_init()
2111 msk->rcvq_space.time = tp->tcp_mstamp; in mptcp_rcv_space_init()
2114 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, in mptcp_rcv_space_init()
2116 if (msk->rcvq_space.space == 0) in mptcp_rcv_space_init()
2117 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; in mptcp_rcv_space_init()
2123 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_accept() local
2127 listener = __mptcp_nmpc_socket(msk); in mptcp_accept()
2133 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); in mptcp_accept()
2138 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); in mptcp_accept()
2160 msk = mptcp_sk(new_mptcp_sock); in mptcp_accept()
2161 msk->first = newsk; in mptcp_accept()
2165 list_add(&subflow->node, &msk->conn_list); in mptcp_accept()
2167 mptcp_rcv_space_init(msk, ssk); in mptcp_accept()
2182 void mptcp_destroy_common(struct mptcp_sock *msk) in mptcp_destroy_common() argument
2184 skb_rbtree_purge(&msk->out_of_order_queue); in mptcp_destroy_common()
2185 mptcp_token_destroy(msk); in mptcp_destroy_common()
2186 mptcp_pm_free_anno_list(msk); in mptcp_destroy_common()
2191 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_destroy() local
2193 if (msk->cached_ext) in mptcp_destroy()
2194 __skb_ext_put(msk->cached_ext); in mptcp_destroy()
2196 mptcp_destroy_common(msk); in mptcp_destroy()
2200 static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname, in mptcp_setsockopt_sol_socket() argument
2203 struct sock *sk = (struct sock *)msk; in mptcp_setsockopt_sol_socket()
2211 ssock = __mptcp_nmpc_socket(msk); in mptcp_setsockopt_sol_socket()
2231 static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, in mptcp_setsockopt_v6() argument
2234 struct sock *sk = (struct sock *)msk; in mptcp_setsockopt_v6()
2241 ssock = __mptcp_nmpc_socket(msk); in mptcp_setsockopt_v6()
2303 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_setsockopt() local
2306 pr_debug("msk=%p", msk); in mptcp_setsockopt()
2312 return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen); in mptcp_setsockopt()
2321 ssk = __mptcp_tcp_fallback(msk); in mptcp_setsockopt()
2327 return mptcp_setsockopt_v6(msk, optname, optval, optlen); in mptcp_setsockopt()
2335 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_getsockopt() local
2338 pr_debug("msk=%p", msk); in mptcp_getsockopt()
2347 ssk = __mptcp_tcp_fallback(msk); in mptcp_getsockopt()
2375 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_release_cb() local
2378 ssk = mptcp_subflow_recv_lookup(msk); in mptcp_release_cb()
2379 if (!ssk || !schedule_work(&msk->work)) in mptcp_release_cb()
2405 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_get_port() local
2408 ssock = __mptcp_nmpc_socket(msk); in mptcp_get_port()
2409 pr_debug("msk=%p, subflow=%p", msk, ssock); in mptcp_get_port()
2419 struct mptcp_sock *msk; in mptcp_finish_connect() local
2425 msk = mptcp_sk(sk); in mptcp_finish_connect()
2437 WRITE_ONCE(msk->remote_key, subflow->remote_key); in mptcp_finish_connect()
2438 WRITE_ONCE(msk->local_key, subflow->local_key); in mptcp_finish_connect()
2439 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); in mptcp_finish_connect()
2440 WRITE_ONCE(msk->ack_seq, ack_seq); in mptcp_finish_connect()
2441 WRITE_ONCE(msk->can_ack, 1); in mptcp_finish_connect()
2442 atomic64_set(&msk->snd_una, msk->write_seq); in mptcp_finish_connect()
2444 mptcp_pm_new_connection(msk, 0); in mptcp_finish_connect()
2446 mptcp_rcv_space_init(msk, ssk); in mptcp_finish_connect()
2461 struct mptcp_sock *msk = mptcp_sk(subflow->conn); in mptcp_finish_join() local
2462 struct sock *parent = (void *)msk; in mptcp_finish_join()
2466 pr_debug("msk=%p, subflow=%p", msk, subflow); in mptcp_finish_join()
2472 if (!msk->pm.server_side) in mptcp_finish_join()
2475 if (!mptcp_pm_allow_new_subflow(msk)) in mptcp_finish_join()
2483 spin_lock_bh(&msk->join_list_lock); in mptcp_finish_join()
2486 list_add_tail(&subflow->node, &msk->join_list); in mptcp_finish_join()
2487 spin_unlock_bh(&msk->join_list_lock); in mptcp_finish_join()
2497 subflow->map_seq = READ_ONCE(msk->ack_seq); in mptcp_finish_join()
2503 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_memory_free() local
2505 return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; in mptcp_memory_free()
2539 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_bind() local
2544 ssock = __mptcp_nmpc_socket(msk); in mptcp_bind()
2559 static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, in mptcp_subflow_early_fallback() argument
2563 __mptcp_do_fallback(msk); in mptcp_subflow_early_fallback()
2569 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_connect() local
2575 if (sock->state != SS_UNCONNECTED && msk->subflow) { in mptcp_stream_connect()
2579 ssock = msk->subflow; in mptcp_stream_connect()
2583 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_connect()
2589 mptcp_token_destroy(msk); in mptcp_stream_connect()
2597 mptcp_subflow_early_fallback(msk, subflow); in mptcp_stream_connect()
2600 mptcp_subflow_early_fallback(msk, subflow); in mptcp_stream_connect()
2621 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_listen() local
2625 pr_debug("msk=%p", msk); in mptcp_listen()
2628 ssock = __mptcp_nmpc_socket(msk); in mptcp_listen()
2634 mptcp_token_destroy(msk); in mptcp_listen()
2651 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_accept() local
2655 pr_debug("msk=%p", msk); in mptcp_stream_accept()
2661 ssock = __mptcp_nmpc_socket(msk); in mptcp_stream_accept()
2665 clear_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_stream_accept()
2671 struct mptcp_sock *msk = mptcp_sk(newsock->sk); in mptcp_stream_accept() local
2677 __mptcp_flush_join_list(msk); in mptcp_stream_accept()
2678 mptcp_for_each_subflow(msk, subflow) { in mptcp_stream_accept()
2687 set_bit(MPTCP_DATA_READY, &msk->flags); in mptcp_stream_accept()
2696 static __poll_t mptcp_check_readable(struct mptcp_sock *msk) in mptcp_check_readable() argument
2698 return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : in mptcp_check_readable()
2706 struct mptcp_sock *msk; in mptcp_poll() local
2710 msk = mptcp_sk(sk); in mptcp_poll()
2714 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); in mptcp_poll()
2716 return mptcp_check_readable(msk); in mptcp_poll()
2719 mask |= mptcp_check_readable(msk); in mptcp_poll()
2720 if (test_bit(MPTCP_SEND_SPACE, &msk->flags)) in mptcp_poll()
2731 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_shutdown() local
2735 pr_debug("sk=%p, how=%d", msk, how); in mptcp_shutdown()
2754 if (__mptcp_check_fallback(msk)) { in mptcp_shutdown()
2758 mptcp_for_each_subflow(msk, subflow) { in mptcp_shutdown()
2768 __mptcp_flush_join_list(msk); in mptcp_shutdown()
2770 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); in mptcp_shutdown()
2771 WRITE_ONCE(msk->snd_data_fin_enable, 1); in mptcp_shutdown()
2773 mptcp_for_each_subflow(msk, subflow) { in mptcp_shutdown()