Lines Matching refs:bbr
87 struct bbr { struct
207 const struct bbr *bbr = inet_csk_ca(sk); in bbr_full_bw_reached() local
209 return bbr->full_bw_reached; in bbr_full_bw_reached()
215 struct bbr *bbr = inet_csk_ca(sk); in bbr_max_bw() local
217 return minmax_get(&bbr->bw); in bbr_max_bw()
223 struct bbr *bbr = inet_csk_ca(sk); in bbr_bw() local
225 return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk); in bbr_bw()
233 struct bbr *bbr = inet_csk_ca(sk); in bbr_extra_acked() local
235 return max(bbr->extra_acked[0], bbr->extra_acked[1]); in bbr_extra_acked()
267 struct bbr *bbr = inet_csk_ca(sk); in bbr_init_pacing_rate_from_rtt() local
273 bbr->has_seen_rtt = 1; in bbr_init_pacing_rate_from_rtt()
286 struct bbr *bbr = inet_csk_ca(sk); in bbr_set_pacing_rate() local
289 if (unlikely(!bbr->has_seen_rtt && tp->srtt_us)) in bbr_set_pacing_rate()
321 struct bbr *bbr = inet_csk_ca(sk); in bbr_save_cwnd() local
323 if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT) in bbr_save_cwnd()
324 bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */ in bbr_save_cwnd()
326 bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd); in bbr_save_cwnd()
332 struct bbr *bbr = inet_csk_ca(sk); in bbr_cwnd_event() local
335 bbr->idle_restart = 1; in bbr_cwnd_event()
336 bbr->ack_epoch_mstamp = tp->tcp_mstamp; in bbr_cwnd_event()
337 bbr->ack_epoch_acked = 0; in bbr_cwnd_event()
341 if (bbr->mode == BBR_PROBE_BW) in bbr_cwnd_event()
343 else if (bbr->mode == BBR_PROBE_RTT) in bbr_cwnd_event()
359 struct bbr *bbr = inet_csk_ca(sk); in bbr_bdp() local
369 if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */ in bbr_bdp()
372 w = (u64)bw * bbr->min_rtt_us; in bbr_bdp()
394 struct bbr *bbr = inet_csk_ca(sk); in bbr_quantization_budget() local
403 if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0) in bbr_quantization_budget()
437 struct bbr *bbr = inet_csk_ca(sk); in bbr_packets_in_net_at_edt() local
446 if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */ in bbr_packets_in_net_at_edt()
481 struct bbr *bbr = inet_csk_ca(sk); in bbr_set_cwnd_to_recover_or_restore() local
482 u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state; in bbr_set_cwnd_to_recover_or_restore()
494 bbr->packet_conservation = 1; in bbr_set_cwnd_to_recover_or_restore()
495 bbr->next_rtt_delivered = tp->delivered; /* start round now */ in bbr_set_cwnd_to_recover_or_restore()
500 cwnd = max(cwnd, bbr->prior_cwnd); in bbr_set_cwnd_to_recover_or_restore()
501 bbr->packet_conservation = 0; in bbr_set_cwnd_to_recover_or_restore()
503 bbr->prev_ca_state = state; in bbr_set_cwnd_to_recover_or_restore()
505 if (bbr->packet_conservation) { in bbr_set_cwnd_to_recover_or_restore()
520 struct bbr *bbr = inet_csk_ca(sk); in bbr_set_cwnd() local
546 if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */ in bbr_set_cwnd()
555 struct bbr *bbr = inet_csk_ca(sk); in bbr_is_next_cycle_phase() local
557 tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp) > in bbr_is_next_cycle_phase()
558 bbr->min_rtt_us; in bbr_is_next_cycle_phase()
564 if (bbr->pacing_gain == BBR_UNIT) in bbr_is_next_cycle_phase()
575 if (bbr->pacing_gain > BBR_UNIT) in bbr_is_next_cycle_phase()
578 inflight >= bbr_inflight(sk, bw, bbr->pacing_gain)); in bbr_is_next_cycle_phase()
591 struct bbr *bbr = inet_csk_ca(sk); in bbr_advance_cycle_phase() local
593 bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1); in bbr_advance_cycle_phase()
594 bbr->cycle_mstamp = tp->delivered_mstamp; in bbr_advance_cycle_phase()
601 struct bbr *bbr = inet_csk_ca(sk); in bbr_update_cycle_phase() local
603 if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs)) in bbr_update_cycle_phase()
609 struct bbr *bbr = inet_csk_ca(sk); in bbr_reset_startup_mode() local
611 bbr->mode = BBR_STARTUP; in bbr_reset_startup_mode()
616 struct bbr *bbr = inet_csk_ca(sk); in bbr_reset_probe_bw_mode() local
618 bbr->mode = BBR_PROBE_BW; in bbr_reset_probe_bw_mode()
619 bbr->cycle_idx = CYCLE_LEN - 1 - prandom_u32_max(bbr_cycle_rand); in bbr_reset_probe_bw_mode()
635 struct bbr *bbr = inet_csk_ca(sk); in bbr_reset_lt_bw_sampling_interval() local
637 bbr->lt_last_stamp = div_u64(tp->delivered_mstamp, USEC_PER_MSEC); in bbr_reset_lt_bw_sampling_interval()
638 bbr->lt_last_delivered = tp->delivered; in bbr_reset_lt_bw_sampling_interval()
639 bbr->lt_last_lost = tp->lost; in bbr_reset_lt_bw_sampling_interval()
640 bbr->lt_rtt_cnt = 0; in bbr_reset_lt_bw_sampling_interval()
646 struct bbr *bbr = inet_csk_ca(sk); in bbr_reset_lt_bw_sampling() local
648 bbr->lt_bw = 0; in bbr_reset_lt_bw_sampling()
649 bbr->lt_use_bw = 0; in bbr_reset_lt_bw_sampling()
650 bbr->lt_is_sampling = false; in bbr_reset_lt_bw_sampling()
657 struct bbr *bbr = inet_csk_ca(sk); in bbr_lt_bw_interval_done() local
660 if (bbr->lt_bw) { /* do we have bw from a previous interval? */ in bbr_lt_bw_interval_done()
662 diff = abs(bw - bbr->lt_bw); in bbr_lt_bw_interval_done()
663 if ((diff * BBR_UNIT <= bbr_lt_bw_ratio * bbr->lt_bw) || in bbr_lt_bw_interval_done()
667 bbr->lt_bw = (bw + bbr->lt_bw) >> 1; /* avg 2 intvls */ in bbr_lt_bw_interval_done()
668 bbr->lt_use_bw = 1; in bbr_lt_bw_interval_done()
669 bbr->pacing_gain = BBR_UNIT; /* try to avoid drops */ in bbr_lt_bw_interval_done()
670 bbr->lt_rtt_cnt = 0; in bbr_lt_bw_interval_done()
674 bbr->lt_bw = bw; in bbr_lt_bw_interval_done()
688 struct bbr *bbr = inet_csk_ca(sk); in bbr_lt_bw_sampling() local
693 if (bbr->lt_use_bw) { /* already using long-term rate, lt_bw? */ in bbr_lt_bw_sampling()
694 if (bbr->mode == BBR_PROBE_BW && bbr->round_start && in bbr_lt_bw_sampling()
695 ++bbr->lt_rtt_cnt >= bbr_lt_bw_max_rtts) { in bbr_lt_bw_sampling()
706 if (!bbr->lt_is_sampling) { in bbr_lt_bw_sampling()
710 bbr->lt_is_sampling = true; in bbr_lt_bw_sampling()
719 if (bbr->round_start) in bbr_lt_bw_sampling()
720 bbr->lt_rtt_cnt++; /* count round trips in this interval */ in bbr_lt_bw_sampling()
721 if (bbr->lt_rtt_cnt < bbr_lt_intvl_min_rtts) in bbr_lt_bw_sampling()
723 if (bbr->lt_rtt_cnt > 4 * bbr_lt_intvl_min_rtts) { in bbr_lt_bw_sampling()
736 lost = tp->lost - bbr->lt_last_lost; in bbr_lt_bw_sampling()
737 delivered = tp->delivered - bbr->lt_last_delivered; in bbr_lt_bw_sampling()
743 t = div_u64(tp->delivered_mstamp, USEC_PER_MSEC) - bbr->lt_last_stamp; in bbr_lt_bw_sampling()
761 struct bbr *bbr = inet_csk_ca(sk); in bbr_update_bw() local
764 bbr->round_start = 0; in bbr_update_bw()
769 if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) { in bbr_update_bw()
770 bbr->next_rtt_delivered = tp->delivered; in bbr_update_bw()
771 bbr->rtt_cnt++; in bbr_update_bw()
772 bbr->round_start = 1; in bbr_update_bw()
773 bbr->packet_conservation = 0; in bbr_update_bw()
797 minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw); in bbr_update_bw()
818 struct bbr *bbr = inet_csk_ca(sk); in bbr_update_ack_aggregation() local
825 if (bbr->round_start) { in bbr_update_ack_aggregation()
826 bbr->extra_acked_win_rtts = min(0x1F, in bbr_update_ack_aggregation()
827 bbr->extra_acked_win_rtts + 1); in bbr_update_ack_aggregation()
828 if (bbr->extra_acked_win_rtts >= bbr_extra_acked_win_rtts) { in bbr_update_ack_aggregation()
829 bbr->extra_acked_win_rtts = 0; in bbr_update_ack_aggregation()
830 bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ? in bbr_update_ack_aggregation()
832 bbr->extra_acked[bbr->extra_acked_win_idx] = 0; in bbr_update_ack_aggregation()
838 bbr->ack_epoch_mstamp); in bbr_update_ack_aggregation()
845 if (bbr->ack_epoch_acked <= expected_acked || in bbr_update_ack_aggregation()
846 (bbr->ack_epoch_acked + rs->acked_sacked >= in bbr_update_ack_aggregation()
848 bbr->ack_epoch_acked = 0; in bbr_update_ack_aggregation()
849 bbr->ack_epoch_mstamp = tp->delivered_mstamp; in bbr_update_ack_aggregation()
854 bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, in bbr_update_ack_aggregation()
855 bbr->ack_epoch_acked + rs->acked_sacked); in bbr_update_ack_aggregation()
856 extra_acked = bbr->ack_epoch_acked - expected_acked; in bbr_update_ack_aggregation()
858 if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) in bbr_update_ack_aggregation()
859 bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked; in bbr_update_ack_aggregation()
873 struct bbr *bbr = inet_csk_ca(sk); in bbr_check_full_bw_reached() local
876 if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited) in bbr_check_full_bw_reached()
879 bw_thresh = (u64)bbr->full_bw * bbr_full_bw_thresh >> BBR_SCALE; in bbr_check_full_bw_reached()
881 bbr->full_bw = bbr_max_bw(sk); in bbr_check_full_bw_reached()
882 bbr->full_bw_cnt = 0; in bbr_check_full_bw_reached()
885 ++bbr->full_bw_cnt; in bbr_check_full_bw_reached()
886 bbr->full_bw_reached = bbr->full_bw_cnt >= bbr_full_bw_cnt; in bbr_check_full_bw_reached()
892 struct bbr *bbr = inet_csk_ca(sk); in bbr_check_drain() local
894 if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { in bbr_check_drain()
895 bbr->mode = BBR_DRAIN; /* drain queue we created */ in bbr_check_drain()
899 if (bbr->mode == BBR_DRAIN && in bbr_check_drain()
908 struct bbr *bbr = inet_csk_ca(sk); in bbr_check_probe_rtt_done() local
910 if (!(bbr->probe_rtt_done_stamp && in bbr_check_probe_rtt_done()
911 after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) in bbr_check_probe_rtt_done()
914 bbr->min_rtt_stamp = tcp_jiffies32; /* wait a while until PROBE_RTT */ in bbr_check_probe_rtt_done()
915 tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); in bbr_check_probe_rtt_done()
941 struct bbr *bbr = inet_csk_ca(sk); in bbr_update_min_rtt() local
946 bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ); in bbr_update_min_rtt()
948 (rs->rtt_us < bbr->min_rtt_us || in bbr_update_min_rtt()
950 bbr->min_rtt_us = rs->rtt_us; in bbr_update_min_rtt()
951 bbr->min_rtt_stamp = tcp_jiffies32; in bbr_update_min_rtt()
955 !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { in bbr_update_min_rtt()
956 bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ in bbr_update_min_rtt()
958 bbr->probe_rtt_done_stamp = 0; in bbr_update_min_rtt()
961 if (bbr->mode == BBR_PROBE_RTT) { in bbr_update_min_rtt()
966 if (!bbr->probe_rtt_done_stamp && in bbr_update_min_rtt()
968 bbr->probe_rtt_done_stamp = tcp_jiffies32 + in bbr_update_min_rtt()
970 bbr->probe_rtt_round_done = 0; in bbr_update_min_rtt()
971 bbr->next_rtt_delivered = tp->delivered; in bbr_update_min_rtt()
972 } else if (bbr->probe_rtt_done_stamp) { in bbr_update_min_rtt()
973 if (bbr->round_start) in bbr_update_min_rtt()
974 bbr->probe_rtt_round_done = 1; in bbr_update_min_rtt()
975 if (bbr->probe_rtt_round_done) in bbr_update_min_rtt()
981 bbr->idle_restart = 0; in bbr_update_min_rtt()
986 struct bbr *bbr = inet_csk_ca(sk); in bbr_update_gains() local
988 switch (bbr->mode) { in bbr_update_gains()
990 bbr->pacing_gain = bbr_high_gain; in bbr_update_gains()
991 bbr->cwnd_gain = bbr_high_gain; in bbr_update_gains()
994 bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */ in bbr_update_gains()
995 bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */ in bbr_update_gains()
998 bbr->pacing_gain = (bbr->lt_use_bw ? in bbr_update_gains()
1000 bbr_pacing_gain[bbr->cycle_idx]); in bbr_update_gains()
1001 bbr->cwnd_gain = bbr_cwnd_gain; in bbr_update_gains()
1004 bbr->pacing_gain = BBR_UNIT; in bbr_update_gains()
1005 bbr->cwnd_gain = BBR_UNIT; in bbr_update_gains()
1008 WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); in bbr_update_gains()
1026 struct bbr *bbr = inet_csk_ca(sk); in bbr_main() local
1032 bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); in bbr_main()
1033 bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain); in bbr_main()
1039 struct bbr *bbr = inet_csk_ca(sk); in bbr_init() local
1041 bbr->prior_cwnd = 0; in bbr_init()
1043 bbr->rtt_cnt = 0; in bbr_init()
1044 bbr->next_rtt_delivered = tp->delivered; in bbr_init()
1045 bbr->prev_ca_state = TCP_CA_Open; in bbr_init()
1046 bbr->packet_conservation = 0; in bbr_init()
1048 bbr->probe_rtt_done_stamp = 0; in bbr_init()
1049 bbr->probe_rtt_round_done = 0; in bbr_init()
1050 bbr->min_rtt_us = tcp_min_rtt(tp); in bbr_init()
1051 bbr->min_rtt_stamp = tcp_jiffies32; in bbr_init()
1053 minmax_reset(&bbr->bw, bbr->rtt_cnt, 0); /* init max bw to 0 */ in bbr_init()
1055 bbr->has_seen_rtt = 0; in bbr_init()
1058 bbr->round_start = 0; in bbr_init()
1059 bbr->idle_restart = 0; in bbr_init()
1060 bbr->full_bw_reached = 0; in bbr_init()
1061 bbr->full_bw = 0; in bbr_init()
1062 bbr->full_bw_cnt = 0; in bbr_init()
1063 bbr->cycle_mstamp = 0; in bbr_init()
1064 bbr->cycle_idx = 0; in bbr_init()
1068 bbr->ack_epoch_mstamp = tp->tcp_mstamp; in bbr_init()
1069 bbr->ack_epoch_acked = 0; in bbr_init()
1070 bbr->extra_acked_win_rtts = 0; in bbr_init()
1071 bbr->extra_acked_win_idx = 0; in bbr_init()
1072 bbr->extra_acked[0] = 0; in bbr_init()
1073 bbr->extra_acked[1] = 0; in bbr_init()
1089 struct bbr *bbr = inet_csk_ca(sk); in bbr_undo_cwnd() local
1091 bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */ in bbr_undo_cwnd()
1092 bbr->full_bw_cnt = 0; in bbr_undo_cwnd()
1110 struct bbr *bbr = inet_csk_ca(sk); in bbr_get_info() local
1114 memset(&info->bbr, 0, sizeof(info->bbr)); in bbr_get_info()
1115 info->bbr.bbr_bw_lo = (u32)bw; in bbr_get_info()
1116 info->bbr.bbr_bw_hi = (u32)(bw >> 32); in bbr_get_info()
1117 info->bbr.bbr_min_rtt = bbr->min_rtt_us; in bbr_get_info()
1118 info->bbr.bbr_pacing_gain = bbr->pacing_gain; in bbr_get_info()
1119 info->bbr.bbr_cwnd_gain = bbr->cwnd_gain; in bbr_get_info()
1121 return sizeof(info->bbr); in bbr_get_info()
1128 struct bbr *bbr = inet_csk_ca(sk); in bbr_set_state() local
1133 bbr->prev_ca_state = TCP_CA_Loss; in bbr_set_state()
1134 bbr->full_bw = 0; in bbr_set_state()
1135 bbr->round_start = 1; /* treat RTO like end of a round */ in bbr_set_state()
1157 BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); in bbr_register()