66 #define for_nexthops(fi) {						\  argument
68 	for (nhsel = 0, nh = (fi)->fib_nh;				\
69 	     nhsel < fib_info_num_path((fi));				\
72 #define change_nexthops(fi) {						\  argument
74 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
75 	     nhsel < fib_info_num_path((fi));				\
82 #define for_nexthops(fi) {						\  argument
83 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
86 #define change_nexthops(fi) {						\  argument
88 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
93 #define endfor_nexthops(fi) }  argument
234 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
236 	if (fi->nh) {  in free_fib_info_rcu()
237 		nexthop_put(fi->nh);  in free_fib_info_rcu()
239 		change_nexthops(fi) {  in free_fib_info_rcu()
240 			fib_nh_release(fi->fib_net, nexthop_nh);  in free_fib_info_rcu()
241 		} endfor_nexthops(fi);  in free_fib_info_rcu()
244 	ip_fib_metrics_put(fi->fib_metrics);  in free_fib_info_rcu()
246 	kfree(fi);  in free_fib_info_rcu()
249 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
251 	if (fi->fib_dead == 0) {  in free_fib_info()
252 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
256 	call_rcu(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
260 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
263 	if (fi && --fi->fib_treeref == 0) {  in fib_release_info()
264 		hlist_del(&fi->fib_hash);  in fib_release_info()
269 		if (fi->fib_prefsrc)  in fib_release_info()
270 			hlist_del(&fi->fib_lhash);  in fib_release_info()
271 		if (fi->nh) {  in fib_release_info()
272 			list_del(&fi->nh_list);  in fib_release_info()
274 			change_nexthops(fi) {  in fib_release_info()
278 			} endfor_nexthops(fi)  in fib_release_info()
280 		fi->fib_dead = 1;  in fib_release_info()
281 		fib_info_put(fi);  in fib_release_info()
286 static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)  in nh_comp()  argument
290 	if (fi->nh || ofi->nh)  in nh_comp()
291 		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;  in nh_comp()
296 	for_nexthops(fi) {  in nh_comp()
319 	} endfor_nexthops(fi);  in nh_comp()
355 static inline unsigned int fib_info_hashfn(struct fib_info *fi)  in fib_info_hashfn()  argument
359 	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,  in fib_info_hashfn()
360 				fi->fib_scope, (__force u32)fi->fib_prefsrc,  in fib_info_hashfn()
361 				fi->fib_priority);  in fib_info_hashfn()
363 	if (fi->nh) {  in fib_info_hashfn()
364 		val ^= fib_devindex_hashfn(fi->nh->id);  in fib_info_hashfn()
366 		for_nexthops(fi) {  in fib_info_hashfn()
368 		} endfor_nexthops(fi)  in fib_info_hashfn()
379 	struct fib_info *fi;  in fib_find_info_nh()  local
389 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info_nh()
390 		if (!net_eq(fi->fib_net, net))  in fib_find_info_nh()
392 		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)  in fib_find_info_nh()
394 		if (cfg->fc_protocol == fi->fib_protocol &&  in fib_find_info_nh()
395 		    cfg->fc_scope == fi->fib_scope &&  in fib_find_info_nh()
396 		    cfg->fc_prefsrc == fi->fib_prefsrc &&  in fib_find_info_nh()
397 		    cfg->fc_priority == fi->fib_priority &&  in fib_find_info_nh()
398 		    cfg->fc_type == fi->fib_type &&  in fib_find_info_nh()
399 		    cfg->fc_table == fi->fib_tb_id &&  in fib_find_info_nh()
400 		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))  in fib_find_info_nh()
401 			return fi;  in fib_find_info_nh()
410 	struct fib_info *fi;  in fib_find_info()  local
416 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
417 		if (!net_eq(fi->fib_net, nfi->fib_net))  in fib_find_info()
419 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
421 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
422 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
423 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
424 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
425 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
426 		    nfi->fib_tb_id == fi->fib_tb_id &&  in fib_find_info()
427 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
429 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
430 		    nh_comp(fi, nfi) == 0)  in fib_find_info()
431 			return fi;  in fib_find_info()
463 static inline size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
471 	unsigned int nhs = fib_info_num_path(fi);  in fib_nlmsg_size()
476 	if (fi->nh)  in fib_nlmsg_size()
491 		for (i = 0; i < fib_info_num_path(fi); i++) {  in fib_nlmsg_size()
492 			struct fib_nh_common *nhc = fib_info_nhc(fi, i);  in fib_nlmsg_size()
524 	fri.fi = fa->fa_info;  in rtmsg_fib()
547 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
551 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_detect_death()
575 		*last_resort = fi;  in fib_detect_death()
688 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
692 	struct net *net = fi->fib_net;  in fib_get_nhs()
697 	change_nexthops(fi) {  in fib_get_nhs()
766 	} endfor_nexthops(fi);  in fib_get_nhs()
769 	nh = fib_info_nh(fi, 0);  in fib_get_nhs()
799 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
804 	if (fib_info_num_path(fi) < 2)  in fib_rebalance()
808 	for_nexthops(fi) {  in fib_rebalance()
817 	} endfor_nexthops(fi);  in fib_rebalance()
820 	change_nexthops(fi) {  in fib_rebalance()
835 	} endfor_nexthops(fi);  in fib_rebalance()
839 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
848 #define fib_rebalance(fi) do { } while (0)  argument
874 int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi,  in fib_nh_match()  argument
882 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
886 		if (fi->nh && cfg->fc_nh_id == fi->nh->id)  in fib_nh_match()
891 	if (fi->nh) {  in fib_nh_match()
900 		nh = fib_info_nh(fi, 0);  in fib_nh_match()
934 	for_nexthops(fi) {  in fib_nh_match()
1002 	} endfor_nexthops(fi);  in fib_nh_match()
1007 bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)  in fib_metrics_match()  argument
1029 			val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);  in fib_metrics_match()
1036 		fi_val = fi->fib_metrics->metrics[type - 1];  in fib_metrics_match()
1305 		struct fib_info *fi;  in fib_info_hash_move()  local
1307 		hlist_for_each_entry_safe(fi, n, head, fib_hash) {  in fib_info_hash_move()
1311 			new_hash = fib_info_hashfn(fi);  in fib_info_hash_move()
1313 			hlist_add_head(&fi->fib_hash, dest);  in fib_info_hash_move()
1321 		struct fib_info *fi;  in fib_info_hash_move()  local
1323 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {  in fib_info_hash_move()
1327 			new_hash = fib_laddr_hashfn(fi->fib_prefsrc);  in fib_info_hash_move()
1329 			hlist_add_head(&fi->fib_lhash, ldest);  in fib_info_hash_move()
1360 	if (res->fi->fib_prefsrc)  in fib_result_prefsrc()
1361 		return res->fi->fib_prefsrc;  in fib_result_prefsrc()
1371 	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);  in fib_result_prefsrc()
1402 	struct fib_info *fi = NULL;  in fib_create_info()  local
1425 			fi = fib_find_info_nh(net, cfg);  in fib_create_info()
1426 			if (fi) {  in fib_create_info()
1427 				fi->fib_treeref++;  in fib_create_info()
1428 				return fi;  in fib_create_info()
1472 	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);  in fib_create_info()
1473 	if (!fi)  in fib_create_info()
1475 	fi->fib_metrics = ip_fib_metrics_init(fi->fib_net, cfg->fc_mx,  in fib_create_info()
1477 	if (IS_ERR(fi->fib_metrics)) {  in fib_create_info()
1478 		err = PTR_ERR(fi->fib_metrics);  in fib_create_info()
1479 		kfree(fi);  in fib_create_info()
1483 	fi->fib_net = net;  in fib_create_info()
1484 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1485 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1486 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1487 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1488 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1489 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1490 	fi->fib_tb_id = cfg->fc_table;  in fib_create_info()
1492 	fi->fib_nhs = nhs;  in fib_create_info()
1499 			fi->nh = nh;  in fib_create_info()
1502 		change_nexthops(fi) {  in fib_create_info()
1503 			nexthop_nh->nh_parent = fi;  in fib_create_info()
1504 		} endfor_nexthops(fi)  in fib_create_info()
1507 			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,  in fib_create_info()
1510 			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);  in fib_create_info()
1542 	if (fi->nh) {  in fib_create_info()
1543 		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);  in fib_create_info()
1547 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1568 		change_nexthops(fi) {  in fib_create_info()
1576 		} endfor_nexthops(fi)  in fib_create_info()
1577 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1578 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1581 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {  in fib_create_info()
1586 	if (!fi->nh) {  in fib_create_info()
1587 		change_nexthops(fi) {  in fib_create_info()
1589 						  fi->fib_scope);  in fib_create_info()
1591 				fi->fib_nh_is_v6 = true;  in fib_create_info()
1592 		} endfor_nexthops(fi)  in fib_create_info()
1594 		fib_rebalance(fi);  in fib_create_info()
1598 	ofi = fib_find_info(fi);  in fib_create_info()
1600 		fi->fib_dead = 1;  in fib_create_info()
1601 		free_fib_info(fi);  in fib_create_info()
1606 	fi->fib_treeref++;  in fib_create_info()
1607 	refcount_set(&fi->fib_clntref, 1);  in fib_create_info()
1610 	hlist_add_head(&fi->fib_hash,  in fib_create_info()
1611 		       &fib_info_hash[fib_info_hashfn(fi)]);  in fib_create_info()
1612 	if (fi->fib_prefsrc) {  in fib_create_info()
1615 		head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];  in fib_create_info()
1616 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1618 	if (fi->nh) {  in fib_create_info()
1619 		list_add(&fi->nh_list, &nh->fi_list);  in fib_create_info()
1621 		change_nexthops(fi) {  in fib_create_info()
1628 		} endfor_nexthops(fi)  in fib_create_info()
1631 	return fi;  in fib_create_info()
1637 	if (fi) {  in fib_create_info()
1638 		fi->fib_dead = 1;  in fib_create_info()
1639 		free_fib_info(fi);  in fib_create_info()
1751 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1759 	if (unlikely(fi->nh)) {  in fib_add_multipath()
1760 		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)  in fib_add_multipath()
1765 	for_nexthops(fi) {  in fib_add_multipath()
1773 	} endfor_nexthops(fi);  in fib_add_multipath()
1784 static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)  in fib_add_multipath()  argument
1793 	unsigned int nhs = fib_info_num_path(fri->fi);  in fib_dump_info()
1794 	struct fib_info *fi = fri->fi;  in fib_dump_info()  local
1815 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1816 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1817 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1822 	if (fi->fib_priority &&  in fib_dump_info()
1823 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1825 	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)  in fib_dump_info()
1828 	if (fi->fib_prefsrc &&  in fib_dump_info()
1829 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1832 	if (fi->nh) {  in fib_dump_info()
1833 		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))  in fib_dump_info()
1835 		if (nexthop_is_blackhole(fi->nh))  in fib_dump_info()
1837 		if (!READ_ONCE(fi->fib_net->ipv4.sysctl_nexthop_compat_mode))  in fib_dump_info()
1842 		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);  in fib_dump_info()
1860 		if (fib_add_multipath(skb, fi) < 0)  in fib_dump_info()
1891 	struct fib_info *fi;  in fib_sync_down_addr()  local
1896 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1897 		if (!net_eq(fi->fib_net, net) ||  in fib_sync_down_addr()
1898 		    fi->fib_tb_id != tb_id)  in fib_sync_down_addr()
1900 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1901 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
2005 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
2008 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
2009 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_down_dev()
2011 		prev_fi = fi;  in fib_sync_down_dev()
2013 		change_nexthops(fi) {  in fib_sync_down_dev()
2034 				dead = fi->fib_nhs;  in fib_sync_down_dev()
2038 		} endfor_nexthops(fi)  in fib_sync_down_dev()
2039 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
2043 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
2046 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
2052 		fib_rebalance(fi);  in fib_sync_down_dev()
2061 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
2067 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
2101 		if (!fi) {  in fib_select_default()
2102 			if (next_fi != res->fi)  in fib_select_default()
2105 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
2107 			fib_result_assign(res, fi);  in fib_select_default()
2111 		fi = next_fi;  in fib_select_default()
2115 	if (order <= 0 || !fi) {  in fib_select_default()
2121 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
2123 		fib_result_assign(res, fi);  in fib_select_default()
2163 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
2166 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
2167 		if (nh->fib_nh_dev != dev || fi == prev_fi)  in fib_sync_up()
2170 		prev_fi = fi;  in fib_sync_up()
2172 		change_nexthops(fi) {  in fib_sync_up()
2186 		} endfor_nexthops(fi)  in fib_sync_up()
2189 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
2193 		fib_rebalance(fi);  in fib_sync_up()
2228 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
2229 	struct net *net = fi->fib_net;  in fib_select_multipath()
2232 	if (unlikely(res->fi->nh)) {  in fib_select_multipath()
2237 	change_nexthops(fi) {  in fib_select_multipath()
2254 	} endfor_nexthops(fi);  in fib_select_multipath()
2265 	if (fib_info_num_path(res->fi) > 1) {  in fib_select_path()