104 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
436 	if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound))  in fib6_select_path()
441 		const struct fib6_nh *nh = sibling->fib6_nh;  in fib6_select_path()
455 	res->nh = match->fib6_nh;  in fib6_select_path()
462 static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,  in __rt6_device_match()
488 	struct fib6_nh		*nh;
491 static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)  in __rt6_nh_dev_match()
501 static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,  in rt6_nh_dev_match()
527 	struct fib6_nh *nh;  in rt6_device_match()
535 			nh = f6i->fib6_nh;  in rt6_device_match()
550 			nh = spf6i->fib6_nh;  in rt6_device_match()
562 		nh = res->f6i->fib6_nh;  in rt6_device_match()
571 		nh = f6i->fib6_nh;  in rt6_device_match()
576 		nh = res->f6i->fib6_nh;  in rt6_device_match()
609 static void rt6_probe(struct fib6_nh *fib6_nh)  in rt6_probe()  argument
626 	if (!fib6_nh->fib_nh_gw_family)  in rt6_probe()
629 	nh_gw = &fib6_nh->fib_nh_gw6;  in rt6_probe()
630 	dev = fib6_nh->fib_nh_dev;  in rt6_probe()
632 	last_probe = READ_ONCE(fib6_nh->last_probe);  in rt6_probe()
653 	if (!work || cmpxchg(&fib6_nh->last_probe,  in rt6_probe()
668 static inline void rt6_probe(struct fib6_nh *fib6_nh)  in rt6_probe()  argument
676 static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)  in rt6_check_neigh()  argument
682 	neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev,  in rt6_check_neigh()
683 					  &fib6_nh->fib_nh_gw6);  in rt6_check_neigh()
704 static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,  in rt6_score_route()
726 static bool find_match(struct fib6_nh *nh, u32 fib6_flags,  in find_match()
768 	struct fib6_nh	*nh;
771 static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)  in rt6_nh_find_match()
791 		struct fib6_nh *nh;  in __find_rr_leaf()
823 			nh = f6i->fib6_nh;  in __find_rr_leaf()
909 		res->nh = res->f6i->fib6_nh;  in rt6_select()
1117 	const struct fib6_nh *nh = res->nh;  in ip6_rt_copy_init()
1576 	const struct fib6_nh *nh = res->nh;  in fib6_mtu()
1604 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,  in fib6_nh_get_excptn_bucket()
1634 static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,  in fib6_nh_excptn_bucket_set_flushed()
1657 	struct fib6_nh *nh = res->nh;  in rt6_insert_exception()
1732 static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)  in fib6_nh_flush_exceptions()
1762 static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_flush_exceptions()
1777 		fib6_nh_flush_exceptions(f6i->fib6_nh, f6i);  in rt6_flush_exceptions()
1826 static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_remove_exception()
1869 static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)  in rt6_nh_remove_exception_rt()
1903 	return fib6_nh_remove_exception(from->fib6_nh,  in rt6_remove_exception_rt()
1910 static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,  in fib6_nh_update_exception()
1936 	struct fib6_nh		*match;
1940 static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_find_match()
1959 	struct fib6_nh *fib6_nh;  in rt6_update_exception_stamp_rt()  local
1977 		fib6_nh = arg.match;  in rt6_update_exception_stamp_rt()
1979 		fib6_nh = from->fib6_nh;  in rt6_update_exception_stamp_rt()
1981 	fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt);  in rt6_update_exception_stamp_rt()
2009 				       const struct fib6_nh *nh, int mtu)  in rt6_exceptions_update_pmtu()
2037 static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,  in fib6_nh_exceptions_clean_tohost()
2114 static void fib6_nh_age_exceptions(const struct fib6_nh *nh,  in fib6_nh_age_exceptions()
2148 static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)  in rt6_nh_age_exceptions()
2169 		fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now);  in rt6_age_exceptions()
2780 			res.nh = res.f6i->fib6_nh;  in __ip6_rt_update_pmtu()
2867 	const struct fib6_nh *nh = res->nh;  in ip6_redirect_nh_match()
2899 static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)  in fib6_nh_redirect_match()
2965 			res.nh = rt->fib6_nh;  in __ip6_route_redirect()
2986 	res.nh = rt->fib6_nh;  in __ip6_route_redirect()
3120 	const struct fib6_nh *nh = res->nh;  in ip6_mtu_from_fib6()
3409 int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,  in fib6_nh_init()  argument
3418 	fib6_nh->fib_nh_family = AF_INET6;  in fib6_nh_init()
3420 	fib6_nh->last_probe = jiffies;  in fib6_nh_init()
3423 		fib6_nh->fib_nh_gw6 = cfg->fc_gateway;  in fib6_nh_init()
3424 		fib6_nh->fib_nh_gw_family = AF_INET6;  in fib6_nh_init()
3451 		fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;  in fib6_nh_init()
3454 	fib6_nh->fib_nh_weight = 1;  in fib6_nh_init()
3483 		fib6_nh->fib_nh_gw6 = cfg->fc_gateway;  in fib6_nh_init()
3484 		fib6_nh->fib_nh_gw_family = AF_INET6;  in fib6_nh_init()
3505 		fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;  in fib6_nh_init()
3507 	err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap,  in fib6_nh_init()
3513 	fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);  in fib6_nh_init()
3514 	if (!fib6_nh->rt6i_pcpu) {  in fib6_nh_init()
3519 	fib6_nh->fib_nh_dev = dev;  in fib6_nh_init()
3520 	fib6_nh->fib_nh_oif = dev->ifindex;  in fib6_nh_init()
3527 		lwtstate_put(fib6_nh->fib_nh_lws);  in fib6_nh_init()
3528 		fib6_nh->fib_nh_lws = NULL;  in fib6_nh_init()
3536 void fib6_nh_release(struct fib6_nh *fib6_nh)  in fib6_nh_release()  argument
3542 	fib6_nh_flush_exceptions(fib6_nh, NULL);  in fib6_nh_release()
3543 	bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL);  in fib6_nh_release()
3545 		rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);  in fib6_nh_release()
3551 	if (fib6_nh->rt6i_pcpu) {  in fib6_nh_release()
3558 			ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);  in fib6_nh_release()
3567 		free_percpu(fib6_nh->rt6i_pcpu);  in fib6_nh_release()
3570 	fib_nh_common_release(&fib6_nh->nh_common);  in fib6_nh_release()
3581 	struct fib6_nh *fib6_nh;  in ip6_route_info_create()  local
3692 		fib6_nh = nexthop_fib6_nh(rt->nh);  in ip6_route_info_create()
3694 		err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack);  in ip6_route_info_create()
3698 		fib6_nh = rt->fib6_nh;  in ip6_route_info_create()
3704 		if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev,  in ip6_route_info_create()
3710 		struct net_device *dev = fib6_nh->fib_nh_dev;  in ip6_route_info_create()
3875 			     struct fib6_nh *nh)  in ip6_del_cached_rt()
3895 static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)  in fib6_nh_del_cached_rt()
3937 			struct fib6_nh *nh;  in ip6_route_del()
3951 					nh = rt->fib6_nh;  in ip6_route_del()
3977 			nh = rt->fib6_nh;  in ip6_route_del()
4111 		res.nh = res.f6i->fib6_nh;  in rt6_do_redirect()
4168 		if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)  in rt6_get_route_info()
4171 		    !rt->fib6_nh->fib_nh_gw_family)  in rt6_get_route_info()
4173 		if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))  in rt6_get_route_info()
4231 		struct fib6_nh *nh;  in rt6_get_dflt_router()
4237 		nh = rt->fib6_nh;  in rt6_get_dflt_router()
4465 	    ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) &&  in fib6_remove_prefsrc()
4493 	struct fib6_nh *nh;  in fib6_clean_tohost()
4499 	nh = rt->fib6_nh;  in fib6_clean_tohost()
4549 	if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||  in rt6_is_dead()
4550 	    (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&  in rt6_is_dead()
4551 	     ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev)))  in rt6_is_dead()
4563 		total += rt->fib6_nh->fib_nh_weight;  in rt6_multipath_total_weight()
4567 			total += iter->fib6_nh->fib_nh_weight;  in rt6_multipath_total_weight()
4578 		*weight += rt->fib6_nh->fib_nh_weight;  in rt6_upper_bound_set()
4582 	atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound);  in rt6_upper_bound_set()
4626 	    rt->fib6_nh->fib_nh_dev == arg->dev) {  in fib6_ifup()
4627 		rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;  in fib6_ifup()
4656 	if (rt->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_uses_dev()
4659 		if (iter->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_uses_dev()
4680 	if (rt->fib6_nh->fib_nh_dev == down_dev ||  in rt6_multipath_dead_count()
4681 	    rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)  in rt6_multipath_dead_count()
4684 		if (iter->fib6_nh->fib_nh_dev == down_dev ||  in rt6_multipath_dead_count()
4685 		    iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)  in rt6_multipath_dead_count()
4697 	if (rt->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_nh_flags_set()
4698 		rt->fib6_nh->fib_nh_flags |= nh_flags;  in rt6_multipath_nh_flags_set()
4700 		if (iter->fib6_nh->fib_nh_dev == dev)  in rt6_multipath_nh_flags_set()
4701 			iter->fib6_nh->fib_nh_flags |= nh_flags;  in rt6_multipath_nh_flags_set()
4716 		return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;  in fib6_ifdown()
4721 			return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;  in fib6_ifdown()
4737 		if (rt->fib6_nh->fib_nh_dev != dev ||  in fib6_ifdown()
4740 		rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;  in fib6_ifdown()
4777 static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)  in fib6_nh_mtu_change()
4828 	return fib6_nh_mtu_change(f6i->fib6_nh, arg);  in rt6_mtu_change_route()
5166 		rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;  in ip6_route_multipath_add()
5377 static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)  in rt6_nh_nlmsg_size()
5404 		struct fib6_nh *nh = f6i->fib6_nh;  in rt6_nlmsg_size()
5449 		struct fib6_nh *fib6_nh;  in rt6_fill_node_nexthop()  local
5451 		fib6_nh = nexthop_fib6_nh(nh);  in rt6_fill_node_nexthop()
5452 		if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6,  in rt6_fill_node_nexthop()
5580 		if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common,  in rt6_fill_node()
5581 				    rt->fib6_nh->fib_nh_weight, AF_INET6,  in rt6_fill_node()
5587 			if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,  in rt6_fill_node()
5588 					    sibling->fib6_nh->fib_nh_weight,  in rt6_fill_node()
5607 		if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6,  in rt6_fill_node()
5641 static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)  in fib6_info_nh_uses_dev()
5662 	if (f6i->fib6_nh->fib_nh_dev == dev)  in fib6_info_uses_dev()
5670 			if (sibling->fib6_nh->fib_nh_dev == dev)  in fib6_info_uses_dev()
5686 static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)  in rt6_nh_dump_exceptions()
5794 			err = rt6_nh_dump_exceptions(rt->fib6_nh, &w);  in rt6_dump_route()
6089 		net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;  in ip6_route_dev_notify()
6432 	init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;  in ip6_route_init_special_entries()