174 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
175 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
531 	struct xfrm_policy *pol;  in xfrm_dst_hash_transfer()  local
537 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {  in xfrm_dst_hash_transfer()
540 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);  in xfrm_dst_hash_transfer()
541 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,  in xfrm_dst_hash_transfer()
542 				pol->family, nhashmask, dbits, sbits);  in xfrm_dst_hash_transfer()
544 			hlist_del_rcu(&pol->bydst);  in xfrm_dst_hash_transfer()
545 			hlist_add_head_rcu(&pol->bydst, ndsttable + h);  in xfrm_dst_hash_transfer()
550 			hlist_del_rcu(&pol->bydst);  in xfrm_dst_hash_transfer()
551 			hlist_add_behind_rcu(&pol->bydst, entry0);  in xfrm_dst_hash_transfer()
553 		entry0 = &pol->bydst;  in xfrm_dst_hash_transfer()
566 	struct xfrm_policy *pol;  in xfrm_idx_hash_transfer()  local
568 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {  in xfrm_idx_hash_transfer()
571 		h = __idx_hash(pol->index, nhashmask);  in xfrm_idx_hash_transfer()
572 		hlist_add_head(&pol->byidx, nidxtable+h);  in xfrm_idx_hash_transfer()
701 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)  in xfrm_policy_inexact_alloc_bin()  argument
705 		.family = pol->family,  in xfrm_policy_inexact_alloc_bin()
706 		.type = pol->type,  in xfrm_policy_inexact_alloc_bin()
708 		.if_id = pol->if_id,  in xfrm_policy_inexact_alloc_bin()
710 	struct net *net = xp_net(pol);  in xfrm_policy_inexact_alloc_bin()
1231 	struct xfrm_policy *pol;  in xfrm_hash_rebuild()  local
1349 		hlist_for_each_entry(pol, chain, bydst) {  in xfrm_hash_rebuild()
1350 			if (policy->priority >= pol->priority)  in xfrm_hash_rebuild()
1351 				newpos = &pol->bydst;  in xfrm_hash_rebuild()
1453 					  struct xfrm_policy *pol)  in xfrm_policy_mark_match()  argument
1455 	return mark->v == pol->mark.v && mark->m == pol->mark.m;  in xfrm_policy_mark_match()
1510 	struct xfrm_policy *pol, *delpol = NULL;  in xfrm_policy_insert_inexact_list()  local
1514 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_policy_insert_inexact_list()
1515 		if (pol->type == policy->type &&  in xfrm_policy_insert_inexact_list()
1516 		    pol->if_id == policy->if_id &&  in xfrm_policy_insert_inexact_list()
1517 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert_inexact_list()
1518 		    xfrm_policy_mark_match(&policy->mark, pol) &&  in xfrm_policy_insert_inexact_list()
1519 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert_inexact_list()
1521 			delpol = pol;  in xfrm_policy_insert_inexact_list()
1522 			if (policy->priority > pol->priority)  in xfrm_policy_insert_inexact_list()
1524 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert_inexact_list()
1525 			newpos = &pol->bydst_inexact_list;  in xfrm_policy_insert_inexact_list()
1537 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_policy_insert_inexact_list()
1538 		pol->pos = i;  in xfrm_policy_insert_inexact_list()
1547 	struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;  in xfrm_policy_insert_list()  local
1549 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_insert_list()
1550 		if (pol->type == policy->type &&  in xfrm_policy_insert_list()
1551 		    pol->if_id == policy->if_id &&  in xfrm_policy_insert_list()
1552 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert_list()
1553 		    xfrm_policy_mark_match(&policy->mark, pol) &&  in xfrm_policy_insert_list()
1554 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert_list()
1558 			delpol = pol;  in xfrm_policy_insert_list()
1559 			if (policy->priority > pol->priority)  in xfrm_policy_insert_list()
1561 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert_list()
1562 			newpos = pol;  in xfrm_policy_insert_list()
1629 	struct xfrm_policy *pol;  in __xfrm_policy_bysel_ctx()  local
1634 	hlist_for_each_entry(pol, chain, bydst) {  in __xfrm_policy_bysel_ctx()
1635 		if (pol->type == type &&  in __xfrm_policy_bysel_ctx()
1636 		    pol->if_id == if_id &&  in __xfrm_policy_bysel_ctx()
1637 		    xfrm_policy_mark_match(mark, pol) &&  in __xfrm_policy_bysel_ctx()
1638 		    !selector_cmp(sel, &pol->selector) &&  in __xfrm_policy_bysel_ctx()
1639 		    xfrm_sec_ctx_match(ctx, pol->security))  in __xfrm_policy_bysel_ctx()
1640 			return pol;  in __xfrm_policy_bysel_ctx()
1652 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_policy_bysel_ctx()  local
1676 		pol = NULL;  in xfrm_policy_bysel_ctx()
1686 			if (!pol || tmp->pos < pol->pos)  in xfrm_policy_bysel_ctx()
1687 				pol = tmp;  in xfrm_policy_bysel_ctx()
1690 		pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,  in xfrm_policy_bysel_ctx()
1694 	if (pol) {  in xfrm_policy_bysel_ctx()
1695 		xfrm_pol_hold(pol);  in xfrm_policy_bysel_ctx()
1697 			*err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_bysel_ctx()
1700 				return pol;  in xfrm_policy_bysel_ctx()
1702 			__xfrm_policy_unlink(pol, dir);  in xfrm_policy_bysel_ctx()
1704 		ret = pol;  in xfrm_policy_bysel_ctx()
1720 	struct xfrm_policy *pol, *ret;  in xfrm_policy_byid()  local
1731 	hlist_for_each_entry(pol, chain, byidx) {  in xfrm_policy_byid()
1732 		if (pol->type == type && pol->index == id &&  in xfrm_policy_byid()
1733 		    pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {  in xfrm_policy_byid()
1734 			xfrm_pol_hold(pol);  in xfrm_policy_byid()
1737 								pol->security);  in xfrm_policy_byid()
1740 					return pol;  in xfrm_policy_byid()
1742 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_byid()
1744 			ret = pol;  in xfrm_policy_byid()
1760 	struct xfrm_policy *pol;  in xfrm_policy_flush_secctx_check()  local
1763 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush_secctx_check()
1764 		if (pol->walk.dead ||  in xfrm_policy_flush_secctx_check()
1765 		    xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||  in xfrm_policy_flush_secctx_check()
1766 		    pol->type != type)  in xfrm_policy_flush_secctx_check()
1769 		err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_flush_secctx_check()
1771 			xfrm_audit_policy_delete(pol, 0, task_valid);  in xfrm_policy_flush_secctx_check()
1788 	struct xfrm_policy *pol;  in xfrm_policy_flush()  local
1797 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {  in xfrm_policy_flush()
1798 		dir = xfrm_policy_id2dir(pol->index);  in xfrm_policy_flush()
1799 		if (pol->walk.dead ||  in xfrm_policy_flush()
1801 		    pol->type != type)  in xfrm_policy_flush()
1804 		__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
1807 		xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
1808 		xfrm_policy_kill(pol);  in xfrm_policy_flush()
1826 	struct xfrm_policy *pol;  in xfrm_policy_walk()  local
1847 		pol = container_of(x, struct xfrm_policy, walk);  in xfrm_policy_walk()
1849 		    walk->type != pol->type)  in xfrm_policy_walk()
1851 		error = func(pol, xfrm_policy_id2dir(pol->index),  in xfrm_policy_walk()
1895 static int xfrm_policy_match(const struct xfrm_policy *pol,  in xfrm_policy_match()  argument
1899 	const struct xfrm_selector *sel = &pol->selector;  in xfrm_policy_match()
1903 	if (pol->family != family ||  in xfrm_policy_match()
1904 	    pol->if_id != if_id ||  in xfrm_policy_match()
1905 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||  in xfrm_policy_match()
1906 	    pol->type != type)  in xfrm_policy_match()
1911 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,  in xfrm_policy_match()
2026 	struct xfrm_policy *pol;  in __xfrm_policy_eval_candidates()  local
2031 	hlist_for_each_entry_rcu(pol, chain, bydst) {  in __xfrm_policy_eval_candidates()
2034 		if (pol->priority > priority)  in __xfrm_policy_eval_candidates()
2037 		err = xfrm_policy_match(pol, fl, type, family, dir, if_id);  in __xfrm_policy_eval_candidates()
2047 			if (pol->priority == priority &&  in __xfrm_policy_eval_candidates()
2048 			    prefer->pos < pol->pos)  in __xfrm_policy_eval_candidates()
2052 		return pol;  in __xfrm_policy_eval_candidates()
2091 	struct xfrm_policy *pol, *ret;  in xfrm_policy_lookup_bytype()  local
2109 	hlist_for_each_entry_rcu(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
2110 		err = xfrm_policy_match(pol, fl, type, family, dir, if_id);  in xfrm_policy_lookup_bytype()
2119 			ret = pol;  in xfrm_policy_lookup_bytype()
2128 	pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,  in xfrm_policy_lookup_bytype()
2130 	if (pol) {  in xfrm_policy_lookup_bytype()
2131 		ret = pol;  in xfrm_policy_lookup_bytype()
2132 		if (IS_ERR(pol))  in xfrm_policy_lookup_bytype()
2153 	struct xfrm_policy *pol;  in xfrm_policy_lookup()  local
2155 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,  in xfrm_policy_lookup()
2157 	if (pol != NULL)  in xfrm_policy_lookup()
2158 		return pol;  in xfrm_policy_lookup()
2168 	struct xfrm_policy *pol;  in xfrm_sk_policy_lookup()  local
2172 	pol = rcu_dereference(sk->sk_policy[dir]);  in xfrm_sk_policy_lookup()
2173 	if (pol != NULL) {  in xfrm_sk_policy_lookup()
2177 		if (pol->family != family) {  in xfrm_sk_policy_lookup()
2178 			pol = NULL;  in xfrm_sk_policy_lookup()
2182 		match = xfrm_selector_match(&pol->selector, fl, family);  in xfrm_sk_policy_lookup()
2184 			if ((sk->sk_mark & pol->mark.m) != pol->mark.v ||  in xfrm_sk_policy_lookup()
2185 			    pol->if_id != if_id) {  in xfrm_sk_policy_lookup()
2186 				pol = NULL;  in xfrm_sk_policy_lookup()
2189 			err = security_xfrm_policy_lookup(pol->security,  in xfrm_sk_policy_lookup()
2193 				if (!xfrm_pol_hold_rcu(pol))  in xfrm_sk_policy_lookup()
2196 				pol = NULL;  in xfrm_sk_policy_lookup()
2198 				pol = ERR_PTR(err);  in xfrm_sk_policy_lookup()
2201 			pol = NULL;  in xfrm_sk_policy_lookup()
2205 	return pol;  in xfrm_sk_policy_lookup()
2208 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)  in __xfrm_policy_link()  argument
2210 	struct net *net = xp_net(pol);  in __xfrm_policy_link()
2212 	list_add(&pol->walk.all, &net->xfrm.policy_all);  in __xfrm_policy_link()
2214 	xfrm_pol_hold(pol);  in __xfrm_policy_link()
2217 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,  in __xfrm_policy_unlink()  argument
2220 	struct net *net = xp_net(pol);  in __xfrm_policy_unlink()
2222 	if (list_empty(&pol->walk.all))  in __xfrm_policy_unlink()
2226 	if (!hlist_unhashed(&pol->bydst)) {  in __xfrm_policy_unlink()
2227 		hlist_del_rcu(&pol->bydst);  in __xfrm_policy_unlink()
2228 		hlist_del_init(&pol->bydst_inexact_list);  in __xfrm_policy_unlink()
2229 		hlist_del(&pol->byidx);  in __xfrm_policy_unlink()
2232 	list_del_init(&pol->walk.all);  in __xfrm_policy_unlink()
2235 	return pol;  in __xfrm_policy_unlink()
2238 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_link()  argument
2240 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_link()
2243 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_unlink()  argument
2245 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_unlink()
2248 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)  in xfrm_policy_delete()  argument
2250 	struct net *net = xp_net(pol);  in xfrm_policy_delete()
2253 	pol = __xfrm_policy_unlink(pol, dir);  in xfrm_policy_delete()
2255 	if (pol) {  in xfrm_policy_delete()
2256 		xfrm_policy_kill(pol);  in xfrm_policy_delete()
2263 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)  in xfrm_sk_policy_insert()  argument
2269 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)  in xfrm_sk_policy_insert()
2276 	if (pol) {  in xfrm_sk_policy_insert()
2277 		pol->curlft.add_time = ktime_get_real_seconds();  in xfrm_sk_policy_insert()
2278 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);  in xfrm_sk_policy_insert()
2279 		xfrm_sk_policy_link(pol, dir);  in xfrm_sk_policy_insert()
2281 	rcu_assign_pointer(sk->sk_policy[dir], pol);  in xfrm_sk_policy_insert()
2283 		if (pol)  in xfrm_sk_policy_insert()
2284 			xfrm_policy_requeue(old_pol, pol);  in xfrm_sk_policy_insert()
2768 	struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);  in xfrm_policy_queue_process()  local
2769 	struct net *net = xp_net(pol);  in xfrm_policy_queue_process()
2770 	struct xfrm_policy_queue *pq = &pol->polq;  in xfrm_policy_queue_process()
2786 	skb->mark = pol->mark.v;  in xfrm_policy_queue_process()
2804 			xfrm_pol_hold(pol);  in xfrm_policy_queue_process()
2822 		skb->mark = pol->mark.v;  in xfrm_policy_queue_process()
2841 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
2847 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
2855 	struct xfrm_policy *pol = xdst->pols[0];  in xdst_queue_output()  local
2856 	struct xfrm_policy_queue *pq = &pol->polq;  in xdst_queue_output()
2880 		xfrm_pol_put(pol);  in xdst_queue_output()
2885 		xfrm_pol_hold(pol);  in xdst_queue_output()
3551 	struct xfrm_policy *pol;  in __xfrm_policy_check()  local
3600 	pol = NULL;  in __xfrm_policy_check()
3603 		pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);  in __xfrm_policy_check()
3604 		if (IS_ERR(pol)) {  in __xfrm_policy_check()
3610 	if (!pol)  in __xfrm_policy_check()
3611 		pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);  in __xfrm_policy_check()
3613 	if (IS_ERR(pol)) {  in __xfrm_policy_check()
3618 	if (!pol) {  in __xfrm_policy_check()
3627 	pol->curlft.use_time = ktime_get_real_seconds();  in __xfrm_policy_check()
3629 	pols[0] = pol;  in __xfrm_policy_check()
3648 	if (pol->action == XFRM_POLICY_ALLOW) {  in __xfrm_policy_check()
3661 			if (pols[pi] != pol &&  in __xfrm_policy_check()
4301 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_migrate_policy_find()  local
4307 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
4308 		if ((if_id == 0 || pol->if_id == if_id) &&  in xfrm_migrate_policy_find()
4309 		    xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
4310 		    pol->type == type) {  in xfrm_migrate_policy_find()
4311 			ret = pol;  in xfrm_migrate_policy_find()
4317 	hlist_for_each_entry(pol, chain, bydst_inexact_list) {  in xfrm_migrate_policy_find()
4318 		if ((pol->priority >= priority) && ret)  in xfrm_migrate_policy_find()
4321 		if ((if_id == 0 || pol->if_id == if_id) &&  in xfrm_migrate_policy_find()
4322 		    xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
4323 		    pol->type == type) {  in xfrm_migrate_policy_find()
4324 			ret = pol;  in xfrm_migrate_policy_find()
4366 static int xfrm_policy_migrate(struct xfrm_policy *pol,  in xfrm_policy_migrate()  argument
4372 	write_lock_bh(&pol->lock);  in xfrm_policy_migrate()
4373 	if (unlikely(pol->walk.dead)) {  in xfrm_policy_migrate()
4375 		write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
4379 	for (i = 0; i < pol->xfrm_nr; i++) {  in xfrm_policy_migrate()
4381 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))  in xfrm_policy_migrate()
4384 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&  in xfrm_policy_migrate()
4385 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)  in xfrm_policy_migrate()
4388 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,  in xfrm_policy_migrate()
4389 			       sizeof(pol->xfrm_vec[i].id.daddr));  in xfrm_policy_migrate()
4390 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,  in xfrm_policy_migrate()
4391 			       sizeof(pol->xfrm_vec[i].saddr));  in xfrm_policy_migrate()
4392 			pol->xfrm_vec[i].encap_family = mp->new_family;  in xfrm_policy_migrate()
4394 			atomic_inc(&pol->genid);  in xfrm_policy_migrate()
4398 	write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
4441 	struct xfrm_policy *pol = NULL;  in xfrm_migrate()  local
4457 	if ((pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id)) == NULL) {  in xfrm_migrate()
4479 	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)  in xfrm_migrate()
4491 	xfrm_pol_put(pol);  in xfrm_migrate()
4498 	if (pol)  in xfrm_migrate()
4499 		xfrm_pol_put(pol);  in xfrm_migrate()