Kernel

Kernel Git Source Tree

Root/net/xfrm/xfrm_policy.c

1	/*
2	* xfrm_policy.c
3	*
4	* Changes:
5	* Mitsuru KANDA @USAGI
6	* Kazunori MIYAZAWA @USAGI
7	* Kunihiro Ishiguro <kunihiro@ipinfusion.com>
8	* IPv6 support
9	* Kazunori MIYAZAWA @USAGI
10	* YOSHIFUJI Hideaki
11	* Split up af-specific portion
12	* Derek Atkins <derek@ihtfp.com> Add the post_input processor
13	*
14	*/
15
16	#include <linux/err.h>
17	#include <linux/slab.h>
18	#include <linux/kmod.h>
19	#include <linux/list.h>
20	#include <linux/spinlock.h>
21	#include <linux/workqueue.h>
22	#include <linux/notifier.h>
23	#include <linux/netdevice.h>
24	#include <linux/netfilter.h>
25	#include <linux/module.h>
26	#include <linux/cache.h>
27	#include <linux/audit.h>
28	#include <net/dst.h>
29	#include <net/flow.h>
30	#include <net/xfrm.h>
31	#include <net/ip.h>
32	#ifdef CONFIG_XFRM_STATISTICS
33	#include <net/snmp.h>
34	#endif
35
36	#include "xfrm_hash.h"
37
38	#define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
39	#define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
40	#define XFRM_MAX_QUEUE_LEN 100
41
42	DEFINE_MUTEX(xfrm_cfg_mutex);
43	EXPORT_SYMBOL(xfrm_cfg_mutex);
44
45	static DEFINE_SPINLOCK(xfrm_policy_sk_bundle_lock);
46	static struct dst_entry *xfrm_policy_sk_bundles;
47	static DEFINE_RWLOCK(xfrm_policy_lock);
48
49	static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
50	static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[NPROTO]
51	__read_mostly;
52
53	static struct kmem_cache *xfrm_dst_cache __read_mostly;
54
55	static void xfrm_init_pmtu(struct dst_entry *dst);
56	static int stale_bundle(struct dst_entry *dst);
57	static int xfrm_bundle_ok(struct xfrm_dst *xdst);
58	static void xfrm_policy_queue_process(unsigned long arg);
59
60	static struct xfrm_policy __xfrm_policy_unlink(struct xfrm_policy pol,
61	int dir);
62
63	static inline bool
64	__xfrm4_selector_match(const struct xfrm_selector sel, const struct flowi fl)
65	{
66	const struct flowi4 *fl4 = &fl->u.ip4;
67
68	return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
69	addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
70	!((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
71	!((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
72	(fl4->flowi4_proto == sel->proto \|\| !sel->proto) &&
73	(fl4->flowi4_oif == sel->ifindex \|\| !sel->ifindex);
74	}
75
76	static inline bool
77	__xfrm6_selector_match(const struct xfrm_selector sel, const struct flowi fl)
78	{
79	const struct flowi6 *fl6 = &fl->u.ip6;
80
81	return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
82	addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
83	!((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
84	!((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
85	(fl6->flowi6_proto == sel->proto \|\| !sel->proto) &&
86	(fl6->flowi6_oif == sel->ifindex \|\| !sel->ifindex);
87	}
88
89	bool xfrm_selector_match(const struct xfrm_selector sel, const struct flowi fl,
90	unsigned short family)
91	{
92	switch (family) {
93	case AF_INET:
94	return __xfrm4_selector_match(sel, fl);
95	case AF_INET6:
96	return __xfrm6_selector_match(sel, fl);
97	}
98	return false;
99	}
100
101	static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
102	{
103	struct xfrm_policy_afinfo *afinfo;
104
105	if (unlikely(family >= NPROTO))
106	return NULL;
107	rcu_read_lock();
108	afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
109	if (unlikely(!afinfo))
110	rcu_read_unlock();
111	return afinfo;
112	}
113
114	static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
115	{
116	rcu_read_unlock();
117	}
118
119	static inline struct dst_entry __xfrm_dst_lookup(struct net net, int tos,
120	const xfrm_address_t *saddr,
121	const xfrm_address_t *daddr,
122	int family)
123	{
124	struct xfrm_policy_afinfo *afinfo;
125	struct dst_entry *dst;
126
127	afinfo = xfrm_policy_get_afinfo(family);
128	if (unlikely(afinfo == NULL))
129	return ERR_PTR(-EAFNOSUPPORT);
130
131	dst = afinfo->dst_lookup(net, tos, saddr, daddr);
132
133	xfrm_policy_put_afinfo(afinfo);
134
135	return dst;
136	}
137
138	static inline struct dst_entry xfrm_dst_lookup(struct xfrm_state x, int tos,
139	xfrm_address_t *prev_saddr,
140	xfrm_address_t *prev_daddr,
141	int family)
142	{
143	struct net *net = xs_net(x);
144	xfrm_address_t *saddr = &x->props.saddr;
145	xfrm_address_t *daddr = &x->id.daddr;
146	struct dst_entry *dst;
147
148	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
149	saddr = x->coaddr;
150	daddr = prev_daddr;
151	}
152	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
153	saddr = prev_saddr;
154	daddr = x->coaddr;
155	}
156
157	dst = __xfrm_dst_lookup(net, tos, saddr, daddr, family);
158
159	if (!IS_ERR(dst)) {
160	if (prev_saddr != saddr)
161	memcpy(prev_saddr, saddr, sizeof(*prev_saddr));
162	if (prev_daddr != daddr)
163	memcpy(prev_daddr, daddr, sizeof(*prev_daddr));
164	}
165
166	return dst;
167	}
168
169	static inline unsigned long make_jiffies(long secs)
170	{
171	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
172	return MAX_SCHEDULE_TIMEOUT-1;
173	else
174	return secs*HZ;
175	}
176
177	static void xfrm_policy_timer(unsigned long data)
178	{
179	struct xfrm_policy xp = (struct xfrm_policy)data;
180	unsigned long now = get_seconds();
181	long next = LONG_MAX;
182	int warn = 0;
183	int dir;
184
185	read_lock(&xp->lock);
186
187	if (unlikely(xp->walk.dead))
188	goto out;
189
190	dir = xfrm_policy_id2dir(xp->index);
191
192	if (xp->lft.hard_add_expires_seconds) {
193	long tmo = xp->lft.hard_add_expires_seconds +
194	xp->curlft.add_time - now;
195	if (tmo <= 0)
196	goto expired;
197	if (tmo < next)
198	next = tmo;
199	}
200	if (xp->lft.hard_use_expires_seconds) {
201	long tmo = xp->lft.hard_use_expires_seconds +
202	(xp->curlft.use_time ? : xp->curlft.add_time) - now;
203	if (tmo <= 0)
204	goto expired;
205	if (tmo < next)
206	next = tmo;
207	}
208	if (xp->lft.soft_add_expires_seconds) {
209	long tmo = xp->lft.soft_add_expires_seconds +
210	xp->curlft.add_time - now;
211	if (tmo <= 0) {
212	warn = 1;
213	tmo = XFRM_KM_TIMEOUT;
214	}
215	if (tmo < next)
216	next = tmo;
217	}
218	if (xp->lft.soft_use_expires_seconds) {
219	long tmo = xp->lft.soft_use_expires_seconds +
220	(xp->curlft.use_time ? : xp->curlft.add_time) - now;
221	if (tmo <= 0) {
222	warn = 1;
223	tmo = XFRM_KM_TIMEOUT;
224	}
225	if (tmo < next)
226	next = tmo;
227	}
228
229	if (warn)
230	km_policy_expired(xp, dir, 0, 0);
231	if (next != LONG_MAX &&
232	!mod_timer(&xp->timer, jiffies + make_jiffies(next)))
233	xfrm_pol_hold(xp);
234
235	out:
236	read_unlock(&xp->lock);
237	xfrm_pol_put(xp);
238	return;
239
240	expired:
241	read_unlock(&xp->lock);
242	if (!xfrm_policy_delete(xp, dir))
243	km_policy_expired(xp, dir, 1, 0);
244	xfrm_pol_put(xp);
245	}
246
247	static struct flow_cache_object xfrm_policy_flo_get(struct flow_cache_object flo)
248	{
249	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
250
251	if (unlikely(pol->walk.dead))
252	flo = NULL;
253	else
254	xfrm_pol_hold(pol);
255
256	return flo;
257	}
258
259	static int xfrm_policy_flo_check(struct flow_cache_object *flo)
260	{
261	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
262
263	return !pol->walk.dead;
264	}
265
266	static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
267	{
268	xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
269	}
270
271	static const struct flow_cache_ops xfrm_policy_fc_ops = {
272	.get = xfrm_policy_flo_get,
273	.check = xfrm_policy_flo_check,
274	.delete = xfrm_policy_flo_delete,
275	};
276
277	/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
278	* SPD calls.
279	*/
280
281	struct xfrm_policy xfrm_policy_alloc(struct net net, gfp_t gfp)
282	{
283	struct xfrm_policy *policy;
284
285	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
286
287	if (policy) {
288	write_pnet(&policy->xp_net, net);
289	INIT_LIST_HEAD(&policy->walk.all);
290	INIT_HLIST_NODE(&policy->bydst);
291	INIT_HLIST_NODE(&policy->byidx);
292	rwlock_init(&policy->lock);
293	atomic_set(&policy->refcnt, 1);
294	skb_queue_head_init(&policy->polq.hold_queue);
295	setup_timer(&policy->timer, xfrm_policy_timer,
296	(unsigned long)policy);
297	setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
298	(unsigned long)policy);
299	policy->flo.ops = &xfrm_policy_fc_ops;
300	}
301	return policy;
302	}
303	EXPORT_SYMBOL(xfrm_policy_alloc);
304
305	/* Destroy xfrm_policy: descendant resources must be released to this moment. */
306
307	void xfrm_policy_destroy(struct xfrm_policy *policy)
308	{
309	BUG_ON(!policy->walk.dead);
310
311	if (del_timer(&policy->timer) \|\| del_timer(&policy->polq.hold_timer))
312	BUG();
313
314	security_xfrm_policy_free(policy->security);
315	kfree(policy);
316	}
317	EXPORT_SYMBOL(xfrm_policy_destroy);
318
319	static void xfrm_queue_purge(struct sk_buff_head *list)
320	{
321	struct sk_buff *skb;
322
323	while ((skb = skb_dequeue(list)) != NULL)
324	kfree_skb(skb);
325	}
326
327	/* Rule must be locked. Release descentant resources, announce
328	* entry dead. The rule must be unlinked from lists to the moment.
329	*/
330
331	static void xfrm_policy_kill(struct xfrm_policy *policy)
332	{
333	policy->walk.dead = 1;
334
335	atomic_inc(&policy->genid);
336
337	if (del_timer(&policy->polq.hold_timer))
338	xfrm_pol_put(policy);
339	xfrm_queue_purge(&policy->polq.hold_queue);
340
341	if (del_timer(&policy->timer))
342	xfrm_pol_put(policy);
343
344	xfrm_pol_put(policy);
345	}
346
347	static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
348
349	static inline unsigned int idx_hash(struct net *net, u32 index)
350	{
351	return __idx_hash(index, net->xfrm.policy_idx_hmask);
352	}
353
354	static struct hlist_head policy_hash_bysel(struct net net,
355	const struct xfrm_selector *sel,
356	unsigned short family, int dir)
357	{
358	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
359	unsigned int hash = __sel_hash(sel, family, hmask);
360
361	return (hash == hmask + 1 ?
362	&net->xfrm.policy_inexact[dir] :
363	net->xfrm.policy_bydst[dir].table + hash);
364	}
365
366	static struct hlist_head policy_hash_direct(struct net net,
367	const xfrm_address_t *daddr,
368	const xfrm_address_t *saddr,
369	unsigned short family, int dir)
370	{
371	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
372	unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
373
374	return net->xfrm.policy_bydst[dir].table + hash;
375	}
376
377	static void xfrm_dst_hash_transfer(struct hlist_head *list,
378	struct hlist_head *ndsttable,
379	unsigned int nhashmask)
380	{
381	struct hlist_node tmp, entry0 = NULL;
382	struct xfrm_policy *pol;
383	unsigned int h0 = 0;
384
385	redo:
386	hlist_for_each_entry_safe(pol, tmp, list, bydst) {
387	unsigned int h;
388
389	h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
390	pol->family, nhashmask);
391	if (!entry0) {
392	hlist_del(&pol->bydst);
393	hlist_add_head(&pol->bydst, ndsttable+h);
394	h0 = h;
395	} else {
396	if (h != h0)
397	continue;
398	hlist_del(&pol->bydst);
399	hlist_add_after(entry0, &pol->bydst);
400	}
401	entry0 = &pol->bydst;
402	}
403	if (!hlist_empty(list)) {
404	entry0 = NULL;
405	goto redo;
406	}
407	}
408
409	static void xfrm_idx_hash_transfer(struct hlist_head *list,
410	struct hlist_head *nidxtable,
411	unsigned int nhashmask)
412	{
413	struct hlist_node *tmp;
414	struct xfrm_policy *pol;
415
416	hlist_for_each_entry_safe(pol, tmp, list, byidx) {
417	unsigned int h;
418
419	h = __idx_hash(pol->index, nhashmask);
420	hlist_add_head(&pol->byidx, nidxtable+h);
421	}
422	}
423
424	static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
425	{
426	return ((old_hmask + 1) << 1) - 1;
427	}
428
429	static void xfrm_bydst_resize(struct net *net, int dir)
430	{
431	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
432	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
433	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
434	struct hlist_head *odst = net->xfrm.policy_bydst[dir].table;
435	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
436	int i;
437
438	if (!ndst)
439	return;
440
441	write_lock_bh(&xfrm_policy_lock);
442
443	for (i = hmask; i >= 0; i--)
444	xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
445
446	net->xfrm.policy_bydst[dir].table = ndst;
447	net->xfrm.policy_bydst[dir].hmask = nhashmask;
448
449	write_unlock_bh(&xfrm_policy_lock);
450
451	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
452	}
453
454	static void xfrm_byidx_resize(struct net *net, int total)
455	{
456	unsigned int hmask = net->xfrm.policy_idx_hmask;
457	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
458	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
459	struct hlist_head *oidx = net->xfrm.policy_byidx;
460	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
461	int i;
462
463	if (!nidx)
464	return;
465
466	write_lock_bh(&xfrm_policy_lock);
467
468	for (i = hmask; i >= 0; i--)
469	xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
470
471	net->xfrm.policy_byidx = nidx;
472	net->xfrm.policy_idx_hmask = nhashmask;
473
474	write_unlock_bh(&xfrm_policy_lock);
475
476	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
477	}
478
479	static inline int xfrm_bydst_should_resize(struct net net, int dir, int total)
480	{
481	unsigned int cnt = net->xfrm.policy_count[dir];
482	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
483
484	if (total)
485	*total += cnt;
486
487	if ((hmask + 1) < xfrm_policy_hashmax &&
488	cnt > hmask)
489	return 1;
490
491	return 0;
492	}
493
494	static inline int xfrm_byidx_should_resize(struct net *net, int total)
495	{
496	unsigned int hmask = net->xfrm.policy_idx_hmask;
497
498	if ((hmask + 1) < xfrm_policy_hashmax &&
499	total > hmask)
500	return 1;
501
502	return 0;
503	}
504
505	void xfrm_spd_getinfo(struct net net, struct xfrmk_spdinfo si)
506	{
507	read_lock_bh(&xfrm_policy_lock);
508	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
509	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
510	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
511	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
512	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
513	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
514	si->spdhcnt = net->xfrm.policy_idx_hmask;
515	si->spdhmcnt = xfrm_policy_hashmax;
516	read_unlock_bh(&xfrm_policy_lock);
517	}
518	EXPORT_SYMBOL(xfrm_spd_getinfo);
519
520	static DEFINE_MUTEX(hash_resize_mutex);
521	static void xfrm_hash_resize(struct work_struct *work)
522	{
523	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
524	int dir, total;
525
526	mutex_lock(&hash_resize_mutex);
527
528	total = 0;
529	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
530	if (xfrm_bydst_should_resize(net, dir, &total))
531	xfrm_bydst_resize(net, dir);
532	}
533	if (xfrm_byidx_should_resize(net, total))
534	xfrm_byidx_resize(net, total);
535
536	mutex_unlock(&hash_resize_mutex);
537	}
538
539	/* Generate new index... KAME seems to generate them ordered by cost
540	* of an absolute inpredictability of ordering of rules. This will not pass. */
541	static u32 xfrm_gen_index(struct net *net, int dir)
542	{
543	static u32 idx_generator;
544
545	for (;;) {
546	struct hlist_head *list;
547	struct xfrm_policy *p;
548	u32 idx;
549	int found;
550
551	idx = (idx_generator \| dir);
552	idx_generator += 8;
553	if (idx == 0)
554	idx = 8;
555	list = net->xfrm.policy_byidx + idx_hash(net, idx);
556	found = 0;
557	hlist_for_each_entry(p, list, byidx) {
558	if (p->index == idx) {
559	found = 1;
560	break;
561	}
562	}
563	if (!found)
564	return idx;
565	}
566	}
567
568	static inline int selector_cmp(struct xfrm_selector s1, struct xfrm_selector s2)
569	{
570	u32 p1 = (u32 ) s1;
571	u32 p2 = (u32 ) s2;
572	int len = sizeof(struct xfrm_selector) / sizeof(u32);
573	int i;
574
575	for (i = 0; i < len; i++) {
576	if (p1[i] != p2[i])
577	return 1;
578	}
579
580	return 0;
581	}
582
583	static void xfrm_policy_requeue(struct xfrm_policy *old,
584	struct xfrm_policy *new)
585	{
586	struct xfrm_policy_queue *pq = &old->polq;
587	struct sk_buff_head list;
588
589	__skb_queue_head_init(&list);
590
591	spin_lock_bh(&pq->hold_queue.lock);
592	skb_queue_splice_init(&pq->hold_queue, &list);
593	if (del_timer(&pq->hold_timer))
594	xfrm_pol_put(old);
595	spin_unlock_bh(&pq->hold_queue.lock);
596
597	if (skb_queue_empty(&list))
598	return;
599
600	pq = &new->polq;
601
602	spin_lock_bh(&pq->hold_queue.lock);
603	skb_queue_splice(&list, &pq->hold_queue);
604	pq->timeout = XFRM_QUEUE_TMO_MIN;
605	if (!mod_timer(&pq->hold_timer, jiffies))
606	xfrm_pol_hold(new);
607	spin_unlock_bh(&pq->hold_queue.lock);
608	}
609
610	static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
611	struct xfrm_policy *pol)
612	{
613	u32 mark = policy->mark.v & policy->mark.m;
614
615	if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
616	return true;
617
618	if ((mark & pol->mark.m) == pol->mark.v &&
619	policy->priority == pol->priority)
620	return true;
621
622	return false;
623	}
624
625	int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
626	{
627	struct net *net = xp_net(policy);
628	struct xfrm_policy *pol;
629	struct xfrm_policy *delpol;
630	struct hlist_head *chain;
631	struct hlist_node *newpos;
632
633	write_lock_bh(&xfrm_policy_lock);
634	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
635	delpol = NULL;
636	newpos = NULL;
637	hlist_for_each_entry(pol, chain, bydst) {
638	if (pol->type == policy->type &&
639	!selector_cmp(&pol->selector, &policy->selector) &&
640	xfrm_policy_mark_match(policy, pol) &&
641	xfrm_sec_ctx_match(pol->security, policy->security) &&
642	!WARN_ON(delpol)) {
643	if (excl) {
644	write_unlock_bh(&xfrm_policy_lock);
645	return -EEXIST;
646	}
647	delpol = pol;
648	if (policy->priority > pol->priority)
649	continue;
650	} else if (policy->priority >= pol->priority) {
651	newpos = &pol->bydst;
652	continue;
653	}
654	if (delpol)
655	break;
656	}
657	if (newpos)
658	hlist_add_after(newpos, &policy->bydst);
659	else
660	hlist_add_head(&policy->bydst, chain);
661	xfrm_pol_hold(policy);
662	net->xfrm.policy_count[dir]++;
663	atomic_inc(&flow_cache_genid);
664
665	/* After previous checking, family can either be AF_INET or AF_INET6 */
666	if (policy->family == AF_INET)
667	rt_genid_bump_ipv4(net);
668	else
669	rt_genid_bump_ipv6(net);
670
671	if (delpol) {
672	xfrm_policy_requeue(delpol, policy);
673	__xfrm_policy_unlink(delpol, dir);
674	}
675	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir);
676	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
677	policy->curlft.add_time = get_seconds();
678	policy->curlft.use_time = 0;
679	if (!mod_timer(&policy->timer, jiffies + HZ))
680	xfrm_pol_hold(policy);
681	list_add(&policy->walk.all, &net->xfrm.policy_all);
682	write_unlock_bh(&xfrm_policy_lock);
683
684	if (delpol)
685	xfrm_policy_kill(delpol);
686	else if (xfrm_bydst_should_resize(net, dir, NULL))
687	schedule_work(&net->xfrm.policy_hash_work);
688
689	return 0;
690	}
691	EXPORT_SYMBOL(xfrm_policy_insert);
692
693	struct xfrm_policy xfrm_policy_bysel_ctx(struct net net, u32 mark, u8 type,
694	int dir, struct xfrm_selector *sel,
695	struct xfrm_sec_ctx *ctx, int delete,
696	int *err)
697	{
698	struct xfrm_policy pol, ret;
699	struct hlist_head *chain;
700
701	*err = 0;
702	write_lock_bh(&xfrm_policy_lock);
703	chain = policy_hash_bysel(net, sel, sel->family, dir);
704	ret = NULL;
705	hlist_for_each_entry(pol, chain, bydst) {
706	if (pol->type == type &&
707	(mark & pol->mark.m) == pol->mark.v &&
708	!selector_cmp(sel, &pol->selector) &&
709	xfrm_sec_ctx_match(ctx, pol->security)) {
710	xfrm_pol_hold(pol);
711	if (delete) {
712	*err = security_xfrm_policy_delete(
713	pol->security);
714	if (*err) {
715	write_unlock_bh(&xfrm_policy_lock);
716	return pol;
717	}
718	__xfrm_policy_unlink(pol, dir);
719	}
720	ret = pol;
721	break;
722	}
723	}
724	write_unlock_bh(&xfrm_policy_lock);
725
726	if (ret && delete)
727	xfrm_policy_kill(ret);
728	return ret;
729	}
730	EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
731
732	struct xfrm_policy xfrm_policy_byid(struct net net, u32 mark, u8 type,
733	int dir, u32 id, int delete, int *err)
734	{
735	struct xfrm_policy pol, ret;
736	struct hlist_head *chain;
737
738	*err = -ENOENT;
739	if (xfrm_policy_id2dir(id) != dir)
740	return NULL;
741
742	*err = 0;
743	write_lock_bh(&xfrm_policy_lock);
744	chain = net->xfrm.policy_byidx + idx_hash(net, id);
745	ret = NULL;
746	hlist_for_each_entry(pol, chain, byidx) {
747	if (pol->type == type && pol->index == id &&
748	(mark & pol->mark.m) == pol->mark.v) {
749	xfrm_pol_hold(pol);
750	if (delete) {
751	*err = security_xfrm_policy_delete(
752	pol->security);
753	if (*err) {
754	write_unlock_bh(&xfrm_policy_lock);
755	return pol;
756	}
757	__xfrm_policy_unlink(pol, dir);
758	}
759	ret = pol;
760	break;
761	}
762	}
763	write_unlock_bh(&xfrm_policy_lock);
764
765	if (ret && delete)
766	xfrm_policy_kill(ret);
767	return ret;
768	}
769	EXPORT_SYMBOL(xfrm_policy_byid);
770
771	#ifdef CONFIG_SECURITY_NETWORK_XFRM
772	static inline int
773	xfrm_policy_flush_secctx_check(struct net net, u8 type, struct xfrm_audit audit_info)
774	{
775	int dir, err = 0;
776
777	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
778	struct xfrm_policy *pol;
779	int i;
780
781	hlist_for_each_entry(pol,
782	&net->xfrm.policy_inexact[dir], bydst) {
783	if (pol->type != type)
784	continue;
785	err = security_xfrm_policy_delete(pol->security);
786	if (err) {
787	xfrm_audit_policy_delete(pol, 0,
788	audit_info->loginuid,
789	audit_info->sessionid,
790	audit_info->secid);
791	return err;
792	}
793	}
794	for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
795	hlist_for_each_entry(pol,
796	net->xfrm.policy_bydst[dir].table + i,
797	bydst) {
798	if (pol->type != type)
799	continue;
800	err = security_xfrm_policy_delete(
801	pol->security);
802	if (err) {
803	xfrm_audit_policy_delete(pol, 0,
804	audit_info->loginuid,
805	audit_info->sessionid,
806	audit_info->secid);
807	return err;
808	}
809	}
810	}
811	}
812	return err;
813	}
814	#else
815	static inline int
816	xfrm_policy_flush_secctx_check(struct net net, u8 type, struct xfrm_audit audit_info)
817	{
818	return 0;
819	}
820	#endif
821
822	int xfrm_policy_flush(struct net net, u8 type, struct xfrm_audit audit_info)
823	{
824	int dir, err = 0, cnt = 0;
825
826	write_lock_bh(&xfrm_policy_lock);
827
828	err = xfrm_policy_flush_secctx_check(net, type, audit_info);
829	if (err)
830	goto out;
831
832	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
833	struct xfrm_policy *pol;
834	int i;
835
836	again1:
837	hlist_for_each_entry(pol,
838	&net->xfrm.policy_inexact[dir], bydst) {
839	if (pol->type != type)
840	continue;
841	__xfrm_policy_unlink(pol, dir);
842	write_unlock_bh(&xfrm_policy_lock);
843	cnt++;
844
845	xfrm_audit_policy_delete(pol, 1, audit_info->loginuid,
846	audit_info->sessionid,
847	audit_info->secid);
848
849	xfrm_policy_kill(pol);
850
851	write_lock_bh(&xfrm_policy_lock);
852	goto again1;
853	}
854
855	for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
856	again2:
857	hlist_for_each_entry(pol,
858	net->xfrm.policy_bydst[dir].table + i,
859	bydst) {
860	if (pol->type != type)
861	continue;
862	__xfrm_policy_unlink(pol, dir);
863	write_unlock_bh(&xfrm_policy_lock);
864	cnt++;
865
866	xfrm_audit_policy_delete(pol, 1,
867	audit_info->loginuid,
868	audit_info->sessionid,
869	audit_info->secid);
870	xfrm_policy_kill(pol);
871
872	write_lock_bh(&xfrm_policy_lock);
873	goto again2;
874	}
875	}
876
877	}
878	if (!cnt)
879	err = -ESRCH;
880	out:
881	write_unlock_bh(&xfrm_policy_lock);
882	return err;
883	}
884	EXPORT_SYMBOL(xfrm_policy_flush);
885
886	int xfrm_policy_walk(struct net net, struct xfrm_policy_walk walk,
887	int (func)(struct xfrm_policy , int, int, void*),
888	void *data)
889	{
890	struct xfrm_policy *pol;
891	struct xfrm_policy_walk_entry *x;
892	int error = 0;
893
894	if (walk->type >= XFRM_POLICY_TYPE_MAX &&
895	walk->type != XFRM_POLICY_TYPE_ANY)
896	return -EINVAL;
897
898	if (list_empty(&walk->walk.all) && walk->seq != 0)
899	return 0;
900
901	write_lock_bh(&xfrm_policy_lock);
902	if (list_empty(&walk->walk.all))
903	x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
904	else
905	x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all);
906	list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
907	if (x->dead)
908	continue;
909	pol = container_of(x, struct xfrm_policy, walk);
910	if (walk->type != XFRM_POLICY_TYPE_ANY &&
911	walk->type != pol->type)
912	continue;
913	error = func(pol, xfrm_policy_id2dir(pol->index),
914	walk->seq, data);
915	if (error) {
916	list_move_tail(&walk->walk.all, &x->all);
917	goto out;
918	}
919	walk->seq++;
920	}
921	if (walk->seq == 0) {
922	error = -ENOENT;
923	goto out;
924	}
925	list_del_init(&walk->walk.all);
926	out:
927	write_unlock_bh(&xfrm_policy_lock);
928	return error;
929	}
930	EXPORT_SYMBOL(xfrm_policy_walk);
931
932	void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
933	{
934	INIT_LIST_HEAD(&walk->walk.all);
935	walk->walk.dead = 1;
936	walk->type = type;
937	walk->seq = 0;
938	}
939	EXPORT_SYMBOL(xfrm_policy_walk_init);
940
941	void xfrm_policy_walk_done(struct xfrm_policy_walk *walk)
942	{
943	if (list_empty(&walk->walk.all))
944	return;
945
946	write_lock_bh(&xfrm_policy_lock);
947	list_del(&walk->walk.all);
948	write_unlock_bh(&xfrm_policy_lock);
949	}
950	EXPORT_SYMBOL(xfrm_policy_walk_done);
951
952	/*
953	* Find policy to apply to this flow.
954	*
955	* Returns 0 if policy found, else an -errno.
956	*/
957	static int xfrm_policy_match(const struct xfrm_policy *pol,
958	const struct flowi *fl,
959	u8 type, u16 family, int dir)
960	{
961	const struct xfrm_selector *sel = &pol->selector;
962	int ret = -ESRCH;
963	bool match;
964
965	if (pol->family != family \|\|
966	(fl->flowi_mark & pol->mark.m) != pol->mark.v \|\|
967	pol->type != type)
968	return ret;
969
970	match = xfrm_selector_match(sel, fl, family);
971	if (match)
972	ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
973	dir);
974
975	return ret;
976	}
977
978	static struct xfrm_policy xfrm_policy_lookup_bytype(struct net net, u8 type,
979	const struct flowi *fl,
980	u16 family, u8 dir)
981	{
982	int err;
983	struct xfrm_policy pol, ret;
984	const xfrm_address_t daddr, saddr;
985	struct hlist_head *chain;
986	u32 priority = ~0U;
987
988	daddr = xfrm_flowi_daddr(fl, family);
989	saddr = xfrm_flowi_saddr(fl, family);
990	if (unlikely(!daddr \|\| !saddr))
991	return NULL;
992
993	read_lock_bh(&xfrm_policy_lock);
994	chain = policy_hash_direct(net, daddr, saddr, family, dir);
995	ret = NULL;
996	hlist_for_each_entry(pol, chain, bydst) {
997	err = xfrm_policy_match(pol, fl, type, family, dir);
998	if (err) {
999	if (err == -ESRCH)
1000	continue;
1001	else {
1002	ret = ERR_PTR(err);
1003	goto fail;
1004	}
1005	} else {
1006	ret = pol;
1007	priority = ret->priority;
1008	break;
1009	}
1010	}
1011	chain = &net->xfrm.policy_inexact[dir];
1012	hlist_for_each_entry(pol, chain, bydst) {
1013	err = xfrm_policy_match(pol, fl, type, family, dir);
1014	if (err) {
1015	if (err == -ESRCH)
1016	continue;
1017	else {
1018	ret = ERR_PTR(err);
1019	goto fail;
1020	}
1021	} else if (pol->priority < priority) {
1022	ret = pol;
1023	break;
1024	}
1025	}
1026	if (ret)
1027	xfrm_pol_hold(ret);
1028	fail:
1029	read_unlock_bh(&xfrm_policy_lock);
1030
1031	return ret;
1032	}
1033
1034	static struct xfrm_policy *
1035	__xfrm_policy_lookup(struct net net, const struct flowi fl, u16 family, u8 dir)
1036	{
1037	#ifdef CONFIG_XFRM_SUB_POLICY
1038	struct xfrm_policy *pol;
1039
1040	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);
1041	if (pol != NULL)
1042	return pol;
1043	#endif
1044	return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);
1045	}
1046
1047	static int flow_to_policy_dir(int dir)
1048	{
1049	if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1050	XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1051	XFRM_POLICY_FWD == FLOW_DIR_FWD)
1052	return dir;
1053
1054	switch (dir) {
1055	default:
1056	case FLOW_DIR_IN:
1057	return XFRM_POLICY_IN;
1058	case FLOW_DIR_OUT:
1059	return XFRM_POLICY_OUT;
1060	case FLOW_DIR_FWD:
1061	return XFRM_POLICY_FWD;
1062	}
1063	}
1064
1065	static struct flow_cache_object *
1066	xfrm_policy_lookup(struct net net, const struct flowi fl, u16 family,
1067	u8 dir, struct flow_cache_object old_obj, void ctx)
1068	{
1069	struct xfrm_policy *pol;
1070
1071	if (old_obj)
1072	xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo));
1073
1074	pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));
1075	if (IS_ERR_OR_NULL(pol))
1076	return ERR_CAST(pol);
1077
1078	/* Resolver returns two references:
1079	* one for cache and one for caller of flow_cache_lookup() */
1080	xfrm_pol_hold(pol);
1081
1082	return &pol->flo;
1083	}
1084
1085	static inline int policy_to_flow_dir(int dir)
1086	{
1087	if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1088	XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1089	XFRM_POLICY_FWD == FLOW_DIR_FWD)
1090	return dir;
1091	switch (dir) {
1092	default:
1093	case XFRM_POLICY_IN:
1094	return FLOW_DIR_IN;
1095	case XFRM_POLICY_OUT:
1096	return FLOW_DIR_OUT;
1097	case XFRM_POLICY_FWD:
1098	return FLOW_DIR_FWD;
1099	}
1100	}
1101
1102	static struct xfrm_policy xfrm_sk_policy_lookup(struct sock sk, int dir,
1103	const struct flowi *fl)
1104	{
1105	struct xfrm_policy *pol;
1106
1107	read_lock_bh(&xfrm_policy_lock);
1108	if ((pol = sk->sk_policy[dir]) != NULL) {
1109	bool match = xfrm_selector_match(&pol->selector, fl,
1110	sk->sk_family);
1111	int err = 0;
1112
1113	if (match) {
1114	if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {
1115	pol = NULL;
1116	goto out;
1117	}
1118	err = security_xfrm_policy_lookup(pol->security,
1119	fl->flowi_secid,
1120	policy_to_flow_dir(dir));
1121	if (!err)
1122	xfrm_pol_hold(pol);
1123	else if (err == -ESRCH)
1124	pol = NULL;
1125	else
1126	pol = ERR_PTR(err);
1127	} else
1128	pol = NULL;
1129	}
1130	out:
1131	read_unlock_bh(&xfrm_policy_lock);
1132	return pol;
1133	}
1134
1135	static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1136	{
1137	struct net *net = xp_net(pol);
1138	struct hlist_head *chain = policy_hash_bysel(net, &pol->selector,
1139	pol->family, dir);
1140
1141	list_add(&pol->walk.all, &net->xfrm.policy_all);
1142	hlist_add_head(&pol->bydst, chain);
1143	hlist_add_head(&pol->byidx, net->xfrm.policy_byidx+idx_hash(net, pol->index));
1144	net->xfrm.policy_count[dir]++;
1145	xfrm_pol_hold(pol);
1146
1147	if (xfrm_bydst_should_resize(net, dir, NULL))
1148	schedule_work(&net->xfrm.policy_hash_work);
1149	}
1150
1151	static struct xfrm_policy __xfrm_policy_unlink(struct xfrm_policy pol,
1152	int dir)
1153	{
1154	struct net *net = xp_net(pol);
1155
1156	if (hlist_unhashed(&pol->bydst))
1157	return NULL;
1158
1159	hlist_del(&pol->bydst);
1160	hlist_del(&pol->byidx);
1161	list_del(&pol->walk.all);
1162	net->xfrm.policy_count[dir]--;
1163
1164	return pol;
1165	}
1166
1167	int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1168	{
1169	write_lock_bh(&xfrm_policy_lock);
1170	pol = __xfrm_policy_unlink(pol, dir);
1171	write_unlock_bh(&xfrm_policy_lock);
1172	if (pol) {
1173	xfrm_policy_kill(pol);
1174	return 0;
1175	}
1176	return -ENOENT;
1177	}
1178	EXPORT_SYMBOL(xfrm_policy_delete);
1179
1180	int xfrm_sk_policy_insert(struct sock sk, int dir, struct xfrm_policy pol)
1181	{
1182	struct net *net = xp_net(pol);
1183	struct xfrm_policy *old_pol;
1184
1185	#ifdef CONFIG_XFRM_SUB_POLICY
1186	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1187	return -EINVAL;
1188	#endif
1189
1190	write_lock_bh(&xfrm_policy_lock);
1191	old_pol = sk->sk_policy[dir];
1192	sk->sk_policy[dir] = pol;
1193	if (pol) {
1194	pol->curlft.add_time = get_seconds();
1195	pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir);
1196	__xfrm_policy_link(pol, XFRM_POLICY_MAX+dir);
1197	}
1198	if (old_pol) {
1199	if (pol)
1200	xfrm_policy_requeue(old_pol, pol);
1201
1202	/* Unlinking succeeds always. This is the only function
1203	* allowed to delete or replace socket policy.
1204	*/
1205	__xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir);
1206	}
1207	write_unlock_bh(&xfrm_policy_lock);
1208
1209	if (old_pol) {
1210	xfrm_policy_kill(old_pol);
1211	}
1212	return 0;
1213	}
1214
1215	static struct xfrm_policy clone_policy(const struct xfrm_policy old, int dir)
1216	{
1217	struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
1218
1219	if (newp) {
1220	newp->selector = old->selector;
1221	if (security_xfrm_policy_clone(old->security,
1222	&newp->security)) {
1223	kfree(newp);
1224	return NULL; /* ENOMEM */
1225	}
1226	newp->lft = old->lft;
1227	newp->curlft = old->curlft;
1228	newp->mark = old->mark;
1229	newp->action = old->action;
1230	newp->flags = old->flags;
1231	newp->xfrm_nr = old->xfrm_nr;
1232	newp->index = old->index;
1233	newp->type = old->type;
1234	memcpy(newp->xfrm_vec, old->xfrm_vec,
1235	newp->xfrm_nr*sizeof(struct xfrm_tmpl));
1236	write_lock_bh(&xfrm_policy_lock);
1237	__xfrm_policy_link(newp, XFRM_POLICY_MAX+dir);
1238	write_unlock_bh(&xfrm_policy_lock);
1239	xfrm_pol_put(newp);
1240	}
1241	return newp;
1242	}
1243
1244	int __xfrm_sk_clone_policy(struct sock *sk)
1245	{
1246	struct xfrm_policy *p0 = sk->sk_policy[0],
1247	*p1 = sk->sk_policy[1];
1248
1249	sk->sk_policy[0] = sk->sk_policy[1] = NULL;
1250	if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
1251	return -ENOMEM;
1252	if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
1253	return -ENOMEM;
1254	return 0;
1255	}
1256
1257	static int
1258	xfrm_get_saddr(struct net net, xfrm_address_t local, xfrm_address_t *remote,
1259	unsigned short family)
1260	{
1261	int err;
1262	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1263
1264	if (unlikely(afinfo == NULL))
1265	return -EINVAL;
1266	err = afinfo->get_saddr(net, local, remote);
1267	xfrm_policy_put_afinfo(afinfo);
1268	return err;
1269	}
1270
1271	/* Resolve list of templates for the flow, given policy. */
1272
1273	static int
1274	xfrm_tmpl_resolve_one(struct xfrm_policy policy, const struct flowi fl,
1275	struct xfrm_state **xfrm, unsigned short family)
1276	{
1277	struct net *net = xp_net(policy);
1278	int nx;
1279	int i, error;
1280	xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1281	xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1282	xfrm_address_t tmp;
1283
1284	for (nx=0, i = 0; i < policy->xfrm_nr; i++) {
1285	struct xfrm_state *x;
1286	xfrm_address_t *remote = daddr;
1287	xfrm_address_t *local = saddr;
1288	struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1289
1290	if (tmpl->mode == XFRM_MODE_TUNNEL \|\|
1291	tmpl->mode == XFRM_MODE_BEET) {
1292	remote = &tmpl->id.daddr;
1293	local = &tmpl->saddr;
1294	if (xfrm_addr_any(local, tmpl->encap_family)) {
1295	error = xfrm_get_saddr(net, &tmp, remote, tmpl->encap_family);
1296	if (error)
1297	goto fail;
1298	local = &tmp;
1299	}
1300	}
1301
1302	x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
1303
1304	if (x && x->km.state == XFRM_STATE_VALID) {
1305	xfrm[nx++] = x;
1306	daddr = remote;
1307	saddr = local;
1308	continue;
1309	}
1310	if (x) {
1311	error = (x->km.state == XFRM_STATE_ERROR ?
1312	-EINVAL : -EAGAIN);
1313	xfrm_state_put(x);
1314	}
1315	else if (error == -ESRCH)
1316	error = -EAGAIN;
1317
1318	if (!tmpl->optional)
1319	goto fail;
1320	}
1321	return nx;
1322
1323	fail:
1324	for (nx--; nx>=0; nx--)
1325	xfrm_state_put(xfrm[nx]);
1326	return error;
1327	}
1328
1329	static int
1330	xfrm_tmpl_resolve(struct xfrm_policy *pols, int npols, const struct flowi fl,
1331	struct xfrm_state **xfrm, unsigned short family)
1332	{
1333	struct xfrm_state *tp[XFRM_MAX_DEPTH];
1334	struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1335	int cnx = 0;
1336	int error;
1337	int ret;
1338	int i;
1339
1340	for (i = 0; i < npols; i++) {
1341	if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1342	error = -ENOBUFS;
1343	goto fail;
1344	}
1345
1346	ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1347	if (ret < 0) {
1348	error = ret;
1349	goto fail;
1350	} else
1351	cnx += ret;
1352	}
1353
1354	/* found states are sorted for outbound processing */
1355	if (npols > 1)
1356	xfrm_state_sort(xfrm, tpp, cnx, family);
1357
1358	return cnx;
1359
1360	fail:
1361	for (cnx--; cnx>=0; cnx--)
1362	xfrm_state_put(tpp[cnx]);
1363	return error;
1364
1365	}
1366
1367	/* Check that the bundle accepts the flow and its components are
1368	* still valid.
1369	*/
1370
1371	static inline int xfrm_get_tos(const struct flowi *fl, int family)
1372	{
1373	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1374	int tos;
1375
1376	if (!afinfo)
1377	return -EINVAL;
1378
1379	tos = afinfo->get_tos(fl);
1380
1381	xfrm_policy_put_afinfo(afinfo);
1382
1383	return tos;
1384	}
1385
1386	static struct flow_cache_object xfrm_bundle_flo_get(struct flow_cache_object flo)
1387	{
1388	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1389	struct dst_entry *dst = &xdst->u.dst;
1390
1391	if (xdst->route == NULL) {
1392	/* Dummy bundle - if it has xfrms we were not
1393	* able to build bundle as template resolution failed.
1394	* It means we need to try again resolving. */
1395	if (xdst->num_xfrms > 0)
1396	return NULL;
1397	} else if (dst->flags & DST_XFRM_QUEUE) {
1398	return NULL;
1399	} else {
1400	/* Real bundle */
1401	if (stale_bundle(dst))
1402	return NULL;
1403	}
1404
1405	dst_hold(dst);
1406	return flo;
1407	}
1408
1409	static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
1410	{
1411	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1412	struct dst_entry *dst = &xdst->u.dst;
1413
1414	if (!xdst->route)
1415	return 0;
1416	if (stale_bundle(dst))
1417	return 0;
1418
1419	return 1;
1420	}
1421
1422	static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
1423	{
1424	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1425	struct dst_entry *dst = &xdst->u.dst;
1426
1427	dst_free(dst);
1428	}
1429
1430	static const struct flow_cache_ops xfrm_bundle_fc_ops = {
1431	.get = xfrm_bundle_flo_get,
1432	.check = xfrm_bundle_flo_check,
1433	.delete = xfrm_bundle_flo_delete,
1434	};
1435
1436	static inline struct xfrm_dst xfrm_alloc_dst(struct net net, int family)
1437	{
1438	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1439	struct dst_ops *dst_ops;
1440	struct xfrm_dst *xdst;
1441
1442	if (!afinfo)
1443	return ERR_PTR(-EINVAL);
1444
1445	switch (family) {
1446	case AF_INET:
1447	dst_ops = &net->xfrm.xfrm4_dst_ops;
1448	break;
1449	#if IS_ENABLED(CONFIG_IPV6)
1450	case AF_INET6:
1451	dst_ops = &net->xfrm.xfrm6_dst_ops;
1452	break;
1453	#endif
1454	default:
1455	BUG();
1456	}
1457	xdst = dst_alloc(dst_ops, NULL, 0, DST_OBSOLETE_NONE, 0);
1458
1459	if (likely(xdst)) {
1460	struct dst_entry *dst = &xdst->u.dst;
1461
1462	memset(dst + 1, 0, sizeof(xdst) - sizeof(dst));
1463	xdst->flo.ops = &xfrm_bundle_fc_ops;
1464	if (afinfo->init_dst)
1465	afinfo->init_dst(net, xdst);
1466	} else
1467	xdst = ERR_PTR(-ENOBUFS);
1468
1469	xfrm_policy_put_afinfo(afinfo);
1470
1471	return xdst;
1472	}
1473
1474	static inline int xfrm_init_path(struct xfrm_dst path, struct dst_entry dst,
1475	int nfheader_len)
1476	{
1477	struct xfrm_policy_afinfo *afinfo =
1478	xfrm_policy_get_afinfo(dst->ops->family);
1479	int err;
1480
1481	if (!afinfo)
1482	return -EINVAL;
1483
1484	err = afinfo->init_path(path, dst, nfheader_len);
1485
1486	xfrm_policy_put_afinfo(afinfo);
1487
1488	return err;
1489	}
1490
1491	static inline int xfrm_fill_dst(struct xfrm_dst xdst, struct net_device dev,
1492	const struct flowi *fl)
1493	{
1494	struct xfrm_policy_afinfo *afinfo =
1495	xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1496	int err;
1497
1498	if (!afinfo)
1499	return -EINVAL;
1500
1501	err = afinfo->fill_dst(xdst, dev, fl);
1502
1503	xfrm_policy_put_afinfo(afinfo);
1504
1505	return err;
1506	}
1507
1508
1509	/* Allocate chain of dst_entry's, attach known xfrm's, calculate
1510	* all the metrics... Shortly, bundle a bundle.
1511	*/
1512
1513	static struct dst_entry xfrm_bundle_create(struct xfrm_policy policy,
1514	struct xfrm_state **xfrm, int nx,
1515	const struct flowi *fl,
1516	struct dst_entry *dst)
1517	{
1518	struct net *net = xp_net(policy);
1519	unsigned long now = jiffies;
1520	struct net_device *dev;
1521	struct xfrm_mode *inner_mode;
1522	struct dst_entry *dst_prev = NULL;
1523	struct dst_entry *dst0 = NULL;
1524	int i = 0;
1525	int err;
1526	int header_len = 0;
1527	int nfheader_len = 0;
1528	int trailer_len = 0;
1529	int tos;
1530	int family = policy->selector.family;
1531	xfrm_address_t saddr, daddr;
1532
1533	xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
1534
1535	tos = xfrm_get_tos(fl, family);
1536	err = tos;
1537	if (tos < 0)
1538	goto put_states;
1539
1540	dst_hold(dst);
1541
1542	for (; i < nx; i++) {
1543	struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
1544	struct dst_entry *dst1 = &xdst->u.dst;
1545
1546	err = PTR_ERR(xdst);
1547	if (IS_ERR(xdst)) {
1548	dst_release(dst);
1549	goto put_states;
1550	}
1551
1552	if (xfrm[i]->sel.family == AF_UNSPEC) {
1553	inner_mode = xfrm_ip2inner_mode(xfrm[i],
1554	xfrm_af2proto(family));
1555	if (!inner_mode) {
1556	err = -EAFNOSUPPORT;
1557	dst_release(dst);
1558	goto put_states;
1559	}
1560	} else
1561	inner_mode = xfrm[i]->inner_mode;
1562
1563	if (!dst_prev)
1564	dst0 = dst1;
1565	else {
1566	dst_prev->child = dst_clone(dst1);
1567	dst1->flags \|= DST_NOHASH;
1568	}
1569
1570	xdst->route = dst;
1571	dst_copy_metrics(dst1, dst);
1572
1573	if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1574	family = xfrm[i]->props.family;
1575	dst = xfrm_dst_lookup(xfrm[i], tos, &saddr, &daddr,
1576	family);
1577	err = PTR_ERR(dst);
1578	if (IS_ERR(dst))
1579	goto put_states;
1580	} else
1581	dst_hold(dst);
1582
1583	dst1->xfrm = xfrm[i];
1584	xdst->xfrm_genid = xfrm[i]->genid;
1585
1586	dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1587	dst1->flags \|= DST_HOST;
1588	dst1->lastuse = now;
1589
1590	dst1->input = dst_discard;
1591	dst1->output = inner_mode->afinfo->output;
1592
1593	dst1->next = dst_prev;
1594	dst_prev = dst1;
1595
1596	header_len += xfrm[i]->props.header_len;
1597	if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1598	nfheader_len += xfrm[i]->props.header_len;
1599	trailer_len += xfrm[i]->props.trailer_len;
1600	}
1601
1602	dst_prev->child = dst;
1603	dst0->path = dst;
1604
1605	err = -ENODEV;
1606	dev = dst->dev;
1607	if (!dev)
1608	goto free_dst;
1609
1610	xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len);
1611	xfrm_init_pmtu(dst_prev);
1612
1613	for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) {
1614	struct xfrm_dst xdst = (struct xfrm_dst )dst_prev;
1615
1616	err = xfrm_fill_dst(xdst, dev, fl);
1617	if (err)
1618	goto free_dst;
1619
1620	dst_prev->header_len = header_len;
1621	dst_prev->trailer_len = trailer_len;
1622	header_len -= xdst->u.dst.xfrm->props.header_len;
1623	trailer_len -= xdst->u.dst.xfrm->props.trailer_len;
1624	}
1625
1626	out:
1627	return dst0;
1628
1629	put_states:
1630	for (; i < nx; i++)
1631	xfrm_state_put(xfrm[i]);
1632	free_dst:
1633	if (dst0)
1634	dst_free(dst0);
1635	dst0 = ERR_PTR(err);
1636	goto out;
1637	}
1638
1639	static int inline
1640	xfrm_dst_alloc_copy(void *target, const void src, int size)
1641	{
1642	if (!*target) {
1643	*target = kmalloc(size, GFP_ATOMIC);
1644	if (!*target)
1645	return -ENOMEM;
1646	}
1647	memcpy(*target, src, size);
1648	return 0;
1649	}
1650
1651	static int inline
1652	xfrm_dst_update_parent(struct dst_entry dst, const struct xfrm_selector sel)
1653	{
1654	#ifdef CONFIG_XFRM_SUB_POLICY
1655	struct xfrm_dst xdst = (struct xfrm_dst )dst;
1656	return xfrm_dst_alloc_copy((void **)&(xdst->partner),
1657	sel, sizeof(*sel));
1658	#else
1659	return 0;
1660	#endif
1661	}
1662
1663	static int inline
1664	xfrm_dst_update_origin(struct dst_entry dst, const struct flowi fl)
1665	{
1666	#ifdef CONFIG_XFRM_SUB_POLICY
1667	struct xfrm_dst xdst = (struct xfrm_dst )dst;
1668	return xfrm_dst_alloc_copy((void *)&(xdst->origin), fl, sizeof(fl));
1669	#else
1670	return 0;
1671	#endif
1672	}
1673
1674	static int xfrm_expand_policies(const struct flowi *fl, u16 family,
1675	struct xfrm_policy **pols,
1676	int num_pols, int num_xfrms)
1677	{
1678	int i;
1679
1680	if (*num_pols == 0 \|\| !pols[0]) {
1681	*num_pols = 0;
1682	*num_xfrms = 0;
1683	return 0;
1684	}
1685	if (IS_ERR(pols[0]))
1686	return PTR_ERR(pols[0]);
1687
1688	*num_xfrms = pols[0]->xfrm_nr;
1689
1690	#ifdef CONFIG_XFRM_SUB_POLICY
1691	if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
1692	pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1693	pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
1694	XFRM_POLICY_TYPE_MAIN,
1695	fl, family,
1696	XFRM_POLICY_OUT);
1697	if (pols[1]) {
1698	if (IS_ERR(pols[1])) {
1699	xfrm_pols_put(pols, *num_pols);
1700	return PTR_ERR(pols[1]);
1701	}
1702	(*num_pols) ++;
1703	(*num_xfrms) += pols[1]->xfrm_nr;
1704	}
1705	}
1706	#endif
1707	for (i = 0; i < *num_pols; i++) {
1708	if (pols[i]->action != XFRM_POLICY_ALLOW) {
1709	*num_xfrms = -1;
1710	break;
1711	}
1712	}
1713
1714	return 0;
1715
1716	}
1717
1718	static struct xfrm_dst *
1719	xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
1720	const struct flowi *fl, u16 family,
1721	struct dst_entry *dst_orig)
1722	{
1723	struct net *net = xp_net(pols[0]);
1724	struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
1725	struct dst_entry *dst;
1726	struct xfrm_dst *xdst;
1727	int err;
1728
1729	/* Try to instantiate a bundle */
1730	err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
1731	if (err <= 0) {
1732	if (err != 0 && err != -EAGAIN)
1733	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1734	return ERR_PTR(err);
1735	}
1736
1737	dst = xfrm_bundle_create(pols[0], xfrm, err, fl, dst_orig);
1738	if (IS_ERR(dst)) {
1739	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
1740	return ERR_CAST(dst);
1741	}
1742
1743	xdst = (struct xfrm_dst *)dst;
1744	xdst->num_xfrms = err;
1745	if (num_pols > 1)
1746	err = xfrm_dst_update_parent(dst, &pols[1]->selector);
1747	else
1748	err = xfrm_dst_update_origin(dst, fl);
1749	if (unlikely(err)) {
1750	dst_free(dst);
1751	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1752	return ERR_PTR(err);
1753	}
1754
1755	xdst->num_pols = num_pols;
1756	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy) num_pols);
1757	xdst->policy_genid = atomic_read(&pols[0]->genid);
1758
1759	return xdst;
1760	}
1761
1762	static void xfrm_policy_queue_process(unsigned long arg)
1763	{
1764	int err = 0;
1765	struct sk_buff *skb;
1766	struct sock *sk;
1767	struct dst_entry *dst;
1768	struct xfrm_policy pol = (struct xfrm_policy )arg;
1769	struct xfrm_policy_queue *pq = &pol->polq;
1770	struct flowi fl;
1771	struct sk_buff_head list;
1772
1773	spin_lock(&pq->hold_queue.lock);
1774	skb = skb_peek(&pq->hold_queue);
1775	if (!skb) {
1776	spin_unlock(&pq->hold_queue.lock);
1777	goto out;
1778	}
1779	dst = skb_dst(skb);
1780	sk = skb->sk;
1781	xfrm_decode_session(skb, &fl, dst->ops->family);
1782	spin_unlock(&pq->hold_queue.lock);
1783
1784	dst_hold(dst->path);
1785	dst = xfrm_lookup(xp_net(pol), dst->path, &fl,
1786	sk, 0);
1787	if (IS_ERR(dst))
1788	goto purge_queue;
1789
1790	if (dst->flags & DST_XFRM_QUEUE) {
1791	dst_release(dst);
1792
1793	if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
1794	goto purge_queue;
1795
1796	pq->timeout = pq->timeout << 1;
1797	if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
1798	xfrm_pol_hold(pol);
1799	goto out;
1800	}
1801
1802	dst_release(dst);
1803
1804	__skb_queue_head_init(&list);
1805
1806	spin_lock(&pq->hold_queue.lock);
1807	pq->timeout = 0;
1808	skb_queue_splice_init(&pq->hold_queue, &list);
1809	spin_unlock(&pq->hold_queue.lock);
1810
1811	while (!skb_queue_empty(&list)) {
1812	skb = __skb_dequeue(&list);
1813
1814	xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
1815	dst_hold(skb_dst(skb)->path);
1816	dst = xfrm_lookup(xp_net(pol), skb_dst(skb)->path,
1817	&fl, skb->sk, 0);
1818	if (IS_ERR(dst)) {
1819	kfree_skb(skb);
1820	continue;
1821	}
1822
1823	nf_reset(skb);
1824	skb_dst_drop(skb);
1825	skb_dst_set(skb, dst);
1826
1827	err = dst_output(skb);
1828	}
1829
1830	out:
1831	xfrm_pol_put(pol);
1832	return;
1833
1834	purge_queue:
1835	pq->timeout = 0;
1836	xfrm_queue_purge(&pq->hold_queue);
1837	xfrm_pol_put(pol);
1838	}
1839
1840	static int xdst_queue_output(struct sk_buff *skb)
1841	{
1842	unsigned long sched_next;
1843	struct dst_entry *dst = skb_dst(skb);
1844	struct xfrm_dst xdst = (struct xfrm_dst ) dst;
1845	struct xfrm_policy *pol = xdst->pols[0];
1846	struct xfrm_policy_queue *pq = &pol->polq;
1847
1848	if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
1849	kfree_skb(skb);
1850	return -EAGAIN;
1851	}
1852
1853	skb_dst_force(skb);
1854
1855	spin_lock_bh(&pq->hold_queue.lock);
1856
1857	if (!pq->timeout)
1858	pq->timeout = XFRM_QUEUE_TMO_MIN;
1859
1860	sched_next = jiffies + pq->timeout;
1861
1862	if (del_timer(&pq->hold_timer)) {
1863	if (time_before(pq->hold_timer.expires, sched_next))
1864	sched_next = pq->hold_timer.expires;
1865	xfrm_pol_put(pol);
1866	}
1867
1868	__skb_queue_tail(&pq->hold_queue, skb);
1869	if (!mod_timer(&pq->hold_timer, sched_next))
1870	xfrm_pol_hold(pol);
1871
1872	spin_unlock_bh(&pq->hold_queue.lock);
1873
1874	return 0;
1875	}
1876
1877	static struct xfrm_dst xfrm_create_dummy_bundle(struct net net,
1878	struct dst_entry *dst,
1879	const struct flowi *fl,
1880	int num_xfrms,
1881	u16 family)
1882	{
1883	int err;
1884	struct net_device *dev;
1885	struct dst_entry *dst1;
1886	struct xfrm_dst *xdst;
1887
1888	xdst = xfrm_alloc_dst(net, family);
1889	if (IS_ERR(xdst))
1890	return xdst;
1891
1892	if (net->xfrm.sysctl_larval_drop \|\| num_xfrms <= 0 \|\|
1893	(fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP))
1894	return xdst;
1895
1896	dst1 = &xdst->u.dst;
1897	dst_hold(dst);
1898	xdst->route = dst;
1899
1900	dst_copy_metrics(dst1, dst);
1901
1902	dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1903	dst1->flags \|= DST_HOST \| DST_XFRM_QUEUE;
1904	dst1->lastuse = jiffies;
1905
1906	dst1->input = dst_discard;
1907	dst1->output = xdst_queue_output;
1908
1909	dst_hold(dst);
1910	dst1->child = dst;
1911	dst1->path = dst;
1912
1913	xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
1914
1915	err = -ENODEV;
1916	dev = dst->dev;
1917	if (!dev)
1918	goto free_dst;
1919
1920	err = xfrm_fill_dst(xdst, dev, fl);
1921	if (err)
1922	goto free_dst;
1923
1924	out:
1925	return xdst;
1926
1927	free_dst:
1928	dst_release(dst1);
1929	xdst = ERR_PTR(err);
1930	goto out;
1931	}
1932
1933	static struct flow_cache_object *
1934	xfrm_bundle_lookup(struct net net, const struct flowi fl, u16 family, u8 dir,
1935	struct flow_cache_object oldflo, void ctx)
1936	{
1937	struct dst_entry dst_orig = (struct dst_entry )ctx;
1938	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
1939	struct xfrm_dst xdst, new_xdst;
1940	int num_pols = 0, num_xfrms = 0, i, err, pol_dead;
1941
1942	/* Check if the policies from old bundle are usable */
1943	xdst = NULL;
1944	if (oldflo) {
1945	xdst = container_of(oldflo, struct xfrm_dst, flo);
1946	num_pols = xdst->num_pols;
1947	num_xfrms = xdst->num_xfrms;
1948	pol_dead = 0;
1949	for (i = 0; i < num_pols; i++) {
1950	pols[i] = xdst->pols[i];
1951	pol_dead \|= pols[i]->walk.dead;
1952	}
1953	if (pol_dead) {
1954	dst_free(&xdst->u.dst);
1955	xdst = NULL;
1956	num_pols = 0;
1957	num_xfrms = 0;
1958	oldflo = NULL;
1959	}
1960	}
1961
1962	/* Resolve policies to use if we couldn't get them from
1963	* previous cache entry */
1964	if (xdst == NULL) {
1965	num_pols = 1;
1966	pols[0] = __xfrm_policy_lookup(net, fl, family,
1967	flow_to_policy_dir(dir));
1968	err = xfrm_expand_policies(fl, family, pols,
1969	&num_pols, &num_xfrms);
1970	if (err < 0)
1971	goto inc_error;
1972	if (num_pols == 0)
1973	return NULL;
1974	if (num_xfrms <= 0)
1975	goto make_dummy_bundle;
1976	}
1977
1978	new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family, dst_orig);
1979	if (IS_ERR(new_xdst)) {
1980	err = PTR_ERR(new_xdst);
1981	if (err != -EAGAIN)
1982	goto error;
1983	if (oldflo == NULL)
1984	goto make_dummy_bundle;
1985	dst_hold(&xdst->u.dst);
1986	return oldflo;
1987	} else if (new_xdst == NULL) {
1988	num_xfrms = 0;
1989	if (oldflo == NULL)
1990	goto make_dummy_bundle;
1991	xdst->num_xfrms = 0;
1992	dst_hold(&xdst->u.dst);
1993	return oldflo;
1994	}
1995
1996	/* Kill the previous bundle */
1997	if (xdst) {
1998	/* The policies were stolen for newly generated bundle */
1999	xdst->num_pols = 0;
2000	dst_free(&xdst->u.dst);
2001	}
2002
2003	/* Flow cache does not have reference, it dst_free()'s,
2004	* but we do need to return one reference for original caller */
2005	dst_hold(&new_xdst->u.dst);
2006	return &new_xdst->flo;
2007
2008	make_dummy_bundle:
2009	/* We found policies, but there's no bundles to instantiate:
2010	* either because the policy blocks, has no transformations or
2011	* we could not build template (no xfrm_states).*/
2012	xdst = xfrm_create_dummy_bundle(net, dst_orig, fl, num_xfrms, family);
2013	if (IS_ERR(xdst)) {
2014	xfrm_pols_put(pols, num_pols);
2015	return ERR_CAST(xdst);
2016	}
2017	xdst->num_pols = num_pols;
2018	xdst->num_xfrms = num_xfrms;
2019	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy) num_pols);
2020
2021	dst_hold(&xdst->u.dst);
2022	return &xdst->flo;
2023
2024	inc_error:
2025	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2026	error:
2027	if (xdst != NULL)
2028	dst_free(&xdst->u.dst);
2029	else
2030	xfrm_pols_put(pols, num_pols);
2031	return ERR_PTR(err);
2032	}
2033
2034	static struct dst_entry make_blackhole(struct net net, u16 family,
2035	struct dst_entry *dst_orig)
2036	{
2037	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2038	struct dst_entry *ret;
2039
2040	if (!afinfo) {
2041	dst_release(dst_orig);
2042	return ERR_PTR(-EINVAL);
2043	} else {
2044	ret = afinfo->blackhole_route(net, dst_orig);
2045	}
2046	xfrm_policy_put_afinfo(afinfo);
2047
2048	return ret;
2049	}
2050
2051	/* Main function: finds/creates a bundle for given flow.
2052	*
2053	* At the moment we eat a raw IP route. Mostly to speed up lookups
2054	* on interfaces with disabled IPsec.
2055	*/
2056	struct dst_entry xfrm_lookup(struct net net, struct dst_entry *dst_orig,
2057	const struct flowi *fl,
2058	struct sock *sk, int flags)
2059	{
2060	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2061	struct flow_cache_object *flo;
2062	struct xfrm_dst *xdst;
2063	struct dst_entry dst, route;
2064	u16 family = dst_orig->ops->family;
2065	u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
2066	int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
2067
2068	restart:
2069	dst = NULL;
2070	xdst = NULL;
2071	route = NULL;
2072
2073	if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
2074	num_pols = 1;
2075	pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
2076	err = xfrm_expand_policies(fl, family, pols,
2077	&num_pols, &num_xfrms);
2078	if (err < 0)
2079	goto dropdst;
2080
2081	if (num_pols) {
2082	if (num_xfrms <= 0) {
2083	drop_pols = num_pols;
2084	goto no_transform;
2085	}
2086
2087	xdst = xfrm_resolve_and_create_bundle(
2088	pols, num_pols, fl,
2089	family, dst_orig);
2090	if (IS_ERR(xdst)) {
2091	xfrm_pols_put(pols, num_pols);
2092	err = PTR_ERR(xdst);
2093	goto dropdst;
2094	} else if (xdst == NULL) {
2095	num_xfrms = 0;
2096	drop_pols = num_pols;
2097	goto no_transform;
2098	}
2099
2100	dst_hold(&xdst->u.dst);
2101
2102	spin_lock_bh(&xfrm_policy_sk_bundle_lock);
2103	xdst->u.dst.next = xfrm_policy_sk_bundles;
2104	xfrm_policy_sk_bundles = &xdst->u.dst;
2105	spin_unlock_bh(&xfrm_policy_sk_bundle_lock);
2106
2107	route = xdst->route;
2108	}
2109	}
2110
2111	if (xdst == NULL) {
2112	/* To accelerate a bit... */
2113	if ((dst_orig->flags & DST_NOXFRM) \|\|
2114	!net->xfrm.policy_count[XFRM_POLICY_OUT])
2115	goto nopol;
2116
2117	flo = flow_cache_lookup(net, fl, family, dir,
2118	xfrm_bundle_lookup, dst_orig);
2119	if (flo == NULL)
2120	goto nopol;
2121	if (IS_ERR(flo)) {
2122	err = PTR_ERR(flo);
2123	goto dropdst;
2124	}
2125	xdst = container_of(flo, struct xfrm_dst, flo);
2126
2127	num_pols = xdst->num_pols;
2128	num_xfrms = xdst->num_xfrms;
2129	memcpy(pols, xdst->pols, sizeof(struct xfrm_policy) num_pols);
2130	route = xdst->route;
2131	}
2132
2133	dst = &xdst->u.dst;
2134	if (route == NULL && num_xfrms > 0) {
2135	/* The only case when xfrm_bundle_lookup() returns a
2136	* bundle with null route, is when the template could
2137	* not be resolved. It means policies are there, but
2138	* bundle could not be created, since we don't yet
2139	* have the xfrm_state's. We need to wait for KM to
2140	* negotiate new SA's or bail out with error.*/
2141	if (net->xfrm.sysctl_larval_drop) {
2142	dst_release(dst);
2143	xfrm_pols_put(pols, drop_pols);
2144	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2145
2146	return make_blackhole(net, family, dst_orig);
2147	}
2148	if (fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP) {
2149	DECLARE_WAITQUEUE(wait, current);
2150
2151	add_wait_queue(&net->xfrm.km_waitq, &wait);
2152	set_current_state(TASK_INTERRUPTIBLE);
2153	schedule();
2154	set_current_state(TASK_RUNNING);
2155	remove_wait_queue(&net->xfrm.km_waitq, &wait);
2156
2157	if (!signal_pending(current)) {
2158	dst_release(dst);
2159	goto restart;
2160	}
2161
2162	err = -ERESTART;
2163	} else
2164	err = -EAGAIN;
2165
2166	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2167	goto error;
2168	}
2169
2170	no_transform:
2171	if (num_pols == 0)
2172	goto nopol;
2173
2174	if ((flags & XFRM_LOOKUP_ICMP) &&
2175	!(pols[0]->flags & XFRM_POLICY_ICMP)) {
2176	err = -ENOENT;
2177	goto error;
2178	}
2179
2180	for (i = 0; i < num_pols; i++)
2181	pols[i]->curlft.use_time = get_seconds();
2182
2183	if (num_xfrms < 0) {
2184	/* Prohibit the flow */
2185	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
2186	err = -EPERM;
2187	goto error;
2188	} else if (num_xfrms > 0) {
2189	/* Flow transformed */
2190	dst_release(dst_orig);
2191	} else {
2192	/* Flow passes untransformed */
2193	dst_release(dst);
2194	dst = dst_orig;
2195	}
2196	ok:
2197	xfrm_pols_put(pols, drop_pols);
2198	if (dst && dst->xfrm &&
2199	dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
2200	dst->flags \|= DST_XFRM_TUNNEL;
2201	return dst;
2202
2203	nopol:
2204	if (!(flags & XFRM_LOOKUP_ICMP)) {
2205	dst = dst_orig;
2206	goto ok;
2207	}
2208	err = -ENOENT;
2209	error:
2210	dst_release(dst);
2211	dropdst:
2212	dst_release(dst_orig);
2213	xfrm_pols_put(pols, drop_pols);
2214	return ERR_PTR(err);
2215	}
2216	EXPORT_SYMBOL(xfrm_lookup);
2217
2218	static inline int
2219	xfrm_secpath_reject(int idx, struct sk_buff skb, const struct flowi fl)
2220	{
2221	struct xfrm_state *x;
2222
2223	if (!skb->sp \|\| idx < 0 \|\| idx >= skb->sp->len)
2224	return 0;
2225	x = skb->sp->xvec[idx];
2226	if (!x->type->reject)
2227	return 0;
2228	return x->type->reject(x, skb, fl);
2229	}
2230
2231	/* When skb is transformed back to its "native" form, we have to
2232	* check policy restrictions. At the moment we make this in maximally
2233	* stupid way. Shame on me. :-) Of course, connected sockets must
2234	* have policy cached at them.
2235	*/
2236
2237	static inline int
2238	xfrm_state_ok(const struct xfrm_tmpl tmpl, const struct xfrm_state x,
2239	unsigned short family)
2240	{
2241	if (xfrm_state_kern(x))
2242	return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
2243	return x->id.proto == tmpl->id.proto &&
2244	(x->id.spi == tmpl->id.spi \|\| !tmpl->id.spi) &&
2245	(x->props.reqid == tmpl->reqid \|\| !tmpl->reqid) &&
2246	x->props.mode == tmpl->mode &&
2247	(tmpl->allalgs \|\| (tmpl->aalgos & (1<<x->props.aalgo)) \|\|
2248	!(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
2249	!(x->props.mode != XFRM_MODE_TRANSPORT &&
2250	xfrm_state_addr_cmp(tmpl, x, family));
2251	}
2252
2253	/*
2254	* 0 or more than 0 is returned when validation is succeeded (either bypass
2255	* because of optional transport mode, or next index of the mathced secpath
2256	* state with the template.
2257	* -1 is returned when no matching template is found.
2258	* Otherwise "-2 - errored_index" is returned.
2259	*/
2260	static inline int
2261	xfrm_policy_ok(const struct xfrm_tmpl tmpl, const struct sec_path sp, int start,
2262	unsigned short family)
2263	{
2264	int idx = start;
2265
2266	if (tmpl->optional) {
2267	if (tmpl->mode == XFRM_MODE_TRANSPORT)
2268	return start;
2269	} else
2270	start = -1;
2271	for (; idx < sp->len; idx++) {
2272	if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
2273	return ++idx;
2274	if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
2275	if (start == -1)
2276	start = -2-idx;
2277	break;
2278	}
2279	}
2280	return start;
2281	}
2282
2283	int __xfrm_decode_session(struct sk_buff skb, struct flowi fl,
2284	unsigned int family, int reverse)
2285	{
2286	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2287	int err;
2288
2289	if (unlikely(afinfo == NULL))
2290	return -EAFNOSUPPORT;
2291
2292	afinfo->decode_session(skb, fl, reverse);
2293	err = security_xfrm_decode_session(skb, &fl->flowi_secid);
2294	xfrm_policy_put_afinfo(afinfo);
2295	return err;
2296	}
2297	EXPORT_SYMBOL(__xfrm_decode_session);
2298
2299	static inline int secpath_has_nontransport(const struct sec_path sp, int k, int idxp)
2300	{
2301	for (; k < sp->len; k++) {
2302	if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
2303	*idxp = k;
2304	return 1;
2305	}
2306	}
2307
2308	return 0;
2309	}
2310
2311	int __xfrm_policy_check(struct sock sk, int dir, struct sk_buff skb,
2312	unsigned short family)
2313	{
2314	struct net *net = dev_net(skb->dev);
2315	struct xfrm_policy *pol;
2316	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2317	int npols = 0;
2318	int xfrm_nr;
2319	int pi;
2320	int reverse;
2321	struct flowi fl;
2322	u8 fl_dir;
2323	int xerr_idx = -1;
2324
2325	reverse = dir & ~XFRM_POLICY_MASK;
2326	dir &= XFRM_POLICY_MASK;
2327	fl_dir = policy_to_flow_dir(dir);
2328
2329	if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
2330	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
2331	return 0;
2332	}
2333
2334	nf_nat_decode_session(skb, &fl, family);
2335
2336	/* First, check used SA against their selectors. */
2337	if (skb->sp) {
2338	int i;
2339
2340	for (i=skb->sp->len-1; i>=0; i--) {
2341	struct xfrm_state *x = skb->sp->xvec[i];
2342	if (!xfrm_selector_match(&x->sel, &fl, family)) {
2343	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
2344	return 0;
2345	}
2346	}
2347	}
2348
2349	pol = NULL;
2350	if (sk && sk->sk_policy[dir]) {
2351	pol = xfrm_sk_policy_lookup(sk, dir, &fl);
2352	if (IS_ERR(pol)) {
2353	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2354	return 0;
2355	}
2356	}
2357
2358	if (!pol) {
2359	struct flow_cache_object *flo;
2360
2361	flo = flow_cache_lookup(net, &fl, family, fl_dir,
2362	xfrm_policy_lookup, NULL);
2363	if (IS_ERR_OR_NULL(flo))
2364	pol = ERR_CAST(flo);
2365	else
2366	pol = container_of(flo, struct xfrm_policy, flo);
2367	}
2368
2369	if (IS_ERR(pol)) {
2370	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2371	return 0;
2372	}
2373
2374	if (!pol) {
2375	if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
2376	xfrm_secpath_reject(xerr_idx, skb, &fl);
2377	XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
2378	return 0;
2379	}
2380	return 1;
2381	}
2382
2383	pol->curlft.use_time = get_seconds();
2384
2385	pols[0] = pol;
2386	npols ++;
2387	#ifdef CONFIG_XFRM_SUB_POLICY
2388	if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2389	pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
2390	&fl, family,
2391	XFRM_POLICY_IN);
2392	if (pols[1]) {
2393	if (IS_ERR(pols[1])) {
2394	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2395	return 0;
2396	}
2397	pols[1]->curlft.use_time = get_seconds();
2398	npols ++;
2399	}
2400	}
2401	#endif
2402
2403	if (pol->action == XFRM_POLICY_ALLOW) {
2404	struct sec_path *sp;
2405	static struct sec_path dummy;
2406	struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
2407	struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
2408	struct xfrm_tmpl **tpp = tp;
2409	int ti = 0;
2410	int i, k;
2411
2412	if ((sp = skb->sp) == NULL)
2413	sp = &dummy;
2414
2415	for (pi = 0; pi < npols; pi++) {
2416	if (pols[pi] != pol &&
2417	pols[pi]->action != XFRM_POLICY_ALLOW) {
2418	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2419	goto reject;
2420	}
2421	if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
2422	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
2423	goto reject_error;
2424	}
2425	for (i = 0; i < pols[pi]->xfrm_nr; i++)
2426	tpp[ti++] = &pols[pi]->xfrm_vec[i];
2427	}
2428	xfrm_nr = ti;
2429	if (npols > 1) {
2430	xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
2431	tpp = stp;
2432	}
2433
2434	/* For each tunnel xfrm, find the first matching tmpl.
2435	* For each tmpl before that, find corresponding xfrm.
2436	* Order is _important_. Later we will implement
2437	* some barriers, but at the moment barriers
2438	* are implied between each two transformations.
2439	*/
2440	for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2441	k = xfrm_policy_ok(tpp[i], sp, k, family);
2442	if (k < 0) {
2443	if (k < -1)
2444	/* "-2 - errored_index" returned */
2445	xerr_idx = -(2+k);
2446	XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2447	goto reject;
2448	}
2449	}
2450
2451	if (secpath_has_nontransport(sp, k, &xerr_idx)) {
2452	XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2453	goto reject;
2454	}
2455
2456	xfrm_pols_put(pols, npols);
2457	return 1;
2458	}
2459	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2460
2461	reject:
2462	xfrm_secpath_reject(xerr_idx, skb, &fl);
2463	reject_error:
2464	xfrm_pols_put(pols, npols);
2465	return 0;
2466	}
2467	EXPORT_SYMBOL(__xfrm_policy_check);
2468
2469	int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2470	{
2471	struct net *net = dev_net(skb->dev);
2472	struct flowi fl;
2473	struct dst_entry *dst;
2474	int res = 1;
2475
2476	if (xfrm_decode_session(skb, &fl, family) < 0) {
2477	XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
2478	return 0;
2479	}
2480
2481	skb_dst_force(skb);
2482
2483	dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, 0);
2484	if (IS_ERR(dst)) {
2485	res = 0;
2486	dst = NULL;
2487	}
2488	skb_dst_set(skb, dst);
2489	return res;
2490	}
2491	EXPORT_SYMBOL(__xfrm_route_forward);
2492
2493	/* Optimize later using cookies and generation ids. */
2494
2495	static struct dst_entry xfrm_dst_check(struct dst_entry dst, u32 cookie)
2496	{
2497	/* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
2498	* to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
2499	* get validated by dst_ops->check on every use. We do this
2500	* because when a normal route referenced by an XFRM dst is
2501	* obsoleted we do not go looking around for all parent
2502	* referencing XFRM dsts so that we can invalidate them. It
2503	* is just too much work. Instead we make the checks here on
2504	* every use. For example:
2505	*
2506	* XFRM dst A --> IPv4 dst X
2507	*
2508	* X is the "xdst->route" of A (X is also the "dst->path" of A
2509	* in this example). If X is marked obsolete, "A" will not
2510	* notice. That's what we are validating here via the
2511	* stale_bundle() check.
2512	*
2513	* When a policy's bundle is pruned, we dst_free() the XFRM
2514	* dst which causes it's ->obsolete field to be set to
2515	* DST_OBSOLETE_DEAD. If an XFRM dst has been pruned like
2516	* this, we want to force a new route lookup.
2517	*/
2518	if (dst->obsolete < 0 && !stale_bundle(dst))
2519	return dst;
2520
2521	return NULL;
2522	}
2523
2524	static int stale_bundle(struct dst_entry *dst)
2525	{
2526	return !xfrm_bundle_ok((struct xfrm_dst *)dst);
2527	}
2528
2529	void xfrm_dst_ifdown(struct dst_entry dst, struct net_device dev)
2530	{
2531	while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
2532	dst->dev = dev_net(dev)->loopback_dev;
2533	dev_hold(dst->dev);
2534	dev_put(dev);
2535	}
2536	}
2537	EXPORT_SYMBOL(xfrm_dst_ifdown);
2538
2539	static void xfrm_link_failure(struct sk_buff *skb)
2540	{
2541	/* Impossible. Such dst must be popped before reaches point of failure. */
2542	}
2543
2544	static struct dst_entry xfrm_negative_advice(struct dst_entry dst)
2545	{
2546	if (dst) {
2547	if (dst->obsolete) {
2548	dst_release(dst);
2549	dst = NULL;
2550	}
2551	}
2552	return dst;
2553	}
2554
2555	static void __xfrm_garbage_collect(struct net *net)
2556	{
2557	struct dst_entry head, next;
2558
2559	spin_lock_bh(&xfrm_policy_sk_bundle_lock);
2560	head = xfrm_policy_sk_bundles;
2561	xfrm_policy_sk_bundles = NULL;
2562	spin_unlock_bh(&xfrm_policy_sk_bundle_lock);
2563
2564	while (head) {
2565	next = head->next;
2566	dst_free(head);
2567	head = next;
2568	}
2569	}
2570
2571	void xfrm_garbage_collect(struct net *net)
2572	{
2573	flow_cache_flush();
2574	__xfrm_garbage_collect(net);
2575	}
2576	EXPORT_SYMBOL(xfrm_garbage_collect);
2577
2578	static void xfrm_garbage_collect_deferred(struct net *net)
2579	{
2580	flow_cache_flush_deferred();
2581	__xfrm_garbage_collect(net);
2582	}
2583
2584	static void xfrm_init_pmtu(struct dst_entry *dst)
2585	{
2586	do {
2587	struct xfrm_dst xdst = (struct xfrm_dst )dst;
2588	u32 pmtu, route_mtu_cached;
2589
2590	pmtu = dst_mtu(dst->child);
2591	xdst->child_mtu_cached = pmtu;
2592
2593	pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2594
2595	route_mtu_cached = dst_mtu(xdst->route);
2596	xdst->route_mtu_cached = route_mtu_cached;
2597
2598	if (pmtu > route_mtu_cached)
2599	pmtu = route_mtu_cached;
2600
2601	dst_metric_set(dst, RTAX_MTU, pmtu);
2602	} while ((dst = dst->next));
2603	}
2604
2605	/* Check that the bundle accepts the flow and its components are
2606	* still valid.
2607	*/
2608
2609	static int xfrm_bundle_ok(struct xfrm_dst *first)
2610	{
2611	struct dst_entry *dst = &first->u.dst;
2612	struct xfrm_dst *last;
2613	u32 mtu;
2614
2615	if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) \|\|
2616	(dst->dev && !netif_running(dst->dev)))
2617	return 0;
2618
2619	if (dst->flags & DST_XFRM_QUEUE)
2620	return 1;
2621
2622	last = NULL;
2623
2624	do {
2625	struct xfrm_dst xdst = (struct xfrm_dst )dst;
2626
2627	if (dst->xfrm->km.state != XFRM_STATE_VALID)
2628	return 0;
2629	if (xdst->xfrm_genid != dst->xfrm->genid)
2630	return 0;
2631	if (xdst->num_pols > 0 &&
2632	xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
2633	return 0;
2634
2635	mtu = dst_mtu(dst->child);
2636	if (xdst->child_mtu_cached != mtu) {
2637	last = xdst;
2638	xdst->child_mtu_cached = mtu;
2639	}
2640
2641	if (!dst_check(xdst->route, xdst->route_cookie))
2642	return 0;
2643	mtu = dst_mtu(xdst->route);
2644	if (xdst->route_mtu_cached != mtu) {
2645	last = xdst;
2646	xdst->route_mtu_cached = mtu;
2647	}
2648
2649	dst = dst->child;
2650	} while (dst->xfrm);
2651
2652	if (likely(!last))
2653	return 1;
2654
2655	mtu = last->child_mtu_cached;
2656	for (;;) {
2657	dst = &last->u.dst;
2658
2659	mtu = xfrm_state_mtu(dst->xfrm, mtu);
2660	if (mtu > last->route_mtu_cached)
2661	mtu = last->route_mtu_cached;
2662	dst_metric_set(dst, RTAX_MTU, mtu);
2663
2664	if (last == first)
2665	break;
2666
2667	last = (struct xfrm_dst *)last->u.dst.next;
2668	last->child_mtu_cached = mtu;
2669	}
2670
2671	return 1;
2672	}
2673
2674	static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
2675	{
2676	return dst_metric_advmss(dst->path);
2677	}
2678
2679	static unsigned int xfrm_mtu(const struct dst_entry *dst)
2680	{
2681	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2682
2683	return mtu ? : dst_mtu(dst->path);
2684	}
2685
2686	static struct neighbour xfrm_neigh_lookup(const struct dst_entry dst,
2687	struct sk_buff *skb,
2688	const void *daddr)
2689	{
2690	return dst->path->ops->neigh_lookup(dst, skb, daddr);
2691	}
2692
2693	int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
2694	{
2695	struct net *net;
2696	int err = 0;
2697	if (unlikely(afinfo == NULL))
2698	return -EINVAL;
2699	if (unlikely(afinfo->family >= NPROTO))
2700	return -EAFNOSUPPORT;
2701	spin_lock(&xfrm_policy_afinfo_lock);
2702	if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
2703	err = -ENOBUFS;
2704	else {
2705	struct dst_ops *dst_ops = afinfo->dst_ops;
2706	if (likely(dst_ops->kmem_cachep == NULL))
2707	dst_ops->kmem_cachep = xfrm_dst_cache;
2708	if (likely(dst_ops->check == NULL))
2709	dst_ops->check = xfrm_dst_check;
2710	if (likely(dst_ops->default_advmss == NULL))
2711	dst_ops->default_advmss = xfrm_default_advmss;
2712	if (likely(dst_ops->mtu == NULL))
2713	dst_ops->mtu = xfrm_mtu;
2714	if (likely(dst_ops->negative_advice == NULL))
2715	dst_ops->negative_advice = xfrm_negative_advice;
2716	if (likely(dst_ops->link_failure == NULL))
2717	dst_ops->link_failure = xfrm_link_failure;
2718	if (likely(dst_ops->neigh_lookup == NULL))
2719	dst_ops->neigh_lookup = xfrm_neigh_lookup;
2720	if (likely(afinfo->garbage_collect == NULL))
2721	afinfo->garbage_collect = xfrm_garbage_collect_deferred;
2722	rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
2723	}
2724	spin_unlock(&xfrm_policy_afinfo_lock);
2725
2726	rtnl_lock();
2727	for_each_net(net) {
2728	struct dst_ops *xfrm_dst_ops;
2729
2730	switch (afinfo->family) {
2731	case AF_INET:
2732	xfrm_dst_ops = &net->xfrm.xfrm4_dst_ops;
2733	break;
2734	#if IS_ENABLED(CONFIG_IPV6)
2735	case AF_INET6:
2736	xfrm_dst_ops = &net->xfrm.xfrm6_dst_ops;
2737	break;
2738	#endif
2739	default:
2740	BUG();
2741	}
2742	xfrm_dst_ops = afinfo->dst_ops;
2743	}
2744	rtnl_unlock();
2745
2746	return err;
2747	}
2748	EXPORT_SYMBOL(xfrm_policy_register_afinfo);
2749
2750	int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
2751	{
2752	int err = 0;
2753	if (unlikely(afinfo == NULL))
2754	return -EINVAL;
2755	if (unlikely(afinfo->family >= NPROTO))
2756	return -EAFNOSUPPORT;
2757	spin_lock(&xfrm_policy_afinfo_lock);
2758	if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
2759	if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
2760	err = -EINVAL;
2761	else
2762	RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
2763	NULL);
2764	}
2765	spin_unlock(&xfrm_policy_afinfo_lock);
2766	if (!err) {
2767	struct dst_ops *dst_ops = afinfo->dst_ops;
2768
2769	synchronize_rcu();
2770
2771	dst_ops->kmem_cachep = NULL;
2772	dst_ops->check = NULL;
2773	dst_ops->negative_advice = NULL;
2774	dst_ops->link_failure = NULL;
2775	afinfo->garbage_collect = NULL;
2776	}
2777	return err;
2778	}
2779	EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
2780
2781	static void __net_init xfrm_dst_ops_init(struct net *net)
2782	{
2783	struct xfrm_policy_afinfo *afinfo;
2784
2785	rcu_read_lock();
2786	afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET]);
2787	if (afinfo)
2788	net->xfrm.xfrm4_dst_ops = *afinfo->dst_ops;
2789	#if IS_ENABLED(CONFIG_IPV6)
2790	afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET6]);
2791	if (afinfo)
2792	net->xfrm.xfrm6_dst_ops = *afinfo->dst_ops;
2793	#endif
2794	rcu_read_unlock();
2795	}
2796
2797	static int xfrm_dev_event(struct notifier_block this, unsigned long event, void ptr)
2798	{
2799	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2800
2801	switch (event) {
2802	case NETDEV_DOWN:
2803	xfrm_garbage_collect(dev_net(dev));
2804	}
2805	return NOTIFY_DONE;
2806	}
2807
2808	static struct notifier_block xfrm_dev_notifier = {
2809	.notifier_call = xfrm_dev_event,
2810	};
2811
2812	#ifdef CONFIG_XFRM_STATISTICS
2813	static int __net_init xfrm_statistics_init(struct net *net)
2814	{
2815	int rv;
2816
2817	if (snmp_mib_init((void __percpu **)net->mib.xfrm_statistics,
2818	sizeof(struct linux_xfrm_mib),
2819	__alignof__(struct linux_xfrm_mib)) < 0)
2820	return -ENOMEM;
2821	rv = xfrm_proc_init(net);
2822	if (rv < 0)
2823	snmp_mib_free((void __percpu **)net->mib.xfrm_statistics);
2824	return rv;
2825	}
2826
2827	static void xfrm_statistics_fini(struct net *net)
2828	{
2829	xfrm_proc_fini(net);
2830	snmp_mib_free((void __percpu **)net->mib.xfrm_statistics);
2831	}
2832	#else
2833	static int __net_init xfrm_statistics_init(struct net *net)
2834	{
2835	return 0;
2836	}
2837
2838	static void xfrm_statistics_fini(struct net *net)
2839	{
2840	}
2841	#endif
2842
2843	static int __net_init xfrm_policy_init(struct net *net)
2844	{
2845	unsigned int hmask, sz;
2846	int dir;
2847
2848	if (net_eq(net, &init_net))
2849	xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
2850	sizeof(struct xfrm_dst),
2851	0, SLAB_HWCACHE_ALIGN\|SLAB_PANIC,
2852	NULL);
2853
2854	hmask = 8 - 1;
2855	sz = (hmask+1) * sizeof(struct hlist_head);
2856
2857	net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
2858	if (!net->xfrm.policy_byidx)
2859	goto out_byidx;
2860	net->xfrm.policy_idx_hmask = hmask;
2861
2862	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2863	struct xfrm_policy_hash *htab;
2864
2865	net->xfrm.policy_count[dir] = 0;
2866	INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
2867
2868	htab = &net->xfrm.policy_bydst[dir];
2869	htab->table = xfrm_hash_alloc(sz);
2870	if (!htab->table)
2871	goto out_bydst;
2872	htab->hmask = hmask;
2873	}
2874
2875	INIT_LIST_HEAD(&net->xfrm.policy_all);
2876	INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
2877	if (net_eq(net, &init_net))
2878	register_netdevice_notifier(&xfrm_dev_notifier);
2879	return 0;
2880
2881	out_bydst:
2882	for (dir--; dir >= 0; dir--) {
2883	struct xfrm_policy_hash *htab;
2884
2885	htab = &net->xfrm.policy_bydst[dir];
2886	xfrm_hash_free(htab->table, sz);
2887	}
2888	xfrm_hash_free(net->xfrm.policy_byidx, sz);
2889	out_byidx:
2890	return -ENOMEM;
2891	}
2892
2893	static void xfrm_policy_fini(struct net *net)
2894	{
2895	struct xfrm_audit audit_info;
2896	unsigned int sz;
2897	int dir;
2898
2899	flush_work(&net->xfrm.policy_hash_work);
2900	#ifdef CONFIG_XFRM_SUB_POLICY
2901	audit_info.loginuid = INVALID_UID;
2902	audit_info.sessionid = -1;
2903	audit_info.secid = 0;
2904	xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, &audit_info);
2905	#endif
2906	audit_info.loginuid = INVALID_UID;
2907	audit_info.sessionid = -1;
2908	audit_info.secid = 0;
2909	xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, &audit_info);
2910
2911	WARN_ON(!list_empty(&net->xfrm.policy_all));
2912
2913	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2914	struct xfrm_policy_hash *htab;
2915
2916	WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
2917
2918	htab = &net->xfrm.policy_bydst[dir];
2919	sz = (htab->hmask + 1) * sizeof(struct hlist_head);
2920	WARN_ON(!hlist_empty(htab->table));
2921	xfrm_hash_free(htab->table, sz);
2922	}
2923
2924	sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
2925	WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
2926	xfrm_hash_free(net->xfrm.policy_byidx, sz);
2927	}
2928
2929	static int __net_init xfrm_net_init(struct net *net)
2930	{
2931	int rv;
2932
2933	rv = xfrm_statistics_init(net);
2934	if (rv < 0)
2935	goto out_statistics;
2936	rv = xfrm_state_init(net);
2937	if (rv < 0)
2938	goto out_state;
2939	rv = xfrm_policy_init(net);
2940	if (rv < 0)
2941	goto out_policy;
2942	xfrm_dst_ops_init(net);
2943	rv = xfrm_sysctl_init(net);
2944	if (rv < 0)
2945	goto out_sysctl;
2946	return 0;
2947
2948	out_sysctl:
2949	xfrm_policy_fini(net);
2950	out_policy:
2951	xfrm_state_fini(net);
2952	out_state:
2953	xfrm_statistics_fini(net);
2954	out_statistics:
2955	return rv;
2956	}
2957
2958	static void __net_exit xfrm_net_exit(struct net *net)
2959	{
2960	xfrm_sysctl_fini(net);
2961	xfrm_policy_fini(net);
2962	xfrm_state_fini(net);
2963	xfrm_statistics_fini(net);
2964	}
2965
2966	static struct pernet_operations __net_initdata xfrm_net_ops = {
2967	.init = xfrm_net_init,
2968	.exit = xfrm_net_exit,
2969	};
2970
2971	void __init xfrm_init(void)
2972	{
2973	register_pernet_subsys(&xfrm_net_ops);
2974	xfrm_input_init();
2975	}
2976
2977	#ifdef CONFIG_AUDITSYSCALL
2978	static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
2979	struct audit_buffer *audit_buf)
2980	{
2981	struct xfrm_sec_ctx *ctx = xp->security;
2982	struct xfrm_selector *sel = &xp->selector;
2983
2984	if (ctx)
2985	audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
2986	ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2987
2988	switch(sel->family) {
2989	case AF_INET:
2990	audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
2991	if (sel->prefixlen_s != 32)
2992	audit_log_format(audit_buf, " src_prefixlen=%d",
2993	sel->prefixlen_s);
2994	audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
2995	if (sel->prefixlen_d != 32)
2996	audit_log_format(audit_buf, " dst_prefixlen=%d",
2997	sel->prefixlen_d);
2998	break;
2999	case AF_INET6:
3000	audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
3001	if (sel->prefixlen_s != 128)
3002	audit_log_format(audit_buf, " src_prefixlen=%d",
3003	sel->prefixlen_s);
3004	audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
3005	if (sel->prefixlen_d != 128)
3006	audit_log_format(audit_buf, " dst_prefixlen=%d",
3007	sel->prefixlen_d);
3008	break;
3009	}
3010	}
3011
3012	void xfrm_audit_policy_add(struct xfrm_policy *xp, int result,
3013	kuid_t auid, u32 sessionid, u32 secid)
3014	{
3015	struct audit_buffer *audit_buf;
3016
3017	audit_buf = xfrm_audit_start("SPD-add");
3018	if (audit_buf == NULL)
3019	return;
3020	xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
3021	audit_log_format(audit_buf, " res=%u", result);
3022	xfrm_audit_common_policyinfo(xp, audit_buf);
3023	audit_log_end(audit_buf);
3024	}
3025	EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
3026
3027	void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
3028	kuid_t auid, u32 sessionid, u32 secid)
3029	{
3030	struct audit_buffer *audit_buf;
3031
3032	audit_buf = xfrm_audit_start("SPD-delete");
3033	if (audit_buf == NULL)
3034	return;
3035	xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
3036	audit_log_format(audit_buf, " res=%u", result);
3037	xfrm_audit_common_policyinfo(xp, audit_buf);
3038	audit_log_end(audit_buf);
3039	}
3040	EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
3041	#endif
3042
3043	#ifdef CONFIG_XFRM_MIGRATE
3044	static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
3045	const struct xfrm_selector *sel_tgt)
3046	{
3047	if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
3048	if (sel_tgt->family == sel_cmp->family &&
3049	xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
3050	sel_cmp->family) &&
3051	xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
3052	sel_cmp->family) &&
3053	sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
3054	sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
3055	return true;
3056	}
3057	} else {
3058	if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
3059	return true;
3060	}
3061	}
3062	return false;
3063	}
3064
3065	static struct xfrm_policy * xfrm_migrate_policy_find(const struct xfrm_selector *sel,
3066	u8 dir, u8 type)
3067	{
3068	struct xfrm_policy pol, ret = NULL;
3069	struct hlist_head *chain;
3070	u32 priority = ~0U;
3071
3072	read_lock_bh(&xfrm_policy_lock);
3073	chain = policy_hash_direct(&init_net, &sel->daddr, &sel->saddr, sel->family, dir);
3074	hlist_for_each_entry(pol, chain, bydst) {
3075	if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3076	pol->type == type) {
3077	ret = pol;
3078	priority = ret->priority;
3079	break;
3080	}
3081	}
3082	chain = &init_net.xfrm.policy_inexact[dir];
3083	hlist_for_each_entry(pol, chain, bydst) {
3084	if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3085	pol->type == type &&
3086	pol->priority < priority) {
3087	ret = pol;
3088	break;
3089	}
3090	}
3091
3092	if (ret)
3093	xfrm_pol_hold(ret);
3094
3095	read_unlock_bh(&xfrm_policy_lock);
3096
3097	return ret;
3098	}
3099
3100	static int migrate_tmpl_match(const struct xfrm_migrate m, const struct xfrm_tmpl t)
3101	{
3102	int match = 0;
3103
3104	if (t->mode == m->mode && t->id.proto == m->proto &&
3105	(m->reqid == 0 \|\| t->reqid == m->reqid)) {
3106	switch (t->mode) {
3107	case XFRM_MODE_TUNNEL:
3108	case XFRM_MODE_BEET:
3109	if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
3110	m->old_family) &&
3111	xfrm_addr_equal(&t->saddr, &m->old_saddr,
3112	m->old_family)) {
3113	match = 1;
3114	}
3115	break;
3116	case XFRM_MODE_TRANSPORT:
3117	/* in case of transport mode, template does not store
3118	any IP addresses, hence we just compare mode and
3119	protocol */
3120	match = 1;
3121	break;
3122	default:
3123	break;
3124	}
3125	}
3126	return match;
3127	}
3128
3129	/* update endpoint address(es) of template(s) */
3130	static int xfrm_policy_migrate(struct xfrm_policy *pol,
3131	struct xfrm_migrate *m, int num_migrate)
3132	{
3133	struct xfrm_migrate *mp;
3134	int i, j, n = 0;
3135
3136	write_lock_bh(&pol->lock);
3137	if (unlikely(pol->walk.dead)) {
3138	/* target policy has been deleted */
3139	write_unlock_bh(&pol->lock);
3140	return -ENOENT;
3141	}
3142
3143	for (i = 0; i < pol->xfrm_nr; i++) {
3144	for (j = 0, mp = m; j < num_migrate; j++, mp++) {
3145	if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
3146	continue;
3147	n++;
3148	if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
3149	pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
3150	continue;
3151	/* update endpoints */
3152	memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
3153	sizeof(pol->xfrm_vec[i].id.daddr));
3154	memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
3155	sizeof(pol->xfrm_vec[i].saddr));
3156	pol->xfrm_vec[i].encap_family = mp->new_family;
3157	/* flush bundles */
3158	atomic_inc(&pol->genid);
3159	}
3160	}
3161
3162	write_unlock_bh(&pol->lock);
3163
3164	if (!n)
3165	return -ENODATA;
3166
3167	return 0;
3168	}
3169
3170	static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
3171	{
3172	int i, j;
3173
3174	if (num_migrate < 1 \|\| num_migrate > XFRM_MAX_DEPTH)
3175	return -EINVAL;
3176
3177	for (i = 0; i < num_migrate; i++) {
3178	if (xfrm_addr_equal(&m[i].old_daddr, &m[i].new_daddr,
3179	m[i].old_family) &&
3180	xfrm_addr_equal(&m[i].old_saddr, &m[i].new_saddr,
3181	m[i].old_family))
3182	return -EINVAL;
3183	if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) \|\|
3184	xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
3185	return -EINVAL;
3186
3187	/* check if there is any duplicated entry */
3188	for (j = i + 1; j < num_migrate; j++) {
3189	if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
3190	sizeof(m[i].old_daddr)) &&
3191	!memcmp(&m[i].old_saddr, &m[j].old_saddr,
3192	sizeof(m[i].old_saddr)) &&
3193	m[i].proto == m[j].proto &&
3194	m[i].mode == m[j].mode &&
3195	m[i].reqid == m[j].reqid &&
3196	m[i].old_family == m[j].old_family)
3197	return -EINVAL;
3198	}
3199	}
3200
3201	return 0;
3202	}
3203
3204	int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
3205	struct xfrm_migrate *m, int num_migrate,
3206	struct xfrm_kmaddress *k)
3207	{
3208	int i, err, nx_cur = 0, nx_new = 0;
3209	struct xfrm_policy *pol = NULL;
3210	struct xfrm_state x, xc;
3211	struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
3212	struct xfrm_state *x_new[XFRM_MAX_DEPTH];
3213	struct xfrm_migrate *mp;
3214
3215	if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
3216	goto out;
3217
3218	/* Stage 1 - find policy */
3219	if ((pol = xfrm_migrate_policy_find(sel, dir, type)) == NULL) {
3220	err = -ENOENT;
3221	goto out;
3222	}
3223
3224	/* Stage 2 - find and update state(s) */
3225	for (i = 0, mp = m; i < num_migrate; i++, mp++) {
3226	if ((x = xfrm_migrate_state_find(mp))) {
3227	x_cur[nx_cur] = x;
3228	nx_cur++;
3229	if ((xc = xfrm_state_migrate(x, mp))) {
3230	x_new[nx_new] = xc;
3231	nx_new++;
3232	} else {
3233	err = -ENODATA;
3234	goto restore_state;
3235	}
3236	}
3237	}
3238
3239	/* Stage 3 - update policy */
3240	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
3241	goto restore_state;
3242
3243	/* Stage 4 - delete old state(s) */
3244	if (nx_cur) {
3245	xfrm_states_put(x_cur, nx_cur);
3246	xfrm_states_delete(x_cur, nx_cur);
3247	}
3248
3249	/* Stage 5 - announce */
3250	km_migrate(sel, dir, type, m, num_migrate, k);
3251
3252	xfrm_pol_put(pol);
3253
3254	return 0;
3255	out:
3256	return err;
3257
3258	restore_state:
3259	if (pol)
3260	xfrm_pol_put(pol);
3261	if (nx_cur)
3262	xfrm_states_put(x_cur, nx_cur);
3263	if (nx_new)
3264	xfrm_states_delete(x_new, nx_new);
3265
3266	return err;
3267	}
3268	EXPORT_SYMBOL(xfrm_migrate);
3269	#endif
3270

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/net/xfrm/xfrm_policy.c

interactive