Kernel

Kernel Git Source Tree

Root/kernel/cpuset.c

1	/*
2	* kernel/cpuset.c
3	*
4	* Processor and Memory placement constraints for sets of tasks.
5	*
6	* Copyright (C) 2003 BULL SA.
7	* Copyright (C) 2004-2007 Silicon Graphics, Inc.
8	* Copyright (C) 2006 Google, Inc
9	*
10	* Portions derived from Patrick Mochel's sysfs code.
11	* sysfs is Copyright (c) 2001-3 Patrick Mochel
12	*
13	* 2003-10-10 Written by Simon Derr.
14	* 2003-10-22 Updates by Stephen Hemminger.
15	* 2004 May-July Rework by Paul Jackson.
16	* 2006 Rework by Paul Menage to use generic cgroups
17	* 2008 Rework of the scheduler domains and CPU hotplug handling
18	* by Max Krasnyansky
19	*
20	* This file is subject to the terms and conditions of the GNU General Public
21	* License. See the file COPYING in the main directory of the Linux
22	* distribution for more details.
23	*/
24
25	#include <linux/cpu.h>
26	#include <linux/cpumask.h>
27	#include <linux/cpuset.h>
28	#include <linux/err.h>
29	#include <linux/errno.h>
30	#include <linux/file.h>
31	#include <linux/fs.h>
32	#include <linux/init.h>
33	#include <linux/interrupt.h>
34	#include <linux/kernel.h>
35	#include <linux/kmod.h>
36	#include <linux/list.h>
37	#include <linux/mempolicy.h>
38	#include <linux/mm.h>
39	#include <linux/memory.h>
40	#include <linux/module.h>
41	#include <linux/mount.h>
42	#include <linux/namei.h>
43	#include <linux/pagemap.h>
44	#include <linux/proc_fs.h>
45	#include <linux/rcupdate.h>
46	#include <linux/sched.h>
47	#include <linux/seq_file.h>
48	#include <linux/security.h>
49	#include <linux/slab.h>
50	#include <linux/spinlock.h>
51	#include <linux/stat.h>
52	#include <linux/string.h>
53	#include <linux/time.h>
54	#include <linux/backing-dev.h>
55	#include <linux/sort.h>
56
57	#include <asm/uaccess.h>
58	#include <asm/atomic.h>
59	#include <linux/mutex.h>
60	#include <linux/workqueue.h>
61	#include <linux/cgroup.h>
62
63	/*
64	* Workqueue for cpuset related tasks.
65	*
66	* Using kevent workqueue may cause deadlock when memory_migrate
67	* is set. So we create a separate workqueue thread for cpuset.
68	*/
69	static struct workqueue_struct *cpuset_wq;
70
71	/*
72	* Tracks how many cpusets are currently defined in system.
73	* When there is only one cpuset (the root cpuset) we can
74	* short circuit some hooks.
75	*/
76	int number_of_cpusets __read_mostly;
77
78	/* Forward declare cgroup structures */
79	struct cgroup_subsys cpuset_subsys;
80	struct cpuset;
81
82	/* See "Frequency meter" comments, below. */
83
84	struct fmeter {
85	int cnt; /* unprocessed events count */
86	int val; /* most recent output value */
87	time_t time; /* clock (secs) when val computed */
88	spinlock_t lock; /* guards read or write of above */
89	};
90
91	struct cpuset {
92	struct cgroup_subsys_state css;
93
94	unsigned long flags; /* "unsigned long" so bitops work */
95	cpumask_var_t cpus_allowed; /* CPUs allowed to tasks in cpuset */
96	nodemask_t mems_allowed; /* Memory Nodes allowed to tasks */
97
98	struct cpuset parent; / my parent */
99
100	struct fmeter fmeter; /* memory_pressure filter */
101
102	/* partition number for rebuild_sched_domains() */
103	int pn;
104
105	/* for custom sched domain */
106	int relax_domain_level;
107
108	/* used for walking a cpuset heirarchy */
109	struct list_head stack_list;
110	};
111
112	/* Retrieve the cpuset for a cgroup */
113	static inline struct cpuset cgroup_cs(struct cgroup cont)
114	{
115	return container_of(cgroup_subsys_state(cont, cpuset_subsys_id),
116	struct cpuset, css);
117	}
118
119	/* Retrieve the cpuset for a task */
120	static inline struct cpuset task_cs(struct task_struct task)
121	{
122	return container_of(task_subsys_state(task, cpuset_subsys_id),
123	struct cpuset, css);
124	}
125
126	/* bits in struct cpuset flags field */
127	typedef enum {
128	CS_CPU_EXCLUSIVE,
129	CS_MEM_EXCLUSIVE,
130	CS_MEM_HARDWALL,
131	CS_MEMORY_MIGRATE,
132	CS_SCHED_LOAD_BALANCE,
133	CS_SPREAD_PAGE,
134	CS_SPREAD_SLAB,
135	} cpuset_flagbits_t;
136
137	/* convenient tests for these bits */
138	static inline int is_cpu_exclusive(const struct cpuset *cs)
139	{
140	return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
141	}
142
143	static inline int is_mem_exclusive(const struct cpuset *cs)
144	{
145	return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
146	}
147
148	static inline int is_mem_hardwall(const struct cpuset *cs)
149	{
150	return test_bit(CS_MEM_HARDWALL, &cs->flags);
151	}
152
153	static inline int is_sched_load_balance(const struct cpuset *cs)
154	{
155	return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
156	}
157
158	static inline int is_memory_migrate(const struct cpuset *cs)
159	{
160	return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
161	}
162
163	static inline int is_spread_page(const struct cpuset *cs)
164	{
165	return test_bit(CS_SPREAD_PAGE, &cs->flags);
166	}
167
168	static inline int is_spread_slab(const struct cpuset *cs)
169	{
170	return test_bit(CS_SPREAD_SLAB, &cs->flags);
171	}
172
173	static struct cpuset top_cpuset = {
174	.flags = ((1 << CS_CPU_EXCLUSIVE) \| (1 << CS_MEM_EXCLUSIVE)),
175	};
176
177	/*
178	* There are two global mutexes guarding cpuset structures. The first
179	* is the main control groups cgroup_mutex, accessed via
180	* cgroup_lock()/cgroup_unlock(). The second is the cpuset-specific
181	* callback_mutex, below. They can nest. It is ok to first take
182	* cgroup_mutex, then nest callback_mutex. We also require taking
183	* task_lock() when dereferencing a task's cpuset pointer. See "The
184	* task_lock() exception", at the end of this comment.
185	*
186	* A task must hold both mutexes to modify cpusets. If a task
187	* holds cgroup_mutex, then it blocks others wanting that mutex,
188	* ensuring that it is the only task able to also acquire callback_mutex
189	* and be able to modify cpusets. It can perform various checks on
190	* the cpuset structure first, knowing nothing will change. It can
191	* also allocate memory while just holding cgroup_mutex. While it is
192	* performing these checks, various callback routines can briefly
193	* acquire callback_mutex to query cpusets. Once it is ready to make
194	* the changes, it takes callback_mutex, blocking everyone else.
195	*
196	* Calls to the kernel memory allocator can not be made while holding
197	* callback_mutex, as that would risk double tripping on callback_mutex
198	* from one of the callbacks into the cpuset code from within
199	* __alloc_pages().
200	*
201	* If a task is only holding callback_mutex, then it has read-only
202	* access to cpusets.
203	*
204	* Now, the task_struct fields mems_allowed and mempolicy may be changed
205	* by other task, we use alloc_lock in the task_struct fields to protect
206	* them.
207	*
208	* The cpuset_common_file_read() handlers only hold callback_mutex across
209	* small pieces of code, such as when reading out possibly multi-word
210	* cpumasks and nodemasks.
211	*
212	* Accessing a task's cpuset should be done in accordance with the
213	* guidelines for accessing subsystem state in kernel/cgroup.c
214	*/
215
216	static DEFINE_MUTEX(callback_mutex);
217
218	/*
219	* cpuset_buffer_lock protects both the cpuset_name and cpuset_nodelist
220	* buffers. They are statically allocated to prevent using excess stack
221	* when calling cpuset_print_task_mems_allowed().
222	*/
223	#define CPUSET_NAME_LEN (128)
224	#define CPUSET_NODELIST_LEN (256)
225	static char cpuset_name[CPUSET_NAME_LEN];
226	static char cpuset_nodelist[CPUSET_NODELIST_LEN];
227	static DEFINE_SPINLOCK(cpuset_buffer_lock);
228
229	/*
230	* This is ugly, but preserves the userspace API for existing cpuset
231	* users. If someone tries to mount the "cpuset" filesystem, we
232	* silently switch it to mount "cgroup" instead
233	*/
234	static int cpuset_get_sb(struct file_system_type *fs_type,
235	int flags, const char *unused_dev_name,
236	void data, struct vfsmount mnt)
237	{
238	struct file_system_type *cgroup_fs = get_fs_type("cgroup");
239	int ret = -ENODEV;
240	if (cgroup_fs) {
241	char mountopts[] =
242	"cpuset,noprefix,"
243	"release_agent=/sbin/cpuset_release_agent";
244	ret = cgroup_fs->get_sb(cgroup_fs, flags,
245	unused_dev_name, mountopts, mnt);
246	put_filesystem(cgroup_fs);
247	}
248	return ret;
249	}
250
251	static struct file_system_type cpuset_fs_type = {
252	.name = "cpuset",
253	.get_sb = cpuset_get_sb,
254	};
255
256	/*
257	* Return in pmask the portion of a cpusets's cpus_allowed that
258	* are online. If none are online, walk up the cpuset hierarchy
259	* until we find one that does have some online cpus. If we get
260	* all the way to the top and still haven't found any online cpus,
261	* return cpu_online_map. Or if passed a NULL cs from an exit'ing
262	* task, return cpu_online_map.
263	*
264	* One way or another, we guarantee to return some non-empty subset
265	* of cpu_online_map.
266	*
267	* Call with callback_mutex held.
268	*/
269
270	static void guarantee_online_cpus(const struct cpuset *cs,
271	struct cpumask *pmask)
272	{
273	while (cs && !cpumask_intersects(cs->cpus_allowed, cpu_online_mask))
274	cs = cs->parent;
275	if (cs)
276	cpumask_and(pmask, cs->cpus_allowed, cpu_online_mask);
277	else
278	cpumask_copy(pmask, cpu_online_mask);
279	BUG_ON(!cpumask_intersects(pmask, cpu_online_mask));
280	}
281
282	/*
283	* Return in *pmask the portion of a cpusets's mems_allowed that
284	* are online, with memory. If none are online with memory, walk
285	* up the cpuset hierarchy until we find one that does have some
286	* online mems. If we get all the way to the top and still haven't
287	* found any online mems, return node_states[N_HIGH_MEMORY].
288	*
289	* One way or another, we guarantee to return some non-empty subset
290	* of node_states[N_HIGH_MEMORY].
291	*
292	* Call with callback_mutex held.
293	*/
294
295	static void guarantee_online_mems(const struct cpuset cs, nodemask_t pmask)
296	{
297	while (cs && !nodes_intersects(cs->mems_allowed,
298	node_states[N_HIGH_MEMORY]))
299	cs = cs->parent;
300	if (cs)
301	nodes_and(*pmask, cs->mems_allowed,
302	node_states[N_HIGH_MEMORY]);
303	else
304	*pmask = node_states[N_HIGH_MEMORY];
305	BUG_ON(!nodes_intersects(*pmask, node_states[N_HIGH_MEMORY]));
306	}
307
308	/*
309	* update task's spread flag if cpuset's page/slab spread flag is set
310	*
311	* Called with callback_mutex/cgroup_mutex held
312	*/
313	static void cpuset_update_task_spread_flag(struct cpuset *cs,
314	struct task_struct *tsk)
315	{
316	if (is_spread_page(cs))
317	tsk->flags \|= PF_SPREAD_PAGE;
318	else
319	tsk->flags &= ~PF_SPREAD_PAGE;
320	if (is_spread_slab(cs))
321	tsk->flags \|= PF_SPREAD_SLAB;
322	else
323	tsk->flags &= ~PF_SPREAD_SLAB;
324	}
325
326	/*
327	* is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
328	*
329	* One cpuset is a subset of another if all its allowed CPUs and
330	* Memory Nodes are a subset of the other, and its exclusive flags
331	* are only set if the other's are set. Call holding cgroup_mutex.
332	*/
333
334	static int is_cpuset_subset(const struct cpuset p, const struct cpuset q)
335	{
336	return cpumask_subset(p->cpus_allowed, q->cpus_allowed) &&
337	nodes_subset(p->mems_allowed, q->mems_allowed) &&
338	is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
339	is_mem_exclusive(p) <= is_mem_exclusive(q);
340	}
341
342	/**
343	* alloc_trial_cpuset - allocate a trial cpuset
344	* @cs: the cpuset that the trial cpuset duplicates
345	*/
346	static struct cpuset alloc_trial_cpuset(const struct cpuset cs)
347	{
348	struct cpuset *trial;
349
350	trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
351	if (!trial)
352	return NULL;
353
354	if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) {
355	kfree(trial);
356	return NULL;
357	}
358	cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
359
360	return trial;
361	}
362
363	/**
364	* free_trial_cpuset - free the trial cpuset
365	* @trial: the trial cpuset to be freed
366	*/
367	static void free_trial_cpuset(struct cpuset *trial)
368	{
369	free_cpumask_var(trial->cpus_allowed);
370	kfree(trial);
371	}
372
373	/*
374	* validate_change() - Used to validate that any proposed cpuset change
375	* follows the structural rules for cpusets.
376	*
377	* If we replaced the flag and mask values of the current cpuset
378	* (cur) with those values in the trial cpuset (trial), would
379	* our various subset and exclusive rules still be valid? Presumes
380	* cgroup_mutex held.
381	*
382	* 'cur' is the address of an actual, in-use cpuset. Operations
383	* such as list traversal that depend on the actual address of the
384	* cpuset in the list must use cur below, not trial.
385	*
386	* 'trial' is the address of bulk structure copy of cur, with
387	* perhaps one or more of the fields cpus_allowed, mems_allowed,
388	* or flags changed to new, trial values.
389	*
390	* Return 0 if valid, -errno if not.
391	*/
392
393	static int validate_change(const struct cpuset cur, const struct cpuset trial)
394	{
395	struct cgroup *cont;
396	struct cpuset c, par;
397
398	/* Each of our child cpusets must be a subset of us */
399	list_for_each_entry(cont, &cur->css.cgroup->children, sibling) {
400	if (!is_cpuset_subset(cgroup_cs(cont), trial))
401	return -EBUSY;
402	}
403
404	/* Remaining checks don't apply to root cpuset */
405	if (cur == &top_cpuset)
406	return 0;
407
408	par = cur->parent;
409
410	/* We must be a subset of our parent cpuset */
411	if (!is_cpuset_subset(trial, par))
412	return -EACCES;
413
414	/*
415	* If either I or some sibling (!= me) is exclusive, we can't
416	* overlap
417	*/
418	list_for_each_entry(cont, &par->css.cgroup->children, sibling) {
419	c = cgroup_cs(cont);
420	if ((is_cpu_exclusive(trial) \|\| is_cpu_exclusive(c)) &&
421	c != cur &&
422	cpumask_intersects(trial->cpus_allowed, c->cpus_allowed))
423	return -EINVAL;
424	if ((is_mem_exclusive(trial) \|\| is_mem_exclusive(c)) &&
425	c != cur &&
426	nodes_intersects(trial->mems_allowed, c->mems_allowed))
427	return -EINVAL;
428	}
429
430	/* Cpusets with tasks can't have empty cpus_allowed or mems_allowed */
431	if (cgroup_task_count(cur->css.cgroup)) {
432	if (cpumask_empty(trial->cpus_allowed) \|\|
433	nodes_empty(trial->mems_allowed)) {
434	return -ENOSPC;
435	}
436	}
437
438	return 0;
439	}
440
441	#ifdef CONFIG_SMP
442	/*
443	* Helper routine for generate_sched_domains().
444	* Do cpusets a, b have overlapping cpus_allowed masks?
445	*/
446	static int cpusets_overlap(struct cpuset a, struct cpuset b)
447	{
448	return cpumask_intersects(a->cpus_allowed, b->cpus_allowed);
449	}
450
451	static void
452	update_domain_attr(struct sched_domain_attr dattr, struct cpuset c)
453	{
454	if (dattr->relax_domain_level < c->relax_domain_level)
455	dattr->relax_domain_level = c->relax_domain_level;
456	return;
457	}
458
459	static void
460	update_domain_attr_tree(struct sched_domain_attr dattr, struct cpuset c)
461	{
462	LIST_HEAD(q);
463
464	list_add(&c->stack_list, &q);
465	while (!list_empty(&q)) {
466	struct cpuset *cp;
467	struct cgroup *cont;
468	struct cpuset *child;
469
470	cp = list_first_entry(&q, struct cpuset, stack_list);
471	list_del(q.next);
472
473	if (cpumask_empty(cp->cpus_allowed))
474	continue;
475
476	if (is_sched_load_balance(cp))
477	update_domain_attr(dattr, cp);
478
479	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
480	child = cgroup_cs(cont);
481	list_add_tail(&child->stack_list, &q);
482	}
483	}
484	}
485
486	/*
487	* generate_sched_domains()
488	*
489	* This function builds a partial partition of the systems CPUs
490	* A 'partial partition' is a set of non-overlapping subsets whose
491	* union is a subset of that set.
492	* The output of this function needs to be passed to kernel/sched.c
493	* partition_sched_domains() routine, which will rebuild the scheduler's
494	* load balancing domains (sched domains) as specified by that partial
495	* partition.
496	*
497	* See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt
498	* for a background explanation of this.
499	*
500	* Does not return errors, on the theory that the callers of this
501	* routine would rather not worry about failures to rebuild sched
502	* domains when operating in the severe memory shortage situations
503	* that could cause allocation failures below.
504	*
505	* Must be called with cgroup_lock held.
506	*
507	* The three key local variables below are:
508	* q - a linked-list queue of cpuset pointers, used to implement a
509	* top-down scan of all cpusets. This scan loads a pointer
510	* to each cpuset marked is_sched_load_balance into the
511	* array 'csa'. For our purposes, rebuilding the schedulers
512	* sched domains, we can ignore !is_sched_load_balance cpusets.
513	* csa - (for CpuSet Array) Array of pointers to all the cpusets
514	* that need to be load balanced, for convenient iterative
515	* access by the subsequent code that finds the best partition,
516	* i.e the set of domains (subsets) of CPUs such that the
517	* cpus_allowed of every cpuset marked is_sched_load_balance
518	* is a subset of one of these domains, while there are as
519	* many such domains as possible, each as small as possible.
520	* doms - Conversion of 'csa' to an array of cpumasks, for passing to
521	* the kernel/sched.c routine partition_sched_domains() in a
522	* convenient format, that can be easily compared to the prior
523	* value to determine what partition elements (sched domains)
524	* were changed (added or removed.)
525	*
526	* Finding the best partition (set of domains):
527	* The triple nested loops below over i, j, k scan over the
528	* load balanced cpusets (using the array of cpuset pointers in
529	* csa[]) looking for pairs of cpusets that have overlapping
530	* cpus_allowed, but which don't have the same 'pn' partition
531	* number and gives them in the same partition number. It keeps
532	* looping on the 'restart' label until it can no longer find
533	* any such pairs.
534	*
535	* The union of the cpus_allowed masks from the set of
536	* all cpusets having the same 'pn' value then form the one
537	* element of the partition (one sched domain) to be passed to
538	* partition_sched_domains().
539	*/
540	/* FIXME: see the FIXME in partition_sched_domains() */
541	static int generate_sched_domains(struct cpumask **domains,
542	struct sched_domain_attr **attributes)
543	{
544	LIST_HEAD(q); /* queue of cpusets to be scanned */
545	struct cpuset cp; / scans q */
546	struct cpuset *csa; / array of all cpuset ptrs */
547	int csn; /* how many cpuset ptrs in csa so far */
548	int i, j, k; /* indices for partition finding loops */
549	struct cpumask doms; / resulting partition; i.e. sched domains */
550	struct sched_domain_attr dattr; / attributes for custom domains */
551	int ndoms = 0; /* number of sched domains in result */
552	int nslot; /* next empty doms[] struct cpumask slot */
553
554	doms = NULL;
555	dattr = NULL;
556	csa = NULL;
557
558	/* Special case for the 99% of systems with one, full, sched domain */
559	if (is_sched_load_balance(&top_cpuset)) {
560	doms = kmalloc(cpumask_size(), GFP_KERNEL);
561	if (!doms)
562	goto done;
563
564	dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
565	if (dattr) {
566	*dattr = SD_ATTR_INIT;
567	update_domain_attr_tree(dattr, &top_cpuset);
568	}
569	cpumask_copy(doms, top_cpuset.cpus_allowed);
570
571	ndoms = 1;
572	goto done;
573	}
574
575	csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);
576	if (!csa)
577	goto done;
578	csn = 0;
579
580	list_add(&top_cpuset.stack_list, &q);
581	while (!list_empty(&q)) {
582	struct cgroup *cont;
583	struct cpuset child; / scans child cpusets of cp */
584
585	cp = list_first_entry(&q, struct cpuset, stack_list);
586	list_del(q.next);
587
588	if (cpumask_empty(cp->cpus_allowed))
589	continue;
590
591	/*
592	* All child cpusets contain a subset of the parent's cpus, so
593	* just skip them, and then we call update_domain_attr_tree()
594	* to calc relax_domain_level of the corresponding sched
595	* domain.
596	*/
597	if (is_sched_load_balance(cp)) {
598	csa[csn++] = cp;
599	continue;
600	}
601
602	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
603	child = cgroup_cs(cont);
604	list_add_tail(&child->stack_list, &q);
605	}
606	}
607
608	for (i = 0; i < csn; i++)
609	csa[i]->pn = i;
610	ndoms = csn;
611
612	restart:
613	/* Find the best partition (set of sched domains) */
614	for (i = 0; i < csn; i++) {
615	struct cpuset *a = csa[i];
616	int apn = a->pn;
617
618	for (j = 0; j < csn; j++) {
619	struct cpuset *b = csa[j];
620	int bpn = b->pn;
621
622	if (apn != bpn && cpusets_overlap(a, b)) {
623	for (k = 0; k < csn; k++) {
624	struct cpuset *c = csa[k];
625
626	if (c->pn == bpn)
627	c->pn = apn;
628	}
629	ndoms--; /* one less element */
630	goto restart;
631	}
632	}
633	}
634
635	/*
636	* Now we know how many domains to create.
637	* Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
638	*/
639	doms = kmalloc(ndoms * cpumask_size(), GFP_KERNEL);
640	if (!doms)
641	goto done;
642
643	/*
644	* The rest of the code, including the scheduler, can deal with
645	* dattr==NULL case. No need to abort if alloc fails.
646	*/
647	dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL);
648
649	for (nslot = 0, i = 0; i < csn; i++) {
650	struct cpuset *a = csa[i];
651	struct cpumask *dp;
652	int apn = a->pn;
653
654	if (apn < 0) {
655	/* Skip completed partitions */
656	continue;
657	}
658
659	dp = doms + nslot;
660
661	if (nslot == ndoms) {
662	static int warnings = 10;
663	if (warnings) {
664	printk(KERN_WARNING
665	"rebuild_sched_domains confused:"
666	" nslot %d, ndoms %d, csn %d, i %d,"
667	" apn %d\n",
668	nslot, ndoms, csn, i, apn);
669	warnings--;
670	}
671	continue;
672	}
673
674	cpumask_clear(dp);
675	if (dattr)
676	*(dattr + nslot) = SD_ATTR_INIT;
677	for (j = i; j < csn; j++) {
678	struct cpuset *b = csa[j];
679
680	if (apn == b->pn) {
681	cpumask_or(dp, dp, b->cpus_allowed);
682	if (dattr)
683	update_domain_attr_tree(dattr + nslot, b);
684
685	/* Done with this partition */
686	b->pn = -1;
687	}
688	}
689	nslot++;
690	}
691	BUG_ON(nslot != ndoms);
692
693	done:
694	kfree(csa);
695
696	/*
697	* Fallback to the default domain if kmalloc() failed.
698	* See comments in partition_sched_domains().
699	*/
700	if (doms == NULL)
701	ndoms = 1;
702
703	*domains = doms;
704	*attributes = dattr;
705	return ndoms;
706	}
707
708	/*
709	* Rebuild scheduler domains.
710	*
711	* Call with neither cgroup_mutex held nor within get_online_cpus().
712	* Takes both cgroup_mutex and get_online_cpus().
713	*
714	* Cannot be directly called from cpuset code handling changes
715	* to the cpuset pseudo-filesystem, because it cannot be called
716	* from code that already holds cgroup_mutex.
717	*/
718	static void do_rebuild_sched_domains(struct work_struct *unused)
719	{
720	struct sched_domain_attr *attr;
721	struct cpumask *doms;
722	int ndoms;
723
724	get_online_cpus();
725
726	/* Generate domain masks and attrs */
727	cgroup_lock();
728	ndoms = generate_sched_domains(&doms, &attr);
729	cgroup_unlock();
730
731	/* Have scheduler rebuild the domains */
732	partition_sched_domains(ndoms, doms, attr);
733
734	put_online_cpus();
735	}
736	#else /* !CONFIG_SMP */
737	static void do_rebuild_sched_domains(struct work_struct *unused)
738	{
739	}
740
741	static int generate_sched_domains(struct cpumask **domains,
742	struct sched_domain_attr **attributes)
743	{
744	*domains = NULL;
745	return 1;
746	}
747	#endif /* CONFIG_SMP */
748
749	static DECLARE_WORK(rebuild_sched_domains_work, do_rebuild_sched_domains);
750
751	/*
752	* Rebuild scheduler domains, asynchronously via workqueue.
753	*
754	* If the flag 'sched_load_balance' of any cpuset with non-empty
755	* 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
756	* which has that flag enabled, or if any cpuset with a non-empty
757	* 'cpus' is removed, then call this routine to rebuild the
758	* scheduler's dynamic sched domains.
759	*
760	* The rebuild_sched_domains() and partition_sched_domains()
761	* routines must nest cgroup_lock() inside get_online_cpus(),
762	* but such cpuset changes as these must nest that locking the
763	* other way, holding cgroup_lock() for much of the code.
764	*
765	* So in order to avoid an ABBA deadlock, the cpuset code handling
766	* these user changes delegates the actual sched domain rebuilding
767	* to a separate workqueue thread, which ends up processing the
768	* above do_rebuild_sched_domains() function.
769	*/
770	static void async_rebuild_sched_domains(void)
771	{
772	queue_work(cpuset_wq, &rebuild_sched_domains_work);
773	}
774
775	/*
776	* Accomplishes the same scheduler domain rebuild as the above
777	* async_rebuild_sched_domains(), however it directly calls the
778	* rebuild routine synchronously rather than calling it via an
779	* asynchronous work thread.
780	*
781	* This can only be called from code that is not holding
782	* cgroup_mutex (not nested in a cgroup_lock() call.)
783	*/
784	void rebuild_sched_domains(void)
785	{
786	do_rebuild_sched_domains(NULL);
787	}
788
789	/**
790	* cpuset_test_cpumask - test a task's cpus_allowed versus its cpuset's
791	* @tsk: task to test
792	* @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner
793	*
794	* Call with cgroup_mutex held. May take callback_mutex during call.
795	* Called for each task in a cgroup by cgroup_scan_tasks().
796	* Return nonzero if this tasks's cpus_allowed mask should be changed (in other
797	* words, if its mask is not equal to its cpuset's mask).
798	*/
799	static int cpuset_test_cpumask(struct task_struct *tsk,
800	struct cgroup_scanner *scan)
801	{
802	return !cpumask_equal(&tsk->cpus_allowed,
803	(cgroup_cs(scan->cg))->cpus_allowed);
804	}
805
806	/**
807	* cpuset_change_cpumask - make a task's cpus_allowed the same as its cpuset's
808	* @tsk: task to test
809	* @scan: struct cgroup_scanner containing the cgroup of the task
810	*
811	* Called by cgroup_scan_tasks() for each task in a cgroup whose
812	* cpus_allowed mask needs to be changed.
813	*
814	* We don't need to re-check for the cgroup/cpuset membership, since we're
815	* holding cgroup_lock() at this point.
816	*/
817	static void cpuset_change_cpumask(struct task_struct *tsk,
818	struct cgroup_scanner *scan)
819	{
820	set_cpus_allowed_ptr(tsk, ((cgroup_cs(scan->cg))->cpus_allowed));
821	}
822
823	/**
824	* update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
825	* @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
826	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
827	*
828	* Called with cgroup_mutex held
829	*
830	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
831	* calling callback functions for each.
832	*
833	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
834	* if @heap != NULL.
835	*/
836	static void update_tasks_cpumask(struct cpuset cs, struct ptr_heap heap)
837	{
838	struct cgroup_scanner scan;
839
840	scan.cg = cs->css.cgroup;
841	scan.test_task = cpuset_test_cpumask;
842	scan.process_task = cpuset_change_cpumask;
843	scan.heap = heap;
844	cgroup_scan_tasks(&scan);
845	}
846
847	/**
848	* update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
849	* @cs: the cpuset to consider
850	* @buf: buffer of cpu numbers written to this cpuset
851	*/
852	static int update_cpumask(struct cpuset cs, struct cpuset trialcs,
853	const char *buf)
854	{
855	struct ptr_heap heap;
856	int retval;
857	int is_load_balanced;
858
859	/* top_cpuset.cpus_allowed tracks cpu_online_map; it's read-only */
860	if (cs == &top_cpuset)
861	return -EACCES;
862
863	/*
864	* An empty cpus_allowed is ok only if the cpuset has no tasks.
865	* Since cpulist_parse() fails on an empty mask, we special case
866	* that parsing. The validate_change() call ensures that cpusets
867	* with tasks have cpus.
868	*/
869	if (!*buf) {
870	cpumask_clear(trialcs->cpus_allowed);
871	} else {
872	retval = cpulist_parse(buf, trialcs->cpus_allowed);
873	if (retval < 0)
874	return retval;
875
876	if (!cpumask_subset(trialcs->cpus_allowed, cpu_active_mask))
877	return -EINVAL;
878	}
879	retval = validate_change(cs, trialcs);
880	if (retval < 0)
881	return retval;
882
883	/* Nothing to do if the cpus didn't change */
884	if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed))
885	return 0;
886
887	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
888	if (retval)
889	return retval;
890
891	is_load_balanced = is_sched_load_balance(trialcs);
892
893	mutex_lock(&callback_mutex);
894	cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
895	mutex_unlock(&callback_mutex);
896
897	/*
898	* Scan tasks in the cpuset, and update the cpumasks of any
899	* that need an update.
900	*/
901	update_tasks_cpumask(cs, &heap);
902
903	heap_free(&heap);
904
905	if (is_load_balanced)
906	async_rebuild_sched_domains();
907	return 0;
908	}
909
910	/*
911	* cpuset_migrate_mm
912	*
913	* Migrate memory region from one set of nodes to another.
914	*
915	* Temporarilly set tasks mems_allowed to target nodes of migration,
916	* so that the migration code can allocate pages on these nodes.
917	*
918	* Call holding cgroup_mutex, so current's cpuset won't change
919	* during this call, as manage_mutex holds off any cpuset_attach()
920	* calls. Therefore we don't need to take task_lock around the
921	* call to guarantee_online_mems(), as we know no one is changing
922	* our task's cpuset.
923	*
924	* Hold callback_mutex around the two modifications of our tasks
925	* mems_allowed to synchronize with cpuset_mems_allowed().
926	*
927	* While the mm_struct we are migrating is typically from some
928	* other task, the task_struct mems_allowed that we are hacking
929	* is for our current task, which must allocate new pages for that
930	* migrating memory region.
931	*/
932
933	static void cpuset_migrate_mm(struct mm_struct mm, const nodemask_t from,
934	const nodemask_t *to)
935	{
936	struct task_struct *tsk = current;
937
938	tsk->mems_allowed = *to;
939
940	do_migrate_pages(mm, from, to, MPOL_MF_MOVE_ALL);
941
942	guarantee_online_mems(task_cs(tsk),&tsk->mems_allowed);
943	}
944
945	/*
946	* cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
947	* @tsk: the task to change
948	* @newmems: new nodes that the task will be set
949	*
950	* In order to avoid seeing no nodes if the old and new nodes are disjoint,
951	* we structure updates as setting all new allowed nodes, then clearing newly
952	* disallowed ones.
953	*
954	* Called with task's alloc_lock held
955	*/
956	static void cpuset_change_task_nodemask(struct task_struct *tsk,
957	nodemask_t *newmems)
958	{
959	nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
960	mpol_rebind_task(tsk, &tsk->mems_allowed);
961	mpol_rebind_task(tsk, newmems);
962	tsk->mems_allowed = *newmems;
963	}
964
965	/*
966	* Update task's mems_allowed and rebind its mempolicy and vmas' mempolicy
967	* of it to cpuset's new mems_allowed, and migrate pages to new nodes if
968	* memory_migrate flag is set. Called with cgroup_mutex held.
969	*/
970	static void cpuset_change_nodemask(struct task_struct *p,
971	struct cgroup_scanner *scan)
972	{
973	struct mm_struct *mm;
974	struct cpuset *cs;
975	int migrate;
976	const nodemask_t *oldmem = scan->data;
977	nodemask_t newmems;
978
979	cs = cgroup_cs(scan->cg);
980	guarantee_online_mems(cs, &newmems);
981
982	task_lock(p);
983	cpuset_change_task_nodemask(p, &newmems);
984	task_unlock(p);
985
986	mm = get_task_mm(p);
987	if (!mm)
988	return;
989
990	migrate = is_memory_migrate(cs);
991
992	mpol_rebind_mm(mm, &cs->mems_allowed);
993	if (migrate)
994	cpuset_migrate_mm(mm, oldmem, &cs->mems_allowed);
995	mmput(mm);
996	}
997
998	static void *cpuset_being_rebound;
999
1000	/**
1001	* update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
1002	* @cs: the cpuset in which each task's mems_allowed mask needs to be changed
1003	* @oldmem: old mems_allowed of cpuset cs
1004	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
1005	*
1006	* Called with cgroup_mutex held
1007	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
1008	* if @heap != NULL.
1009	*/
1010	static void update_tasks_nodemask(struct cpuset cs, const nodemask_t oldmem,
1011	struct ptr_heap *heap)
1012	{
1013	struct cgroup_scanner scan;
1014
1015	cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
1016
1017	scan.cg = cs->css.cgroup;
1018	scan.test_task = NULL;
1019	scan.process_task = cpuset_change_nodemask;
1020	scan.heap = heap;
1021	scan.data = (nodemask_t *)oldmem;
1022
1023	/*
1024	* The mpol_rebind_mm() call takes mmap_sem, which we couldn't
1025	* take while holding tasklist_lock. Forks can happen - the
1026	* mpol_dup() cpuset_being_rebound check will catch such forks,
1027	* and rebind their vma mempolicies too. Because we still hold
1028	* the global cgroup_mutex, we know that no other rebind effort
1029	* will be contending for the global variable cpuset_being_rebound.
1030	* It's ok if we rebind the same mm twice; mpol_rebind_mm()
1031	* is idempotent. Also migrate pages in each mm to new nodes.
1032	*/
1033	cgroup_scan_tasks(&scan);
1034
1035	/* We're done rebinding vmas to this cpuset's new mems_allowed. */
1036	cpuset_being_rebound = NULL;
1037	}
1038
1039	/*
1040	* Handle user request to change the 'mems' memory placement
1041	* of a cpuset. Needs to validate the request, update the
1042	* cpusets mems_allowed, and for each task in the cpuset,
1043	* update mems_allowed and rebind task's mempolicy and any vma
1044	* mempolicies and if the cpuset is marked 'memory_migrate',
1045	* migrate the tasks pages to the new memory.
1046	*
1047	* Call with cgroup_mutex held. May take callback_mutex during call.
1048	* Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
1049	* lock each such tasks mm->mmap_sem, scan its vma's and rebind
1050	* their mempolicies to the cpusets new mems_allowed.
1051	*/
1052	static int update_nodemask(struct cpuset cs, struct cpuset trialcs,
1053	const char *buf)
1054	{
1055	nodemask_t oldmem;
1056	int retval;
1057	struct ptr_heap heap;
1058
1059	/*
1060	* top_cpuset.mems_allowed tracks node_stats[N_HIGH_MEMORY];
1061	* it's read-only
1062	*/
1063	if (cs == &top_cpuset)
1064	return -EACCES;
1065
1066	/*
1067	* An empty mems_allowed is ok iff there are no tasks in the cpuset.
1068	* Since nodelist_parse() fails on an empty mask, we special case
1069	* that parsing. The validate_change() call ensures that cpusets
1070	* with tasks have memory.
1071	*/
1072	if (!*buf) {
1073	nodes_clear(trialcs->mems_allowed);
1074	} else {
1075	retval = nodelist_parse(buf, trialcs->mems_allowed);
1076	if (retval < 0)
1077	goto done;
1078
1079	if (!nodes_subset(trialcs->mems_allowed,
1080	node_states[N_HIGH_MEMORY]))
1081	return -EINVAL;
1082	}
1083	oldmem = cs->mems_allowed;
1084	if (nodes_equal(oldmem, trialcs->mems_allowed)) {
1085	retval = 0; /* Too easy - nothing to do */
1086	goto done;
1087	}
1088	retval = validate_change(cs, trialcs);
1089	if (retval < 0)
1090	goto done;
1091
1092	retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
1093	if (retval < 0)
1094	goto done;
1095
1096	mutex_lock(&callback_mutex);
1097	cs->mems_allowed = trialcs->mems_allowed;
1098	mutex_unlock(&callback_mutex);
1099
1100	update_tasks_nodemask(cs, &oldmem, &heap);
1101
1102	heap_free(&heap);
1103	done:
1104	return retval;
1105	}
1106
1107	int current_cpuset_is_being_rebound(void)
1108	{
1109	return task_cs(current) == cpuset_being_rebound;
1110	}
1111
1112	static int update_relax_domain_level(struct cpuset *cs, s64 val)
1113	{
1114	#ifdef CONFIG_SMP
1115	if (val < -1 \|\| val >= SD_LV_MAX)
1116	return -EINVAL;
1117	#endif
1118
1119	if (val != cs->relax_domain_level) {
1120	cs->relax_domain_level = val;
1121	if (!cpumask_empty(cs->cpus_allowed) &&
1122	is_sched_load_balance(cs))
1123	async_rebuild_sched_domains();
1124	}
1125
1126	return 0;
1127	}
1128
1129	/*
1130	* cpuset_change_flag - make a task's spread flags the same as its cpuset's
1131	* @tsk: task to be updated
1132	* @scan: struct cgroup_scanner containing the cgroup of the task
1133	*
1134	* Called by cgroup_scan_tasks() for each task in a cgroup.
1135	*
1136	* We don't need to re-check for the cgroup/cpuset membership, since we're
1137	* holding cgroup_lock() at this point.
1138	*/
1139	static void cpuset_change_flag(struct task_struct *tsk,
1140	struct cgroup_scanner *scan)
1141	{
1142	cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
1143	}
1144
1145	/*
1146	* update_tasks_flags - update the spread flags of tasks in the cpuset.
1147	* @cs: the cpuset in which each task's spread flags needs to be changed
1148	* @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
1149	*
1150	* Called with cgroup_mutex held
1151	*
1152	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
1153	* calling callback functions for each.
1154	*
1155	* No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
1156	* if @heap != NULL.
1157	*/
1158	static void update_tasks_flags(struct cpuset cs, struct ptr_heap heap)
1159	{
1160	struct cgroup_scanner scan;
1161
1162	scan.cg = cs->css.cgroup;
1163	scan.test_task = NULL;
1164	scan.process_task = cpuset_change_flag;
1165	scan.heap = heap;
1166	cgroup_scan_tasks(&scan);
1167	}
1168
1169	/*
1170	* update_flag - read a 0 or a 1 in a file and update associated flag
1171	* bit: the bit to update (see cpuset_flagbits_t)
1172	* cs: the cpuset to update
1173	* turning_on: whether the flag is being set or cleared
1174	*
1175	* Call with cgroup_mutex held.
1176	*/
1177
1178	static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1179	int turning_on)
1180	{
1181	struct cpuset *trialcs;
1182	int balance_flag_changed;
1183	int spread_flag_changed;
1184	struct ptr_heap heap;
1185	int err;
1186
1187	trialcs = alloc_trial_cpuset(cs);
1188	if (!trialcs)
1189	return -ENOMEM;
1190
1191	if (turning_on)
1192	set_bit(bit, &trialcs->flags);
1193	else
1194	clear_bit(bit, &trialcs->flags);
1195
1196	err = validate_change(cs, trialcs);
1197	if (err < 0)
1198	goto out;
1199
1200	err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
1201	if (err < 0)
1202	goto out;
1203
1204	balance_flag_changed = (is_sched_load_balance(cs) !=
1205	is_sched_load_balance(trialcs));
1206
1207	spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
1208	\|\| (is_spread_page(cs) != is_spread_page(trialcs)));
1209
1210	mutex_lock(&callback_mutex);
1211	cs->flags = trialcs->flags;
1212	mutex_unlock(&callback_mutex);
1213
1214	if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
1215	async_rebuild_sched_domains();
1216
1217	if (spread_flag_changed)
1218	update_tasks_flags(cs, &heap);
1219	heap_free(&heap);
1220	out:
1221	free_trial_cpuset(trialcs);
1222	return err;
1223	}
1224
1225	/*
1226	* Frequency meter - How fast is some event occurring?
1227	*
1228	* These routines manage a digitally filtered, constant time based,
1229	* event frequency meter. There are four routines:
1230	* fmeter_init() - initialize a frequency meter.
1231	* fmeter_markevent() - called each time the event happens.
1232	* fmeter_getrate() - returns the recent rate of such events.
1233	* fmeter_update() - internal routine used to update fmeter.
1234	*
1235	* A common data structure is passed to each of these routines,
1236	* which is used to keep track of the state required to manage the
1237	* frequency meter and its digital filter.
1238	*
1239	* The filter works on the number of events marked per unit time.
1240	* The filter is single-pole low-pass recursive (IIR). The time unit
1241	* is 1 second. Arithmetic is done using 32-bit integers scaled to
1242	* simulate 3 decimal digits of precision (multiplied by 1000).
1243	*
1244	* With an FM_COEF of 933, and a time base of 1 second, the filter
1245	* has a half-life of 10 seconds, meaning that if the events quit
1246	* happening, then the rate returned from the fmeter_getrate()
1247	* will be cut in half each 10 seconds, until it converges to zero.
1248	*
1249	* It is not worth doing a real infinitely recursive filter. If more
1250	* than FM_MAXTICKS ticks have elapsed since the last filter event,
1251	* just compute FM_MAXTICKS ticks worth, by which point the level
1252	* will be stable.
1253	*
1254	* Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
1255	* arithmetic overflow in the fmeter_update() routine.
1256	*
1257	* Given the simple 32 bit integer arithmetic used, this meter works
1258	* best for reporting rates between one per millisecond (msec) and
1259	* one per 32 (approx) seconds. At constant rates faster than one
1260	* per msec it maxes out at values just under 1,000,000. At constant
1261	* rates between one per msec, and one per second it will stabilize
1262	* to a value N*1000, where N is the rate of events per second.
1263	* At constant rates between one per second and one per 32 seconds,
1264	* it will be choppy, moving up on the seconds that have an event,
1265	* and then decaying until the next event. At rates slower than
1266	* about one in 32 seconds, it decays all the way back to zero between
1267	* each event.
1268	*/
1269
1270	#define FM_COEF 933 /* coefficient for half-life of 10 secs */
1271	#define FM_MAXTICKS ((time_t)99) /* useless computing more ticks than this */
1272	#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
1273	#define FM_SCALE 1000 /* faux fixed point scale */
1274
1275	/* Initialize a frequency meter */
1276	static void fmeter_init(struct fmeter *fmp)
1277	{
1278	fmp->cnt = 0;
1279	fmp->val = 0;
1280	fmp->time = 0;
1281	spin_lock_init(&fmp->lock);
1282	}
1283
1284	/* Internal meter update - process cnt events and update value */
1285	static void fmeter_update(struct fmeter *fmp)
1286	{
1287	time_t now = get_seconds();
1288	time_t ticks = now - fmp->time;
1289
1290	if (ticks == 0)
1291	return;
1292
1293	ticks = min(FM_MAXTICKS, ticks);
1294	while (ticks-- > 0)
1295	fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
1296	fmp->time = now;
1297
1298	fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
1299	fmp->cnt = 0;
1300	}
1301
1302	/* Process any previous ticks, then bump cnt by one (times scale). */
1303	static void fmeter_markevent(struct fmeter *fmp)
1304	{
1305	spin_lock(&fmp->lock);
1306	fmeter_update(fmp);
1307	fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
1308	spin_unlock(&fmp->lock);
1309	}
1310
1311	/* Process any previous ticks, then return current value. */
1312	static int fmeter_getrate(struct fmeter *fmp)
1313	{
1314	int val;
1315
1316	spin_lock(&fmp->lock);
1317	fmeter_update(fmp);
1318	val = fmp->val;
1319	spin_unlock(&fmp->lock);
1320	return val;
1321	}
1322
1323	/* Protected by cgroup_lock */
1324	static cpumask_var_t cpus_attach;
1325
1326	/* Called by cgroups to determine if a cpuset is usable; cgroup_mutex held */
1327	static int cpuset_can_attach(struct cgroup_subsys ss, struct cgroup cont,
1328	struct task_struct *tsk, bool threadgroup)
1329	{
1330	int ret;
1331	struct cpuset *cs = cgroup_cs(cont);
1332
1333	if (cpumask_empty(cs->cpus_allowed) \|\| nodes_empty(cs->mems_allowed))
1334	return -ENOSPC;
1335
1336	/*
1337	* Kthreads bound to specific cpus cannot be moved to a new cpuset; we
1338	* cannot change their cpu affinity and isolating such threads by their
1339	* set of allowed nodes is unnecessary. Thus, cpusets are not
1340	* applicable for such threads. This prevents checking for success of
1341	* set_cpus_allowed_ptr() on all attached tasks before cpus_allowed may
1342	* be changed.
1343	*/
1344	if (tsk->flags & PF_THREAD_BOUND)
1345	return -EINVAL;
1346
1347	ret = security_task_setscheduler(tsk, 0, NULL);
1348	if (ret)
1349	return ret;
1350	if (threadgroup) {
1351	struct task_struct *c;
1352
1353	rcu_read_lock();
1354	list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
1355	ret = security_task_setscheduler(c, 0, NULL);
1356	if (ret) {
1357	rcu_read_unlock();
1358	return ret;
1359	}
1360	}
1361	rcu_read_unlock();
1362	}
1363	return 0;
1364	}
1365
1366	static void cpuset_attach_task(struct task_struct tsk, nodemask_t to,
1367	struct cpuset *cs)
1368	{
1369	int err;
1370	/*
1371	* can_attach beforehand should guarantee that this doesn't fail.
1372	* TODO: have a better way to handle failure here
1373	*/
1374	err = set_cpus_allowed_ptr(tsk, cpus_attach);
1375	WARN_ON_ONCE(err);
1376
1377	task_lock(tsk);
1378	cpuset_change_task_nodemask(tsk, to);
1379	task_unlock(tsk);
1380	cpuset_update_task_spread_flag(cs, tsk);
1381
1382	}
1383
1384	static void cpuset_attach(struct cgroup_subsys ss, struct cgroup cont,
1385	struct cgroup oldcont, struct task_struct tsk,
1386	bool threadgroup)
1387	{
1388	nodemask_t from, to;
1389	struct mm_struct *mm;
1390	struct cpuset *cs = cgroup_cs(cont);
1391	struct cpuset *oldcs = cgroup_cs(oldcont);
1392
1393	if (cs == &top_cpuset) {
1394	cpumask_copy(cpus_attach, cpu_possible_mask);
1395	to = node_possible_map;
1396	} else {
1397	guarantee_online_cpus(cs, cpus_attach);
1398	guarantee_online_mems(cs, &to);
1399	}
1400
1401	/* do per-task migration stuff possibly for each in the threadgroup */
1402	cpuset_attach_task(tsk, &to, cs);
1403	if (threadgroup) {
1404	struct task_struct *c;
1405	rcu_read_lock();
1406	list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
1407	cpuset_attach_task(c, &to, cs);
1408	}
1409	rcu_read_unlock();
1410	}
1411
1412	/* change mm; only needs to be done once even if threadgroup */
1413	from = oldcs->mems_allowed;
1414	to = cs->mems_allowed;
1415	mm = get_task_mm(tsk);
1416	if (mm) {
1417	mpol_rebind_mm(mm, &to);
1418	if (is_memory_migrate(cs))
1419	cpuset_migrate_mm(mm, &from, &to);
1420	mmput(mm);
1421	}
1422	}
1423
1424	/* The various types of files and directories in a cpuset file system */
1425
1426	typedef enum {
1427	FILE_MEMORY_MIGRATE,
1428	FILE_CPULIST,
1429	FILE_MEMLIST,
1430	FILE_CPU_EXCLUSIVE,
1431	FILE_MEM_EXCLUSIVE,
1432	FILE_MEM_HARDWALL,
1433	FILE_SCHED_LOAD_BALANCE,
1434	FILE_SCHED_RELAX_DOMAIN_LEVEL,
1435	FILE_MEMORY_PRESSURE_ENABLED,
1436	FILE_MEMORY_PRESSURE,
1437	FILE_SPREAD_PAGE,
1438	FILE_SPREAD_SLAB,
1439	} cpuset_filetype_t;
1440
1441	static int cpuset_write_u64(struct cgroup cgrp, struct cftype cft, u64 val)
1442	{
1443	int retval = 0;
1444	struct cpuset *cs = cgroup_cs(cgrp);
1445	cpuset_filetype_t type = cft->private;
1446
1447	if (!cgroup_lock_live_group(cgrp))
1448	return -ENODEV;
1449
1450	switch (type) {
1451	case FILE_CPU_EXCLUSIVE:
1452	retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
1453	break;
1454	case FILE_MEM_EXCLUSIVE:
1455	retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
1456	break;
1457	case FILE_MEM_HARDWALL:
1458	retval = update_flag(CS_MEM_HARDWALL, cs, val);
1459	break;
1460	case FILE_SCHED_LOAD_BALANCE:
1461	retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
1462	break;
1463	case FILE_MEMORY_MIGRATE:
1464	retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
1465	break;
1466	case FILE_MEMORY_PRESSURE_ENABLED:
1467	cpuset_memory_pressure_enabled = !!val;
1468	break;
1469	case FILE_MEMORY_PRESSURE:
1470	retval = -EACCES;
1471	break;
1472	case FILE_SPREAD_PAGE:
1473	retval = update_flag(CS_SPREAD_PAGE, cs, val);
1474	break;
1475	case FILE_SPREAD_SLAB:
1476	retval = update_flag(CS_SPREAD_SLAB, cs, val);
1477	break;
1478	default:
1479	retval = -EINVAL;
1480	break;
1481	}
1482	cgroup_unlock();
1483	return retval;
1484	}
1485
1486	static int cpuset_write_s64(struct cgroup cgrp, struct cftype cft, s64 val)
1487	{
1488	int retval = 0;
1489	struct cpuset *cs = cgroup_cs(cgrp);
1490	cpuset_filetype_t type = cft->private;
1491
1492	if (!cgroup_lock_live_group(cgrp))
1493	return -ENODEV;
1494
1495	switch (type) {
1496	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1497	retval = update_relax_domain_level(cs, val);
1498	break;
1499	default:
1500	retval = -EINVAL;
1501	break;
1502	}
1503	cgroup_unlock();
1504	return retval;
1505	}
1506
1507	/*
1508	* Common handling for a write to a "cpus" or "mems" file.
1509	*/
1510	static int cpuset_write_resmask(struct cgroup cgrp, struct cftype cft,
1511	const char *buf)
1512	{
1513	int retval = 0;
1514	struct cpuset *cs = cgroup_cs(cgrp);
1515	struct cpuset *trialcs;
1516
1517	if (!cgroup_lock_live_group(cgrp))
1518	return -ENODEV;
1519
1520	trialcs = alloc_trial_cpuset(cs);
1521	if (!trialcs)
1522	return -ENOMEM;
1523
1524	switch (cft->private) {
1525	case FILE_CPULIST:
1526	retval = update_cpumask(cs, trialcs, buf);
1527	break;
1528	case FILE_MEMLIST:
1529	retval = update_nodemask(cs, trialcs, buf);
1530	break;
1531	default:
1532	retval = -EINVAL;
1533	break;
1534	}
1535
1536	free_trial_cpuset(trialcs);
1537	cgroup_unlock();
1538	return retval;
1539	}
1540
1541	/*
1542	* These ascii lists should be read in a single call, by using a user
1543	* buffer large enough to hold the entire map. If read in smaller
1544	* chunks, there is no guarantee of atomicity. Since the display format
1545	* used, list of ranges of sequential numbers, is variable length,
1546	* and since these maps can change value dynamically, one could read
1547	* gibberish by doing partial reads while a list was changing.
1548	* A single large read to a buffer that crosses a page boundary is
1549	* ok, because the result being copied to user land is not recomputed
1550	* across a page fault.
1551	*/
1552
1553	static int cpuset_sprintf_cpulist(char page, struct cpuset cs)
1554	{
1555	int ret;
1556
1557	mutex_lock(&callback_mutex);
1558	ret = cpulist_scnprintf(page, PAGE_SIZE, cs->cpus_allowed);
1559	mutex_unlock(&callback_mutex);
1560
1561	return ret;
1562	}
1563
1564	static int cpuset_sprintf_memlist(char page, struct cpuset cs)
1565	{
1566	nodemask_t mask;
1567
1568	mutex_lock(&callback_mutex);
1569	mask = cs->mems_allowed;
1570	mutex_unlock(&callback_mutex);
1571
1572	return nodelist_scnprintf(page, PAGE_SIZE, mask);
1573	}
1574
1575	static ssize_t cpuset_common_file_read(struct cgroup *cont,
1576	struct cftype *cft,
1577	struct file *file,
1578	char __user *buf,
1579	size_t nbytes, loff_t *ppos)
1580	{
1581	struct cpuset *cs = cgroup_cs(cont);
1582	cpuset_filetype_t type = cft->private;
1583	char *page;
1584	ssize_t retval = 0;
1585	char *s;
1586
1587	if (!(page = (char *)__get_free_page(GFP_TEMPORARY)))
1588	return -ENOMEM;
1589
1590	s = page;
1591
1592	switch (type) {
1593	case FILE_CPULIST:
1594	s += cpuset_sprintf_cpulist(s, cs);
1595	break;
1596	case FILE_MEMLIST:
1597	s += cpuset_sprintf_memlist(s, cs);
1598	break;
1599	default:
1600	retval = -EINVAL;
1601	goto out;
1602	}
1603	*s++ = '\n';
1604
1605	retval = simple_read_from_buffer(buf, nbytes, ppos, page, s - page);
1606	out:
1607	free_page((unsigned long)page);
1608	return retval;
1609	}
1610
1611	static u64 cpuset_read_u64(struct cgroup cont, struct cftype cft)
1612	{
1613	struct cpuset *cs = cgroup_cs(cont);
1614	cpuset_filetype_t type = cft->private;
1615	switch (type) {
1616	case FILE_CPU_EXCLUSIVE:
1617	return is_cpu_exclusive(cs);
1618	case FILE_MEM_EXCLUSIVE:
1619	return is_mem_exclusive(cs);
1620	case FILE_MEM_HARDWALL:
1621	return is_mem_hardwall(cs);
1622	case FILE_SCHED_LOAD_BALANCE:
1623	return is_sched_load_balance(cs);
1624	case FILE_MEMORY_MIGRATE:
1625	return is_memory_migrate(cs);
1626	case FILE_MEMORY_PRESSURE_ENABLED:
1627	return cpuset_memory_pressure_enabled;
1628	case FILE_MEMORY_PRESSURE:
1629	return fmeter_getrate(&cs->fmeter);
1630	case FILE_SPREAD_PAGE:
1631	return is_spread_page(cs);
1632	case FILE_SPREAD_SLAB:
1633	return is_spread_slab(cs);
1634	default:
1635	BUG();
1636	}
1637
1638	/* Unreachable but makes gcc happy */
1639	return 0;
1640	}
1641
1642	static s64 cpuset_read_s64(struct cgroup cont, struct cftype cft)
1643	{
1644	struct cpuset *cs = cgroup_cs(cont);
1645	cpuset_filetype_t type = cft->private;
1646	switch (type) {
1647	case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1648	return cs->relax_domain_level;
1649	default:
1650	BUG();
1651	}
1652
1653	/* Unrechable but makes gcc happy */
1654	return 0;
1655	}
1656
1657
1658	/*
1659	* for the common functions, 'private' gives the type of file
1660	*/
1661
1662	static struct cftype files[] = {
1663	{
1664	.name = "cpus",
1665	.read = cpuset_common_file_read,
1666	.write_string = cpuset_write_resmask,
1667	.max_write_len = (100U + 6 * NR_CPUS),
1668	.private = FILE_CPULIST,
1669	},
1670
1671	{
1672	.name = "mems",
1673	.read = cpuset_common_file_read,
1674	.write_string = cpuset_write_resmask,
1675	.max_write_len = (100U + 6 * MAX_NUMNODES),
1676	.private = FILE_MEMLIST,
1677	},
1678
1679	{
1680	.name = "cpu_exclusive",
1681	.read_u64 = cpuset_read_u64,
1682	.write_u64 = cpuset_write_u64,
1683	.private = FILE_CPU_EXCLUSIVE,
1684	},
1685
1686	{
1687	.name = "mem_exclusive",
1688	.read_u64 = cpuset_read_u64,
1689	.write_u64 = cpuset_write_u64,
1690	.private = FILE_MEM_EXCLUSIVE,
1691	},
1692
1693	{
1694	.name = "mem_hardwall",
1695	.read_u64 = cpuset_read_u64,
1696	.write_u64 = cpuset_write_u64,
1697	.private = FILE_MEM_HARDWALL,
1698	},
1699
1700	{
1701	.name = "sched_load_balance",
1702	.read_u64 = cpuset_read_u64,
1703	.write_u64 = cpuset_write_u64,
1704	.private = FILE_SCHED_LOAD_BALANCE,
1705	},
1706
1707	{
1708	.name = "sched_relax_domain_level",
1709	.read_s64 = cpuset_read_s64,
1710	.write_s64 = cpuset_write_s64,
1711	.private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
1712	},
1713
1714	{
1715	.name = "memory_migrate",
1716	.read_u64 = cpuset_read_u64,
1717	.write_u64 = cpuset_write_u64,
1718	.private = FILE_MEMORY_MIGRATE,
1719	},
1720
1721	{
1722	.name = "memory_pressure",
1723	.read_u64 = cpuset_read_u64,
1724	.write_u64 = cpuset_write_u64,
1725	.private = FILE_MEMORY_PRESSURE,
1726	.mode = S_IRUGO,
1727	},
1728
1729	{
1730	.name = "memory_spread_page",
1731	.read_u64 = cpuset_read_u64,
1732	.write_u64 = cpuset_write_u64,
1733	.private = FILE_SPREAD_PAGE,
1734	},
1735
1736	{
1737	.name = "memory_spread_slab",
1738	.read_u64 = cpuset_read_u64,
1739	.write_u64 = cpuset_write_u64,
1740	.private = FILE_SPREAD_SLAB,
1741	},
1742	};
1743
1744	static struct cftype cft_memory_pressure_enabled = {
1745	.name = "memory_pressure_enabled",
1746	.read_u64 = cpuset_read_u64,
1747	.write_u64 = cpuset_write_u64,
1748	.private = FILE_MEMORY_PRESSURE_ENABLED,
1749	};
1750
1751	static int cpuset_populate(struct cgroup_subsys ss, struct cgroup cont)
1752	{
1753	int err;
1754
1755	err = cgroup_add_files(cont, ss, files, ARRAY_SIZE(files));
1756	if (err)
1757	return err;
1758	/* memory_pressure_enabled is in root cpuset only */
1759	if (!cont->parent)
1760	err = cgroup_add_file(cont, ss,
1761	&cft_memory_pressure_enabled);
1762	return err;
1763	}
1764
1765	/*
1766	* post_clone() is called at the end of cgroup_clone().
1767	* 'cgroup' was just created automatically as a result of
1768	* a cgroup_clone(), and the current task is about to
1769	* be moved into 'cgroup'.
1770	*
1771	* Currently we refuse to set up the cgroup - thereby
1772	* refusing the task to be entered, and as a result refusing
1773	* the sys_unshare() or clone() which initiated it - if any
1774	* sibling cpusets have exclusive cpus or mem.
1775	*
1776	* If this becomes a problem for some users who wish to
1777	* allow that scenario, then cpuset_post_clone() could be
1778	* changed to grant parent->cpus_allowed-sibling_cpus_exclusive
1779	* (and likewise for mems) to the new cgroup. Called with cgroup_mutex
1780	* held.
1781	*/
1782	static void cpuset_post_clone(struct cgroup_subsys *ss,
1783	struct cgroup *cgroup)
1784	{
1785	struct cgroup parent, child;
1786	struct cpuset cs, parent_cs;
1787
1788	parent = cgroup->parent;
1789	list_for_each_entry(child, &parent->children, sibling) {
1790	cs = cgroup_cs(child);
1791	if (is_mem_exclusive(cs) \|\| is_cpu_exclusive(cs))
1792	return;
1793	}
1794	cs = cgroup_cs(cgroup);
1795	parent_cs = cgroup_cs(parent);
1796
1797	cs->mems_allowed = parent_cs->mems_allowed;
1798	cpumask_copy(cs->cpus_allowed, parent_cs->cpus_allowed);
1799	return;
1800	}
1801
1802	/*
1803	* cpuset_create - create a cpuset
1804	* ss: cpuset cgroup subsystem
1805	* cont: control group that the new cpuset will be part of
1806	*/
1807
1808	static struct cgroup_subsys_state *cpuset_create(
1809	struct cgroup_subsys *ss,
1810	struct cgroup *cont)
1811	{
1812	struct cpuset *cs;
1813	struct cpuset *parent;
1814
1815	if (!cont->parent) {
1816	return &top_cpuset.css;
1817	}
1818	parent = cgroup_cs(cont->parent);
1819	cs = kmalloc(sizeof(*cs), GFP_KERNEL);
1820	if (!cs)
1821	return ERR_PTR(-ENOMEM);
1822	if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) {
1823	kfree(cs);
1824	return ERR_PTR(-ENOMEM);
1825	}
1826
1827	cs->flags = 0;
1828	if (is_spread_page(parent))
1829	set_bit(CS_SPREAD_PAGE, &cs->flags);
1830	if (is_spread_slab(parent))
1831	set_bit(CS_SPREAD_SLAB, &cs->flags);
1832	set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
1833	cpumask_clear(cs->cpus_allowed);
1834	nodes_clear(cs->mems_allowed);
1835	fmeter_init(&cs->fmeter);
1836	cs->relax_domain_level = -1;
1837
1838	cs->parent = parent;
1839	number_of_cpusets++;
1840	return &cs->css ;
1841	}
1842
1843	/*
1844	* If the cpuset being removed has its flag 'sched_load_balance'
1845	* enabled, then simulate turning sched_load_balance off, which
1846	* will call async_rebuild_sched_domains().
1847	*/
1848
1849	static void cpuset_destroy(struct cgroup_subsys ss, struct cgroup cont)
1850	{
1851	struct cpuset *cs = cgroup_cs(cont);
1852
1853	if (is_sched_load_balance(cs))
1854	update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
1855
1856	number_of_cpusets--;
1857	free_cpumask_var(cs->cpus_allowed);
1858	kfree(cs);
1859	}
1860
1861	struct cgroup_subsys cpuset_subsys = {
1862	.name = "cpuset",
1863	.create = cpuset_create,
1864	.destroy = cpuset_destroy,
1865	.can_attach = cpuset_can_attach,
1866	.attach = cpuset_attach,
1867	.populate = cpuset_populate,
1868	.post_clone = cpuset_post_clone,
1869	.subsys_id = cpuset_subsys_id,
1870	.early_init = 1,
1871	};
1872
1873	/**
1874	* cpuset_init - initialize cpusets at system boot
1875	*
1876	* Description: Initialize top_cpuset and the cpuset internal file system,
1877	**/
1878
1879	int __init cpuset_init(void)
1880	{
1881	int err = 0;
1882
1883	if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL))
1884	BUG();
1885
1886	cpumask_setall(top_cpuset.cpus_allowed);
1887	nodes_setall(top_cpuset.mems_allowed);
1888
1889	fmeter_init(&top_cpuset.fmeter);
1890	set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
1891	top_cpuset.relax_domain_level = -1;
1892
1893	err = register_filesystem(&cpuset_fs_type);
1894	if (err < 0)
1895	return err;
1896
1897	if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
1898	BUG();
1899
1900	number_of_cpusets = 1;
1901	return 0;
1902	}
1903
1904	/**
1905	* cpuset_do_move_task - move a given task to another cpuset
1906	* @tsk: pointer to task_struct the task to move
1907	* @scan: struct cgroup_scanner contained in its struct cpuset_hotplug_scanner
1908	*
1909	* Called by cgroup_scan_tasks() for each task in a cgroup.
1910	* Return nonzero to stop the walk through the tasks.
1911	*/
1912	static void cpuset_do_move_task(struct task_struct *tsk,
1913	struct cgroup_scanner *scan)
1914	{
1915	struct cgroup *new_cgroup = scan->data;
1916
1917	cgroup_attach_task(new_cgroup, tsk);
1918	}
1919
1920	/**
1921	* move_member_tasks_to_cpuset - move tasks from one cpuset to another
1922	* @from: cpuset in which the tasks currently reside
1923	* @to: cpuset to which the tasks will be moved
1924	*
1925	* Called with cgroup_mutex held
1926	* callback_mutex must not be held, as cpuset_attach() will take it.
1927	*
1928	* The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
1929	* calling callback functions for each.
1930	*/
1931	static void move_member_tasks_to_cpuset(struct cpuset from, struct cpuset to)
1932	{
1933	struct cgroup_scanner scan;
1934
1935	scan.cg = from->css.cgroup;
1936	scan.test_task = NULL; /* select all tasks in cgroup */
1937	scan.process_task = cpuset_do_move_task;
1938	scan.heap = NULL;
1939	scan.data = to->css.cgroup;
1940
1941	if (cgroup_scan_tasks(&scan))
1942	printk(KERN_ERR "move_member_tasks_to_cpuset: "
1943	"cgroup_scan_tasks failed\n");
1944	}
1945
1946	/*
1947	* If CPU and/or memory hotplug handlers, below, unplug any CPUs
1948	* or memory nodes, we need to walk over the cpuset hierarchy,
1949	* removing that CPU or node from all cpusets. If this removes the
1950	* last CPU or node from a cpuset, then move the tasks in the empty
1951	* cpuset to its next-highest non-empty parent.
1952	*
1953	* Called with cgroup_mutex held
1954	* callback_mutex must not be held, as cpuset_attach() will take it.
1955	*/
1956	static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
1957	{
1958	struct cpuset *parent;
1959
1960	/*
1961	* The cgroup's css_sets list is in use if there are tasks
1962	* in the cpuset; the list is empty if there are none;
1963	* the cs->css.refcnt seems always 0.
1964	*/
1965	if (list_empty(&cs->css.cgroup->css_sets))
1966	return;
1967
1968	/*
1969	* Find its next-highest non-empty parent, (top cpuset
1970	* has online cpus, so can't be empty).
1971	*/
1972	parent = cs->parent;
1973	while (cpumask_empty(parent->cpus_allowed) \|\|
1974	nodes_empty(parent->mems_allowed))
1975	parent = parent->parent;
1976
1977	move_member_tasks_to_cpuset(cs, parent);
1978	}
1979
1980	/*
1981	* Walk the specified cpuset subtree and look for empty cpusets.
1982	* The tasks of such cpuset must be moved to a parent cpuset.
1983	*
1984	* Called with cgroup_mutex held. We take callback_mutex to modify
1985	* cpus_allowed and mems_allowed.
1986	*
1987	* This walk processes the tree from top to bottom, completing one layer
1988	* before dropping down to the next. It always processes a node before
1989	* any of its children.
1990	*
1991	* For now, since we lack memory hot unplug, we'll never see a cpuset
1992	* that has tasks along with an empty 'mems'. But if we did see such
1993	* a cpuset, we'd handle it just like we do if its 'cpus' was empty.
1994	*/
1995	static void scan_for_empty_cpusets(struct cpuset *root)
1996	{
1997	LIST_HEAD(queue);
1998	struct cpuset cp; / scans cpusets being updated */
1999	struct cpuset child; / scans child cpusets of cp */
2000	struct cgroup *cont;
2001	nodemask_t oldmems;
2002
2003	list_add_tail((struct list_head *)&root->stack_list, &queue);
2004
2005	while (!list_empty(&queue)) {
2006	cp = list_first_entry(&queue, struct cpuset, stack_list);
2007	list_del(queue.next);
2008	list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
2009	child = cgroup_cs(cont);
2010	list_add_tail(&child->stack_list, &queue);
2011	}
2012
2013	/* Continue past cpusets with all cpus, mems online */
2014	if (cpumask_subset(cp->cpus_allowed, cpu_active_mask) &&
2015	nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY]))
2016	continue;
2017
2018	oldmems = cp->mems_allowed;
2019
2020	/* Remove offline cpus and mems from this cpuset. */
2021	mutex_lock(&callback_mutex);
2022	cpumask_and(cp->cpus_allowed, cp->cpus_allowed,
2023	cpu_active_mask);
2024	nodes_and(cp->mems_allowed, cp->mems_allowed,
2025	node_states[N_HIGH_MEMORY]);
2026	mutex_unlock(&callback_mutex);
2027
2028	/* Move tasks from the empty cpuset to a parent */
2029	if (cpumask_empty(cp->cpus_allowed) \|\|
2030	nodes_empty(cp->mems_allowed))
2031	remove_tasks_in_empty_cpuset(cp);
2032	else {
2033	update_tasks_cpumask(cp, NULL);
2034	update_tasks_nodemask(cp, &oldmems, NULL);
2035	}
2036	}
2037	}
2038
2039	/*
2040	* The top_cpuset tracks what CPUs and Memory Nodes are online,
2041	* period. This is necessary in order to make cpusets transparent
2042	* (of no affect) on systems that are actively using CPU hotplug
2043	* but making no active use of cpusets.
2044	*
2045	* This routine ensures that top_cpuset.cpus_allowed tracks
2046	* cpu_online_map on each CPU hotplug (cpuhp) event.
2047	*
2048	* Called within get_online_cpus(). Needs to call cgroup_lock()
2049	* before calling generate_sched_domains().
2050	*/
2051	static int cpuset_track_online_cpus(struct notifier_block *unused_nb,
2052	unsigned long phase, void *unused_cpu)
2053	{
2054	struct sched_domain_attr *attr;
2055	struct cpumask *doms;
2056	int ndoms;
2057
2058	switch (phase) {
2059	case CPU_ONLINE:
2060	case CPU_ONLINE_FROZEN:
2061	case CPU_DOWN_PREPARE:
2062	case CPU_DOWN_PREPARE_FROZEN:
2063	case CPU_DOWN_FAILED:
2064	case CPU_DOWN_FAILED_FROZEN:
2065	break;
2066
2067	default:
2068	return NOTIFY_DONE;
2069	}
2070
2071	cgroup_lock();
2072	mutex_lock(&callback_mutex);
2073	cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
2074	mutex_unlock(&callback_mutex);
2075	scan_for_empty_cpusets(&top_cpuset);
2076	ndoms = generate_sched_domains(&doms, &attr);
2077	cgroup_unlock();
2078
2079	/* Have scheduler rebuild the domains */
2080	partition_sched_domains(ndoms, doms, attr);
2081
2082	return NOTIFY_OK;
2083	}
2084
2085	#ifdef CONFIG_MEMORY_HOTPLUG
2086	/*
2087	* Keep top_cpuset.mems_allowed tracking node_states[N_HIGH_MEMORY].
2088	* Call this routine anytime after node_states[N_HIGH_MEMORY] changes.
2089	* See also the previous routine cpuset_track_online_cpus().
2090	*/
2091	static int cpuset_track_online_nodes(struct notifier_block *self,
2092	unsigned long action, void *arg)
2093	{
2094	cgroup_lock();
2095	switch (action) {
2096	case MEM_ONLINE:
2097	case MEM_OFFLINE:
2098	mutex_lock(&callback_mutex);
2099	top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
2100	mutex_unlock(&callback_mutex);
2101	if (action == MEM_OFFLINE)
2102	scan_for_empty_cpusets(&top_cpuset);
2103	break;
2104	default:
2105	break;
2106	}
2107	cgroup_unlock();
2108	return NOTIFY_OK;
2109	}
2110	#endif
2111
2112	/**
2113	* cpuset_init_smp - initialize cpus_allowed
2114	*
2115	* Description: Finish top cpuset after cpu, node maps are initialized
2116	**/
2117
2118	void __init cpuset_init_smp(void)
2119	{
2120	cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
2121	top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY];
2122
2123	hotcpu_notifier(cpuset_track_online_cpus, 0);
2124	hotplug_memory_notifier(cpuset_track_online_nodes, 10);
2125
2126	cpuset_wq = create_singlethread_workqueue("cpuset");
2127	BUG_ON(!cpuset_wq);
2128	}
2129
2130	/**
2131	* cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
2132	* @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
2133	* @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
2134	*
2135	* Description: Returns the cpumask_var_t cpus_allowed of the cpuset
2136	* attached to the specified @tsk. Guaranteed to return some non-empty
2137	* subset of cpu_online_map, even if this means going outside the
2138	* tasks cpuset.
2139	**/
2140
2141	void cpuset_cpus_allowed(struct task_struct tsk, struct cpumask pmask)
2142	{
2143	mutex_lock(&callback_mutex);
2144	cpuset_cpus_allowed_locked(tsk, pmask);
2145	mutex_unlock(&callback_mutex);
2146	}
2147
2148	/**
2149	* cpuset_cpus_allowed_locked - return cpus_allowed mask from a tasks cpuset.
2150	* Must be called with callback_mutex held.
2151	**/
2152	void cpuset_cpus_allowed_locked(struct task_struct tsk, struct cpumask pmask)
2153	{
2154	task_lock(tsk);
2155	guarantee_online_cpus(task_cs(tsk), pmask);
2156	task_unlock(tsk);
2157	}
2158
2159	void cpuset_init_current_mems_allowed(void)
2160	{
2161	nodes_setall(current->mems_allowed);
2162	}
2163
2164	/**
2165	* cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
2166	* @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
2167	*
2168	* Description: Returns the nodemask_t mems_allowed of the cpuset
2169	* attached to the specified @tsk. Guaranteed to return some non-empty
2170	* subset of node_states[N_HIGH_MEMORY], even if this means going outside the
2171	* tasks cpuset.
2172	**/
2173
2174	nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
2175	{
2176	nodemask_t mask;
2177
2178	mutex_lock(&callback_mutex);
2179	task_lock(tsk);
2180	guarantee_online_mems(task_cs(tsk), &mask);
2181	task_unlock(tsk);
2182	mutex_unlock(&callback_mutex);
2183
2184	return mask;
2185	}
2186
2187	/**
2188	* cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
2189	* @nodemask: the nodemask to be checked
2190	*
2191	* Are any of the nodes in the nodemask allowed in current->mems_allowed?
2192	*/
2193	int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
2194	{
2195	return nodes_intersects(*nodemask, current->mems_allowed);
2196	}
2197
2198	/*
2199	* nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
2200	* mem_hardwall ancestor to the specified cpuset. Call holding
2201	* callback_mutex. If no ancestor is mem_exclusive or mem_hardwall
2202	* (an unusual configuration), then returns the root cpuset.
2203	*/
2204	static const struct cpuset nearest_hardwall_ancestor(const struct cpuset cs)
2205	{
2206	while (!(is_mem_exclusive(cs) \|\| is_mem_hardwall(cs)) && cs->parent)
2207	cs = cs->parent;
2208	return cs;
2209	}
2210
2211	/**
2212	* cpuset_node_allowed_softwall - Can we allocate on a memory node?
2213	* @node: is this an allowed node?
2214	* @gfp_mask: memory allocation flags
2215	*
2216	* If we're in interrupt, yes, we can always allocate. If __GFP_THISNODE is
2217	* set, yes, we can always allocate. If node is in our task's mems_allowed,
2218	* yes. If it's not a __GFP_HARDWALL request and this node is in the nearest
2219	* hardwalled cpuset ancestor to this task's cpuset, yes. If the task has been
2220	* OOM killed and has access to memory reserves as specified by the TIF_MEMDIE
2221	* flag, yes.
2222	* Otherwise, no.
2223	*
2224	* If __GFP_HARDWALL is set, cpuset_node_allowed_softwall() reduces to
2225	* cpuset_node_allowed_hardwall(). Otherwise, cpuset_node_allowed_softwall()
2226	* might sleep, and might allow a node from an enclosing cpuset.
2227	*
2228	* cpuset_node_allowed_hardwall() only handles the simpler case of hardwall
2229	* cpusets, and never sleeps.
2230	*
2231	* The __GFP_THISNODE placement logic is really handled elsewhere,
2232	* by forcibly using a zonelist starting at a specified node, and by
2233	* (in get_page_from_freelist()) refusing to consider the zones for
2234	* any node on the zonelist except the first. By the time any such
2235	* calls get to this routine, we should just shut up and say 'yes'.
2236	*
2237	* GFP_USER allocations are marked with the __GFP_HARDWALL bit,
2238	* and do not allow allocations outside the current tasks cpuset
2239	* unless the task has been OOM killed as is marked TIF_MEMDIE.
2240	* GFP_KERNEL allocations are not so marked, so can escape to the
2241	* nearest enclosing hardwalled ancestor cpuset.
2242	*
2243	* Scanning up parent cpusets requires callback_mutex. The
2244	* __alloc_pages() routine only calls here with __GFP_HARDWALL bit
2245	* _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
2246	* current tasks mems_allowed came up empty on the first pass over
2247	* the zonelist. So only GFP_KERNEL allocations, if all nodes in the
2248	* cpuset are short of memory, might require taking the callback_mutex
2249	* mutex.
2250	*
2251	* The first call here from mm/page_alloc:get_page_from_freelist()
2252	* has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
2253	* so no allocation on a node outside the cpuset is allowed (unless
2254	* in interrupt, of course).
2255	*
2256	* The second pass through get_page_from_freelist() doesn't even call
2257	* here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
2258	* variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
2259	* in alloc_flags. That logic and the checks below have the combined
2260	* affect that:
2261	* in_interrupt - any node ok (current task context irrelevant)
2262	* GFP_ATOMIC - any node ok
2263	* TIF_MEMDIE - any node ok
2264	* GFP_KERNEL - any node in enclosing hardwalled cpuset ok
2265	* GFP_USER - only nodes in current tasks mems allowed ok.
2266	*
2267	* Rule:
2268	* Don't call cpuset_node_allowed_softwall if you can't sleep, unless you
2269	* pass in the __GFP_HARDWALL flag set in gfp_flag, which disables
2270	* the code that might scan up ancestor cpusets and sleep.
2271	*/
2272	int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
2273	{
2274	const struct cpuset cs; / current cpuset ancestors */
2275	int allowed; /* is allocation in zone z allowed? */
2276
2277	if (in_interrupt() \|\| (gfp_mask & __GFP_THISNODE))
2278	return 1;
2279	might_sleep_if(!(gfp_mask & __GFP_HARDWALL));
2280	if (node_isset(node, current->mems_allowed))
2281	return 1;
2282	/*
2283	* Allow tasks that have access to memory reserves because they have
2284	* been OOM killed to get memory anywhere.
2285	*/
2286	if (unlikely(test_thread_flag(TIF_MEMDIE)))
2287	return 1;
2288	if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
2289	return 0;
2290
2291	if (current->flags & PF_EXITING) /* Let dying task have memory */
2292	return 1;
2293
2294	/* Not hardwall and node outside mems_allowed: scan up cpusets */
2295	mutex_lock(&callback_mutex);
2296
2297	task_lock(current);
2298	cs = nearest_hardwall_ancestor(task_cs(current));
2299	task_unlock(current);
2300
2301	allowed = node_isset(node, cs->mems_allowed);
2302	mutex_unlock(&callback_mutex);
2303	return allowed;
2304	}
2305
2306	/*
2307	* cpuset_node_allowed_hardwall - Can we allocate on a memory node?
2308	* @node: is this an allowed node?
2309	* @gfp_mask: memory allocation flags
2310	*
2311	* If we're in interrupt, yes, we can always allocate. If __GFP_THISNODE is
2312	* set, yes, we can always allocate. If node is in our task's mems_allowed,
2313	* yes. If the task has been OOM killed and has access to memory reserves as
2314	* specified by the TIF_MEMDIE flag, yes.
2315	* Otherwise, no.
2316	*
2317	* The __GFP_THISNODE placement logic is really handled elsewhere,
2318	* by forcibly using a zonelist starting at a specified node, and by
2319	* (in get_page_from_freelist()) refusing to consider the zones for
2320	* any node on the zonelist except the first. By the time any such
2321	* calls get to this routine, we should just shut up and say 'yes'.
2322	*
2323	* Unlike the cpuset_node_allowed_softwall() variant, above,
2324	* this variant requires that the node be in the current task's
2325	* mems_allowed or that we're in interrupt. It does not scan up the
2326	* cpuset hierarchy for the nearest enclosing mem_exclusive cpuset.
2327	* It never sleeps.
2328	*/
2329	int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
2330	{
2331	if (in_interrupt() \|\| (gfp_mask & __GFP_THISNODE))
2332	return 1;
2333	if (node_isset(node, current->mems_allowed))
2334	return 1;
2335	/*
2336	* Allow tasks that have access to memory reserves because they have
2337	* been OOM killed to get memory anywhere.
2338	*/
2339	if (unlikely(test_thread_flag(TIF_MEMDIE)))
2340	return 1;
2341	return 0;
2342	}
2343
2344	/**
2345	* cpuset_lock - lock out any changes to cpuset structures
2346	*
2347	* The out of memory (oom) code needs to mutex_lock cpusets
2348	* from being changed while it scans the tasklist looking for a
2349	* task in an overlapping cpuset. Expose callback_mutex via this
2350	* cpuset_lock() routine, so the oom code can lock it, before
2351	* locking the task list. The tasklist_lock is a spinlock, so
2352	* must be taken inside callback_mutex.
2353	*/
2354
2355	void cpuset_lock(void)
2356	{
2357	mutex_lock(&callback_mutex);
2358	}
2359
2360	/**
2361	* cpuset_unlock - release lock on cpuset changes
2362	*
2363	* Undo the lock taken in a previous cpuset_lock() call.
2364	*/
2365
2366	void cpuset_unlock(void)
2367	{
2368	mutex_unlock(&callback_mutex);
2369	}
2370
2371	/**
2372	* cpuset_mem_spread_node() - On which node to begin search for a page
2373	*
2374	* If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
2375	* tasks in a cpuset with is_spread_page or is_spread_slab set),
2376	* and if the memory allocation used cpuset_mem_spread_node()
2377	* to determine on which node to start looking, as it will for
2378	* certain page cache or slab cache pages such as used for file
2379	* system buffers and inode caches, then instead of starting on the
2380	* local node to look for a free page, rather spread the starting
2381	* node around the tasks mems_allowed nodes.
2382	*
2383	* We don't have to worry about the returned node being offline
2384	* because "it can't happen", and even if it did, it would be ok.
2385	*
2386	* The routines calling guarantee_online_mems() are careful to
2387	* only set nodes in task->mems_allowed that are online. So it
2388	* should not be possible for the following code to return an
2389	* offline node. But if it did, that would be ok, as this routine
2390	* is not returning the node where the allocation must be, only
2391	* the node where the search should start. The zonelist passed to
2392	* __alloc_pages() will include all nodes. If the slab allocator
2393	* is passed an offline node, it will fall back to the local node.
2394	* See kmem_cache_alloc_node().
2395	*/
2396
2397	int cpuset_mem_spread_node(void)
2398	{
2399	int node;
2400
2401	node = next_node(current->cpuset_mem_spread_rotor, current->mems_allowed);
2402	if (node == MAX_NUMNODES)
2403	node = first_node(current->mems_allowed);
2404	current->cpuset_mem_spread_rotor = node;
2405	return node;
2406	}
2407	EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
2408
2409	/**
2410	* cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
2411	* @tsk1: pointer to task_struct of some task.
2412	* @tsk2: pointer to task_struct of some other task.
2413	*
2414	* Description: Return true if @tsk1's mems_allowed intersects the
2415	* mems_allowed of @tsk2. Used by the OOM killer to determine if
2416	* one of the task's memory usage might impact the memory available
2417	* to the other.
2418	**/
2419
2420	int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
2421	const struct task_struct *tsk2)
2422	{
2423	return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
2424	}
2425
2426	/**
2427	* cpuset_print_task_mems_allowed - prints task's cpuset and mems_allowed
2428	* @task: pointer to task_struct of some task.
2429	*
2430	* Description: Prints @task's name, cpuset name, and cached copy of its
2431	* mems_allowed to the kernel log. Must hold task_lock(task) to allow
2432	* dereferencing task_cs(task).
2433	*/
2434	void cpuset_print_task_mems_allowed(struct task_struct *tsk)
2435	{
2436	struct dentry *dentry;
2437
2438	dentry = task_cs(tsk)->css.cgroup->dentry;
2439	spin_lock(&cpuset_buffer_lock);
2440	snprintf(cpuset_name, CPUSET_NAME_LEN,
2441	dentry ? (const char *)dentry->d_name.name : "/");
2442	nodelist_scnprintf(cpuset_nodelist, CPUSET_NODELIST_LEN,
2443	tsk->mems_allowed);
2444	printk(KERN_INFO "%s cpuset=%s mems_allowed=%s\n",
2445	tsk->comm, cpuset_name, cpuset_nodelist);
2446	spin_unlock(&cpuset_buffer_lock);
2447	}
2448
2449	/*
2450	* Collection of memory_pressure is suppressed unless
2451	* this flag is enabled by writing "1" to the special
2452	* cpuset file 'memory_pressure_enabled' in the root cpuset.
2453	*/
2454
2455	int cpuset_memory_pressure_enabled __read_mostly;
2456
2457	/**
2458	* cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
2459	*
2460	* Keep a running average of the rate of synchronous (direct)
2461	* page reclaim efforts initiated by tasks in each cpuset.
2462	*
2463	* This represents the rate at which some task in the cpuset
2464	* ran low on memory on all nodes it was allowed to use, and
2465	* had to enter the kernels page reclaim code in an effort to
2466	* create more free memory by tossing clean pages or swapping
2467	* or writing dirty pages.
2468	*
2469	* Display to user space in the per-cpuset read-only file
2470	* "memory_pressure". Value displayed is an integer
2471	* representing the recent rate of entry into the synchronous
2472	* (direct) page reclaim by any task attached to the cpuset.
2473	**/
2474
2475	void __cpuset_memory_pressure_bump(void)
2476	{
2477	task_lock(current);
2478	fmeter_markevent(&task_cs(current)->fmeter);
2479	task_unlock(current);
2480	}
2481
2482	#ifdef CONFIG_PROC_PID_CPUSET
2483	/*
2484	* proc_cpuset_show()
2485	* - Print tasks cpuset path into seq_file.
2486	* - Used for /proc/<pid>/cpuset.
2487	* - No need to task_lock(tsk) on this tsk->cpuset reference, as it
2488	* doesn't really matter if tsk->cpuset changes after we read it,
2489	* and we take cgroup_mutex, keeping cpuset_attach() from changing it
2490	* anyway.
2491	*/
2492	static int proc_cpuset_show(struct seq_file m, void unused_v)
2493	{
2494	struct pid *pid;
2495	struct task_struct *tsk;
2496	char *buf;
2497	struct cgroup_subsys_state *css;
2498	int retval;
2499
2500	retval = -ENOMEM;
2501	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
2502	if (!buf)
2503	goto out;
2504
2505	retval = -ESRCH;
2506	pid = m->private;
2507	tsk = get_pid_task(pid, PIDTYPE_PID);
2508	if (!tsk)
2509	goto out_free;
2510
2511	retval = -EINVAL;
2512	cgroup_lock();
2513	css = task_subsys_state(tsk, cpuset_subsys_id);
2514	retval = cgroup_path(css->cgroup, buf, PAGE_SIZE);
2515	if (retval < 0)
2516	goto out_unlock;
2517	seq_puts(m, buf);
2518	seq_putc(m, '\n');
2519	out_unlock:
2520	cgroup_unlock();
2521	put_task_struct(tsk);
2522	out_free:
2523	kfree(buf);
2524	out:
2525	return retval;
2526	}
2527
2528	static int cpuset_open(struct inode inode, struct file file)
2529	{
2530	struct pid *pid = PROC_I(inode)->pid;
2531	return single_open(file, proc_cpuset_show, pid);
2532	}
2533
2534	const struct file_operations proc_cpuset_operations = {
2535	.open = cpuset_open,
2536	.read = seq_read,
2537	.llseek = seq_lseek,
2538	.release = single_release,
2539	};
2540	#endif /* CONFIG_PROC_PID_CPUSET */
2541
2542	/* Display task cpus_allowed, mems_allowed in /proc/<pid>/status file. */
2543	void cpuset_task_status_allowed(struct seq_file m, struct task_struct task)
2544	{
2545	seq_printf(m, "Cpus_allowed:\t");
2546	seq_cpumask(m, &task->cpus_allowed);
2547	seq_printf(m, "\n");
2548	seq_printf(m, "Cpus_allowed_list:\t");
2549	seq_cpumask_list(m, &task->cpus_allowed);
2550	seq_printf(m, "\n");
2551	seq_printf(m, "Mems_allowed:\t");
2552	seq_nodemask(m, &task->mems_allowed);
2553	seq_printf(m, "\n");
2554	seq_printf(m, "Mems_allowed_list:\t");
2555	seq_nodemask_list(m, &task->mems_allowed);
2556	seq_printf(m, "\n");
2557	}
2558

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/kernel/cpuset.c

interactive