Kernel

Kernel Git Source Tree

Root/kernel/sched.c

1	/*
2	* kernel/sched.c
3	*
4	* Kernel scheduler and related syscalls
5	*
6	* Copyright (C) 1991-2002 Linus Torvalds
7	*
8	* 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
9	* make semaphores SMP safe
10	* 1998-11-19 Implemented schedule_timeout() and related stuff
11	* by Andrea Arcangeli
12	* 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
13	* hybrid priority-list and round-robin design with
14	* an array-switch method of distributing timeslices
15	* and per-CPU runqueues. Cleanups and useful suggestions
16	* by Davide Libenzi, preemptible kernel bits by Robert Love.
17	* 2003-09-03 Interactivity tuning by Con Kolivas.
18	* 2004-04-02 Scheduler domains code by Nick Piggin
19	* 2007-04-15 Work begun on replacing all interactivity tuning with a
20	* fair scheduling design by Con Kolivas.
21	* 2007-05-05 Load balancing (smp-nice) and other improvements
22	* by Peter Williams
23	* 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
24	* 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
25	* 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
26	* Thomas Gleixner, Mike Kravetz
27	*/
28
29	#include <linux/mm.h>
30	#include <linux/module.h>
31	#include <linux/nmi.h>
32	#include <linux/init.h>
33	#include <linux/uaccess.h>
34	#include <linux/highmem.h>
35	#include <linux/smp_lock.h>
36	#include <asm/mmu_context.h>
37	#include <linux/interrupt.h>
38	#include <linux/capability.h>
39	#include <linux/completion.h>
40	#include <linux/kernel_stat.h>
41	#include <linux/debug_locks.h>
42	#include <linux/perf_event.h>
43	#include <linux/security.h>
44	#include <linux/notifier.h>
45	#include <linux/profile.h>
46	#include <linux/freezer.h>
47	#include <linux/vmalloc.h>
48	#include <linux/blkdev.h>
49	#include <linux/delay.h>
50	#include <linux/pid_namespace.h>
51	#include <linux/smp.h>
52	#include <linux/threads.h>
53	#include <linux/timer.h>
54	#include <linux/rcupdate.h>
55	#include <linux/cpu.h>
56	#include <linux/cpuset.h>
57	#include <linux/percpu.h>
58	#include <linux/kthread.h>
59	#include <linux/proc_fs.h>
60	#include <linux/seq_file.h>
61	#include <linux/sysctl.h>
62	#include <linux/syscalls.h>
63	#include <linux/times.h>
64	#include <linux/tsacct_kern.h>
65	#include <linux/kprobes.h>
66	#include <linux/delayacct.h>
67	#include <linux/unistd.h>
68	#include <linux/pagemap.h>
69	#include <linux/hrtimer.h>
70	#include <linux/tick.h>
71	#include <linux/debugfs.h>
72	#include <linux/ctype.h>
73	#include <linux/ftrace.h>
74
75	#include <asm/tlb.h>
76	#include <asm/irq_regs.h>
77
78	#include "sched_cpupri.h"
79
80	#define CREATE_TRACE_POINTS
81	#include <trace/events/sched.h>
82
83	/*
84	* Convert user-nice values [ -20 ... 0 ... 19 ]
85	* to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
86	* and back.
87	*/
88	#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
89	#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
90	#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
91
92	/*
93	* 'User priority' is the nice value converted to something we
94	* can work with better when scaling various scheduler parameters,
95	* it's a [ 0 ... 39 ] range.
96	*/
97	#define USER_PRIO(p) ((p)-MAX_RT_PRIO)
98	#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
99	#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
100
101	/*
102	* Helpers for converting nanosecond timing to jiffy resolution
103	*/
104	#define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
105
106	#define NICE_0_LOAD SCHED_LOAD_SCALE
107	#define NICE_0_SHIFT SCHED_LOAD_SHIFT
108
109	/*
110	* These are the 'tuning knobs' of the scheduler:
111	*
112	* default timeslice is 100 msecs (used only for SCHED_RR tasks).
113	* Timeslices get refilled after they expire.
114	*/
115	#define DEF_TIMESLICE (100 * HZ / 1000)
116
117	/*
118	* single value that denotes runtime == period, ie unlimited time.
119	*/
120	#define RUNTIME_INF ((u64)~0ULL)
121
122	static inline int rt_policy(int policy)
123	{
124	if (unlikely(policy == SCHED_FIFO \|\| policy == SCHED_RR))
125	return 1;
126	return 0;
127	}
128
129	static inline int task_has_rt_policy(struct task_struct *p)
130	{
131	return rt_policy(p->policy);
132	}
133
134	/*
135	* This is the priority-queue data structure of the RT scheduling class:
136	*/
137	struct rt_prio_array {
138	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
139	struct list_head queue[MAX_RT_PRIO];
140	};
141
142	struct rt_bandwidth {
143	/* nests inside the rq lock: */
144	spinlock_t rt_runtime_lock;
145	ktime_t rt_period;
146	u64 rt_runtime;
147	struct hrtimer rt_period_timer;
148	};
149
150	static struct rt_bandwidth def_rt_bandwidth;
151
152	static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
153
154	static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
155	{
156	struct rt_bandwidth *rt_b =
157	container_of(timer, struct rt_bandwidth, rt_period_timer);
158	ktime_t now;
159	int overrun;
160	int idle = 0;
161
162	for (;;) {
163	now = hrtimer_cb_get_time(timer);
164	overrun = hrtimer_forward(timer, now, rt_b->rt_period);
165
166	if (!overrun)
167	break;
168
169	idle = do_sched_rt_period_timer(rt_b, overrun);
170	}
171
172	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
173	}
174
175	static
176	void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
177	{
178	rt_b->rt_period = ns_to_ktime(period);
179	rt_b->rt_runtime = runtime;
180
181	spin_lock_init(&rt_b->rt_runtime_lock);
182
183	hrtimer_init(&rt_b->rt_period_timer,
184	CLOCK_MONOTONIC, HRTIMER_MODE_REL);
185	rt_b->rt_period_timer.function = sched_rt_period_timer;
186	}
187
188	static inline int rt_bandwidth_enabled(void)
189	{
190	return sysctl_sched_rt_runtime >= 0;
191	}
192
193	static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
194	{
195	ktime_t now;
196
197	if (!rt_bandwidth_enabled() \|\| rt_b->rt_runtime == RUNTIME_INF)
198	return;
199
200	if (hrtimer_active(&rt_b->rt_period_timer))
201	return;
202
203	spin_lock(&rt_b->rt_runtime_lock);
204	for (;;) {
205	unsigned long delta;
206	ktime_t soft, hard;
207
208	if (hrtimer_active(&rt_b->rt_period_timer))
209	break;
210
211	now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
212	hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
213
214	soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
215	hard = hrtimer_get_expires(&rt_b->rt_period_timer);
216	delta = ktime_to_ns(ktime_sub(hard, soft));
217	__hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
218	HRTIMER_MODE_ABS_PINNED, 0);
219	}
220	spin_unlock(&rt_b->rt_runtime_lock);
221	}
222
223	#ifdef CONFIG_RT_GROUP_SCHED
224	static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
225	{
226	hrtimer_cancel(&rt_b->rt_period_timer);
227	}
228	#endif
229
230	/*
231	* sched_domains_mutex serializes calls to arch_init_sched_domains,
232	* detach_destroy_domains and partition_sched_domains.
233	*/
234	static DEFINE_MUTEX(sched_domains_mutex);
235
236	#ifdef CONFIG_GROUP_SCHED
237
238	#include <linux/cgroup.h>
239
240	struct cfs_rq;
241
242	static LIST_HEAD(task_groups);
243
244	/* task group related information */
245	struct task_group {
246	#ifdef CONFIG_CGROUP_SCHED
247	struct cgroup_subsys_state css;
248	#endif
249
250	#ifdef CONFIG_USER_SCHED
251	uid_t uid;
252	#endif
253
254	#ifdef CONFIG_FAIR_GROUP_SCHED
255	/* schedulable entities of this group on each cpu */
256	struct sched_entity **se;
257	/* runqueue "owned" by this group on each cpu */
258	struct cfs_rq **cfs_rq;
259	unsigned long shares;
260	#endif
261
262	#ifdef CONFIG_RT_GROUP_SCHED
263	struct sched_rt_entity **rt_se;
264	struct rt_rq **rt_rq;
265
266	struct rt_bandwidth rt_bandwidth;
267	#endif
268
269	struct rcu_head rcu;
270	struct list_head list;
271
272	struct task_group *parent;
273	struct list_head siblings;
274	struct list_head children;
275	};
276
277	#ifdef CONFIG_USER_SCHED
278
279	/* Helper function to pass uid information to create_sched_user() */
280	void set_tg_uid(struct user_struct *user)
281	{
282	user->tg->uid = user->uid;
283	}
284
285	/*
286	* Root task group.
287	* Every UID task group (including init_task_group aka UID-0) will
288	* be a child to this group.
289	*/
290	struct task_group root_task_group;
291
292	#ifdef CONFIG_FAIR_GROUP_SCHED
293	/* Default task group's sched entity on each cpu */
294	static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
295	/* Default task group's cfs_rq on each cpu */
296	static DEFINE_PER_CPU_SHARED_ALIGNED(struct cfs_rq, init_tg_cfs_rq);
297	#endif /* CONFIG_FAIR_GROUP_SCHED */
298
299	#ifdef CONFIG_RT_GROUP_SCHED
300	static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
301	static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq);
302	#endif /* CONFIG_RT_GROUP_SCHED */
303	#else /* !CONFIG_USER_SCHED */
304	#define root_task_group init_task_group
305	#endif /* CONFIG_USER_SCHED */
306
307	/* task_group_lock serializes add/remove of task groups and also changes to
308	* a task group's cpu shares.
309	*/
310	static DEFINE_SPINLOCK(task_group_lock);
311
312	#ifdef CONFIG_FAIR_GROUP_SCHED
313
314	#ifdef CONFIG_SMP
315	static int root_task_group_empty(void)
316	{
317	return list_empty(&root_task_group.children);
318	}
319	#endif
320
321	#ifdef CONFIG_USER_SCHED
322	# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
323	#else /* !CONFIG_USER_SCHED */
324	# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
325	#endif /* CONFIG_USER_SCHED */
326
327	/*
328	* A weight of 0 or 1 can cause arithmetics problems.
329	* A weight of a cfs_rq is the sum of weights of which entities
330	* are queued on this cfs_rq, so a weight of a entity should not be
331	* too large, so as the shares value of a task group.
332	* (The default weight is 1024 - so there's no practical
333	* limitation from this.)
334	*/
335	#define MIN_SHARES 2
336	#define MAX_SHARES (1UL << 18)
337
338	static int init_task_group_load = INIT_TASK_GROUP_LOAD;
339	#endif
340
341	/* Default task group.
342	* Every task in system belong to this group at bootup.
343	*/
344	struct task_group init_task_group;
345
346	/* return group to which a task belongs */
347	static inline struct task_group task_group(struct task_struct p)
348	{
349	struct task_group *tg;
350
351	#ifdef CONFIG_USER_SCHED
352	rcu_read_lock();
353	tg = __task_cred(p)->user->tg;
354	rcu_read_unlock();
355	#elif defined(CONFIG_CGROUP_SCHED)
356	tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
357	struct task_group, css);
358	#else
359	tg = &init_task_group;
360	#endif
361	return tg;
362	}
363
364	/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
365	static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
366	{
367	#ifdef CONFIG_FAIR_GROUP_SCHED
368	p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
369	p->se.parent = task_group(p)->se[cpu];
370	#endif
371
372	#ifdef CONFIG_RT_GROUP_SCHED
373	p->rt.rt_rq = task_group(p)->rt_rq[cpu];
374	p->rt.parent = task_group(p)->rt_se[cpu];
375	#endif
376	}
377
378	#else
379
380	static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
381	static inline struct task_group task_group(struct task_struct p)
382	{
383	return NULL;
384	}
385
386	#endif /* CONFIG_GROUP_SCHED */
387
388	/* CFS-related fields in a runqueue */
389	struct cfs_rq {
390	struct load_weight load;
391	unsigned long nr_running;
392
393	u64 exec_clock;
394	u64 min_vruntime;
395
396	struct rb_root tasks_timeline;
397	struct rb_node *rb_leftmost;
398
399	struct list_head tasks;
400	struct list_head *balance_iterator;
401
402	/*
403	* 'curr' points to currently running entity on this cfs_rq.
404	* It is set to NULL otherwise (i.e when none are currently running).
405	*/
406	struct sched_entity curr, next, *last;
407
408	unsigned int nr_spread_over;
409
410	#ifdef CONFIG_FAIR_GROUP_SCHED
411	struct rq rq; / cpu runqueue to which this cfs_rq is attached */
412
413	/*
414	* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
415	* a hierarchy). Non-leaf lrqs hold other higher schedulable entities
416	* (like users, containers etc.)
417	*
418	* leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
419	* list is used during load balance.
420	*/
421	struct list_head leaf_cfs_rq_list;
422	struct task_group tg; / group that "owns" this runqueue */
423
424	#ifdef CONFIG_SMP
425	/*
426	* the part of load.weight contributed by tasks
427	*/
428	unsigned long task_weight;
429
430	/*
431	* h_load = weight * f(tg)
432	*
433	* Where f(tg) is the recursive weight fraction assigned to
434	* this group.
435	*/
436	unsigned long h_load;
437
438	/*
439	* this cpu's part of tg->shares
440	*/
441	unsigned long shares;
442
443	/*
444	* load.weight at the time we set shares
445	*/
446	unsigned long rq_weight;
447	#endif
448	#endif
449	};
450
451	/* Real-Time classes' related field in a runqueue: */
452	struct rt_rq {
453	struct rt_prio_array active;
454	unsigned long rt_nr_running;
455	#if defined CONFIG_SMP \|\| defined CONFIG_RT_GROUP_SCHED
456	struct {
457	int curr; /* highest queued rt task prio */
458	#ifdef CONFIG_SMP
459	int next; /* next highest */
460	#endif
461	} highest_prio;
462	#endif
463	#ifdef CONFIG_SMP
464	unsigned long rt_nr_migratory;
465	unsigned long rt_nr_total;
466	int overloaded;
467	struct plist_head pushable_tasks;
468	#endif
469	int rt_throttled;
470	u64 rt_time;
471	u64 rt_runtime;
472	/* Nests inside the rq lock: */
473	spinlock_t rt_runtime_lock;
474
475	#ifdef CONFIG_RT_GROUP_SCHED
476	unsigned long rt_nr_boosted;
477
478	struct rq *rq;
479	struct list_head leaf_rt_rq_list;
480	struct task_group *tg;
481	struct sched_rt_entity *rt_se;
482	#endif
483	};
484
485	#ifdef CONFIG_SMP
486
487	/*
488	* We add the notion of a root-domain which will be used to define per-domain
489	* variables. Each exclusive cpuset essentially defines an island domain by
490	* fully partitioning the member cpus from any other cpuset. Whenever a new
491	* exclusive cpuset is created, we also create and attach a new root-domain
492	* object.
493	*
494	*/
495	struct root_domain {
496	atomic_t refcount;
497	cpumask_var_t span;
498	cpumask_var_t online;
499
500	/*
501	* The "RT overload" flag: it gets set if a CPU has more than
502	* one runnable RT task.
503	*/
504	cpumask_var_t rto_mask;
505	atomic_t rto_count;
506	#ifdef CONFIG_SMP
507	struct cpupri cpupri;
508	#endif
509	};
510
511	/*
512	* By default the system creates a single root-domain with all cpus as
513	* members (mimicking the global state we have today).
514	*/
515	static struct root_domain def_root_domain;
516
517	#endif
518
519	/*
520	* This is the main, per-CPU runqueue data structure.
521	*
522	* Locking rule: those places that want to lock multiple runqueues
523	* (such as the load balancing or the thread migration code), lock
524	* acquire operations must be ordered by ascending &runqueue.
525	*/
526	struct rq {
527	/* runqueue lock: */
528	spinlock_t lock;
529
530	/*
531	* nr_running and cpu_load should be in the same cacheline because
532	* remote CPUs use both these fields when doing load calculation.
533	*/
534	unsigned long nr_running;
535	#define CPU_LOAD_IDX_MAX 5
536	unsigned long cpu_load[CPU_LOAD_IDX_MAX];
537	#ifdef CONFIG_NO_HZ
538	unsigned long last_tick_seen;
539	unsigned char in_nohz_recently;
540	#endif
541	/* capture load from all tasks on this cpu: */
542	struct load_weight load;
543	unsigned long nr_load_updates;
544	u64 nr_switches;
545	u64 nr_migrations_in;
546
547	struct cfs_rq cfs;
548	struct rt_rq rt;
549
550	#ifdef CONFIG_FAIR_GROUP_SCHED
551	/* list of leaf cfs_rq on this cpu: */
552	struct list_head leaf_cfs_rq_list;
553	#endif
554	#ifdef CONFIG_RT_GROUP_SCHED
555	struct list_head leaf_rt_rq_list;
556	#endif
557
558	/*
559	* This is part of a global counter where only the total sum
560	* over all CPUs matters. A task can increase this counter on
561	* one CPU and if it got migrated afterwards it may decrease
562	* it on another CPU. Always updated under the runqueue lock:
563	*/
564	unsigned long nr_uninterruptible;
565
566	struct task_struct curr, idle;
567	unsigned long next_balance;
568	struct mm_struct *prev_mm;
569
570	u64 clock;
571
572	atomic_t nr_iowait;
573
574	#ifdef CONFIG_SMP
575	struct root_domain *rd;
576	struct sched_domain *sd;
577
578	unsigned char idle_at_tick;
579	/* For active balancing */
580	int post_schedule;
581	int active_balance;
582	int push_cpu;
583	/* cpu of this runqueue: */
584	int cpu;
585	int online;
586
587	unsigned long avg_load_per_task;
588
589	struct task_struct *migration_thread;
590	struct list_head migration_queue;
591
592	u64 rt_avg;
593	u64 age_stamp;
594	u64 idle_stamp;
595	u64 avg_idle;
596	#endif
597
598	/* calc_load related fields */
599	unsigned long calc_load_update;
600	long calc_load_active;
601
602	#ifdef CONFIG_SCHED_HRTICK
603	#ifdef CONFIG_SMP
604	int hrtick_csd_pending;
605	struct call_single_data hrtick_csd;
606	#endif
607	struct hrtimer hrtick_timer;
608	#endif
609
610	#ifdef CONFIG_SCHEDSTATS
611	/* latency stats */
612	struct sched_info rq_sched_info;
613	unsigned long long rq_cpu_time;
614	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
615
616	/* sys_sched_yield() stats */
617	unsigned int yld_count;
618
619	/* schedule() stats */
620	unsigned int sched_switch;
621	unsigned int sched_count;
622	unsigned int sched_goidle;
623
624	/* try_to_wake_up() stats */
625	unsigned int ttwu_count;
626	unsigned int ttwu_local;
627
628	/* BKL stats */
629	unsigned int bkl_count;
630	#endif
631	};
632
633	static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
634
635	static inline
636	void check_preempt_curr(struct rq rq, struct task_struct p, int flags)
637	{
638	rq->curr->sched_class->check_preempt_curr(rq, p, flags);
639	}
640
641	static inline int cpu_of(struct rq *rq)
642	{
643	#ifdef CONFIG_SMP
644	return rq->cpu;
645	#else
646	return 0;
647	#endif
648	}
649
650	/*
651	* The domain tree (rq->sd) is protected by RCU's quiescent state transition.
652	* See detach_destroy_domains: synchronize_sched for details.
653	*
654	* The domain tree of any CPU may only be accessed from within
655	* preempt-disabled sections.
656	*/
657	#define for_each_domain(cpu, __sd) \
658	for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
659
660	#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
661	#define this_rq() (&__get_cpu_var(runqueues))
662	#define task_rq(p) cpu_rq(task_cpu(p))
663	#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
664	#define raw_rq() (&__raw_get_cpu_var(runqueues))
665
666	inline void update_rq_clock(struct rq *rq)
667	{
668	rq->clock = sched_clock_cpu(cpu_of(rq));
669	}
670
671	/*
672	* Tunables that become constants when CONFIG_SCHED_DEBUG is off:
673	*/
674	#ifdef CONFIG_SCHED_DEBUG
675	# define const_debug __read_mostly
676	#else
677	# define const_debug static const
678	#endif
679
680	/**
681	* runqueue_is_locked
682	* @cpu: the processor in question.
683	*
684	* Returns true if the current cpu runqueue is locked.
685	* This interface allows printk to be called with the runqueue lock
686	* held and know whether or not it is OK to wake up the klogd.
687	*/
688	int runqueue_is_locked(int cpu)
689	{
690	return spin_is_locked(&cpu_rq(cpu)->lock);
691	}
692
693	/*
694	* Debugging: various feature bits
695	*/
696
697	#define SCHED_FEAT(name, enabled) \
698	__SCHED_FEAT_##name ,
699
700	enum {
701	#include "sched_features.h"
702	};
703
704	#undef SCHED_FEAT
705
706	#define SCHED_FEAT(name, enabled) \
707	(1UL << __SCHED_FEAT_##name) * enabled \|
708
709	const_debug unsigned int sysctl_sched_features =
710	#include "sched_features.h"
711	0;
712
713	#undef SCHED_FEAT
714
715	#ifdef CONFIG_SCHED_DEBUG
716	#define SCHED_FEAT(name, enabled) \
717	#name ,
718
719	static __read_mostly char *sched_feat_names[] = {
720	#include "sched_features.h"
721	NULL
722	};
723
724	#undef SCHED_FEAT
725
726	static int sched_feat_show(struct seq_file m, void v)
727	{
728	int i;
729
730	for (i = 0; sched_feat_names[i]; i++) {
731	if (!(sysctl_sched_features & (1UL << i)))
732	seq_puts(m, "NO_");
733	seq_printf(m, "%s ", sched_feat_names[i]);
734	}
735	seq_puts(m, "\n");
736
737	return 0;
738	}
739
740	static ssize_t
741	sched_feat_write(struct file filp, const char __user ubuf,
742	size_t cnt, loff_t *ppos)
743	{
744	char buf[64];
745	char *cmp = buf;
746	int neg = 0;
747	int i;
748
749	if (cnt > 63)
750	cnt = 63;
751
752	if (copy_from_user(&buf, ubuf, cnt))
753	return -EFAULT;
754
755	buf[cnt] = 0;
756
757	if (strncmp(buf, "NO_", 3) == 0) {
758	neg = 1;
759	cmp += 3;
760	}
761
762	for (i = 0; sched_feat_names[i]; i++) {
763	int len = strlen(sched_feat_names[i]);
764
765	if (strncmp(cmp, sched_feat_names[i], len) == 0) {
766	if (neg)
767	sysctl_sched_features &= ~(1UL << i);
768	else
769	sysctl_sched_features \|= (1UL << i);
770	break;
771	}
772	}
773
774	if (!sched_feat_names[i])
775	return -EINVAL;
776
777	filp->f_pos += cnt;
778
779	return cnt;
780	}
781
782	static int sched_feat_open(struct inode inode, struct file filp)
783	{
784	return single_open(filp, sched_feat_show, NULL);
785	}
786
787	static const struct file_operations sched_feat_fops = {
788	.open = sched_feat_open,
789	.write = sched_feat_write,
790	.read = seq_read,
791	.llseek = seq_lseek,
792	.release = single_release,
793	};
794
795	static __init int sched_init_debug(void)
796	{
797	debugfs_create_file("sched_features", 0644, NULL, NULL,
798	&sched_feat_fops);
799
800	return 0;
801	}
802	late_initcall(sched_init_debug);
803
804	#endif
805
806	#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
807
808	/*
809	* Number of tasks to iterate in a single balance run.
810	* Limited because this is done with IRQs disabled.
811	*/
812	const_debug unsigned int sysctl_sched_nr_migrate = 32;
813
814	/*
815	* ratelimit for updating the group shares.
816	* default: 0.25ms
817	*/
818	unsigned int sysctl_sched_shares_ratelimit = 250000;
819	unsigned int normalized_sysctl_sched_shares_ratelimit = 250000;
820
821	/*
822	* Inject some fuzzyness into changing the per-cpu group shares
823	* this avoids remote rq-locks at the expense of fairness.
824	* default: 4
825	*/
826	unsigned int sysctl_sched_shares_thresh = 4;
827
828	/*
829	* period over which we average the RT time consumption, measured
830	* in ms.
831	*
832	* default: 1s
833	*/
834	const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
835
836	/*
837	* period over which we measure -rt task cpu usage in us.
838	* default: 1s
839	*/
840	unsigned int sysctl_sched_rt_period = 1000000;
841
842	static __read_mostly int scheduler_running;
843
844	/*
845	* part of the period that we allow rt tasks to run in us.
846	* default: 0.95s
847	*/
848	int sysctl_sched_rt_runtime = 950000;
849
850	static inline u64 global_rt_period(void)
851	{
852	return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
853	}
854
855	static inline u64 global_rt_runtime(void)
856	{
857	if (sysctl_sched_rt_runtime < 0)
858	return RUNTIME_INF;
859
860	return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
861	}
862
863	#ifndef prepare_arch_switch
864	# define prepare_arch_switch(next) do { } while (0)
865	#endif
866	#ifndef finish_arch_switch
867	# define finish_arch_switch(prev) do { } while (0)
868	#endif
869
870	static inline int task_current(struct rq rq, struct task_struct p)
871	{
872	return rq->curr == p;
873	}
874
875	#ifndef __ARCH_WANT_UNLOCKED_CTXSW
876	static inline int task_running(struct rq rq, struct task_struct p)
877	{
878	return task_current(rq, p);
879	}
880
881	static inline void prepare_lock_switch(struct rq rq, struct task_struct next)
882	{
883	}
884
885	static inline void finish_lock_switch(struct rq rq, struct task_struct prev)
886	{
887	#ifdef CONFIG_DEBUG_SPINLOCK
888	/* this is a valid case when another task releases the spinlock */
889	rq->lock.owner = current;
890	#endif
891	/*
892	* If we are tracking spinlock dependencies then we have to
893	* fix up the runqueue lock - which gets 'carried over' from
894	* prev into current:
895	*/
896	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
897
898	spin_unlock_irq(&rq->lock);
899	}
900
901	#else /* __ARCH_WANT_UNLOCKED_CTXSW */
902	static inline int task_running(struct rq rq, struct task_struct p)
903	{
904	#ifdef CONFIG_SMP
905	return p->oncpu;
906	#else
907	return task_current(rq, p);
908	#endif
909	}
910
911	static inline void prepare_lock_switch(struct rq rq, struct task_struct next)
912	{
913	#ifdef CONFIG_SMP
914	/*
915	* We can optimise this out completely for !SMP, because the
916	* SMP rebalancing from interrupt is the only thing that cares
917	* here.
918	*/
919	next->oncpu = 1;
920	#endif
921	#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
922	spin_unlock_irq(&rq->lock);
923	#else
924	spin_unlock(&rq->lock);
925	#endif
926	}
927
928	static inline void finish_lock_switch(struct rq rq, struct task_struct prev)
929	{
930	#ifdef CONFIG_SMP
931	/*
932	* After ->oncpu is cleared, the task can be moved to a different CPU.
933	* We must ensure this doesn't happen until the switch is completely
934	* finished.
935	*/
936	smp_wmb();
937	prev->oncpu = 0;
938	#endif
939	#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
940	local_irq_enable();
941	#endif
942	}
943	#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
944
945	/*
946	* __task_rq_lock - lock the runqueue a given task resides on.
947	* Must be called interrupts disabled.
948	*/
949	static inline struct rq __task_rq_lock(struct task_struct p)
950	__acquires(rq->lock)
951	{
952	for (;;) {
953	struct rq *rq = task_rq(p);
954	spin_lock(&rq->lock);
955	if (likely(rq == task_rq(p)))
956	return rq;
957	spin_unlock(&rq->lock);
958	}
959	}
960
961	/*
962	* task_rq_lock - lock the runqueue a given task resides on and disable
963	* interrupts. Note the ordering: we can safely lookup the task_rq without
964	* explicitly disabling preemption.
965	*/
966	static struct rq task_rq_lock(struct task_struct p, unsigned long *flags)
967	__acquires(rq->lock)
968	{
969	struct rq *rq;
970
971	for (;;) {
972	local_irq_save(*flags);
973	rq = task_rq(p);
974	spin_lock(&rq->lock);
975	if (likely(rq == task_rq(p)))
976	return rq;
977	spin_unlock_irqrestore(&rq->lock, *flags);
978	}
979	}
980
981	void task_rq_unlock_wait(struct task_struct *p)
982	{
983	struct rq *rq = task_rq(p);
984
985	smp_mb(); /* spin-unlock-wait is not a full memory barrier */
986	spin_unlock_wait(&rq->lock);
987	}
988
989	static void __task_rq_unlock(struct rq *rq)
990	__releases(rq->lock)
991	{
992	spin_unlock(&rq->lock);
993	}
994
995	static inline void task_rq_unlock(struct rq rq, unsigned long flags)
996	__releases(rq->lock)
997	{
998	spin_unlock_irqrestore(&rq->lock, *flags);
999	}
1000
1001	/*
1002	* this_rq_lock - lock this runqueue and disable interrupts.
1003	*/
1004	static struct rq *this_rq_lock(void)
1005	__acquires(rq->lock)
1006	{
1007	struct rq *rq;
1008
1009	local_irq_disable();
1010	rq = this_rq();
1011	spin_lock(&rq->lock);
1012
1013	return rq;
1014	}
1015
1016	#ifdef CONFIG_SCHED_HRTICK
1017	/*
1018	* Use HR-timers to deliver accurate preemption points.
1019	*
1020	* Its all a bit involved since we cannot program an hrt while holding the
1021	* rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
1022	* reschedule event.
1023	*
1024	* When we get rescheduled we reprogram the hrtick_timer outside of the
1025	* rq->lock.
1026	*/
1027
1028	/*
1029	* Use hrtick when:
1030	* - enabled by features
1031	* - hrtimer is actually high res
1032	*/
1033	static inline int hrtick_enabled(struct rq *rq)
1034	{
1035	if (!sched_feat(HRTICK))
1036	return 0;
1037	if (!cpu_active(cpu_of(rq)))
1038	return 0;
1039	return hrtimer_is_hres_active(&rq->hrtick_timer);
1040	}
1041
1042	static void hrtick_clear(struct rq *rq)
1043	{
1044	if (hrtimer_active(&rq->hrtick_timer))
1045	hrtimer_cancel(&rq->hrtick_timer);
1046	}
1047
1048	/*
1049	* High-resolution timer tick.
1050	* Runs from hardirq context with interrupts disabled.
1051	*/
1052	static enum hrtimer_restart hrtick(struct hrtimer *timer)
1053	{
1054	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
1055
1056	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
1057
1058	spin_lock(&rq->lock);
1059	update_rq_clock(rq);
1060	rq->curr->sched_class->task_tick(rq, rq->curr, 1);
1061	spin_unlock(&rq->lock);
1062
1063	return HRTIMER_NORESTART;
1064	}
1065
1066	#ifdef CONFIG_SMP
1067	/*
1068	* called from hardirq (IPI) context
1069	*/
1070	static void __hrtick_start(void *arg)
1071	{
1072	struct rq *rq = arg;
1073
1074	spin_lock(&rq->lock);
1075	hrtimer_restart(&rq->hrtick_timer);
1076	rq->hrtick_csd_pending = 0;
1077	spin_unlock(&rq->lock);
1078	}
1079
1080	/*
1081	* Called to set the hrtick timer state.
1082	*
1083	* called with rq->lock held and irqs disabled
1084	*/
1085	static void hrtick_start(struct rq *rq, u64 delay)
1086	{
1087	struct hrtimer *timer = &rq->hrtick_timer;
1088	ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
1089
1090	hrtimer_set_expires(timer, time);
1091
1092	if (rq == this_rq()) {
1093	hrtimer_restart(timer);
1094	} else if (!rq->hrtick_csd_pending) {
1095	__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
1096	rq->hrtick_csd_pending = 1;
1097	}
1098	}
1099
1100	static int
1101	hotplug_hrtick(struct notifier_block nfb, unsigned long action, void hcpu)
1102	{
1103	int cpu = (int)(long)hcpu;
1104
1105	switch (action) {
1106	case CPU_UP_CANCELED:
1107	case CPU_UP_CANCELED_FROZEN:
1108	case CPU_DOWN_PREPARE:
1109	case CPU_DOWN_PREPARE_FROZEN:
1110	case CPU_DEAD:
1111	case CPU_DEAD_FROZEN:
1112	hrtick_clear(cpu_rq(cpu));
1113	return NOTIFY_OK;
1114	}
1115
1116	return NOTIFY_DONE;
1117	}
1118
1119	static __init void init_hrtick(void)
1120	{
1121	hotcpu_notifier(hotplug_hrtick, 0);
1122	}
1123	#else
1124	/*
1125	* Called to set the hrtick timer state.
1126	*
1127	* called with rq->lock held and irqs disabled
1128	*/
1129	static void hrtick_start(struct rq *rq, u64 delay)
1130	{
1131	__hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
1132	HRTIMER_MODE_REL_PINNED, 0);
1133	}
1134
1135	static inline void init_hrtick(void)
1136	{
1137	}
1138	#endif /* CONFIG_SMP */
1139
1140	static void init_rq_hrtick(struct rq *rq)
1141	{
1142	#ifdef CONFIG_SMP
1143	rq->hrtick_csd_pending = 0;
1144
1145	rq->hrtick_csd.flags = 0;
1146	rq->hrtick_csd.func = __hrtick_start;
1147	rq->hrtick_csd.info = rq;
1148	#endif
1149
1150	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1151	rq->hrtick_timer.function = hrtick;
1152	}
1153	#else /* CONFIG_SCHED_HRTICK */
1154	static inline void hrtick_clear(struct rq *rq)
1155	{
1156	}
1157
1158	static inline void init_rq_hrtick(struct rq *rq)
1159	{
1160	}
1161
1162	static inline void init_hrtick(void)
1163	{
1164	}
1165	#endif /* CONFIG_SCHED_HRTICK */
1166
1167	/*
1168	* resched_task - mark a task 'to be rescheduled now'.
1169	*
1170	* On UP this means the setting of the need_resched flag, on SMP it
1171	* might also involve a cross-CPU call to trigger the scheduler on
1172	* the target CPU.
1173	*/
1174	#ifdef CONFIG_SMP
1175
1176	#ifndef tsk_is_polling
1177	#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
1178	#endif
1179
1180	static void resched_task(struct task_struct *p)
1181	{
1182	int cpu;
1183
1184	assert_spin_locked(&task_rq(p)->lock);
1185
1186	if (test_tsk_need_resched(p))
1187	return;
1188
1189	set_tsk_need_resched(p);
1190
1191	cpu = task_cpu(p);
1192	if (cpu == smp_processor_id())
1193	return;
1194
1195	/* NEED_RESCHED must be visible before we test polling */
1196	smp_mb();
1197	if (!tsk_is_polling(p))
1198	smp_send_reschedule(cpu);
1199	}
1200
1201	static void resched_cpu(int cpu)
1202	{
1203	struct rq *rq = cpu_rq(cpu);
1204	unsigned long flags;
1205
1206	if (!spin_trylock_irqsave(&rq->lock, flags))
1207	return;
1208	resched_task(cpu_curr(cpu));
1209	spin_unlock_irqrestore(&rq->lock, flags);
1210	}
1211
1212	#ifdef CONFIG_NO_HZ
1213	/*
1214	* When add_timer_on() enqueues a timer into the timer wheel of an
1215	* idle CPU then this timer might expire before the next timer event
1216	* which is scheduled to wake up that CPU. In case of a completely
1217	* idle system the next event might even be infinite time into the
1218	* future. wake_up_idle_cpu() ensures that the CPU is woken up and
1219	* leaves the inner idle loop so the newly added timer is taken into
1220	* account when the CPU goes back to idle and evaluates the timer
1221	* wheel for the next timer event.
1222	*/
1223	void wake_up_idle_cpu(int cpu)
1224	{
1225	struct rq *rq = cpu_rq(cpu);
1226
1227	if (cpu == smp_processor_id())
1228	return;
1229
1230	/*
1231	* This is safe, as this function is called with the timer
1232	* wheel base lock of (cpu) held. When the CPU is on the way
1233	* to idle and has not yet set rq->curr to idle then it will
1234	* be serialized on the timer wheel base lock and take the new
1235	* timer into account automatically.
1236	*/
1237	if (rq->curr != rq->idle)
1238	return;
1239
1240	/*
1241	* We can set TIF_RESCHED on the idle task of the other CPU
1242	* lockless. The worst case is that the other CPU runs the
1243	* idle task through an additional NOOP schedule()
1244	*/
1245	set_tsk_need_resched(rq->idle);
1246
1247	/* NEED_RESCHED must be visible before we test polling */
1248	smp_mb();
1249	if (!tsk_is_polling(rq->idle))
1250	smp_send_reschedule(cpu);
1251	}
1252	#endif /* CONFIG_NO_HZ */
1253
1254	static u64 sched_avg_period(void)
1255	{
1256	return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
1257	}
1258
1259	static void sched_avg_update(struct rq *rq)
1260	{
1261	s64 period = sched_avg_period();
1262
1263	while ((s64)(rq->clock - rq->age_stamp) > period) {
1264	rq->age_stamp += period;
1265	rq->rt_avg /= 2;
1266	}
1267	}
1268
1269	static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
1270	{
1271	rq->rt_avg += rt_delta;
1272	sched_avg_update(rq);
1273	}
1274
1275	#else /* !CONFIG_SMP */
1276	static void resched_task(struct task_struct *p)
1277	{
1278	assert_spin_locked(&task_rq(p)->lock);
1279	set_tsk_need_resched(p);
1280	}
1281
1282	static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
1283	{
1284	}
1285	#endif /* CONFIG_SMP */
1286
1287	#if BITS_PER_LONG == 32
1288	# define WMULT_CONST (~0UL)
1289	#else
1290	# define WMULT_CONST (1UL << 32)
1291	#endif
1292
1293	#define WMULT_SHIFT 32
1294
1295	/*
1296	* Shift right and round:
1297	*/
1298	#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
1299
1300	/*
1301	* delta *= weight / lw
1302	*/
1303	static unsigned long
1304	calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1305	struct load_weight *lw)
1306	{
1307	u64 tmp;
1308
1309	if (!lw->inv_weight) {
1310	if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
1311	lw->inv_weight = 1;
1312	else
1313	lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
1314	/ (lw->weight+1);
1315	}
1316
1317	tmp = (u64)delta_exec * weight;
1318	/*
1319	* Check whether we'd overflow the 64-bit multiplication:
1320	*/
1321	if (unlikely(tmp > WMULT_CONST))
1322	tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
1323	WMULT_SHIFT/2);
1324	else
1325	tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
1326
1327	return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1328	}
1329
1330	static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1331	{
1332	lw->weight += inc;
1333	lw->inv_weight = 0;
1334	}
1335
1336	static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
1337	{
1338	lw->weight -= dec;
1339	lw->inv_weight = 0;
1340	}
1341
1342	/*
1343	* To aid in avoiding the subversion of "niceness" due to uneven distribution
1344	* of tasks with abnormal "nice" values across CPUs the contribution that
1345	* each task makes to its run queue's load is weighted according to its
1346	* scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
1347	* scaled version of the new time slice allocation that they receive on time
1348	* slice expiry etc.
1349	*/
1350
1351	#define WEIGHT_IDLEPRIO 3
1352	#define WMULT_IDLEPRIO 1431655765
1353
1354	/*
1355	* Nice levels are multiplicative, with a gentle 10% change for every
1356	* nice level changed. I.e. when a CPU-bound task goes from nice 0 to
1357	* nice 1, it will get ~10% less CPU time than another CPU-bound task
1358	* that remained on nice 0.
1359	*
1360	* The "10% effect" is relative and cumulative: from _any_ nice level,
1361	* if you go up 1 level, it's -10% CPU usage, if you go down 1 level
1362	* it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
1363	* If a task goes up by ~10% and another task goes down by ~10% then
1364	* the relative distance between them is ~25%.)
1365	*/
1366	static const int prio_to_weight[40] = {
1367	/* -20 */ 88761, 71755, 56483, 46273, 36291,
1368	/* -15 */ 29154, 23254, 18705, 14949, 11916,
1369	/* -10 */ 9548, 7620, 6100, 4904, 3906,
1370	/* -5 */ 3121, 2501, 1991, 1586, 1277,
1371	/* 0 */ 1024, 820, 655, 526, 423,
1372	/* 5 */ 335, 272, 215, 172, 137,
1373	/* 10 */ 110, 87, 70, 56, 45,
1374	/* 15 */ 36, 29, 23, 18, 15,
1375	};
1376
1377	/*
1378	* Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
1379	*
1380	* In cases where the weight does not change often, we can use the
1381	* precalculated inverse to speed up arithmetics by turning divisions
1382	* into multiplications:
1383	*/
1384	static const u32 prio_to_wmult[40] = {
1385	/* -20 */ 48388, 59856, 76040, 92818, 118348,
1386	/* -15 */ 147320, 184698, 229616, 287308, 360437,
1387	/* -10 */ 449829, 563644, 704093, 875809, 1099582,
1388	/* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
1389	/* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
1390	/* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
1391	/* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
1392	/* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
1393	};
1394
1395	static void activate_task(struct rq rq, struct task_struct p, int wakeup);
1396
1397	/*
1398	* runqueue iterator, to support SMP load-balancing between different
1399	* scheduling classes, without having to expose their internal data
1400	* structures to the load-balancing proper:
1401	*/
1402	struct rq_iterator {
1403	void *arg;
1404	struct task_struct (start)(void *);
1405	struct task_struct (next)(void *);
1406	};
1407
1408	#ifdef CONFIG_SMP
1409	static unsigned long
1410	balance_tasks(struct rq this_rq, int this_cpu, struct rq busiest,
1411	unsigned long max_load_move, struct sched_domain *sd,
1412	enum cpu_idle_type idle, int *all_pinned,
1413	int this_best_prio, struct rq_iterator iterator);
1414
1415	static int
1416	iter_move_one_task(struct rq this_rq, int this_cpu, struct rq busiest,
1417	struct sched_domain *sd, enum cpu_idle_type idle,
1418	struct rq_iterator *iterator);
1419	#endif
1420
1421	/* Time spent by the tasks of the cpu accounting group executing in ... */
1422	enum cpuacct_stat_index {
1423	CPUACCT_STAT_USER, /* ... user mode */
1424	CPUACCT_STAT_SYSTEM, /* ... kernel mode */
1425
1426	CPUACCT_STAT_NSTATS,
1427	};
1428
1429	#ifdef CONFIG_CGROUP_CPUACCT
1430	static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
1431	static void cpuacct_update_stats(struct task_struct *tsk,
1432	enum cpuacct_stat_index idx, cputime_t val);
1433	#else
1434	static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
1435	static inline void cpuacct_update_stats(struct task_struct *tsk,
1436	enum cpuacct_stat_index idx, cputime_t val) {}
1437	#endif
1438
1439	static inline void inc_cpu_load(struct rq *rq, unsigned long load)
1440	{
1441	update_load_add(&rq->load, load);
1442	}
1443
1444	static inline void dec_cpu_load(struct rq *rq, unsigned long load)
1445	{
1446	update_load_sub(&rq->load, load);
1447	}
1448
1449	#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) \|\| defined(CONFIG_RT_GROUP_SCHED)
1450	typedef int (tg_visitor)(struct task_group , void *);
1451
1452	/*
1453	* Iterate the full tree, calling @down when first entering a node and @up when
1454	* leaving it for the final time.
1455	*/
1456	static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
1457	{
1458	struct task_group parent, child;
1459	int ret;
1460
1461	rcu_read_lock();
1462	parent = &root_task_group;
1463	down:
1464	ret = (*down)(parent, data);
1465	if (ret)
1466	goto out_unlock;
1467	list_for_each_entry_rcu(child, &parent->children, siblings) {
1468	parent = child;
1469	goto down;
1470
1471	up:
1472	continue;
1473	}
1474	ret = (*up)(parent, data);
1475	if (ret)
1476	goto out_unlock;
1477
1478	child = parent;
1479	parent = parent->parent;
1480	if (parent)
1481	goto up;
1482	out_unlock:
1483	rcu_read_unlock();
1484
1485	return ret;
1486	}
1487
1488	static int tg_nop(struct task_group tg, void data)
1489	{
1490	return 0;
1491	}
1492	#endif
1493
1494	#ifdef CONFIG_SMP
1495	/* Used instead of source_load when we know the type == 0 */
1496	static unsigned long weighted_cpuload(const int cpu)
1497	{
1498	return cpu_rq(cpu)->load.weight;
1499	}
1500
1501	/*
1502	* Return a low guess at the load of a migration-source cpu weighted
1503	* according to the scheduling class and "nice" value.
1504	*
1505	* We want to under-estimate the load of migration sources, to
1506	* balance conservatively.
1507	*/
1508	static unsigned long source_load(int cpu, int type)
1509	{
1510	struct rq *rq = cpu_rq(cpu);
1511	unsigned long total = weighted_cpuload(cpu);
1512
1513	if (type == 0 \|\| !sched_feat(LB_BIAS))
1514	return total;
1515
1516	return min(rq->cpu_load[type-1], total);
1517	}
1518
1519	/*
1520	* Return a high guess at the load of a migration-target cpu weighted
1521	* according to the scheduling class and "nice" value.
1522	*/
1523	static unsigned long target_load(int cpu, int type)
1524	{
1525	struct rq *rq = cpu_rq(cpu);
1526	unsigned long total = weighted_cpuload(cpu);
1527
1528	if (type == 0 \|\| !sched_feat(LB_BIAS))
1529	return total;
1530
1531	return max(rq->cpu_load[type-1], total);
1532	}
1533
1534	static struct sched_group *group_of(int cpu)
1535	{
1536	struct sched_domain *sd = rcu_dereference(cpu_rq(cpu)->sd);
1537
1538	if (!sd)
1539	return NULL;
1540
1541	return sd->groups;
1542	}
1543
1544	static unsigned long power_of(int cpu)
1545	{
1546	struct sched_group *group = group_of(cpu);
1547
1548	if (!group)
1549	return SCHED_LOAD_SCALE;
1550
1551	return group->cpu_power;
1552	}
1553
1554	static int task_hot(struct task_struct p, u64 now, struct sched_domain sd);
1555
1556	static unsigned long cpu_avg_load_per_task(int cpu)
1557	{
1558	struct rq *rq = cpu_rq(cpu);
1559	unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
1560
1561	if (nr_running)
1562	rq->avg_load_per_task = rq->load.weight / nr_running;
1563	else
1564	rq->avg_load_per_task = 0;
1565
1566	return rq->avg_load_per_task;
1567	}
1568
1569	#ifdef CONFIG_FAIR_GROUP_SCHED
1570
1571	static __read_mostly unsigned long *update_shares_data;
1572
1573	static void __set_se_shares(struct sched_entity *se, unsigned long shares);
1574
1575	/*
1576	* Calculate and set the cpu's group shares.
1577	*/
1578	static void update_group_shares_cpu(struct task_group *tg, int cpu,
1579	unsigned long sd_shares,
1580	unsigned long sd_rq_weight,
1581	unsigned long *usd_rq_weight)
1582	{
1583	unsigned long shares, rq_weight;
1584	int boost = 0;
1585
1586	rq_weight = usd_rq_weight[cpu];
1587	if (!rq_weight) {
1588	boost = 1;
1589	rq_weight = NICE_0_LOAD;
1590	}
1591
1592	/*
1593	* \Sum_j shares_j * rq_weight_i
1594	* shares_i = -----------------------------
1595	* \Sum_j rq_weight_j
1596	*/
1597	shares = (sd_shares * rq_weight) / sd_rq_weight;
1598	shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1599
1600	if (abs(shares - tg->se[cpu]->load.weight) >
1601	sysctl_sched_shares_thresh) {
1602	struct rq *rq = cpu_rq(cpu);
1603	unsigned long flags;
1604
1605	spin_lock_irqsave(&rq->lock, flags);
1606	tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight;
1607	tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1608	__set_se_shares(tg->se[cpu], shares);
1609	spin_unlock_irqrestore(&rq->lock, flags);
1610	}
1611	}
1612
1613	/*
1614	* Re-compute the task group their per cpu shares over the given domain.
1615	* This needs to be done in a bottom-up fashion because the rq weight of a
1616	* parent group depends on the shares of its child groups.
1617	*/
1618	static int tg_shares_up(struct task_group tg, void data)
1619	{
1620	unsigned long weight, rq_weight = 0, shares = 0;
1621	unsigned long *usd_rq_weight;
1622	struct sched_domain *sd = data;
1623	unsigned long flags;
1624	int i;
1625
1626	if (!tg->se[0])
1627	return 0;
1628
1629	local_irq_save(flags);
1630	usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id());
1631
1632	for_each_cpu(i, sched_domain_span(sd)) {
1633	weight = tg->cfs_rq[i]->load.weight;
1634	usd_rq_weight[i] = weight;
1635
1636	/*
1637	* If there are currently no tasks on the cpu pretend there
1638	* is one of average load so that when a new task gets to
1639	* run here it will not get delayed by group starvation.
1640	*/
1641	if (!weight)
1642	weight = NICE_0_LOAD;
1643
1644	rq_weight += weight;
1645	shares += tg->cfs_rq[i]->shares;
1646	}
1647
1648	if ((!shares && rq_weight) \|\| shares > tg->shares)
1649	shares = tg->shares;
1650
1651	if (!sd->parent \|\| !(sd->parent->flags & SD_LOAD_BALANCE))
1652	shares = tg->shares;
1653
1654	for_each_cpu(i, sched_domain_span(sd))
1655	update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight);
1656
1657	local_irq_restore(flags);
1658
1659	return 0;
1660	}
1661
1662	/*
1663	* Compute the cpu's hierarchical load factor for each task group.
1664	* This needs to be done in a top-down fashion because the load of a child
1665	* group is a fraction of its parents load.
1666	*/
1667	static int tg_load_down(struct task_group tg, void data)
1668	{
1669	unsigned long load;
1670	long cpu = (long)data;
1671
1672	if (!tg->parent) {
1673	load = cpu_rq(cpu)->load.weight;
1674	} else {
1675	load = tg->parent->cfs_rq[cpu]->h_load;
1676	load *= tg->cfs_rq[cpu]->shares;
1677	load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
1678	}
1679
1680	tg->cfs_rq[cpu]->h_load = load;
1681
1682	return 0;
1683	}
1684
1685	static void update_shares(struct sched_domain *sd)
1686	{
1687	s64 elapsed;
1688	u64 now;
1689
1690	if (root_task_group_empty())
1691	return;
1692
1693	now = cpu_clock(raw_smp_processor_id());
1694	elapsed = now - sd->last_update;
1695
1696	if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
1697	sd->last_update = now;
1698	walk_tg_tree(tg_nop, tg_shares_up, sd);
1699	}
1700	}
1701
1702	static void update_shares_locked(struct rq rq, struct sched_domain sd)
1703	{
1704	if (root_task_group_empty())
1705	return;
1706
1707	spin_unlock(&rq->lock);
1708	update_shares(sd);
1709	spin_lock(&rq->lock);
1710	}
1711
1712	static void update_h_load(long cpu)
1713	{
1714	if (root_task_group_empty())
1715	return;
1716
1717	walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
1718	}
1719
1720	#else
1721
1722	static inline void update_shares(struct sched_domain *sd)
1723	{
1724	}
1725
1726	static inline void update_shares_locked(struct rq rq, struct sched_domain sd)
1727	{
1728	}
1729
1730	#endif
1731
1732	#ifdef CONFIG_PREEMPT
1733
1734	static void double_rq_lock(struct rq rq1, struct rq rq2);
1735
1736	/*
1737	* fair double_lock_balance: Safely acquires both rq->locks in a fair
1738	* way at the expense of forcing extra atomic operations in all
1739	* invocations. This assures that the double_lock is acquired using the
1740	* same underlying policy as the spinlock_t on this architecture, which
1741	* reduces latency compared to the unfair variant below. However, it
1742	* also adds more overhead and therefore may reduce throughput.
1743	*/
1744	static inline int _double_lock_balance(struct rq this_rq, struct rq busiest)
1745	__releases(this_rq->lock)
1746	__acquires(busiest->lock)
1747	__acquires(this_rq->lock)
1748	{
1749	spin_unlock(&this_rq->lock);
1750	double_rq_lock(this_rq, busiest);
1751
1752	return 1;
1753	}
1754
1755	#else
1756	/*
1757	* Unfair double_lock_balance: Optimizes throughput at the expense of
1758	* latency by eliminating extra atomic operations when the locks are
1759	* already in proper order on entry. This favors lower cpu-ids and will
1760	* grant the double lock to lower cpus over higher ids under contention,
1761	* regardless of entry order into the function.
1762	*/
1763	static int _double_lock_balance(struct rq this_rq, struct rq busiest)
1764	__releases(this_rq->lock)
1765	__acquires(busiest->lock)
1766	__acquires(this_rq->lock)
1767	{
1768	int ret = 0;
1769
1770	if (unlikely(!spin_trylock(&busiest->lock))) {
1771	if (busiest < this_rq) {
1772	spin_unlock(&this_rq->lock);
1773	spin_lock(&busiest->lock);
1774	spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
1775	ret = 1;
1776	} else
1777	spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
1778	}
1779	return ret;
1780	}
1781
1782	#endif /* CONFIG_PREEMPT */
1783
1784	/*
1785	* double_lock_balance - lock the busiest runqueue, this_rq is locked already.
1786	*/
1787	static int double_lock_balance(struct rq this_rq, struct rq busiest)
1788	{
1789	if (unlikely(!irqs_disabled())) {
1790	/* printk() doesn't work good under rq->lock */
1791	spin_unlock(&this_rq->lock);
1792	BUG_ON(1);
1793	}
1794
1795	return _double_lock_balance(this_rq, busiest);
1796	}
1797
1798	static inline void double_unlock_balance(struct rq this_rq, struct rq busiest)
1799	__releases(busiest->lock)
1800	{
1801	spin_unlock(&busiest->lock);
1802	lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
1803	}
1804	#endif
1805
1806	#ifdef CONFIG_FAIR_GROUP_SCHED
1807	static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1808	{
1809	#ifdef CONFIG_SMP
1810	cfs_rq->shares = shares;
1811	#endif
1812	}
1813	#endif
1814
1815	static void calc_load_account_active(struct rq *this_rq);
1816	static void update_sysctl(void);
1817
1818	#include "sched_stats.h"
1819	#include "sched_idletask.c"
1820	#include "sched_fair.c"
1821	#include "sched_rt.c"
1822	#ifdef CONFIG_SCHED_DEBUG
1823	# include "sched_debug.c"
1824	#endif
1825
1826	#define sched_class_highest (&rt_sched_class)
1827	#define for_each_class(class) \
1828	for (class = sched_class_highest; class; class = class->next)
1829
1830	static void inc_nr_running(struct rq *rq)
1831	{
1832	rq->nr_running++;
1833	}
1834
1835	static void dec_nr_running(struct rq *rq)
1836	{
1837	rq->nr_running--;
1838	}
1839
1840	static void set_load_weight(struct task_struct *p)
1841	{
1842	if (task_has_rt_policy(p)) {
1843	p->se.load.weight = prio_to_weight[0] * 2;
1844	p->se.load.inv_weight = prio_to_wmult[0] >> 1;
1845	return;
1846	}
1847
1848	/*
1849	* SCHED_IDLE tasks get minimal weight:
1850	*/
1851	if (p->policy == SCHED_IDLE) {
1852	p->se.load.weight = WEIGHT_IDLEPRIO;
1853	p->se.load.inv_weight = WMULT_IDLEPRIO;
1854	return;
1855	}
1856
1857	p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
1858	p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
1859	}
1860
1861	static void update_avg(u64 *avg, u64 sample)
1862	{
1863	s64 diff = sample - *avg;
1864	*avg += diff >> 3;
1865	}
1866
1867	static void enqueue_task(struct rq rq, struct task_struct p, int wakeup)
1868	{
1869	if (wakeup)
1870	p->se.start_runtime = p->se.sum_exec_runtime;
1871
1872	sched_info_queued(p);
1873	p->sched_class->enqueue_task(rq, p, wakeup);
1874	p->se.on_rq = 1;
1875	}
1876
1877	static void dequeue_task(struct rq rq, struct task_struct p, int sleep)
1878	{
1879	if (sleep) {
1880	if (p->se.last_wakeup) {
1881	update_avg(&p->se.avg_overlap,
1882	p->se.sum_exec_runtime - p->se.last_wakeup);
1883	p->se.last_wakeup = 0;
1884	} else {
1885	update_avg(&p->se.avg_wakeup,
1886	sysctl_sched_wakeup_granularity);
1887	}
1888	}
1889
1890	sched_info_dequeued(p);
1891	p->sched_class->dequeue_task(rq, p, sleep);
1892	p->se.on_rq = 0;
1893	}
1894
1895	/*
1896	* __normal_prio - return the priority that is based on the static prio
1897	*/
1898	static inline int __normal_prio(struct task_struct *p)
1899	{
1900	return p->static_prio;
1901	}
1902
1903	/*
1904	* Calculate the expected normal priority: i.e. priority
1905	* without taking RT-inheritance into account. Might be
1906	* boosted by interactivity modifiers. Changes upon fork,
1907	* setprio syscalls, and whenever the interactivity
1908	* estimator recalculates.
1909	*/
1910	static inline int normal_prio(struct task_struct *p)
1911	{
1912	int prio;
1913
1914	if (task_has_rt_policy(p))
1915	prio = MAX_RT_PRIO-1 - p->rt_priority;
1916	else
1917	prio = __normal_prio(p);
1918	return prio;
1919	}
1920
1921	/*
1922	* Calculate the current priority, i.e. the priority
1923	* taken into account by the scheduler. This value might
1924	* be boosted by RT tasks, or might be boosted by
1925	* interactivity modifiers. Will be RT if the task got
1926	* RT-boosted. If not then it returns p->normal_prio.
1927	*/
1928	static int effective_prio(struct task_struct *p)
1929	{
1930	p->normal_prio = normal_prio(p);
1931	/*
1932	* If we are RT tasks or we were boosted to RT priority,
1933	* keep the priority unchanged. Otherwise, update priority
1934	* to the normal priority:
1935	*/
1936	if (!rt_prio(p->prio))
1937	return p->normal_prio;
1938	return p->prio;
1939	}
1940
1941	/*
1942	* activate_task - move a task to the runqueue.
1943	*/
1944	static void activate_task(struct rq rq, struct task_struct p, int wakeup)
1945	{
1946	if (task_contributes_to_load(p))
1947	rq->nr_uninterruptible--;
1948
1949	enqueue_task(rq, p, wakeup);
1950	inc_nr_running(rq);
1951	}
1952
1953	/*
1954	* deactivate_task - remove a task from the runqueue.
1955	*/
1956	static void deactivate_task(struct rq rq, struct task_struct p, int sleep)
1957	{
1958	if (task_contributes_to_load(p))
1959	rq->nr_uninterruptible++;
1960
1961	dequeue_task(rq, p, sleep);
1962	dec_nr_running(rq);
1963	}
1964
1965	/**
1966	* task_curr - is this task currently executing on a CPU?
1967	* @p: the task in question.
1968	*/
1969	inline int task_curr(const struct task_struct *p)
1970	{
1971	return cpu_curr(task_cpu(p)) == p;
1972	}
1973
1974	static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1975	{
1976	set_task_rq(p, cpu);
1977	#ifdef CONFIG_SMP
1978	/*
1979	* After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
1980	* successfuly executed on another CPU. We must ensure that updates of
1981	* per-task data have been completed by this moment.
1982	*/
1983	smp_wmb();
1984	task_thread_info(p)->cpu = cpu;
1985	#endif
1986	}
1987
1988	static inline void check_class_changed(struct rq rq, struct task_struct p,
1989	const struct sched_class *prev_class,
1990	int oldprio, int running)
1991	{
1992	if (prev_class != p->sched_class) {
1993	if (prev_class->switched_from)
1994	prev_class->switched_from(rq, p, running);
1995	p->sched_class->switched_to(rq, p, running);
1996	} else
1997	p->sched_class->prio_changed(rq, p, oldprio, running);
1998	}
1999
2000	/**
2001	* kthread_bind - bind a just-created kthread to a cpu.
2002	* @p: thread created by kthread_create().
2003	* @cpu: cpu (might not be online, must be possible) for @k to run on.
2004	*
2005	* Description: This function is equivalent to set_cpus_allowed(),
2006	* except that @cpu doesn't need to be online, and the thread must be
2007	* stopped (i.e., just returned from kthread_create()).
2008	*
2009	* Function lives here instead of kthread.c because it messes with
2010	* scheduler internals which require locking.
2011	*/
2012	void kthread_bind(struct task_struct *p, unsigned int cpu)
2013	{
2014	struct rq *rq = cpu_rq(cpu);
2015	unsigned long flags;
2016
2017	/* Must have done schedule() in kthread() before we set_task_cpu */
2018	if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) {
2019	WARN_ON(1);
2020	return;
2021	}
2022
2023	spin_lock_irqsave(&rq->lock, flags);
2024	set_task_cpu(p, cpu);
2025	p->cpus_allowed = cpumask_of_cpu(cpu);
2026	p->rt.nr_cpus_allowed = 1;
2027	p->flags \|= PF_THREAD_BOUND;
2028	spin_unlock_irqrestore(&rq->lock, flags);
2029	}
2030	EXPORT_SYMBOL(kthread_bind);
2031
2032	#ifdef CONFIG_SMP
2033	/*
2034	* Is this task likely cache-hot:
2035	*/
2036	static int
2037	task_hot(struct task_struct p, u64 now, struct sched_domain sd)
2038	{
2039	s64 delta;
2040
2041	if (p->sched_class != &fair_sched_class)
2042	return 0;
2043
2044	/*
2045	* Buddy candidates are cache hot:
2046	*/
2047	if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
2048	(&p->se == cfs_rq_of(&p->se)->next \|\|
2049	&p->se == cfs_rq_of(&p->se)->last))
2050	return 1;
2051
2052	if (sysctl_sched_migration_cost == -1)
2053	return 1;
2054	if (sysctl_sched_migration_cost == 0)
2055	return 0;
2056
2057	delta = now - p->se.exec_start;
2058
2059	return delta < (s64)sysctl_sched_migration_cost;
2060	}
2061
2062
2063	void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
2064	{
2065	int old_cpu = task_cpu(p);
2066	struct rq old_rq = cpu_rq(old_cpu), new_rq = cpu_rq(new_cpu);
2067	struct cfs_rq *old_cfsrq = task_cfs_rq(p),
2068	*new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
2069	u64 clock_offset;
2070
2071	clock_offset = old_rq->clock - new_rq->clock;
2072
2073	trace_sched_migrate_task(p, new_cpu);
2074
2075	#ifdef CONFIG_SCHEDSTATS
2076	if (p->se.wait_start)
2077	p->se.wait_start -= clock_offset;
2078	if (p->se.sleep_start)
2079	p->se.sleep_start -= clock_offset;
2080	if (p->se.block_start)
2081	p->se.block_start -= clock_offset;
2082	#endif
2083	if (old_cpu != new_cpu) {
2084	p->se.nr_migrations++;
2085	new_rq->nr_migrations_in++;
2086	#ifdef CONFIG_SCHEDSTATS
2087	if (task_hot(p, old_rq->clock, NULL))
2088	schedstat_inc(p, se.nr_forced2_migrations);
2089	#endif
2090	perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS,
2091	1, 1, NULL, 0);
2092	}
2093	p->se.vruntime -= old_cfsrq->min_vruntime -
2094	new_cfsrq->min_vruntime;
2095
2096	__set_task_cpu(p, new_cpu);
2097	}
2098
2099	struct migration_req {
2100	struct list_head list;
2101
2102	struct task_struct *task;
2103	int dest_cpu;
2104
2105	struct completion done;
2106	};
2107
2108	/*
2109	* The task's runqueue lock must be held.
2110	* Returns true if you have to wait for migration thread.
2111	*/
2112	static int
2113	migrate_task(struct task_struct p, int dest_cpu, struct migration_req req)
2114	{
2115	struct rq *rq = task_rq(p);
2116
2117	/*
2118	* If the task is not on a runqueue (and not running), then
2119	* it is sufficient to simply update the task's cpu field.
2120	*/
2121	if (!p->se.on_rq && !task_running(rq, p)) {
2122	set_task_cpu(p, dest_cpu);
2123	return 0;
2124	}
2125
2126	init_completion(&req->done);
2127	req->task = p;
2128	req->dest_cpu = dest_cpu;
2129	list_add(&req->list, &rq->migration_queue);
2130
2131	return 1;
2132	}
2133
2134	/*
2135	* wait_task_context_switch - wait for a thread to complete at least one
2136	* context switch.
2137	*
2138	* @p must not be current.
2139	*/
2140	void wait_task_context_switch(struct task_struct *p)
2141	{
2142	unsigned long nvcsw, nivcsw, flags;
2143	int running;
2144	struct rq *rq;
2145
2146	nvcsw = p->nvcsw;
2147	nivcsw = p->nivcsw;
2148	for (;;) {
2149	/*
2150	* The runqueue is assigned before the actual context
2151	* switch. We need to take the runqueue lock.
2152	*
2153	* We could check initially without the lock but it is
2154	* very likely that we need to take the lock in every
2155	* iteration.
2156	*/
2157	rq = task_rq_lock(p, &flags);
2158	running = task_running(rq, p);
2159	task_rq_unlock(rq, &flags);
2160
2161	if (likely(!running))
2162	break;
2163	/*
2164	* The switch count is incremented before the actual
2165	* context switch. We thus wait for two switches to be
2166	* sure at least one completed.
2167	*/
2168	if ((p->nvcsw - nvcsw) > 1)
2169	break;
2170	if ((p->nivcsw - nivcsw) > 1)
2171	break;
2172
2173	cpu_relax();
2174	}
2175	}
2176
2177	/*
2178	* wait_task_inactive - wait for a thread to unschedule.
2179	*
2180	* If @match_state is nonzero, it's the @p->state value just checked and
2181	* not expected to change. If it changes, i.e. @p might have woken up,
2182	* then return zero. When we succeed in waiting for @p to be off its CPU,
2183	* we return a positive number (its total switch count). If a second call
2184	* a short while later returns the same number, the caller can be sure that
2185	* @p has remained unscheduled the whole time.
2186	*
2187	* The caller must ensure that the task will unschedule sometime soon,
2188	* else this function might spin for a long time. This function can't
2189	* be called with interrupts off, or it may introduce deadlock with
2190	* smp_call_function() if an IPI is sent by the same process we are
2191	* waiting to become inactive.
2192	*/
2193	unsigned long wait_task_inactive(struct task_struct *p, long match_state)
2194	{
2195	unsigned long flags;
2196	int running, on_rq;
2197	unsigned long ncsw;
2198	struct rq *rq;
2199
2200	for (;;) {
2201	/*
2202	* We do the initial early heuristics without holding
2203	* any task-queue locks at all. We'll only try to get
2204	* the runqueue lock when things look like they will
2205	* work out!
2206	*/
2207	rq = task_rq(p);
2208
2209	/*
2210	* If the task is actively running on another CPU
2211	* still, just relax and busy-wait without holding
2212	* any locks.
2213	*
2214	* NOTE! Since we don't hold any locks, it's not
2215	* even sure that "rq" stays as the right runqueue!
2216	* But we don't care, since "task_running()" will
2217	* return false if the runqueue has changed and p
2218	* is actually now running somewhere else!
2219	*/
2220	while (task_running(rq, p)) {
2221	if (match_state && unlikely(p->state != match_state))
2222	return 0;
2223	cpu_relax();
2224	}
2225
2226	/*
2227	* Ok, time to look more closely! We need the rq
2228	* lock now, to be sure. If we're wrong, we'll
2229	* just go back and repeat.
2230	*/
2231	rq = task_rq_lock(p, &flags);
2232	trace_sched_wait_task(rq, p);
2233	running = task_running(rq, p);
2234	on_rq = p->se.on_rq;
2235	ncsw = 0;
2236	if (!match_state \|\| p->state == match_state)
2237	ncsw = p->nvcsw \| LONG_MIN; /* sets MSB */
2238	task_rq_unlock(rq, &flags);
2239
2240	/*
2241	* If it changed from the expected state, bail out now.
2242	*/
2243	if (unlikely(!ncsw))
2244	break;
2245
2246	/*
2247	* Was it really running after all now that we
2248	* checked with the proper locks actually held?
2249	*
2250	* Oops. Go back and try again..
2251	*/
2252	if (unlikely(running)) {
2253	cpu_relax();
2254	continue;
2255	}
2256
2257	/*
2258	* It's not enough that it's not actively running,
2259	* it must be off the runqueue _entirely_, and not
2260	* preempted!
2261	*
2262	* So if it was still runnable (but just not actively
2263	* running right now), it's preempted, and we should
2264	* yield - it could be a while.
2265	*/
2266	if (unlikely(on_rq)) {
2267	schedule_timeout_uninterruptible(1);
2268	continue;
2269	}
2270
2271	/*
2272	* Ahh, all good. It wasn't running, and it wasn't
2273	* runnable, which means that it will never become
2274	* running in the future either. We're all done!
2275	*/
2276	break;
2277	}
2278
2279	return ncsw;
2280	}
2281
2282	/***
2283	* kick_process - kick a running thread to enter/exit the kernel
2284	* @p: the to-be-kicked thread
2285	*
2286	* Cause a process which is running on another CPU to enter
2287	* kernel-mode, without any delay. (to get signals handled.)
2288	*
2289	* NOTE: this function doesnt have to take the runqueue lock,
2290	* because all it wants to ensure is that the remote task enters
2291	* the kernel. If the IPI races and the task has been migrated
2292	* to another CPU then no harm is done and the purpose has been
2293	* achieved as well.
2294	*/
2295	void kick_process(struct task_struct *p)
2296	{
2297	int cpu;
2298
2299	preempt_disable();
2300	cpu = task_cpu(p);
2301	if ((cpu != smp_processor_id()) && task_curr(p))
2302	smp_send_reschedule(cpu);
2303	preempt_enable();
2304	}
2305	EXPORT_SYMBOL_GPL(kick_process);
2306	#endif /* CONFIG_SMP */
2307
2308	/**
2309	* task_oncpu_function_call - call a function on the cpu on which a task runs
2310	* @p: the task to evaluate
2311	* @func: the function to be called
2312	* @info: the function call argument
2313	*
2314	* Calls the function @func when the task is currently running. This might
2315	* be on the current CPU, which just calls the function directly
2316	*/
2317	void task_oncpu_function_call(struct task_struct *p,
2318	void (func) (void info), void *info)
2319	{
2320	int cpu;
2321
2322	preempt_disable();
2323	cpu = task_cpu(p);
2324	if (task_curr(p))
2325	smp_call_function_single(cpu, func, info, 1);
2326	preempt_enable();
2327	}
2328
2329	/***
2330	* try_to_wake_up - wake up a thread
2331	* @p: the to-be-woken-up thread
2332	* @state: the mask of task states that can be woken
2333	* @sync: do a synchronous wakeup?
2334	*
2335	* Put it on the run-queue if it's not already there. The "current"
2336	* thread is always on the run-queue (except when the actual
2337	* re-schedule is in progress), and as such you're allowed to do
2338	* the simpler "current->state = TASK_RUNNING" to mark yourself
2339	* runnable without the overhead of this.
2340	*
2341	* returns failure only if the task is already active.
2342	*/
2343	static int try_to_wake_up(struct task_struct *p, unsigned int state,
2344	int wake_flags)
2345	{
2346	int cpu, orig_cpu, this_cpu, success = 0;
2347	unsigned long flags;
2348	struct rq rq, orig_rq;
2349
2350	if (!sched_feat(SYNC_WAKEUPS))
2351	wake_flags &= ~WF_SYNC;
2352
2353	this_cpu = get_cpu();
2354
2355	smp_wmb();
2356	rq = orig_rq = task_rq_lock(p, &flags);
2357	update_rq_clock(rq);
2358	if (!(p->state & state))
2359	goto out;
2360
2361	if (p->se.on_rq)
2362	goto out_running;
2363
2364	cpu = task_cpu(p);
2365	orig_cpu = cpu;
2366
2367	#ifdef CONFIG_SMP
2368	if (unlikely(task_running(rq, p)))
2369	goto out_activate;
2370
2371	/*
2372	* In order to handle concurrent wakeups and release the rq->lock
2373	* we put the task in TASK_WAKING state.
2374	*
2375	* First fix up the nr_uninterruptible count:
2376	*/
2377	if (task_contributes_to_load(p))
2378	rq->nr_uninterruptible--;
2379	p->state = TASK_WAKING;
2380	task_rq_unlock(rq, &flags);
2381
2382	cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
2383	if (cpu != orig_cpu)
2384	set_task_cpu(p, cpu);
2385
2386	rq = task_rq_lock(p, &flags);
2387
2388	if (rq != orig_rq)
2389	update_rq_clock(rq);
2390
2391	WARN_ON(p->state != TASK_WAKING);
2392	cpu = task_cpu(p);
2393
2394	#ifdef CONFIG_SCHEDSTATS
2395	schedstat_inc(rq, ttwu_count);
2396	if (cpu == this_cpu)
2397	schedstat_inc(rq, ttwu_local);
2398	else {
2399	struct sched_domain *sd;
2400	for_each_domain(this_cpu, sd) {
2401	if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2402	schedstat_inc(sd, ttwu_wake_remote);
2403	break;
2404	}
2405	}
2406	}
2407	#endif /* CONFIG_SCHEDSTATS */
2408
2409	out_activate:
2410	#endif /* CONFIG_SMP */
2411	schedstat_inc(p, se.nr_wakeups);
2412	if (wake_flags & WF_SYNC)
2413	schedstat_inc(p, se.nr_wakeups_sync);
2414	if (orig_cpu != cpu)
2415	schedstat_inc(p, se.nr_wakeups_migrate);
2416	if (cpu == this_cpu)
2417	schedstat_inc(p, se.nr_wakeups_local);
2418	else
2419	schedstat_inc(p, se.nr_wakeups_remote);
2420	activate_task(rq, p, 1);
2421	success = 1;
2422
2423	/*
2424	* Only attribute actual wakeups done by this task.
2425	*/
2426	if (!in_interrupt()) {
2427	struct sched_entity *se = &current->se;
2428	u64 sample = se->sum_exec_runtime;
2429
2430	if (se->last_wakeup)
2431	sample -= se->last_wakeup;
2432	else
2433	sample -= se->start_runtime;
2434	update_avg(&se->avg_wakeup, sample);
2435
2436	se->last_wakeup = se->sum_exec_runtime;
2437	}
2438
2439	out_running:
2440	trace_sched_wakeup(rq, p, success);
2441	check_preempt_curr(rq, p, wake_flags);
2442
2443	p->state = TASK_RUNNING;
2444	#ifdef CONFIG_SMP
2445	if (p->sched_class->task_wake_up)
2446	p->sched_class->task_wake_up(rq, p);
2447
2448	if (unlikely(rq->idle_stamp)) {
2449	u64 delta = rq->clock - rq->idle_stamp;
2450	u64 max = 2*sysctl_sched_migration_cost;
2451
2452	if (delta > max)
2453	rq->avg_idle = max;
2454	else
2455	update_avg(&rq->avg_idle, delta);
2456	rq->idle_stamp = 0;
2457	}
2458	#endif
2459	out:
2460	task_rq_unlock(rq, &flags);
2461	put_cpu();
2462
2463	return success;
2464	}
2465
2466	/**
2467	* wake_up_process - Wake up a specific process
2468	* @p: The process to be woken up.
2469	*
2470	* Attempt to wake up the nominated process and move it to the set of runnable
2471	* processes. Returns 1 if the process was woken up, 0 if it was already
2472	* running.
2473	*
2474	* It may be assumed that this function implies a write memory barrier before
2475	* changing the task state if and only if any tasks are woken up.
2476	*/
2477	int wake_up_process(struct task_struct *p)
2478	{
2479	return try_to_wake_up(p, TASK_ALL, 0);
2480	}
2481	EXPORT_SYMBOL(wake_up_process);
2482
2483	int wake_up_state(struct task_struct *p, unsigned int state)
2484	{
2485	return try_to_wake_up(p, state, 0);
2486	}
2487
2488	/*
2489	* Perform scheduler related setup for a newly forked process p.
2490	* p is forked by current.
2491	*
2492	* __sched_fork() is basic setup used by init_idle() too:
2493	*/
2494	static void __sched_fork(struct task_struct *p)
2495	{
2496	p->se.exec_start = 0;
2497	p->se.sum_exec_runtime = 0;
2498	p->se.prev_sum_exec_runtime = 0;
2499	p->se.nr_migrations = 0;
2500	p->se.last_wakeup = 0;
2501	p->se.avg_overlap = 0;
2502	p->se.start_runtime = 0;
2503	p->se.avg_wakeup = sysctl_sched_wakeup_granularity;
2504	p->se.avg_running = 0;
2505
2506	#ifdef CONFIG_SCHEDSTATS
2507	p->se.wait_start = 0;
2508	p->se.wait_max = 0;
2509	p->se.wait_count = 0;
2510	p->se.wait_sum = 0;
2511
2512	p->se.sleep_start = 0;
2513	p->se.sleep_max = 0;
2514	p->se.sum_sleep_runtime = 0;
2515
2516	p->se.block_start = 0;
2517	p->se.block_max = 0;
2518	p->se.exec_max = 0;
2519	p->se.slice_max = 0;
2520
2521	p->se.nr_migrations_cold = 0;
2522	p->se.nr_failed_migrations_affine = 0;
2523	p->se.nr_failed_migrations_running = 0;
2524	p->se.nr_failed_migrations_hot = 0;
2525	p->se.nr_forced_migrations = 0;
2526	p->se.nr_forced2_migrations = 0;
2527
2528	p->se.nr_wakeups = 0;
2529	p->se.nr_wakeups_sync = 0;
2530	p->se.nr_wakeups_migrate = 0;
2531	p->se.nr_wakeups_local = 0;
2532	p->se.nr_wakeups_remote = 0;
2533	p->se.nr_wakeups_affine = 0;
2534	p->se.nr_wakeups_affine_attempts = 0;
2535	p->se.nr_wakeups_passive = 0;
2536	p->se.nr_wakeups_idle = 0;
2537
2538	#endif
2539
2540	INIT_LIST_HEAD(&p->rt.run_list);
2541	p->se.on_rq = 0;
2542	INIT_LIST_HEAD(&p->se.group_node);
2543
2544	#ifdef CONFIG_PREEMPT_NOTIFIERS
2545	INIT_HLIST_HEAD(&p->preempt_notifiers);
2546	#endif
2547
2548	/*
2549	* We mark the process as running here, but have not actually
2550	* inserted it onto the runqueue yet. This guarantees that
2551	* nobody will actually run it, and a signal or other external
2552	* event cannot wake it up and insert it on the runqueue either.
2553	*/
2554	p->state = TASK_RUNNING;
2555	}
2556
2557	/*
2558	* fork()/clone()-time setup:
2559	*/
2560	void sched_fork(struct task_struct *p, int clone_flags)
2561	{
2562	int cpu = get_cpu();
2563
2564	__sched_fork(p);
2565
2566	/*
2567	* Revert to default priority/policy on fork if requested.
2568	*/
2569	if (unlikely(p->sched_reset_on_fork)) {
2570	if (p->policy == SCHED_FIFO \|\| p->policy == SCHED_RR) {
2571	p->policy = SCHED_NORMAL;
2572	p->normal_prio = p->static_prio;
2573	}
2574
2575	if (PRIO_TO_NICE(p->static_prio) < 0) {
2576	p->static_prio = NICE_TO_PRIO(0);
2577	p->normal_prio = p->static_prio;
2578	set_load_weight(p);
2579	}
2580
2581	/*
2582	* We don't need the reset flag anymore after the fork. It has
2583	* fulfilled its duty:
2584	*/
2585	p->sched_reset_on_fork = 0;
2586	}
2587
2588	/*
2589	* Make sure we do not leak PI boosting priority to the child.
2590	*/
2591	p->prio = current->normal_prio;
2592
2593	if (!rt_prio(p->prio))
2594	p->sched_class = &fair_sched_class;
2595
2596	#ifdef CONFIG_SMP
2597	cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0);
2598	#endif
2599	set_task_cpu(p, cpu);
2600
2601	#if defined(CONFIG_SCHEDSTATS) \|\| defined(CONFIG_TASK_DELAY_ACCT)
2602	if (likely(sched_info_on()))
2603	memset(&p->sched_info, 0, sizeof(p->sched_info));
2604	#endif
2605	#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
2606	p->oncpu = 0;
2607	#endif
2608	#ifdef CONFIG_PREEMPT
2609	/* Want to start with kernel preemption disabled. */
2610	task_thread_info(p)->preempt_count = 1;
2611	#endif
2612	plist_node_init(&p->pushable_tasks, MAX_PRIO);
2613
2614	put_cpu();
2615	}
2616
2617	/*
2618	* wake_up_new_task - wake up a newly created task for the first time.
2619	*
2620	* This function will do some initial scheduler statistics housekeeping
2621	* that must be done for every newly created context, then puts the task
2622	* on the runqueue and wakes it.
2623	*/
2624	void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2625	{
2626	unsigned long flags;
2627	struct rq *rq;
2628
2629	rq = task_rq_lock(p, &flags);
2630	BUG_ON(p->state != TASK_RUNNING);
2631	update_rq_clock(rq);
2632
2633	if (!p->sched_class->task_new \|\| !current->se.on_rq) {
2634	activate_task(rq, p, 0);
2635	} else {
2636	/*
2637	* Let the scheduling class do new task startup
2638	* management (if any):
2639	*/
2640	p->sched_class->task_new(rq, p);
2641	inc_nr_running(rq);
2642	}
2643	trace_sched_wakeup_new(rq, p, 1);
2644	check_preempt_curr(rq, p, WF_FORK);
2645	#ifdef CONFIG_SMP
2646	if (p->sched_class->task_wake_up)
2647	p->sched_class->task_wake_up(rq, p);
2648	#endif
2649	task_rq_unlock(rq, &flags);
2650	}
2651
2652	#ifdef CONFIG_PREEMPT_NOTIFIERS
2653
2654	/**
2655	* preempt_notifier_register - tell me when current is being preempted & rescheduled
2656	* @notifier: notifier struct to register
2657	*/
2658	void preempt_notifier_register(struct preempt_notifier *notifier)
2659	{
2660	hlist_add_head(&notifier->link, &current->preempt_notifiers);
2661	}
2662	EXPORT_SYMBOL_GPL(preempt_notifier_register);
2663
2664	/**
2665	* preempt_notifier_unregister - no longer interested in preemption notifications
2666	* @notifier: notifier struct to unregister
2667	*
2668	* This is safe to call from within a preemption notifier.
2669	*/
2670	void preempt_notifier_unregister(struct preempt_notifier *notifier)
2671	{
2672	hlist_del(&notifier->link);
2673	}
2674	EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2675
2676	static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2677	{
2678	struct preempt_notifier *notifier;
2679	struct hlist_node *node;
2680
2681	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
2682	notifier->ops->sched_in(notifier, raw_smp_processor_id());
2683	}
2684
2685	static void
2686	fire_sched_out_preempt_notifiers(struct task_struct *curr,
2687	struct task_struct *next)
2688	{
2689	struct preempt_notifier *notifier;
2690	struct hlist_node *node;
2691
2692	hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
2693	notifier->ops->sched_out(notifier, next);
2694	}
2695
2696	#else /* !CONFIG_PREEMPT_NOTIFIERS */
2697
2698	static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2699	{
2700	}
2701
2702	static void
2703	fire_sched_out_preempt_notifiers(struct task_struct *curr,
2704	struct task_struct *next)
2705	{
2706	}
2707
2708	#endif /* CONFIG_PREEMPT_NOTIFIERS */
2709
2710	/**
2711	* prepare_task_switch - prepare to switch tasks
2712	* @rq: the runqueue preparing to switch
2713	* @prev: the current task that is being switched out
2714	* @next: the task we are going to switch to.
2715	*
2716	* This is called with the rq lock held and interrupts off. It must
2717	* be paired with a subsequent finish_task_switch after the context
2718	* switch.
2719	*
2720	* prepare_task_switch sets up locking and calls architecture specific
2721	* hooks.
2722	*/
2723	static inline void
2724	prepare_task_switch(struct rq rq, struct task_struct prev,
2725	struct task_struct *next)
2726	{
2727	fire_sched_out_preempt_notifiers(prev, next);
2728	prepare_lock_switch(rq, next);
2729	prepare_arch_switch(next);
2730	}
2731
2732	/**
2733	* finish_task_switch - clean up after a task-switch
2734	* @rq: runqueue associated with task-switch
2735	* @prev: the thread we just switched away from.
2736	*
2737	* finish_task_switch must be called after the context switch, paired
2738	* with a prepare_task_switch call before the context switch.
2739	* finish_task_switch will reconcile locking set up by prepare_task_switch,
2740	* and do any other architecture-specific cleanup actions.
2741	*
2742	* Note that we may have delayed dropping an mm in context_switch(). If
2743	* so, we finish that here outside of the runqueue lock. (Doing it
2744	* with the lock held can cause deadlocks; see schedule() for
2745	* details.)
2746	*/
2747	static void finish_task_switch(struct rq rq, struct task_struct prev)
2748	__releases(rq->lock)
2749	{
2750	struct mm_struct *mm = rq->prev_mm;
2751	long prev_state;
2752
2753	rq->prev_mm = NULL;
2754
2755	/*
2756	* A task struct has one reference for the use as "current".
2757	* If a task dies, then it sets TASK_DEAD in tsk->state and calls
2758	* schedule one last time. The schedule call will never return, and
2759	* the scheduled task must drop that reference.
2760	* The test for TASK_DEAD must occur while the runqueue locks are
2761	* still held, otherwise prev could be scheduled on another cpu, die
2762	* there before we look at prev->state, and then the reference would
2763	* be dropped twice.
2764	* Manfred Spraul <manfred@colorfullife.com>
2765	*/
2766	prev_state = prev->state;
2767	finish_arch_switch(prev);
2768	perf_event_task_sched_in(current, cpu_of(rq));
2769	finish_lock_switch(rq, prev);
2770
2771	fire_sched_in_preempt_notifiers(current);
2772	if (mm)
2773	mmdrop(mm);
2774	if (unlikely(prev_state == TASK_DEAD)) {
2775	/*
2776	* Remove function-return probe instances associated with this
2777	* task and put them back on the free list.
2778	*/
2779	kprobe_flush_task(prev);
2780	put_task_struct(prev);
2781	}
2782	}
2783
2784	#ifdef CONFIG_SMP
2785
2786	/* assumes rq->lock is held */
2787	static inline void pre_schedule(struct rq rq, struct task_struct prev)
2788	{
2789	if (prev->sched_class->pre_schedule)
2790	prev->sched_class->pre_schedule(rq, prev);
2791	}
2792
2793	/* rq->lock is NOT held, but preemption is disabled */
2794	static inline void post_schedule(struct rq *rq)
2795	{
2796	if (rq->post_schedule) {
2797	unsigned long flags;
2798
2799	spin_lock_irqsave(&rq->lock, flags);
2800	if (rq->curr->sched_class->post_schedule)
2801	rq->curr->sched_class->post_schedule(rq);
2802	spin_unlock_irqrestore(&rq->lock, flags);
2803
2804	rq->post_schedule = 0;
2805	}
2806	}
2807
2808	#else
2809
2810	static inline void pre_schedule(struct rq rq, struct task_struct p)
2811	{
2812	}
2813
2814	static inline void post_schedule(struct rq *rq)
2815	{
2816	}
2817
2818	#endif
2819
2820	/**
2821	* schedule_tail - first thing a freshly forked thread must call.
2822	* @prev: the thread we just switched away from.
2823	*/
2824	asmlinkage void schedule_tail(struct task_struct *prev)
2825	__releases(rq->lock)
2826	{
2827	struct rq *rq = this_rq();
2828
2829	finish_task_switch(rq, prev);
2830
2831	/*
2832	* FIXME: do we need to worry about rq being invalidated by the
2833	* task_switch?
2834	*/
2835	post_schedule(rq);
2836
2837	#ifdef __ARCH_WANT_UNLOCKED_CTXSW
2838	/* In this case, finish_task_switch does not reenable preemption */
2839	preempt_enable();
2840	#endif
2841	if (current->set_child_tid)
2842	put_user(task_pid_vnr(current), current->set_child_tid);
2843	}
2844
2845	/*
2846	* context_switch - switch to the new MM and the new
2847	* thread's register state.
2848	*/
2849	static inline void
2850	context_switch(struct rq rq, struct task_struct prev,
2851	struct task_struct *next)
2852	{
2853	struct mm_struct mm, oldmm;
2854
2855	prepare_task_switch(rq, prev, next);
2856	trace_sched_switch(rq, prev, next);
2857	mm = next->mm;
2858	oldmm = prev->active_mm;
2859	/*
2860	* For paravirt, this is coupled with an exit in switch_to to
2861	* combine the page table reload and the switch backend into
2862	* one hypercall.
2863	*/
2864	arch_start_context_switch(prev);
2865
2866	if (unlikely(!mm)) {
2867	next->active_mm = oldmm;
2868	atomic_inc(&oldmm->mm_count);
2869	enter_lazy_tlb(oldmm, next);
2870	} else
2871	switch_mm(oldmm, mm, next);
2872
2873	if (unlikely(!prev->mm)) {
2874	prev->active_mm = NULL;
2875	rq->prev_mm = oldmm;
2876	}
2877	/*
2878	* Since the runqueue lock will be released by the next
2879	* task (which is an invalid locking op but in the case
2880	* of the scheduler it's an obvious special-case), so we
2881	* do an early lockdep release here:
2882	*/
2883	#ifndef __ARCH_WANT_UNLOCKED_CTXSW
2884	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
2885	#endif
2886
2887	/* Here we just switch the register state and the stack. */
2888	switch_to(prev, next, prev);
2889
2890	barrier();
2891	/*
2892	* this_rq must be evaluated again because prev may have moved
2893	* CPUs since it called schedule(), thus the 'rq' on its stack
2894	* frame will be invalid.
2895	*/
2896	finish_task_switch(this_rq(), prev);
2897	}
2898
2899	/*
2900	* nr_running, nr_uninterruptible and nr_context_switches:
2901	*
2902	* externally visible scheduler statistics: current number of runnable
2903	* threads, current number of uninterruptible-sleeping threads, total
2904	* number of context switches performed since bootup.
2905	*/
2906	unsigned long nr_running(void)
2907	{
2908	unsigned long i, sum = 0;
2909
2910	for_each_online_cpu(i)
2911	sum += cpu_rq(i)->nr_running;
2912
2913	return sum;
2914	}
2915
2916	unsigned long nr_uninterruptible(void)
2917	{
2918	unsigned long i, sum = 0;
2919
2920	for_each_possible_cpu(i)
2921	sum += cpu_rq(i)->nr_uninterruptible;
2922
2923	/*
2924	* Since we read the counters lockless, it might be slightly
2925	* inaccurate. Do not allow it to go below zero though:
2926	*/
2927	if (unlikely((long)sum < 0))
2928	sum = 0;
2929
2930	return sum;
2931	}
2932
2933	unsigned long long nr_context_switches(void)
2934	{
2935	int i;
2936	unsigned long long sum = 0;
2937
2938	for_each_possible_cpu(i)
2939	sum += cpu_rq(i)->nr_switches;
2940
2941	return sum;
2942	}
2943
2944	unsigned long nr_iowait(void)
2945	{
2946	unsigned long i, sum = 0;
2947
2948	for_each_possible_cpu(i)
2949	sum += atomic_read(&cpu_rq(i)->nr_iowait);
2950
2951	return sum;
2952	}
2953
2954	unsigned long nr_iowait_cpu(void)
2955	{
2956	struct rq *this = this_rq();
2957	return atomic_read(&this->nr_iowait);
2958	}
2959
2960	unsigned long this_cpu_load(void)
2961	{
2962	struct rq *this = this_rq();
2963	return this->cpu_load[0];
2964	}
2965
2966
2967	/* Variables and functions for calc_load */
2968	static atomic_long_t calc_load_tasks;
2969	static unsigned long calc_load_update;
2970	unsigned long avenrun[3];
2971	EXPORT_SYMBOL(avenrun);
2972
2973	/**
2974	* get_avenrun - get the load average array
2975	* @loads: pointer to dest load array
2976	* @offset: offset to add
2977	* @shift: shift count to shift the result left
2978	*
2979	* These values are estimates at best, so no need for locking.
2980	*/
2981	void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
2982	{
2983	loads[0] = (avenrun[0] + offset) << shift;
2984	loads[1] = (avenrun[1] + offset) << shift;
2985	loads[2] = (avenrun[2] + offset) << shift;
2986	}
2987
2988	static unsigned long
2989	calc_load(unsigned long load, unsigned long exp, unsigned long active)
2990	{
2991	load *= exp;
2992	load += active * (FIXED_1 - exp);
2993	return load >> FSHIFT;
2994	}
2995
2996	/*
2997	* calc_load - update the avenrun load estimates 10 ticks after the
2998	* CPUs have updated calc_load_tasks.
2999	*/
3000	void calc_global_load(void)
3001	{
3002	unsigned long upd = calc_load_update + 10;
3003	long active;
3004
3005	if (time_before(jiffies, upd))
3006	return;
3007
3008	active = atomic_long_read(&calc_load_tasks);
3009	active = active > 0 ? active * FIXED_1 : 0;
3010
3011	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
3012	avenrun[1] = calc_load(avenrun[1], EXP_5, active);
3013	avenrun[2] = calc_load(avenrun[2], EXP_15, active);
3014
3015	calc_load_update += LOAD_FREQ;
3016	}
3017
3018	/*
3019	* Either called from update_cpu_load() or from a cpu going idle
3020	*/
3021	static void calc_load_account_active(struct rq *this_rq)
3022	{
3023	long nr_active, delta;
3024
3025	nr_active = this_rq->nr_running;
3026	nr_active += (long) this_rq->nr_uninterruptible;
3027
3028	if (nr_active != this_rq->calc_load_active) {
3029	delta = nr_active - this_rq->calc_load_active;
3030	this_rq->calc_load_active = nr_active;
3031	atomic_long_add(delta, &calc_load_tasks);
3032	}
3033	}
3034
3035	/*
3036	* Externally visible per-cpu scheduler statistics:
3037	* cpu_nr_migrations(cpu) - number of migrations into that cpu
3038	*/
3039	u64 cpu_nr_migrations(int cpu)
3040	{
3041	return cpu_rq(cpu)->nr_migrations_in;
3042	}
3043
3044	/*
3045	* Update rq->cpu_load[] statistics. This function is usually called every
3046	* scheduler tick (TICK_NSEC).
3047	*/
3048	static void update_cpu_load(struct rq *this_rq)
3049	{
3050	unsigned long this_load = this_rq->load.weight;
3051	int i, scale;
3052
3053	this_rq->nr_load_updates++;
3054
3055	/* Update our load: */
3056	for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
3057	unsigned long old_load, new_load;
3058
3059	/* scale is effectively 1 << i now, and >> i divides by scale */
3060
3061	old_load = this_rq->cpu_load[i];
3062	new_load = this_load;
3063	/*
3064	* Round up the averaging division if load is increasing. This
3065	* prevents us from getting stuck on 9 if the load is 10, for
3066	* example.
3067	*/
3068	if (new_load > old_load)
3069	new_load += scale-1;
3070	this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
3071	}
3072
3073	if (time_after_eq(jiffies, this_rq->calc_load_update)) {
3074	this_rq->calc_load_update += LOAD_FREQ;
3075	calc_load_account_active(this_rq);
3076	}
3077	}
3078
3079	#ifdef CONFIG_SMP
3080
3081	/*
3082	* double_rq_lock - safely lock two runqueues
3083	*
3084	* Note this does not disable interrupts like task_rq_lock,
3085	* you need to do so manually before calling.
3086	*/
3087	static void double_rq_lock(struct rq rq1, struct rq rq2)
3088	__acquires(rq1->lock)
3089	__acquires(rq2->lock)
3090	{
3091	BUG_ON(!irqs_disabled());
3092	if (rq1 == rq2) {
3093	spin_lock(&rq1->lock);
3094	__acquire(rq2->lock); /* Fake it out ;) */
3095	} else {
3096	if (rq1 < rq2) {
3097	spin_lock(&rq1->lock);
3098	spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
3099	} else {
3100	spin_lock(&rq2->lock);
3101	spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
3102	}
3103	}
3104	update_rq_clock(rq1);
3105	update_rq_clock(rq2);
3106	}
3107
3108	/*
3109	* double_rq_unlock - safely unlock two runqueues
3110	*
3111	* Note this does not restore interrupts like task_rq_unlock,
3112	* you need to do so manually after calling.
3113	*/
3114	static void double_rq_unlock(struct rq rq1, struct rq rq2)
3115	__releases(rq1->lock)
3116	__releases(rq2->lock)
3117	{
3118	spin_unlock(&rq1->lock);
3119	if (rq1 != rq2)
3120	spin_unlock(&rq2->lock);
3121	else
3122	__release(rq2->lock);
3123	}
3124
3125	/*
3126	* If dest_cpu is allowed for this process, migrate the task to it.
3127	* This is accomplished by forcing the cpu_allowed mask to only
3128	* allow dest_cpu, which will force the cpu onto dest_cpu. Then
3129	* the cpu_allowed mask is restored.
3130	*/
3131	static void sched_migrate_task(struct task_struct *p, int dest_cpu)
3132	{
3133	struct migration_req req;
3134	unsigned long flags;
3135	struct rq *rq;
3136
3137	rq = task_rq_lock(p, &flags);
3138	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
3139	\|\| unlikely(!cpu_active(dest_cpu)))
3140	goto out;
3141
3142	/* force the process onto the specified CPU */
3143	if (migrate_task(p, dest_cpu, &req)) {
3144	/* Need to wait for migration thread (might exit: take ref). */
3145	struct task_struct *mt = rq->migration_thread;
3146
3147	get_task_struct(mt);
3148	task_rq_unlock(rq, &flags);
3149	wake_up_process(mt);
3150	put_task_struct(mt);
3151	wait_for_completion(&req.done);
3152
3153	return;
3154	}
3155	out:
3156	task_rq_unlock(rq, &flags);
3157	}
3158
3159	/*
3160	* sched_exec - execve() is a valuable balancing opportunity, because at
3161	* this point the task has the smallest effective memory and cache footprint.
3162	*/
3163	void sched_exec(void)
3164	{
3165	int new_cpu, this_cpu = get_cpu();
3166	new_cpu = current->sched_class->select_task_rq(current, SD_BALANCE_EXEC, 0);
3167	put_cpu();
3168	if (new_cpu != this_cpu)
3169	sched_migrate_task(current, new_cpu);
3170	}
3171
3172	/*
3173	* pull_task - move a task from a remote runqueue to the local runqueue.
3174	* Both runqueues must be locked.
3175	*/
3176	static void pull_task(struct rq src_rq, struct task_struct p,
3177	struct rq *this_rq, int this_cpu)
3178	{
3179	deactivate_task(src_rq, p, 0);
3180	set_task_cpu(p, this_cpu);
3181	activate_task(this_rq, p, 0);
3182	check_preempt_curr(this_rq, p, 0);
3183	}
3184
3185	/*
3186	* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
3187	*/
3188	static
3189	int can_migrate_task(struct task_struct p, struct rq rq, int this_cpu,
3190	struct sched_domain *sd, enum cpu_idle_type idle,
3191	int *all_pinned)
3192	{
3193	int tsk_cache_hot = 0;
3194	/*
3195	* We do not migrate tasks that are:
3196	* 1) running (obviously), or
3197	* 2) cannot be migrated to this CPU due to cpus_allowed, or
3198	* 3) are cache-hot on their current CPU.
3199	*/
3200	if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
3201	schedstat_inc(p, se.nr_failed_migrations_affine);
3202	return 0;
3203	}
3204	*all_pinned = 0;
3205
3206	if (task_running(rq, p)) {
3207	schedstat_inc(p, se.nr_failed_migrations_running);
3208	return 0;
3209	}
3210
3211	/*
3212	* Aggressive migration if:
3213	* 1) task is cache cold, or
3214	* 2) too many balance attempts have failed.
3215	*/
3216
3217	tsk_cache_hot = task_hot(p, rq->clock, sd);
3218	if (!tsk_cache_hot \|\|
3219	sd->nr_balance_failed > sd->cache_nice_tries) {
3220	#ifdef CONFIG_SCHEDSTATS
3221	if (tsk_cache_hot) {
3222	schedstat_inc(sd, lb_hot_gained[idle]);
3223	schedstat_inc(p, se.nr_forced_migrations);
3224	}
3225	#endif
3226	return 1;
3227	}
3228
3229	if (tsk_cache_hot) {
3230	schedstat_inc(p, se.nr_failed_migrations_hot);
3231	return 0;
3232	}
3233	return 1;
3234	}
3235
3236	static unsigned long
3237	balance_tasks(struct rq this_rq, int this_cpu, struct rq busiest,
3238	unsigned long max_load_move, struct sched_domain *sd,
3239	enum cpu_idle_type idle, int *all_pinned,
3240	int this_best_prio, struct rq_iterator iterator)
3241	{
3242	int loops = 0, pulled = 0, pinned = 0;
3243	struct task_struct *p;
3244	long rem_load_move = max_load_move;
3245
3246	if (max_load_move == 0)
3247	goto out;
3248
3249	pinned = 1;
3250
3251	/*
3252	* Start the load-balancing iterator:
3253	*/
3254	p = iterator->start(iterator->arg);
3255	next:
3256	if (!p \|\| loops++ > sysctl_sched_nr_migrate)
3257	goto out;
3258
3259	if ((p->se.load.weight >> 1) > rem_load_move \|\|
3260	!can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
3261	p = iterator->next(iterator->arg);
3262	goto next;
3263	}
3264
3265	pull_task(busiest, p, this_rq, this_cpu);
3266	pulled++;
3267	rem_load_move -= p->se.load.weight;
3268
3269	#ifdef CONFIG_PREEMPT
3270	/*
3271	* NEWIDLE balancing is a source of latency, so preemptible kernels
3272	* will stop after the first task is pulled to minimize the critical
3273	* section.
3274	*/
3275	if (idle == CPU_NEWLY_IDLE)
3276	goto out;
3277	#endif
3278
3279	/*
3280	* We only want to steal up to the prescribed amount of weighted load.
3281	*/
3282	if (rem_load_move > 0) {
3283	if (p->prio < *this_best_prio)
3284	*this_best_prio = p->prio;
3285	p = iterator->next(iterator->arg);
3286	goto next;
3287	}
3288	out:
3289	/*
3290	* Right now, this is one of only two places pull_task() is called,
3291	* so we can safely collect pull_task() stats here rather than
3292	* inside pull_task().
3293	*/
3294	schedstat_add(sd, lb_gained[idle], pulled);
3295
3296	if (all_pinned)
3297	*all_pinned = pinned;
3298
3299	return max_load_move - rem_load_move;
3300	}
3301
3302	/*
3303	* move_tasks tries to move up to max_load_move weighted load from busiest to
3304	* this_rq, as part of a balancing operation within domain "sd".
3305	* Returns 1 if successful and 0 otherwise.
3306	*
3307	* Called with both runqueues locked.
3308	*/
3309	static int move_tasks(struct rq this_rq, int this_cpu, struct rq busiest,
3310	unsigned long max_load_move,
3311	struct sched_domain *sd, enum cpu_idle_type idle,
3312	int *all_pinned)
3313	{
3314	const struct sched_class *class = sched_class_highest;
3315	unsigned long total_load_moved = 0;
3316	int this_best_prio = this_rq->curr->prio;
3317
3318	do {
3319	total_load_moved +=
3320	class->load_balance(this_rq, this_cpu, busiest,
3321	max_load_move - total_load_moved,
3322	sd, idle, all_pinned, &this_best_prio);
3323	class = class->next;
3324
3325	#ifdef CONFIG_PREEMPT
3326	/*
3327	* NEWIDLE balancing is a source of latency, so preemptible
3328	* kernels will stop after the first task is pulled to minimize
3329	* the critical section.
3330	*/
3331	if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
3332	break;
3333	#endif
3334	} while (class && max_load_move > total_load_moved);
3335
3336	return total_load_moved > 0;
3337	}
3338
3339	static int
3340	iter_move_one_task(struct rq this_rq, int this_cpu, struct rq busiest,
3341	struct sched_domain *sd, enum cpu_idle_type idle,
3342	struct rq_iterator *iterator)
3343	{
3344	struct task_struct *p = iterator->start(iterator->arg);
3345	int pinned = 0;
3346
3347	while (p) {
3348	if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
3349	pull_task(busiest, p, this_rq, this_cpu);
3350	/*
3351	* Right now, this is only the second place pull_task()
3352	* is called, so we can safely collect pull_task()
3353	* stats here rather than inside pull_task().
3354	*/
3355	schedstat_inc(sd, lb_gained[idle]);
3356
3357	return 1;
3358	}
3359	p = iterator->next(iterator->arg);
3360	}
3361
3362	return 0;
3363	}
3364
3365	/*
3366	* move_one_task tries to move exactly one task from busiest to this_rq, as
3367	* part of active balancing operations within "domain".
3368	* Returns 1 if successful and 0 otherwise.
3369	*
3370	* Called with both runqueues locked.
3371	*/
3372	static int move_one_task(struct rq this_rq, int this_cpu, struct rq busiest,
3373	struct sched_domain *sd, enum cpu_idle_type idle)
3374	{
3375	const struct sched_class *class;
3376
3377	for_each_class(class) {
3378	if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
3379	return 1;
3380	}
3381
3382	return 0;
3383	}
3384	/******** Helpers for find_busiest_group **********************/
3385	/*
3386	* sd_lb_stats - Structure to store the statistics of a sched_domain
3387	* during load balancing.
3388	*/
3389	struct sd_lb_stats {
3390	struct sched_group busiest; / Busiest group in this sd */
3391	struct sched_group this; / Local group in this sd */
3392	unsigned long total_load; /* Total load of all groups in sd */
3393	unsigned long total_pwr; /* Total power of all groups in sd */
3394	unsigned long avg_load; /* Average load across all groups in sd */
3395
3396	/** Statistics of this group */
3397	unsigned long this_load;
3398	unsigned long this_load_per_task;
3399	unsigned long this_nr_running;
3400
3401	/* Statistics of the busiest group */
3402	unsigned long max_load;
3403	unsigned long busiest_load_per_task;
3404	unsigned long busiest_nr_running;
3405
3406	int group_imb; /* Is there imbalance in this sd */
3407	#if defined(CONFIG_SCHED_MC) \|\| defined(CONFIG_SCHED_SMT)
3408	int power_savings_balance; /* Is powersave balance needed for this sd */
3409	struct sched_group group_min; / Least loaded group in sd */
3410	struct sched_group group_leader; / Group which relieves group_min */
3411	unsigned long min_load_per_task; /* load_per_task in group_min */
3412	unsigned long leader_nr_running; /* Nr running of group_leader */
3413	unsigned long min_nr_running; /* Nr running of group_min */
3414	#endif
3415	};
3416
3417	/*
3418	* sg_lb_stats - stats of a sched_group required for load_balancing
3419	*/
3420	struct sg_lb_stats {
3421	unsigned long avg_load; /Avg load across the CPUs of the group /
3422	unsigned long group_load; /* Total load over the CPUs of the group */
3423	unsigned long sum_nr_running; /* Nr tasks running in the group */
3424	unsigned long sum_weighted_load; /* Weighted load of group's tasks */
3425	unsigned long group_capacity;
3426	int group_imb; /* Is there an imbalance in the group ? */
3427	};
3428
3429	/**
3430	* group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
3431	* @group: The group whose first cpu is to be returned.
3432	*/
3433	static inline unsigned int group_first_cpu(struct sched_group *group)
3434	{
3435	return cpumask_first(sched_group_cpus(group));
3436	}
3437
3438	/**
3439	* get_sd_load_idx - Obtain the load index for a given sched domain.
3440	* @sd: The sched_domain whose load_idx is to be obtained.
3441	* @idle: The Idle status of the CPU for whose sd load_icx is obtained.
3442	*/
3443	static inline int get_sd_load_idx(struct sched_domain *sd,
3444	enum cpu_idle_type idle)
3445	{
3446	int load_idx;
3447
3448	switch (idle) {
3449	case CPU_NOT_IDLE:
3450	load_idx = sd->busy_idx;
3451	break;
3452
3453	case CPU_NEWLY_IDLE:
3454	load_idx = sd->newidle_idx;
3455	break;
3456	default:
3457	load_idx = sd->idle_idx;
3458	break;
3459	}
3460
3461	return load_idx;
3462	}
3463
3464
3465	#if defined(CONFIG_SCHED_MC) \|\| defined(CONFIG_SCHED_SMT)
3466	/**
3467	* init_sd_power_savings_stats - Initialize power savings statistics for
3468	* the given sched_domain, during load balancing.
3469	*
3470	* @sd: Sched domain whose power-savings statistics are to be initialized.
3471	* @sds: Variable containing the statistics for sd.
3472	* @idle: Idle status of the CPU at which we're performing load-balancing.
3473	*/
3474	static inline void init_sd_power_savings_stats(struct sched_domain *sd,
3475	struct sd_lb_stats *sds, enum cpu_idle_type idle)
3476	{
3477	/*
3478	* Busy processors will not participate in power savings
3479	* balance.
3480	*/
3481	if (idle == CPU_NOT_IDLE \|\| !(sd->flags & SD_POWERSAVINGS_BALANCE))
3482	sds->power_savings_balance = 0;
3483	else {
3484	sds->power_savings_balance = 1;
3485	sds->min_nr_running = ULONG_MAX;
3486	sds->leader_nr_running = 0;
3487	}
3488	}
3489
3490	/**
3491	* update_sd_power_savings_stats - Update the power saving stats for a
3492	* sched_domain while performing load balancing.
3493	*
3494	* @group: sched_group belonging to the sched_domain under consideration.
3495	* @sds: Variable containing the statistics of the sched_domain
3496	* @local_group: Does group contain the CPU for which we're performing
3497	* load balancing ?
3498	* @sgs: Variable containing the statistics of the group.
3499	*/
3500	static inline void update_sd_power_savings_stats(struct sched_group *group,
3501	struct sd_lb_stats sds, int local_group, struct sg_lb_stats sgs)
3502	{
3503
3504	if (!sds->power_savings_balance)
3505	return;
3506
3507	/*
3508	* If the local group is idle or completely loaded
3509	* no need to do power savings balance at this domain
3510	*/
3511	if (local_group && (sds->this_nr_running >= sgs->group_capacity \|\|
3512	!sds->this_nr_running))
3513	sds->power_savings_balance = 0;
3514
3515	/*
3516	* If a group is already running at full capacity or idle,
3517	* don't include that group in power savings calculations
3518	*/
3519	if (!sds->power_savings_balance \|\|
3520	sgs->sum_nr_running >= sgs->group_capacity \|\|
3521	!sgs->sum_nr_running)
3522	return;
3523
3524	/*
3525	* Calculate the group which has the least non-idle load.
3526	* This is the group from where we need to pick up the load
3527	* for saving power
3528	*/
3529	if ((sgs->sum_nr_running < sds->min_nr_running) \|\|
3530	(sgs->sum_nr_running == sds->min_nr_running &&
3531	group_first_cpu(group) > group_first_cpu(sds->group_min))) {
3532	sds->group_min = group;
3533	sds->min_nr_running = sgs->sum_nr_running;
3534	sds->min_load_per_task = sgs->sum_weighted_load /
3535	sgs->sum_nr_running;
3536	}
3537
3538	/*
3539	* Calculate the group which is almost near its
3540	* capacity but still has some space to pick up some load
3541	* from other group and save more power
3542	*/
3543	if (sgs->sum_nr_running + 1 > sgs->group_capacity)
3544	return;
3545
3546	if (sgs->sum_nr_running > sds->leader_nr_running \|\|
3547	(sgs->sum_nr_running == sds->leader_nr_running &&
3548	group_first_cpu(group) < group_first_cpu(sds->group_leader))) {
3549	sds->group_leader = group;
3550	sds->leader_nr_running = sgs->sum_nr_running;
3551	}
3552	}
3553
3554	/**
3555	* check_power_save_busiest_group - see if there is potential for some power-savings balance
3556	* @sds: Variable containing the statistics of the sched_domain
3557	* under consideration.
3558	* @this_cpu: Cpu at which we're currently performing load-balancing.
3559	* @imbalance: Variable to store the imbalance.
3560	*
3561	* Description:
3562	* Check if we have potential to perform some power-savings balance.
3563	* If yes, set the busiest group to be the least loaded group in the
3564	* sched_domain, so that it's CPUs can be put to idle.
3565	*
3566	* Returns 1 if there is potential to perform power-savings balance.
3567	* Else returns 0.
3568	*/
3569	static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
3570	int this_cpu, unsigned long *imbalance)
3571	{
3572	if (!sds->power_savings_balance)
3573	return 0;
3574
3575	if (sds->this != sds->group_leader \|\|
3576	sds->group_leader == sds->group_min)
3577	return 0;
3578
3579	*imbalance = sds->min_load_per_task;
3580	sds->busiest = sds->group_min;
3581
3582	return 1;
3583
3584	}
3585	#else /* CONFIG_SCHED_MC \|\| CONFIG_SCHED_SMT */
3586	static inline void init_sd_power_savings_stats(struct sched_domain *sd,
3587	struct sd_lb_stats *sds, enum cpu_idle_type idle)
3588	{
3589	return;
3590	}
3591
3592	static inline void update_sd_power_savings_stats(struct sched_group *group,
3593	struct sd_lb_stats sds, int local_group, struct sg_lb_stats sgs)
3594	{
3595	return;
3596	}
3597
3598	static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
3599	int this_cpu, unsigned long *imbalance)
3600	{
3601	return 0;
3602	}
3603	#endif /* CONFIG_SCHED_MC \|\| CONFIG_SCHED_SMT */
3604
3605
3606	unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
3607	{
3608	return SCHED_LOAD_SCALE;
3609	}
3610
3611	unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
3612	{
3613	return default_scale_freq_power(sd, cpu);
3614	}
3615
3616	unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
3617	{
3618	unsigned long weight = cpumask_weight(sched_domain_span(sd));
3619	unsigned long smt_gain = sd->smt_gain;
3620
3621	smt_gain /= weight;
3622
3623	return smt_gain;
3624	}
3625
3626	unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
3627	{
3628	return default_scale_smt_power(sd, cpu);
3629	}
3630
3631	unsigned long scale_rt_power(int cpu)
3632	{
3633	struct rq *rq = cpu_rq(cpu);
3634	u64 total, available;
3635
3636	sched_avg_update(rq);
3637
3638	total = sched_avg_period() + (rq->clock - rq->age_stamp);
3639	available = total - rq->rt_avg;
3640
3641	if (unlikely((s64)total < SCHED_LOAD_SCALE))
3642	total = SCHED_LOAD_SCALE;
3643
3644	total >>= SCHED_LOAD_SHIFT;
3645
3646	return div_u64(available, total);
3647	}
3648
3649	static void update_cpu_power(struct sched_domain *sd, int cpu)
3650	{
3651	unsigned long weight = cpumask_weight(sched_domain_span(sd));
3652	unsigned long power = SCHED_LOAD_SCALE;
3653	struct sched_group *sdg = sd->groups;
3654
3655	if (sched_feat(ARCH_POWER))
3656	power *= arch_scale_freq_power(sd, cpu);
3657	else
3658	power *= default_scale_freq_power(sd, cpu);
3659
3660	power >>= SCHED_LOAD_SHIFT;
3661
3662	if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
3663	if (sched_feat(ARCH_POWER))
3664	power *= arch_scale_smt_power(sd, cpu);
3665	else
3666	power *= default_scale_smt_power(sd, cpu);
3667
3668	power >>= SCHED_LOAD_SHIFT;
3669	}
3670
3671	power *= scale_rt_power(cpu);
3672	power >>= SCHED_LOAD_SHIFT;
3673
3674	if (!power)
3675	power = 1;
3676
3677	sdg->cpu_power = power;
3678	}
3679
3680	static void update_group_power(struct sched_domain *sd, int cpu)
3681	{
3682	struct sched_domain *child = sd->child;
3683	struct sched_group group, sdg = sd->groups;
3684	unsigned long power;
3685
3686	if (!child) {
3687	update_cpu_power(sd, cpu);
3688	return;
3689	}
3690
3691	power = 0;
3692
3693	group = child->groups;
3694	do {
3695	power += group->cpu_power;
3696	group = group->next;
3697	} while (group != child->groups);
3698
3699	sdg->cpu_power = power;
3700	}
3701
3702	/**
3703	* update_sg_lb_stats - Update sched_group's statistics for load balancing.
3704	* @sd: The sched_domain whose statistics are to be updated.
3705	* @group: sched_group whose statistics are to be updated.
3706	* @this_cpu: Cpu for which load balance is currently performed.
3707	* @idle: Idle status of this_cpu
3708	* @load_idx: Load index of sched_domain of this_cpu for load calc.
3709	* @sd_idle: Idle status of the sched_domain containing group.
3710	* @local_group: Does group contain this_cpu.
3711	* @cpus: Set of cpus considered for load balancing.
3712	* @balance: Should we balance.
3713	* @sgs: variable to hold the statistics for this group.
3714	*/
3715	static inline void update_sg_lb_stats(struct sched_domain *sd,
3716	struct sched_group *group, int this_cpu,
3717	enum cpu_idle_type idle, int load_idx, int *sd_idle,
3718	int local_group, const struct cpumask *cpus,
3719	int balance, struct sg_lb_stats sgs)
3720	{
3721	unsigned long load, max_cpu_load, min_cpu_load;
3722	int i;
3723	unsigned int balance_cpu = -1, first_idle_cpu = 0;
3724	unsigned long sum_avg_load_per_task;
3725	unsigned long avg_load_per_task;
3726
3727	if (local_group) {
3728	balance_cpu = group_first_cpu(group);
3729	if (balance_cpu == this_cpu)
3730	update_group_power(sd, this_cpu);
3731	}
3732
3733	/* Tally up the load of all CPUs in the group */
3734	sum_avg_load_per_task = avg_load_per_task = 0;
3735	max_cpu_load = 0;
3736	min_cpu_load = ~0UL;
3737
3738	for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3739	struct rq *rq = cpu_rq(i);
3740
3741	if (*sd_idle && rq->nr_running)
3742	*sd_idle = 0;
3743
3744	/* Bias balancing toward cpus of our domain */
3745	if (local_group) {
3746	if (idle_cpu(i) && !first_idle_cpu) {
3747	first_idle_cpu = 1;
3748	balance_cpu = i;
3749	}
3750
3751	load = target_load(i, load_idx);
3752	} else {
3753	load = source_load(i, load_idx);
3754	if (load > max_cpu_load)
3755	max_cpu_load = load;
3756	if (min_cpu_load > load)
3757	min_cpu_load = load;
3758	}
3759
3760	sgs->group_load += load;
3761	sgs->sum_nr_running += rq->nr_running;
3762	sgs->sum_weighted_load += weighted_cpuload(i);
3763
3764	sum_avg_load_per_task += cpu_avg_load_per_task(i);
3765	}
3766
3767	/*
3768	* First idle cpu or the first cpu(busiest) in this sched group
3769	* is eligible for doing load balancing at this and above
3770	* domains. In the newly idle case, we will allow all the cpu's
3771	* to do the newly idle load balance.
3772	*/
3773	if (idle != CPU_NEWLY_IDLE && local_group &&
3774	balance_cpu != this_cpu && balance) {
3775	*balance = 0;
3776	return;
3777	}
3778
3779	/* Adjust by relative CPU power of the group */
3780	sgs->avg_load = (sgs->group_load * SCHED_LOAD_SCALE) / group->cpu_power;
3781
3782
3783	/*
3784	* Consider the group unbalanced when the imbalance is larger
3785	* than the average weight of two tasks.
3786	*
3787	* APZ: with cgroup the avg task weight can vary wildly and
3788	* might not be a suitable number - should we keep a
3789	* normalized nr_running number somewhere that negates
3790	* the hierarchy?
3791	*/
3792	avg_load_per_task = (sum_avg_load_per_task * SCHED_LOAD_SCALE) /
3793	group->cpu_power;
3794
3795	if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
3796	sgs->group_imb = 1;
3797
3798	sgs->group_capacity =
3799	DIV_ROUND_CLOSEST(group->cpu_power, SCHED_LOAD_SCALE);
3800	}
3801
3802	/**
3803	* update_sd_lb_stats - Update sched_group's statistics for load balancing.
3804	* @sd: sched_domain whose statistics are to be updated.
3805	* @this_cpu: Cpu for which load balance is currently performed.
3806	* @idle: Idle status of this_cpu
3807	* @sd_idle: Idle status of the sched_domain containing group.
3808	* @cpus: Set of cpus considered for load balancing.
3809	* @balance: Should we balance.
3810	* @sds: variable to hold the statistics for this sched_domain.
3811	*/
3812	static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
3813	enum cpu_idle_type idle, int *sd_idle,
3814	const struct cpumask cpus, int balance,
3815	struct sd_lb_stats *sds)
3816	{
3817	struct sched_domain *child = sd->child;
3818	struct sched_group *group = sd->groups;
3819	struct sg_lb_stats sgs;
3820	int load_idx, prefer_sibling = 0;
3821
3822	if (child && child->flags & SD_PREFER_SIBLING)
3823	prefer_sibling = 1;
3824
3825	init_sd_power_savings_stats(sd, sds, idle);
3826	load_idx = get_sd_load_idx(sd, idle);
3827
3828	do {
3829	int local_group;
3830
3831	local_group = cpumask_test_cpu(this_cpu,
3832	sched_group_cpus(group));
3833	memset(&sgs, 0, sizeof(sgs));
3834	update_sg_lb_stats(sd, group, this_cpu, idle, load_idx, sd_idle,
3835	local_group, cpus, balance, &sgs);
3836
3837	if (local_group && balance && !(*balance))
3838	return;
3839
3840	sds->total_load += sgs.group_load;
3841	sds->total_pwr += group->cpu_power;
3842
3843	/*
3844	* In case the child domain prefers tasks go to siblings
3845	* first, lower the group capacity to one so that we'll try
3846	* and move all the excess tasks away.
3847	*/
3848	if (prefer_sibling)
3849	sgs.group_capacity = min(sgs.group_capacity, 1UL);
3850
3851	if (local_group) {
3852	sds->this_load = sgs.avg_load;
3853	sds->this = group;
3854	sds->this_nr_running = sgs.sum_nr_running;
3855	sds->this_load_per_task = sgs.sum_weighted_load;
3856	} else if (sgs.avg_load > sds->max_load &&
3857	(sgs.sum_nr_running > sgs.group_capacity \|\|
3858	sgs.group_imb)) {
3859	sds->max_load = sgs.avg_load;
3860	sds->busiest = group;
3861	sds->busiest_nr_running = sgs.sum_nr_running;
3862	sds->busiest_load_per_task = sgs.sum_weighted_load;
3863	sds->group_imb = sgs.group_imb;
3864	}
3865
3866	update_sd_power_savings_stats(group, sds, local_group, &sgs);
3867	group = group->next;
3868	} while (group != sd->groups);
3869	}
3870
3871	/**
3872	* fix_small_imbalance - Calculate the minor imbalance that exists
3873	* amongst the groups of a sched_domain, during
3874	* load balancing.
3875	* @sds: Statistics of the sched_domain whose imbalance is to be calculated.
3876	* @this_cpu: The cpu at whose sched_domain we're performing load-balance.
3877	* @imbalance: Variable to store the imbalance.
3878	*/
3879	static inline void fix_small_imbalance(struct sd_lb_stats *sds,
3880	int this_cpu, unsigned long *imbalance)
3881	{
3882	unsigned long tmp, pwr_now = 0, pwr_move = 0;
3883	unsigned int imbn = 2;
3884
3885	if (sds->this_nr_running) {
3886	sds->this_load_per_task /= sds->this_nr_running;
3887	if (sds->busiest_load_per_task >
3888	sds->this_load_per_task)
3889	imbn = 1;
3890	} else
3891	sds->this_load_per_task =
3892	cpu_avg_load_per_task(this_cpu);
3893
3894	if (sds->max_load - sds->this_load + sds->busiest_load_per_task >=
3895	sds->busiest_load_per_task * imbn) {
3896	*imbalance = sds->busiest_load_per_task;
3897	return;
3898	}
3899
3900	/*
3901	* OK, we don't have enough imbalance to justify moving tasks,
3902	* however we may be able to increase total CPU power used by
3903	* moving them.
3904	*/
3905
3906	pwr_now += sds->busiest->cpu_power *
3907	min(sds->busiest_load_per_task, sds->max_load);
3908	pwr_now += sds->this->cpu_power *
3909	min(sds->this_load_per_task, sds->this_load);
3910	pwr_now /= SCHED_LOAD_SCALE;
3911
3912	/* Amount of load we'd subtract */
3913	tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
3914	sds->busiest->cpu_power;
3915	if (sds->max_load > tmp)
3916	pwr_move += sds->busiest->cpu_power *
3917	min(sds->busiest_load_per_task, sds->max_load - tmp);
3918
3919	/* Amount of load we'd add */
3920	if (sds->max_load * sds->busiest->cpu_power <
3921	sds->busiest_load_per_task * SCHED_LOAD_SCALE)
3922	tmp = (sds->max_load * sds->busiest->cpu_power) /
3923	sds->this->cpu_power;
3924	else
3925	tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
3926	sds->this->cpu_power;
3927	pwr_move += sds->this->cpu_power *
3928	min(sds->this_load_per_task, sds->this_load + tmp);
3929	pwr_move /= SCHED_LOAD_SCALE;
3930
3931	/* Move if we gain throughput */
3932	if (pwr_move > pwr_now)
3933	*imbalance = sds->busiest_load_per_task;
3934	}
3935
3936	/**
3937	* calculate_imbalance - Calculate the amount of imbalance present within the
3938	* groups of a given sched_domain during load balance.
3939	* @sds: statistics of the sched_domain whose imbalance is to be calculated.
3940	* @this_cpu: Cpu for which currently load balance is being performed.
3941	* @imbalance: The variable to store the imbalance.
3942	*/
3943	static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
3944	unsigned long *imbalance)
3945	{
3946	unsigned long max_pull;
3947	/*
3948	* In the presence of smp nice balancing, certain scenarios can have
3949	* max load less than avg load(as we skip the groups at or below
3950	* its cpu_power, while calculating max_load..)
3951	*/
3952	if (sds->max_load < sds->avg_load) {
3953	*imbalance = 0;
3954	return fix_small_imbalance(sds, this_cpu, imbalance);
3955	}
3956
3957	/* Don't want to pull so many tasks that a group would go idle */
3958	max_pull = min(sds->max_load - sds->avg_load,
3959	sds->max_load - sds->busiest_load_per_task);
3960
3961	/* How much load to actually move to equalise the imbalance */
3962	imbalance = min(max_pull sds->busiest->cpu_power,
3963	(sds->avg_load - sds->this_load) * sds->this->cpu_power)
3964	/ SCHED_LOAD_SCALE;
3965
3966	/*
3967	* if *imbalance is less than the average load per runnable task
3968	* there is no gaurantee that any tasks will be moved so we'll have
3969	* a think about bumping its value to force at least one task to be
3970	* moved
3971	*/
3972	if (*imbalance < sds->busiest_load_per_task)
3973	return fix_small_imbalance(sds, this_cpu, imbalance);
3974
3975	}
3976	/***** find_busiest_group() helpers end here *******************/
3977
3978	/**
3979	* find_busiest_group - Returns the busiest group within the sched_domain
3980	* if there is an imbalance. If there isn't an imbalance, and
3981	* the user has opted for power-savings, it returns a group whose
3982	* CPUs can be put to idle by rebalancing those tasks elsewhere, if
3983	* such a group exists.
3984	*
3985	* Also calculates the amount of weighted load which should be moved
3986	* to restore balance.
3987	*
3988	* @sd: The sched_domain whose busiest group is to be returned.
3989	* @this_cpu: The cpu for which load balancing is currently being performed.
3990	* @imbalance: Variable which stores amount of weighted load which should
3991	* be moved to restore balance/put a group to idle.
3992	* @idle: The idle status of this_cpu.
3993	* @sd_idle: The idleness of sd
3994	* @cpus: The set of CPUs under consideration for load-balancing.
3995	* @balance: Pointer to a variable indicating if this_cpu
3996	* is the appropriate cpu to perform load balancing at this_level.
3997	*
3998	* Returns: - the busiest group if imbalance exists.
3999	* - If no imbalance and user has opted for power-savings balance,
4000	* return the least loaded group whose CPUs can be
4001	* put to idle by rebalancing its tasks onto our group.
4002	*/
4003	static struct sched_group *
4004	find_busiest_group(struct sched_domain *sd, int this_cpu,
4005	unsigned long *imbalance, enum cpu_idle_type idle,
4006	int sd_idle, const struct cpumask cpus, int *balance)
4007	{
4008	struct sd_lb_stats sds;
4009
4010	memset(&sds, 0, sizeof(sds));
4011
4012	/*
4013	* Compute the various statistics relavent for load balancing at
4014	* this level.
4015	*/
4016	update_sd_lb_stats(sd, this_cpu, idle, sd_idle, cpus,
4017	balance, &sds);
4018
4019	/* Cases where imbalance does not exist from POV of this_cpu */
4020	/* 1) this_cpu is not the appropriate cpu to perform load balancing
4021	* at this level.
4022	* 2) There is no busy sibling group to pull from.
4023	* 3) This group is the busiest group.
4024	* 4) This group is more busy than the avg busieness at this
4025	* sched_domain.
4026	* 5) The imbalance is within the specified limit.
4027	* 6) Any rebalance would lead to ping-pong
4028	*/
4029	if (balance && !(*balance))
4030	goto ret;
4031
4032	if (!sds.busiest \|\| sds.busiest_nr_running == 0)
4033	goto out_balanced;
4034
4035	if (sds.this_load >= sds.max_load)
4036	goto out_balanced;
4037
4038	sds.avg_load = (SCHED_LOAD_SCALE * sds.total_load) / sds.total_pwr;
4039
4040	if (sds.this_load >= sds.avg_load)
4041	goto out_balanced;
4042
4043	if (100 * sds.max_load <= sd->imbalance_pct * sds.this_load)
4044	goto out_balanced;
4045
4046	sds.busiest_load_per_task /= sds.busiest_nr_running;
4047	if (sds.group_imb)
4048	sds.busiest_load_per_task =
4049	min(sds.busiest_load_per_task, sds.avg_load);
4050
4051	/*
4052	* We're trying to get all the cpus to the average_load, so we don't
4053	* want to push ourselves above the average load, nor do we wish to
4054	* reduce the max loaded cpu below the average load, as either of these
4055	* actions would just result in more rebalancing later, and ping-pong
4056	* tasks around. Thus we look for the minimum possible imbalance.
4057	* Negative imbalances (we are more loaded than anyone else) will
4058	* be counted as no imbalance for these purposes -- we can't fix that
4059	* by pulling tasks to us. Be careful of negative numbers as they'll
4060	* appear as very large values with unsigned longs.
4061	*/
4062	if (sds.max_load <= sds.busiest_load_per_task)
4063	goto out_balanced;
4064
4065	/* Looks like there is an imbalance. Compute it */
4066	calculate_imbalance(&sds, this_cpu, imbalance);
4067	return sds.busiest;
4068
4069	out_balanced:
4070	/*
4071	* There is no obvious imbalance. But check if we can do some balancing
4072	* to save power.
4073	*/
4074	if (check_power_save_busiest_group(&sds, this_cpu, imbalance))
4075	return sds.busiest;
4076	ret:
4077	*imbalance = 0;
4078	return NULL;
4079	}
4080
4081	/*
4082	* find_busiest_queue - find the busiest runqueue among the cpus in group.
4083	*/
4084	static struct rq *
4085	find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
4086	unsigned long imbalance, const struct cpumask *cpus)
4087	{
4088	struct rq busiest = NULL, rq;
4089	unsigned long max_load = 0;
4090	int i;
4091
4092	for_each_cpu(i, sched_group_cpus(group)) {
4093	unsigned long power = power_of(i);
4094	unsigned long capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
4095	unsigned long wl;
4096
4097	if (!cpumask_test_cpu(i, cpus))
4098	continue;
4099
4100	rq = cpu_rq(i);
4101	wl = weighted_cpuload(i);
4102
4103	/*
4104	* When comparing with imbalance, use weighted_cpuload()
4105	* which is not scaled with the cpu power.
4106	*/
4107	if (capacity && rq->nr_running == 1 && wl > imbalance)
4108	continue;
4109
4110	/*
4111	* For the load comparisons with the other cpu's, consider
4112	* the weighted_cpuload() scaled with the cpu power, so that
4113	* the load can be moved away from the cpu that is potentially
4114	* running at a lower capacity.
4115	*/
4116	wl = (wl * SCHED_LOAD_SCALE) / power;
4117
4118	if (wl > max_load) {
4119	max_load = wl;
4120	busiest = rq;
4121	}
4122	}
4123
4124	return busiest;
4125	}
4126
4127	/*
4128	* Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
4129	* so long as it is large enough.
4130	*/
4131	#define MAX_PINNED_INTERVAL 512
4132
4133	/* Working cpumask for load_balance and load_balance_newidle. */
4134	static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
4135
4136	/*
4137	* Check this_cpu to ensure it is balanced within domain. Attempt to move
4138	* tasks if there is an imbalance.
4139	*/
4140	static int load_balance(int this_cpu, struct rq *this_rq,
4141	struct sched_domain *sd, enum cpu_idle_type idle,
4142	int *balance)
4143	{
4144	int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
4145	struct sched_group *group;
4146	unsigned long imbalance;
4147	struct rq *busiest;
4148	unsigned long flags;
4149	struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
4150
4151	cpumask_copy(cpus, cpu_active_mask);
4152
4153	/*
4154	* When power savings policy is enabled for the parent domain, idle
4155	* sibling can pick up load irrespective of busy siblings. In this case,
4156	* let the state of idle sibling percolate up as CPU_IDLE, instead of
4157	* portraying it as CPU_NOT_IDLE.
4158	*/
4159	if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
4160	!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
4161	sd_idle = 1;
4162
4163	schedstat_inc(sd, lb_count[idle]);
4164
4165	redo:
4166	update_shares(sd);
4167	group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
4168	cpus, balance);
4169
4170	if (*balance == 0)
4171	goto out_balanced;
4172
4173	if (!group) {
4174	schedstat_inc(sd, lb_nobusyg[idle]);
4175	goto out_balanced;
4176	}
4177
4178	busiest = find_busiest_queue(group, idle, imbalance, cpus);
4179	if (!busiest) {
4180	schedstat_inc(sd, lb_nobusyq[idle]);
4181	goto out_balanced;
4182	}
4183
4184	BUG_ON(busiest == this_rq);
4185
4186	schedstat_add(sd, lb_imbalance[idle], imbalance);
4187
4188	ld_moved = 0;
4189	if (busiest->nr_running > 1) {
4190	/*
4191	* Attempt to move tasks. If find_busiest_group has found
4192	* an imbalance but busiest->nr_running <= 1, the group is
4193	* still unbalanced. ld_moved simply stays zero, so it is
4194	* correctly treated as an imbalance.
4195	*/
4196	local_irq_save(flags);
4197	double_rq_lock(this_rq, busiest);
4198	ld_moved = move_tasks(this_rq, this_cpu, busiest,
4199	imbalance, sd, idle, &all_pinned);
4200	double_rq_unlock(this_rq, busiest);
4201	local_irq_restore(flags);
4202
4203	/*
4204	* some other cpu did the load balance for us.
4205	*/
4206	if (ld_moved && this_cpu != smp_processor_id())
4207	resched_cpu(this_cpu);
4208
4209	/* All tasks on this runqueue were pinned by CPU affinity */
4210	if (unlikely(all_pinned)) {
4211	cpumask_clear_cpu(cpu_of(busiest), cpus);
4212	if (!cpumask_empty(cpus))
4213	goto redo;
4214	goto out_balanced;
4215	}
4216	}
4217
4218	if (!ld_moved) {
4219	schedstat_inc(sd, lb_failed[idle]);
4220	sd->nr_balance_failed++;
4221
4222	if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
4223
4224	spin_lock_irqsave(&busiest->lock, flags);
4225
4226	/* don't kick the migration_thread, if the curr
4227	* task on busiest cpu can't be moved to this_cpu
4228	*/
4229	if (!cpumask_test_cpu(this_cpu,
4230	&busiest->curr->cpus_allowed)) {
4231	spin_unlock_irqrestore(&busiest->lock, flags);
4232	all_pinned = 1;
4233	goto out_one_pinned;
4234	}
4235
4236	if (!busiest->active_balance) {
4237	busiest->active_balance = 1;
4238	busiest->push_cpu = this_cpu;
4239	active_balance = 1;
4240	}
4241	spin_unlock_irqrestore(&busiest->lock, flags);
4242	if (active_balance)
4243	wake_up_process(busiest->migration_thread);
4244
4245	/*
4246	* We've kicked active balancing, reset the failure
4247	* counter.
4248	*/
4249	sd->nr_balance_failed = sd->cache_nice_tries+1;
4250	}
4251	} else
4252	sd->nr_balance_failed = 0;
4253
4254	if (likely(!active_balance)) {
4255	/* We were unbalanced, so reset the balancing interval */
4256	sd->balance_interval = sd->min_interval;
4257	} else {
4258	/*
4259	* If we've begun active balancing, start to back off. This
4260	* case may not be covered by the all_pinned logic if there
4261	* is only 1 task on the busy runqueue (because we don't call
4262	* move_tasks).
4263	*/
4264	if (sd->balance_interval < sd->max_interval)
4265	sd->balance_interval *= 2;
4266	}
4267
4268	if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
4269	!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
4270	ld_moved = -1;
4271
4272	goto out;
4273
4274	out_balanced:
4275	schedstat_inc(sd, lb_balanced[idle]);
4276
4277	sd->nr_balance_failed = 0;
4278
4279	out_one_pinned:
4280	/* tune up the balancing interval */
4281	if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) \|\|
4282	(sd->balance_interval < sd->max_interval))
4283	sd->balance_interval *= 2;
4284
4285	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
4286	!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
4287	ld_moved = -1;
4288	else
4289	ld_moved = 0;
4290	out:
4291	if (ld_moved)
4292	update_shares(sd);
4293	return ld_moved;
4294	}
4295
4296	/*
4297	* Check this_cpu to ensure it is balanced within domain. Attempt to move
4298	* tasks if there is an imbalance.
4299	*
4300	* Called from schedule when this_rq is about to become idle (CPU_NEWLY_IDLE).
4301	* this_rq is locked.
4302	*/
4303	static int
4304	load_balance_newidle(int this_cpu, struct rq this_rq, struct sched_domain sd)
4305	{
4306	struct sched_group *group;
4307	struct rq *busiest = NULL;
4308	unsigned long imbalance;
4309	int ld_moved = 0;
4310	int sd_idle = 0;
4311	int all_pinned = 0;
4312	struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
4313
4314	cpumask_copy(cpus, cpu_active_mask);
4315
4316	/*
4317	* When power savings policy is enabled for the parent domain, idle
4318	* sibling can pick up load irrespective of busy siblings. In this case,
4319	* let the state of idle sibling percolate up as IDLE, instead of
4320	* portraying it as CPU_NOT_IDLE.
4321	*/
4322	if (sd->flags & SD_SHARE_CPUPOWER &&
4323	!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
4324	sd_idle = 1;
4325
4326	schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
4327	redo:
4328	update_shares_locked(this_rq, sd);
4329	group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
4330	&sd_idle, cpus, NULL);
4331	if (!group) {
4332	schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
4333	goto out_balanced;
4334	}
4335
4336	busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
4337	if (!busiest) {
4338	schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
4339	goto out_balanced;
4340	}
4341
4342	BUG_ON(busiest == this_rq);
4343
4344	schedstat_add(sd, lb_imbalance[CPU_NEWLY_IDLE], imbalance);
4345
4346	ld_moved = 0;
4347	if (busiest->nr_running > 1) {
4348	/* Attempt to move tasks */
4349	double_lock_balance(this_rq, busiest);
4350	/* this_rq->clock is already updated */
4351	update_rq_clock(busiest);
4352	ld_moved = move_tasks(this_rq, this_cpu, busiest,
4353	imbalance, sd, CPU_NEWLY_IDLE,
4354	&all_pinned);
4355	double_unlock_balance(this_rq, busiest);
4356
4357	if (unlikely(all_pinned)) {
4358	cpumask_clear_cpu(cpu_of(busiest), cpus);
4359	if (!cpumask_empty(cpus))
4360	goto redo;
4361	}
4362	}
4363
4364	if (!ld_moved) {
4365	int active_balance = 0;
4366
4367	schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
4368	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
4369	!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
4370	return -1;
4371
4372	if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
4373	return -1;
4374
4375	if (sd->nr_balance_failed++ < 2)
4376	return -1;
4377
4378	/*
4379	* The only task running in a non-idle cpu can be moved to this
4380	* cpu in an attempt to completely freeup the other CPU
4381	* package. The same method used to move task in load_balance()
4382	* have been extended for load_balance_newidle() to speedup
4383	* consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
4384	*
4385	* The package power saving logic comes from
4386	* find_busiest_group(). If there are no imbalance, then
4387	* f_b_g() will return NULL. However when sched_mc={1,2} then
4388	* f_b_g() will select a group from which a running task may be
4389	* pulled to this cpu in order to make the other package idle.
4390	* If there is no opportunity to make a package idle and if
4391	* there are no imbalance, then f_b_g() will return NULL and no
4392	* action will be taken in load_balance_newidle().
4393	*
4394	* Under normal task pull operation due to imbalance, there
4395	* will be more than one task in the source run queue and
4396	* move_tasks() will succeed. ld_moved will be true and this
4397	* active balance code will not be triggered.
4398	*/
4399
4400	/* Lock busiest in correct order while this_rq is held */
4401	double_lock_balance(this_rq, busiest);
4402
4403	/*
4404	* don't kick the migration_thread, if the curr
4405	* task on busiest cpu can't be moved to this_cpu
4406	*/
4407	if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
4408	double_unlock_balance(this_rq, busiest);
4409	all_pinned = 1;
4410	return ld_moved;
4411	}
4412
4413	if (!busiest->active_balance) {
4414	busiest->active_balance = 1;
4415	busiest->push_cpu = this_cpu;
4416	active_balance = 1;
4417	}
4418
4419	double_unlock_balance(this_rq, busiest);
4420	/*
4421	* Should not call ttwu while holding a rq->lock
4422	*/
4423	spin_unlock(&this_rq->lock);
4424	if (active_balance)
4425	wake_up_process(busiest->migration_thread);
4426	spin_lock(&this_rq->lock);
4427
4428	} else
4429	sd->nr_balance_failed = 0;
4430
4431	update_shares_locked(this_rq, sd);
4432	return ld_moved;
4433
4434	out_balanced:
4435	schedstat_inc(sd, lb_balanced[CPU_NEWLY_IDLE]);
4436	if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
4437	!test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
4438	return -1;
4439	sd->nr_balance_failed = 0;
4440
4441	return 0;
4442	}
4443
4444	/*
4445	* idle_balance is called by schedule() if this_cpu is about to become
4446	* idle. Attempts to pull tasks from other CPUs.
4447	*/
4448	static void idle_balance(int this_cpu, struct rq *this_rq)
4449	{
4450	struct sched_domain *sd;
4451	int pulled_task = 0;
4452	unsigned long next_balance = jiffies + HZ;
4453
4454	this_rq->idle_stamp = this_rq->clock;
4455
4456	if (this_rq->avg_idle < sysctl_sched_migration_cost)
4457	return;
4458
4459	for_each_domain(this_cpu, sd) {
4460	unsigned long interval;
4461
4462	if (!(sd->flags & SD_LOAD_BALANCE))
4463	continue;
4464
4465	if (sd->flags & SD_BALANCE_NEWIDLE)
4466	/* If we've pulled tasks over stop searching: */
4467	pulled_task = load_balance_newidle(this_cpu, this_rq,
4468	sd);
4469
4470	interval = msecs_to_jiffies(sd->balance_interval);
4471	if (time_after(next_balance, sd->last_balance + interval))
4472	next_balance = sd->last_balance + interval;
4473	if (pulled_task) {
4474	this_rq->idle_stamp = 0;
4475	break;
4476	}
4477	}
4478	if (pulled_task \|\| time_after(jiffies, this_rq->next_balance)) {
4479	/*
4480	* We are going idle. next_balance may be set based on
4481	* a busy processor. So reset next_balance.
4482	*/
4483	this_rq->next_balance = next_balance;
4484	}
4485	}
4486
4487	/*
4488	* active_load_balance is run by migration threads. It pushes running tasks
4489	* off the busiest CPU onto idle CPUs. It requires at least 1 task to be
4490	* running on each physical CPU where possible, and avoids physical /
4491	* logical imbalances.
4492	*
4493	* Called with busiest_rq locked.
4494	*/
4495	static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
4496	{
4497	int target_cpu = busiest_rq->push_cpu;
4498	struct sched_domain *sd;
4499	struct rq *target_rq;
4500
4501	/* Is there any task to move? */
4502	if (busiest_rq->nr_running <= 1)
4503	return;
4504
4505	target_rq = cpu_rq(target_cpu);
4506
4507	/*
4508	* This condition is "impossible", if it occurs
4509	* we need to fix it. Originally reported by
4510	* Bjorn Helgaas on a 128-cpu setup.
4511	*/
4512	BUG_ON(busiest_rq == target_rq);
4513
4514	/* move a task from busiest_rq to target_rq */
4515	double_lock_balance(busiest_rq, target_rq);
4516	update_rq_clock(busiest_rq);
4517	update_rq_clock(target_rq);
4518
4519	/* Search for an sd spanning us and the target CPU. */
4520	for_each_domain(target_cpu, sd) {
4521	if ((sd->flags & SD_LOAD_BALANCE) &&
4522	cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
4523	break;
4524	}
4525
4526	if (likely(sd)) {
4527	schedstat_inc(sd, alb_count);
4528
4529	if (move_one_task(target_rq, target_cpu, busiest_rq,
4530	sd, CPU_IDLE))
4531	schedstat_inc(sd, alb_pushed);
4532	else
4533	schedstat_inc(sd, alb_failed);
4534	}
4535	double_unlock_balance(busiest_rq, target_rq);
4536	}
4537
4538	#ifdef CONFIG_NO_HZ
4539	static struct {
4540	atomic_t load_balancer;
4541	cpumask_var_t cpu_mask;
4542	cpumask_var_t ilb_grp_nohz_mask;
4543	} nohz ____cacheline_aligned = {
4544	.load_balancer = ATOMIC_INIT(-1),
4545	};
4546
4547	int get_nohz_load_balancer(void)
4548	{
4549	return atomic_read(&nohz.load_balancer);
4550	}
4551
4552	#if defined(CONFIG_SCHED_MC) \|\| defined(CONFIG_SCHED_SMT)
4553	/**
4554	* lowest_flag_domain - Return lowest sched_domain containing flag.
4555	* @cpu: The cpu whose lowest level of sched domain is to
4556	* be returned.
4557	* @flag: The flag to check for the lowest sched_domain
4558	* for the given cpu.
4559	*
4560	* Returns the lowest sched_domain of a cpu which contains the given flag.
4561	*/
4562	static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
4563	{
4564	struct sched_domain *sd;
4565
4566	for_each_domain(cpu, sd)
4567	if (sd && (sd->flags & flag))
4568	break;
4569
4570	return sd;
4571	}
4572
4573	/**
4574	* for_each_flag_domain - Iterates over sched_domains containing the flag.
4575	* @cpu: The cpu whose domains we're iterating over.
4576	* @sd: variable holding the value of the power_savings_sd
4577	* for cpu.
4578	* @flag: The flag to filter the sched_domains to be iterated.
4579	*
4580	* Iterates over all the scheduler domains for a given cpu that has the 'flag'
4581	* set, starting from the lowest sched_domain to the highest.
4582	*/
4583	#define for_each_flag_domain(cpu, sd, flag) \
4584	for (sd = lowest_flag_domain(cpu, flag); \
4585	(sd && (sd->flags & flag)); sd = sd->parent)
4586
4587	/**
4588	* is_semi_idle_group - Checks if the given sched_group is semi-idle.
4589	* @ilb_group: group to be checked for semi-idleness
4590	*
4591	* Returns: 1 if the group is semi-idle. 0 otherwise.
4592	*
4593	* We define a sched_group to be semi idle if it has atleast one idle-CPU
4594	* and atleast one non-idle CPU. This helper function checks if the given
4595	* sched_group is semi-idle or not.
4596	*/
4597	static inline int is_semi_idle_group(struct sched_group *ilb_group)
4598	{
4599	cpumask_and(nohz.ilb_grp_nohz_mask, nohz.cpu_mask,
4600	sched_group_cpus(ilb_group));
4601
4602	/*
4603	* A sched_group is semi-idle when it has atleast one busy cpu
4604	* and atleast one idle cpu.
4605	*/
4606	if (cpumask_empty(nohz.ilb_grp_nohz_mask))
4607	return 0;
4608
4609	if (cpumask_equal(nohz.ilb_grp_nohz_mask, sched_group_cpus(ilb_group)))
4610	return 0;
4611
4612	return 1;
4613	}
4614	/**
4615	* find_new_ilb - Finds the optimum idle load balancer for nomination.
4616	* @cpu: The cpu which is nominating a new idle_load_balancer.
4617	*
4618	* Returns: Returns the id of the idle load balancer if it exists,
4619	* Else, returns >= nr_cpu_ids.
4620	*
4621	* This algorithm picks the idle load balancer such that it belongs to a
4622	* semi-idle powersavings sched_domain. The idea is to try and avoid
4623	* completely idle packages/cores just for the purpose of idle load balancing
4624	* when there are other idle cpu's which are better suited for that job.
4625	*/
4626	static int find_new_ilb(int cpu)
4627	{
4628	struct sched_domain *sd;
4629	struct sched_group *ilb_group;
4630
4631	/*
4632	* Have idle load balancer selection from semi-idle packages only
4633	* when power-aware load balancing is enabled
4634	*/
4635	if (!(sched_smt_power_savings \|\| sched_mc_power_savings))
4636	goto out_done;
4637
4638	/*
4639	* Optimize for the case when we have no idle CPUs or only one
4640	* idle CPU. Don't walk the sched_domain hierarchy in such cases
4641	*/
4642	if (cpumask_weight(nohz.cpu_mask) < 2)
4643	goto out_done;
4644
4645	for_each_flag_domain(cpu, sd, SD_POWERSAVINGS_BALANCE) {
4646	ilb_group = sd->groups;
4647
4648	do {
4649	if (is_semi_idle_group(ilb_group))
4650	return cpumask_first(nohz.ilb_grp_nohz_mask);
4651
4652	ilb_group = ilb_group->next;
4653
4654	} while (ilb_group != sd->groups);
4655	}
4656
4657	out_done:
4658	return cpumask_first(nohz.cpu_mask);
4659	}
4660	#else /* (CONFIG_SCHED_MC \|\| CONFIG_SCHED_SMT) */
4661	static inline int find_new_ilb(int call_cpu)
4662	{
4663	return cpumask_first(nohz.cpu_mask);
4664	}
4665	#endif
4666
4667	/*
4668	* This routine will try to nominate the ilb (idle load balancing)
4669	* owner among the cpus whose ticks are stopped. ilb owner will do the idle
4670	* load balancing on behalf of all those cpus. If all the cpus in the system
4671	* go into this tickless mode, then there will be no ilb owner (as there is
4672	* no need for one) and all the cpus will sleep till the next wakeup event
4673	* arrives...
4674	*
4675	* For the ilb owner, tick is not stopped. And this tick will be used
4676	* for idle load balancing. ilb owner will still be part of
4677	* nohz.cpu_mask..
4678	*
4679	* While stopping the tick, this cpu will become the ilb owner if there
4680	* is no other owner. And will be the owner till that cpu becomes busy
4681	* or if all cpus in the system stop their ticks at which point
4682	* there is no need for ilb owner.
4683	*
4684	* When the ilb owner becomes busy, it nominates another owner, during the
4685	* next busy scheduler_tick()
4686	*/
4687	int select_nohz_load_balancer(int stop_tick)
4688	{
4689	int cpu = smp_processor_id();
4690
4691	if (stop_tick) {
4692	cpu_rq(cpu)->in_nohz_recently = 1;
4693
4694	if (!cpu_active(cpu)) {
4695	if (atomic_read(&nohz.load_balancer) != cpu)
4696	return 0;
4697
4698	/*
4699	* If we are going offline and still the leader,
4700	* give up!
4701	*/
4702	if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
4703	BUG();
4704
4705	return 0;
4706	}
4707
4708	cpumask_set_cpu(cpu, nohz.cpu_mask);
4709
4710	/* time for ilb owner also to sleep */
4711	if (cpumask_weight(nohz.cpu_mask) == num_active_cpus()) {
4712	if (atomic_read(&nohz.load_balancer) == cpu)
4713	atomic_set(&nohz.load_balancer, -1);
4714	return 0;
4715	}
4716
4717	if (atomic_read(&nohz.load_balancer) == -1) {
4718	/* make me the ilb owner */
4719	if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
4720	return 1;
4721	} else if (atomic_read(&nohz.load_balancer) == cpu) {
4722	int new_ilb;
4723
4724	if (!(sched_smt_power_savings \|\|
4725	sched_mc_power_savings))
4726	return 1;
4727	/*
4728	* Check to see if there is a more power-efficient
4729	* ilb.
4730	*/
4731	new_ilb = find_new_ilb(cpu);
4732	if (new_ilb < nr_cpu_ids && new_ilb != cpu) {
4733	atomic_set(&nohz.load_balancer, -1);
4734	resched_cpu(new_ilb);
4735	return 0;
4736	}
4737	return 1;
4738	}
4739	} else {
4740	if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
4741	return 0;
4742
4743	cpumask_clear_cpu(cpu, nohz.cpu_mask);
4744
4745	if (atomic_read(&nohz.load_balancer) == cpu)
4746	if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
4747	BUG();
4748	}
4749	return 0;
4750	}
4751	#endif
4752
4753	static DEFINE_SPINLOCK(balancing);
4754
4755	/*
4756	* It checks each scheduling domain to see if it is due to be balanced,
4757	* and initiates a balancing operation if so.
4758	*
4759	* Balancing parameters are set up in arch_init_sched_domains.
4760	*/
4761	static void rebalance_domains(int cpu, enum cpu_idle_type idle)
4762	{
4763	int balance = 1;
4764	struct rq *rq = cpu_rq(cpu);
4765	unsigned long interval;
4766	struct sched_domain *sd;
4767	/* Earliest time when we have to do rebalance again */
4768	unsigned long next_balance = jiffies + 60*HZ;
4769	int update_next_balance = 0;
4770	int need_serialize;
4771
4772	for_each_domain(cpu, sd) {
4773	if (!(sd->flags & SD_LOAD_BALANCE))
4774	continue;
4775
4776	interval = sd->balance_interval;
4777	if (idle != CPU_IDLE)
4778	interval *= sd->busy_factor;
4779
4780	/* scale ms to jiffies */
4781	interval = msecs_to_jiffies(interval);
4782	if (unlikely(!interval))
4783	interval = 1;
4784	if (interval > HZ*NR_CPUS/10)
4785	interval = HZ*NR_CPUS/10;
4786
4787	need_serialize = sd->flags & SD_SERIALIZE;
4788
4789	if (need_serialize) {
4790	if (!spin_trylock(&balancing))
4791	goto out;
4792	}
4793
4794	if (time_after_eq(jiffies, sd->last_balance + interval)) {
4795	if (load_balance(cpu, rq, sd, idle, &balance)) {
4796	/*
4797	* We've pulled tasks over so either we're no
4798	* longer idle, or one of our SMT siblings is
4799	* not idle.
4800	*/
4801	idle = CPU_NOT_IDLE;
4802	}
4803	sd->last_balance = jiffies;
4804	}
4805	if (need_serialize)
4806	spin_unlock(&balancing);
4807	out:
4808	if (time_after(next_balance, sd->last_balance + interval)) {
4809	next_balance = sd->last_balance + interval;
4810	update_next_balance = 1;
4811	}
4812
4813	/*
4814	* Stop the load balance at this level. There is another
4815	* CPU in our sched group which is doing load balancing more
4816	* actively.
4817	*/
4818	if (!balance)
4819	break;
4820	}
4821
4822	/*
4823	* next_balance will be updated only when there is a need.
4824	* When the cpu is attached to null domain for ex, it will not be
4825	* updated.
4826	*/
4827	if (likely(update_next_balance))
4828	rq->next_balance = next_balance;
4829	}
4830
4831	/*
4832	* run_rebalance_domains is triggered when needed from the scheduler tick.
4833	* In CONFIG_NO_HZ case, the idle load balance owner will do the
4834	* rebalancing for all the cpus for whom scheduler ticks are stopped.
4835	*/
4836	static void run_rebalance_domains(struct softirq_action *h)
4837	{
4838	int this_cpu = smp_processor_id();
4839	struct rq *this_rq = cpu_rq(this_cpu);
4840	enum cpu_idle_type idle = this_rq->idle_at_tick ?
4841	CPU_IDLE : CPU_NOT_IDLE;
4842
4843	rebalance_domains(this_cpu, idle);
4844
4845	#ifdef CONFIG_NO_HZ
4846	/*
4847	* If this cpu is the owner for idle load balancing, then do the
4848	* balancing on behalf of the other idle cpus whose ticks are
4849	* stopped.
4850	*/
4851	if (this_rq->idle_at_tick &&
4852	atomic_read(&nohz.load_balancer) == this_cpu) {
4853	struct rq *rq;
4854	int balance_cpu;
4855
4856	for_each_cpu(balance_cpu, nohz.cpu_mask) {
4857	if (balance_cpu == this_cpu)
4858	continue;
4859
4860	/*
4861	* If this cpu gets work to do, stop the load balancing
4862	* work being done for other cpus. Next load
4863	* balancing owner will pick it up.
4864	*/
4865	if (need_resched())
4866	break;
4867
4868	rebalance_domains(balance_cpu, CPU_IDLE);
4869
4870	rq = cpu_rq(balance_cpu);
4871	if (time_after(this_rq->next_balance, rq->next_balance))
4872	this_rq->next_balance = rq->next_balance;
4873	}
4874	}
4875	#endif
4876	}
4877
4878	static inline int on_null_domain(int cpu)
4879	{
4880	return !rcu_dereference(cpu_rq(cpu)->sd);
4881	}
4882
4883	/*
4884	* Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
4885	*
4886	* In case of CONFIG_NO_HZ, this is the place where we nominate a new
4887	* idle load balancing owner or decide to stop the periodic load balancing,
4888	* if the whole system is idle.
4889	*/
4890	static inline void trigger_load_balance(struct rq *rq, int cpu)
4891	{
4892	#ifdef CONFIG_NO_HZ
4893	/*
4894	* If we were in the nohz mode recently and busy at the current
4895	* scheduler tick, then check if we need to nominate new idle
4896	* load balancer.
4897	*/
4898	if (rq->in_nohz_recently && !rq->idle_at_tick) {
4899	rq->in_nohz_recently = 0;
4900
4901	if (atomic_read(&nohz.load_balancer) == cpu) {
4902	cpumask_clear_cpu(cpu, nohz.cpu_mask);
4903	atomic_set(&nohz.load_balancer, -1);
4904	}
4905
4906	if (atomic_read(&nohz.load_balancer) == -1) {
4907	int ilb = find_new_ilb(cpu);
4908
4909	if (ilb < nr_cpu_ids)
4910	resched_cpu(ilb);
4911	}
4912	}
4913
4914	/*
4915	* If this cpu is idle and doing idle load balancing for all the
4916	* cpus with ticks stopped, is it time for that to stop?
4917	*/
4918	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4919	cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4920	resched_cpu(cpu);
4921	return;
4922	}
4923
4924	/*
4925	* If this cpu is idle and the idle load balancing is done by
4926	* someone else, then no need raise the SCHED_SOFTIRQ
4927	*/
4928	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4929	cpumask_test_cpu(cpu, nohz.cpu_mask))
4930	return;
4931	#endif
4932	/* Don't need to rebalance while attached to NULL domain */
4933	if (time_after_eq(jiffies, rq->next_balance) &&
4934	likely(!on_null_domain(cpu)))
4935	raise_softirq(SCHED_SOFTIRQ);
4936	}
4937
4938	#else /* CONFIG_SMP */
4939
4940	/*
4941	* on UP we do not need to balance between CPUs:
4942	*/
4943	static inline void idle_balance(int cpu, struct rq *rq)
4944	{
4945	}
4946
4947	#endif
4948
4949	DEFINE_PER_CPU(struct kernel_stat, kstat);
4950
4951	EXPORT_PER_CPU_SYMBOL(kstat);
4952
4953	/*
4954	* Return any ns on the sched_clock that have not yet been accounted in
4955	* @p in case that task is currently running.
4956	*
4957	* Called with task_rq_lock() held on @rq.
4958	*/
4959	static u64 do_task_delta_exec(struct task_struct p, struct rq rq)
4960	{
4961	u64 ns = 0;
4962
4963	if (task_current(rq, p)) {
4964	update_rq_clock(rq);
4965	ns = rq->clock - p->se.exec_start;
4966	if ((s64)ns < 0)
4967	ns = 0;
4968	}
4969
4970	return ns;
4971	}
4972
4973	unsigned long long task_delta_exec(struct task_struct *p)
4974	{
4975	unsigned long flags;
4976	struct rq *rq;
4977	u64 ns = 0;
4978
4979	rq = task_rq_lock(p, &flags);
4980	ns = do_task_delta_exec(p, rq);
4981	task_rq_unlock(rq, &flags);
4982
4983	return ns;
4984	}
4985
4986	/*
4987	* Return accounted runtime for the task.
4988	* In case the task is currently running, return the runtime plus current's
4989	* pending runtime that have not been accounted yet.
4990	*/
4991	unsigned long long task_sched_runtime(struct task_struct *p)
4992	{
4993	unsigned long flags;
4994	struct rq *rq;
4995	u64 ns = 0;
4996
4997	rq = task_rq_lock(p, &flags);
4998	ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
4999	task_rq_unlock(rq, &flags);
5000
5001	return ns;
5002	}
5003
5004	/*
5005	* Return sum_exec_runtime for the thread group.
5006	* In case the task is currently running, return the sum plus current's
5007	* pending runtime that have not been accounted yet.
5008	*
5009	* Note that the thread group might have other running tasks as well,
5010	* so the return value not includes other pending runtime that other
5011	* running tasks might have.
5012	*/
5013	unsigned long long thread_group_sched_runtime(struct task_struct *p)
5014	{
5015	struct task_cputime totals;
5016	unsigned long flags;
5017	struct rq *rq;
5018	u64 ns;
5019
5020	rq = task_rq_lock(p, &flags);
5021	thread_group_cputime(p, &totals);
5022	ns = totals.sum_exec_runtime + do_task_delta_exec(p, rq);
5023	task_rq_unlock(rq, &flags);
5024
5025	return ns;
5026	}
5027
5028	/*
5029	* Account user cpu time to a process.
5030	* @p: the process that the cpu time gets accounted to
5031	* @cputime: the cpu time spent in user space since the last update
5032	* @cputime_scaled: cputime scaled by cpu frequency
5033	*/
5034	void account_user_time(struct task_struct *p, cputime_t cputime,
5035	cputime_t cputime_scaled)
5036	{
5037	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
5038	cputime64_t tmp;
5039
5040	/* Add user time to process. */
5041	p->utime = cputime_add(p->utime, cputime);
5042	p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
5043	account_group_user_time(p, cputime);
5044
5045	/* Add user time to cpustat. */
5046	tmp = cputime_to_cputime64(cputime);
5047	if (TASK_NICE(p) > 0)
5048	cpustat->nice = cputime64_add(cpustat->nice, tmp);
5049	else
5050	cpustat->user = cputime64_add(cpustat->user, tmp);
5051
5052	cpuacct_update_stats(p, CPUACCT_STAT_USER, cputime);
5053	/* Account for user time used */
5054	acct_update_integrals(p);
5055	}
5056
5057	/*
5058	* Account guest cpu time to a process.
5059	* @p: the process that the cpu time gets accounted to
5060	* @cputime: the cpu time spent in virtual machine since the last update
5061	* @cputime_scaled: cputime scaled by cpu frequency
5062	*/
5063	static void account_guest_time(struct task_struct *p, cputime_t cputime,
5064	cputime_t cputime_scaled)
5065	{
5066	cputime64_t tmp;
5067	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
5068
5069	tmp = cputime_to_cputime64(cputime);
5070
5071	/* Add guest time to process. */
5072	p->utime = cputime_add(p->utime, cputime);
5073	p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
5074	account_group_user_time(p, cputime);
5075	p->gtime = cputime_add(p->gtime, cputime);
5076
5077	/* Add guest time to cpustat. */
5078	cpustat->user = cputime64_add(cpustat->user, tmp);
5079	cpustat->guest = cputime64_add(cpustat->guest, tmp);
5080	}
5081
5082	/*
5083	* Account system cpu time to a process.
5084	* @p: the process that the cpu time gets accounted to
5085	* @hardirq_offset: the offset to subtract from hardirq_count()
5086	* @cputime: the cpu time spent in kernel space since the last update
5087	* @cputime_scaled: cputime scaled by cpu frequency
5088	*/
5089	void account_system_time(struct task_struct *p, int hardirq_offset,
5090	cputime_t cputime, cputime_t cputime_scaled)
5091	{
5092	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
5093	cputime64_t tmp;
5094
5095	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
5096	account_guest_time(p, cputime, cputime_scaled);
5097	return;
5098	}
5099
5100	/* Add system time to process. */
5101	p->stime = cputime_add(p->stime, cputime);
5102	p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
5103	account_group_system_time(p, cputime);
5104
5105	/* Add system time to cpustat. */
5106	tmp = cputime_to_cputime64(cputime);
5107	if (hardirq_count() - hardirq_offset)
5108	cpustat->irq = cputime64_add(cpustat->irq, tmp);
5109	else if (softirq_count())
5110	cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
5111	else
5112	cpustat->system = cputime64_add(cpustat->system, tmp);
5113
5114	cpuacct_update_stats(p, CPUACCT_STAT_SYSTEM, cputime);
5115
5116	/* Account for system time used */
5117	acct_update_integrals(p);
5118	}
5119
5120	/*
5121	* Account for involuntary wait time.
5122	* @steal: the cpu time spent in involuntary wait
5123	*/
5124	void account_steal_time(cputime_t cputime)
5125	{
5126	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
5127	cputime64_t cputime64 = cputime_to_cputime64(cputime);
5128
5129	cpustat->steal = cputime64_add(cpustat->steal, cputime64);
5130	}
5131
5132	/*
5133	* Account for idle time.
5134	* @cputime: the cpu time spent in idle wait
5135	*/
5136	void account_idle_time(cputime_t cputime)
5137	{
5138	struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
5139	cputime64_t cputime64 = cputime_to_cputime64(cputime);
5140	struct rq *rq = this_rq();
5141
5142	if (atomic_read(&rq->nr_iowait) > 0)
5143	cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
5144	else
5145	cpustat->idle = cputime64_add(cpustat->idle, cputime64);
5146	}
5147
5148	#ifndef CONFIG_VIRT_CPU_ACCOUNTING
5149
5150	/*
5151	* Account a single tick of cpu time.
5152	* @p: the process that the cpu time gets accounted to
5153	* @user_tick: indicates if the tick is a user or a system tick
5154	*/
5155	void account_process_tick(struct task_struct *p, int user_tick)
5156	{
5157	cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
5158	struct rq *rq = this_rq();
5159
5160	if (user_tick)
5161	account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
5162	else if ((p != rq->idle) \|\| (irq_count() != HARDIRQ_OFFSET))
5163	account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
5164	one_jiffy_scaled);
5165	else
5166	account_idle_time(cputime_one_jiffy);
5167	}
5168
5169	/*
5170	* Account multiple ticks of steal time.
5171	* @p: the process from which the cpu time has been stolen
5172	* @ticks: number of stolen ticks
5173	*/
5174	void account_steal_ticks(unsigned long ticks)
5175	{
5176	account_steal_time(jiffies_to_cputime(ticks));
5177	}
5178
5179	/*
5180	* Account multiple ticks of idle time.
5181	* @ticks: number of stolen ticks
5182	*/
5183	void account_idle_ticks(unsigned long ticks)
5184	{
5185	account_idle_time(jiffies_to_cputime(ticks));
5186	}
5187
5188	#endif
5189
5190	/*
5191	* Use precise platform statistics if available:
5192	*/
5193	#ifdef CONFIG_VIRT_CPU_ACCOUNTING
5194	cputime_t task_utime(struct task_struct *p)
5195	{
5196	return p->utime;
5197	}
5198
5199	cputime_t task_stime(struct task_struct *p)
5200	{
5201	return p->stime;
5202	}
5203	#else
5204	cputime_t task_utime(struct task_struct *p)
5205	{
5206	clock_t utime = cputime_to_clock_t(p->utime),
5207	total = utime + cputime_to_clock_t(p->stime);
5208	u64 temp;
5209
5210	/*
5211	* Use CFS's precise accounting:
5212	*/
5213	temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);
5214
5215	if (total) {
5216	temp *= utime;
5217	do_div(temp, total);
5218	}
5219	utime = (clock_t)temp;
5220
5221	p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
5222	return p->prev_utime;
5223	}
5224
5225	cputime_t task_stime(struct task_struct *p)
5226	{
5227	clock_t stime;
5228
5229	/*
5230	* Use CFS's precise accounting. (we subtract utime from
5231	* the total, to make sure the total observed by userspace
5232	* grows monotonically - apps rely on that):
5233	*/
5234	stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
5235	cputime_to_clock_t(task_utime(p));
5236
5237	if (stime >= 0)
5238	p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));
5239
5240	return p->prev_stime;
5241	}
5242	#endif
5243
5244	inline cputime_t task_gtime(struct task_struct *p)
5245	{
5246	return p->gtime;
5247	}
5248
5249	/*
5250	* This function gets called by the timer code, with HZ frequency.
5251	* We call it with interrupts disabled.
5252	*
5253	* It also gets called by the fork code, when changing the parent's
5254	* timeslices.
5255	*/
5256	void scheduler_tick(void)
5257	{
5258	int cpu = smp_processor_id();
5259	struct rq *rq = cpu_rq(cpu);
5260	struct task_struct *curr = rq->curr;
5261
5262	sched_clock_tick();
5263
5264	spin_lock(&rq->lock);
5265	update_rq_clock(rq);
5266	update_cpu_load(rq);
5267	curr->sched_class->task_tick(rq, curr, 0);
5268	spin_unlock(&rq->lock);
5269
5270	perf_event_task_tick(curr, cpu);
5271
5272	#ifdef CONFIG_SMP
5273	rq->idle_at_tick = idle_cpu(cpu);
5274	trigger_load_balance(rq, cpu);
5275	#endif
5276	}
5277
5278	notrace unsigned long get_parent_ip(unsigned long addr)
5279	{
5280	if (in_lock_functions(addr)) {
5281	addr = CALLER_ADDR2;
5282	if (in_lock_functions(addr))
5283	addr = CALLER_ADDR3;
5284	}
5285	return addr;
5286	}
5287
5288	#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) \|\| \
5289	defined(CONFIG_PREEMPT_TRACER))
5290
5291	void __kprobes add_preempt_count(int val)
5292	{
5293	#ifdef CONFIG_DEBUG_PREEMPT
5294	/*
5295	* Underflow?
5296	*/
5297	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
5298	return;
5299	#endif
5300	preempt_count() += val;
5301	#ifdef CONFIG_DEBUG_PREEMPT
5302	/*
5303	* Spinlock count overflowing soon?
5304	*/
5305	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
5306	PREEMPT_MASK - 10);
5307	#endif
5308	if (preempt_count() == val)
5309	trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
5310	}
5311	EXPORT_SYMBOL(add_preempt_count);
5312
5313	void __kprobes sub_preempt_count(int val)
5314	{
5315	#ifdef CONFIG_DEBUG_PREEMPT
5316	/*
5317	* Underflow?
5318	*/
5319	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
5320	return;
5321	/*
5322	* Is the spinlock portion underflowing?
5323	*/
5324	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
5325	!(preempt_count() & PREEMPT_MASK)))
5326	return;
5327	#endif
5328
5329	if (preempt_count() == val)
5330	trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
5331	preempt_count() -= val;
5332	}
5333	EXPORT_SYMBOL(sub_preempt_count);
5334
5335	#endif
5336
5337	/*
5338	* Print scheduling while atomic bug:
5339	*/
5340	static noinline void __schedule_bug(struct task_struct *prev)
5341	{
5342	struct pt_regs *regs = get_irq_regs();
5343
5344	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
5345	prev->comm, prev->pid, preempt_count());
5346
5347	debug_show_held_locks(prev);
5348	print_modules();
5349	if (irqs_disabled())
5350	print_irqtrace_events(prev);
5351
5352	if (regs)
5353	show_regs(regs);
5354	else
5355	dump_stack();
5356	}
5357
5358	/*
5359	* Various schedule()-time debugging checks and statistics:
5360	*/
5361	static inline void schedule_debug(struct task_struct *prev)
5362	{
5363	/*
5364	* Test if we are atomic. Since do_exit() needs to call into
5365	* schedule() atomically, we ignore that path for now.
5366	* Otherwise, whine if we are scheduling when we should not be.
5367	*/
5368	if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
5369	__schedule_bug(prev);
5370
5371	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
5372
5373	schedstat_inc(this_rq(), sched_count);
5374	#ifdef CONFIG_SCHEDSTATS
5375	if (unlikely(prev->lock_depth >= 0)) {
5376	schedstat_inc(this_rq(), bkl_count);
5377	schedstat_inc(prev, sched_info.bkl_count);
5378	}
5379	#endif
5380	}
5381
5382	static void put_prev_task(struct rq rq, struct task_struct p)
5383	{
5384	u64 runtime = p->se.sum_exec_runtime - p->se.prev_sum_exec_runtime;
5385
5386	update_avg(&p->se.avg_running, runtime);
5387
5388	if (p->state == TASK_RUNNING) {
5389	/*
5390	* In order to avoid avg_overlap growing stale when we are
5391	* indeed overlapping and hence not getting put to sleep, grow
5392	* the avg_overlap on preemption.
5393	*
5394	* We use the average preemption runtime because that
5395	* correlates to the amount of cache footprint a task can
5396	* build up.
5397	*/
5398	runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
5399	update_avg(&p->se.avg_overlap, runtime);
5400	} else {
5401	update_avg(&p->se.avg_running, 0);
5402	}
5403	p->sched_class->put_prev_task(rq, p);
5404	}
5405
5406	/*
5407	* Pick up the highest-prio task:
5408	*/
5409	static inline struct task_struct *
5410	pick_next_task(struct rq *rq)
5411	{
5412	const struct sched_class *class;
5413	struct task_struct *p;
5414
5415	/*
5416	* Optimization: we know that if all tasks are in
5417	* the fair class we can call that function directly:
5418	*/
5419	if (likely(rq->nr_running == rq->cfs.nr_running)) {
5420	p = fair_sched_class.pick_next_task(rq);
5421	if (likely(p))
5422	return p;
5423	}
5424
5425	class = sched_class_highest;
5426	for ( ; ; ) {
5427	p = class->pick_next_task(rq);
5428	if (p)
5429	return p;
5430	/*
5431	* Will never be NULL as the idle class always
5432	* returns a non-NULL p:
5433	*/
5434	class = class->next;
5435	}
5436	}
5437
5438	/*
5439	* schedule() is the main scheduler function.
5440	*/
5441	asmlinkage void __sched schedule(void)
5442	{
5443	struct task_struct prev, next;
5444	unsigned long *switch_count;
5445	struct rq *rq;
5446	int cpu;
5447
5448	need_resched:
5449	preempt_disable();
5450	cpu = smp_processor_id();
5451	rq = cpu_rq(cpu);
5452	rcu_sched_qs(cpu);
5453	prev = rq->curr;
5454	switch_count = &prev->nivcsw;
5455
5456	release_kernel_lock(prev);
5457	need_resched_nonpreemptible:
5458
5459	schedule_debug(prev);
5460
5461	if (sched_feat(HRTICK))
5462	hrtick_clear(rq);
5463
5464	spin_lock_irq(&rq->lock);
5465	update_rq_clock(rq);
5466	clear_tsk_need_resched(prev);
5467
5468	if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
5469	if (unlikely(signal_pending_state(prev->state, prev)))
5470	prev->state = TASK_RUNNING;
5471	else
5472	deactivate_task(rq, prev, 1);
5473	switch_count = &prev->nvcsw;
5474	}
5475
5476	pre_schedule(rq, prev);
5477
5478	if (unlikely(!rq->nr_running))
5479	idle_balance(cpu, rq);
5480
5481	put_prev_task(rq, prev);
5482	next = pick_next_task(rq);
5483
5484	if (likely(prev != next)) {
5485	sched_info_switch(prev, next);
5486	perf_event_task_sched_out(prev, next, cpu);
5487
5488	rq->nr_switches++;
5489	rq->curr = next;
5490	++*switch_count;
5491
5492	context_switch(rq, prev, next); /* unlocks the rq */
5493	/*
5494	* the context switch might have flipped the stack from under
5495	* us, hence refresh the local variables.
5496	*/
5497	cpu = smp_processor_id();
5498	rq = cpu_rq(cpu);
5499	} else
5500	spin_unlock_irq(&rq->lock);
5501
5502	post_schedule(rq);
5503
5504	if (unlikely(reacquire_kernel_lock(current) < 0))
5505	goto need_resched_nonpreemptible;
5506
5507	preempt_enable_no_resched();
5508	if (need_resched())
5509	goto need_resched;
5510	}
5511	EXPORT_SYMBOL(schedule);
5512
5513	#ifdef CONFIG_SMP
5514	/*
5515	* Look out! "owner" is an entirely speculative pointer
5516	* access and not reliable.
5517	*/
5518	int mutex_spin_on_owner(struct mutex lock, struct thread_info owner)
5519	{
5520	unsigned int cpu;
5521	struct rq *rq;
5522
5523	if (!sched_feat(OWNER_SPIN))
5524	return 0;
5525
5526	#ifdef CONFIG_DEBUG_PAGEALLOC
5527	/*
5528	* Need to access the cpu field knowing that
5529	* DEBUG_PAGEALLOC could have unmapped it if
5530	* the mutex owner just released it and exited.
5531	*/
5532	if (probe_kernel_address(&owner->cpu, cpu))
5533	goto out;
5534	#else
5535	cpu = owner->cpu;
5536	#endif
5537
5538	/*
5539	* Even if the access succeeded (likely case),
5540	* the cpu field may no longer be valid.
5541	*/
5542	if (cpu >= nr_cpumask_bits)
5543	goto out;
5544
5545	/*
5546	* We need to validate that we can do a
5547	* get_cpu() and that we have the percpu area.
5548	*/
5549	if (!cpu_online(cpu))
5550	goto out;
5551
5552	rq = cpu_rq(cpu);
5553
5554	for (;;) {
5555	/*
5556	* Owner changed, break to re-assess state.
5557	*/
5558	if (lock->owner != owner)
5559	break;
5560
5561	/*
5562	* Is that owner really running on that cpu?
5563	*/
5564	if (task_thread_info(rq->curr) != owner \|\| need_resched())
5565	return 0;
5566
5567	cpu_relax();
5568	}
5569	out:
5570	return 1;
5571	}
5572	#endif
5573
5574	#ifdef CONFIG_PREEMPT
5575	/*
5576	* this is the entry point to schedule() from in-kernel preemption
5577	* off of preempt_enable. Kernel preemptions off return from interrupt
5578	* occur there and call schedule directly.
5579	*/
5580	asmlinkage void __sched preempt_schedule(void)
5581	{
5582	struct thread_info *ti = current_thread_info();
5583
5584	/*
5585	* If there is a non-zero preempt_count or interrupts are disabled,
5586	* we do not want to preempt the current task. Just return..
5587	*/
5588	if (likely(ti->preempt_count \|\| irqs_disabled()))
5589	return;
5590
5591	do {
5592	add_preempt_count(PREEMPT_ACTIVE);
5593	schedule();
5594	sub_preempt_count(PREEMPT_ACTIVE);
5595
5596	/*
5597	* Check again in case we missed a preemption opportunity
5598	* between schedule and now.
5599	*/
5600	barrier();
5601	} while (need_resched());
5602	}
5603	EXPORT_SYMBOL(preempt_schedule);
5604
5605	/*
5606	* this is the entry point to schedule() from kernel preemption
5607	* off of irq context.
5608	* Note, that this is called and return with irqs disabled. This will
5609	* protect us against recursive calling from irq.
5610	*/
5611	asmlinkage void __sched preempt_schedule_irq(void)
5612	{
5613	struct thread_info *ti = current_thread_info();
5614
5615	/* Catch callers which need to be fixed */
5616	BUG_ON(ti->preempt_count \|\| !irqs_disabled());
5617
5618	do {
5619	add_preempt_count(PREEMPT_ACTIVE);
5620	local_irq_enable();
5621	schedule();
5622	local_irq_disable();
5623	sub_preempt_count(PREEMPT_ACTIVE);
5624
5625	/*
5626	* Check again in case we missed a preemption opportunity
5627	* between schedule and now.
5628	*/
5629	barrier();
5630	} while (need_resched());
5631	}
5632
5633	#endif /* CONFIG_PREEMPT */
5634
5635	int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
5636	void *key)
5637	{
5638	return try_to_wake_up(curr->private, mode, wake_flags);
5639	}
5640	EXPORT_SYMBOL(default_wake_function);
5641
5642	/*
5643	* The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
5644	* wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
5645	* number) then we wake all the non-exclusive tasks and one exclusive task.
5646	*
5647	* There are circumstances in which we can try to wake a task which has already
5648	* started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
5649	* zero in this (rare) case, and we handle it by continuing to scan the queue.
5650	*/
5651	static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
5652	int nr_exclusive, int wake_flags, void *key)
5653	{
5654	wait_queue_t curr, next;
5655
5656	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
5657	unsigned flags = curr->flags;
5658
5659	if (curr->func(curr, mode, wake_flags, key) &&
5660	(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
5661	break;
5662	}
5663	}
5664
5665	/**
5666	* __wake_up - wake up threads blocked on a waitqueue.
5667	* @q: the waitqueue
5668	* @mode: which threads
5669	* @nr_exclusive: how many wake-one or wake-many threads to wake up
5670	* @key: is directly passed to the wakeup function
5671	*
5672	* It may be assumed that this function implies a write memory barrier before
5673	* changing the task state if and only if any tasks are woken up.
5674	*/
5675	void __wake_up(wait_queue_head_t *q, unsigned int mode,
5676	int nr_exclusive, void *key)
5677	{
5678	unsigned long flags;
5679
5680	spin_lock_irqsave(&q->lock, flags);
5681	__wake_up_common(q, mode, nr_exclusive, 0, key);
5682	spin_unlock_irqrestore(&q->lock, flags);
5683	}
5684	EXPORT_SYMBOL(__wake_up);
5685
5686	/*
5687	* Same as __wake_up but called with the spinlock in wait_queue_head_t held.
5688	*/
5689	void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
5690	{
5691	__wake_up_common(q, mode, 1, 0, NULL);
5692	}
5693
5694	void __wake_up_locked_key(wait_queue_head_t q, unsigned int mode, void key)
5695	{
5696	__wake_up_common(q, mode, 1, 0, key);
5697	}
5698
5699	/**
5700	* __wake_up_sync_key - wake up threads blocked on a waitqueue.
5701	* @q: the waitqueue
5702	* @mode: which threads
5703	* @nr_exclusive: how many wake-one or wake-many threads to wake up
5704	* @key: opaque value to be passed to wakeup targets
5705	*
5706	* The sync wakeup differs that the waker knows that it will schedule
5707	* away soon, so while the target thread will be woken up, it will not
5708	* be migrated to another CPU - ie. the two threads are 'synchronized'
5709	* with each other. This can prevent needless bouncing between CPUs.
5710	*
5711	* On UP it can prevent extra preemption.
5712	*
5713	* It may be assumed that this function implies a write memory barrier before
5714	* changing the task state if and only if any tasks are woken up.
5715	*/
5716	void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
5717	int nr_exclusive, void *key)
5718	{
5719	unsigned long flags;
5720	int wake_flags = WF_SYNC;
5721
5722	if (unlikely(!q))
5723	return;
5724
5725	if (unlikely(!nr_exclusive))
5726	wake_flags = 0;
5727
5728	spin_lock_irqsave(&q->lock, flags);
5729	__wake_up_common(q, mode, nr_exclusive, wake_flags, key);
5730	spin_unlock_irqrestore(&q->lock, flags);
5731	}
5732	EXPORT_SYMBOL_GPL(__wake_up_sync_key);
5733
5734	/*
5735	* __wake_up_sync - see __wake_up_sync_key()
5736	*/
5737	void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
5738	{
5739	__wake_up_sync_key(q, mode, nr_exclusive, NULL);
5740	}
5741	EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
5742
5743	/**
5744	* complete: - signals a single thread waiting on this completion
5745	* @x: holds the state of this particular completion
5746	*
5747	* This will wake up a single thread waiting on this completion. Threads will be
5748	* awakened in the same order in which they were queued.
5749	*
5750	* See also complete_all(), wait_for_completion() and related routines.
5751	*
5752	* It may be assumed that this function implies a write memory barrier before
5753	* changing the task state if and only if any tasks are woken up.
5754	*/
5755	void complete(struct completion *x)
5756	{
5757	unsigned long flags;
5758
5759	spin_lock_irqsave(&x->wait.lock, flags);
5760	x->done++;
5761	__wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
5762	spin_unlock_irqrestore(&x->wait.lock, flags);
5763	}
5764	EXPORT_SYMBOL(complete);
5765
5766	/**
5767	* complete_all: - signals all threads waiting on this completion
5768	* @x: holds the state of this particular completion
5769	*
5770	* This will wake up all threads waiting on this particular completion event.
5771	*
5772	* It may be assumed that this function implies a write memory barrier before
5773	* changing the task state if and only if any tasks are woken up.
5774	*/
5775	void complete_all(struct completion *x)
5776	{
5777	unsigned long flags;
5778
5779	spin_lock_irqsave(&x->wait.lock, flags);
5780	x->done += UINT_MAX/2;
5781	__wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
5782	spin_unlock_irqrestore(&x->wait.lock, flags);
5783	}
5784	EXPORT_SYMBOL(complete_all);
5785
5786	static inline long __sched
5787	do_wait_for_common(struct completion *x, long timeout, int state)
5788	{
5789	if (!x->done) {
5790	DECLARE_WAITQUEUE(wait, current);
5791
5792	wait.flags \|= WQ_FLAG_EXCLUSIVE;
5793	__add_wait_queue_tail(&x->wait, &wait);
5794	do {
5795	if (signal_pending_state(state, current)) {
5796	timeout = -ERESTARTSYS;
5797	break;
5798	}
5799	__set_current_state(state);
5800	spin_unlock_irq(&x->wait.lock);
5801	timeout = schedule_timeout(timeout);
5802	spin_lock_irq(&x->wait.lock);
5803	} while (!x->done && timeout);
5804	__remove_wait_queue(&x->wait, &wait);
5805	if (!x->done)
5806	return timeout;
5807	}
5808	x->done--;
5809	return timeout ?: 1;
5810	}
5811
5812	static long __sched
5813	wait_for_common(struct completion *x, long timeout, int state)
5814	{
5815	might_sleep();
5816
5817	spin_lock_irq(&x->wait.lock);
5818	timeout = do_wait_for_common(x, timeout, state);
5819	spin_unlock_irq(&x->wait.lock);
5820	return timeout;
5821	}
5822
5823	/**
5824	* wait_for_completion: - waits for completion of a task
5825	* @x: holds the state of this particular completion
5826	*
5827	* This waits to be signaled for completion of a specific task. It is NOT
5828	* interruptible and there is no timeout.
5829	*
5830	* See also similar routines (i.e. wait_for_completion_timeout()) with timeout
5831	* and interrupt capability. Also see complete().
5832	*/
5833	void __sched wait_for_completion(struct completion *x)
5834	{
5835	wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
5836	}
5837	EXPORT_SYMBOL(wait_for_completion);
5838
5839	/**
5840	* wait_for_completion_timeout: - waits for completion of a task (w/timeout)
5841	* @x: holds the state of this particular completion
5842	* @timeout: timeout value in jiffies
5843	*
5844	* This waits for either a completion of a specific task to be signaled or for a
5845	* specified timeout to expire. The timeout is in jiffies. It is not
5846	* interruptible.
5847	*/
5848	unsigned long __sched
5849	wait_for_completion_timeout(struct completion *x, unsigned long timeout)
5850	{
5851	return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
5852	}
5853	EXPORT_SYMBOL(wait_for_completion_timeout);
5854
5855	/**
5856	* wait_for_completion_interruptible: - waits for completion of a task (w/intr)
5857	* @x: holds the state of this particular completion
5858	*
5859	* This waits for completion of a specific task to be signaled. It is
5860	* interruptible.
5861	*/
5862	int __sched wait_for_completion_interruptible(struct completion *x)
5863	{
5864	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
5865	if (t == -ERESTARTSYS)
5866	return t;
5867	return 0;
5868	}
5869	EXPORT_SYMBOL(wait_for_completion_interruptible);
5870
5871	/**
5872	* wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
5873	* @x: holds the state of this particular completion
5874	* @timeout: timeout value in jiffies
5875	*
5876	* This waits for either a completion of a specific task to be signaled or for a
5877	* specified timeout to expire. It is interruptible. The timeout is in jiffies.
5878	*/
5879	unsigned long __sched
5880	wait_for_completion_interruptible_timeout(struct completion *x,
5881	unsigned long timeout)
5882	{
5883	return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
5884	}
5885	EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
5886
5887	/**
5888	* wait_for_completion_killable: - waits for completion of a task (killable)
5889	* @x: holds the state of this particular completion
5890	*
5891	* This waits to be signaled for completion of a specific task. It can be
5892	* interrupted by a kill signal.
5893	*/
5894	int __sched wait_for_completion_killable(struct completion *x)
5895	{
5896	long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
5897	if (t == -ERESTARTSYS)
5898	return t;
5899	return 0;
5900	}
5901	EXPORT_SYMBOL(wait_for_completion_killable);
5902
5903	/**
5904	* try_wait_for_completion - try to decrement a completion without blocking
5905	* @x: completion structure
5906	*
5907	* Returns: 0 if a decrement cannot be done without blocking
5908	* 1 if a decrement succeeded.
5909	*
5910	* If a completion is being used as a counting completion,
5911	* attempt to decrement the counter without blocking. This
5912	* enables us to avoid waiting if the resource the completion
5913	* is protecting is not available.
5914	*/
5915	bool try_wait_for_completion(struct completion *x)
5916	{
5917	int ret = 1;
5918
5919	spin_lock_irq(&x->wait.lock);
5920	if (!x->done)
5921	ret = 0;
5922	else
5923	x->done--;
5924	spin_unlock_irq(&x->wait.lock);
5925	return ret;
5926	}
5927	EXPORT_SYMBOL(try_wait_for_completion);
5928
5929	/**
5930	* completion_done - Test to see if a completion has any waiters
5931	* @x: completion structure
5932	*
5933	* Returns: 0 if there are waiters (wait_for_completion() in progress)
5934	* 1 if there are no waiters.
5935	*
5936	*/
5937	bool completion_done(struct completion *x)
5938	{
5939	int ret = 1;
5940
5941	spin_lock_irq(&x->wait.lock);
5942	if (!x->done)
5943	ret = 0;
5944	spin_unlock_irq(&x->wait.lock);
5945	return ret;
5946	}
5947	EXPORT_SYMBOL(completion_done);
5948
5949	static long __sched
5950	sleep_on_common(wait_queue_head_t *q, int state, long timeout)
5951	{
5952	unsigned long flags;
5953	wait_queue_t wait;
5954
5955	init_waitqueue_entry(&wait, current);
5956
5957	__set_current_state(state);
5958
5959	spin_lock_irqsave(&q->lock, flags);
5960	__add_wait_queue(q, &wait);
5961	spin_unlock(&q->lock);
5962	timeout = schedule_timeout(timeout);
5963	spin_lock_irq(&q->lock);
5964	__remove_wait_queue(q, &wait);
5965	spin_unlock_irqrestore(&q->lock, flags);
5966
5967	return timeout;
5968	}
5969
5970	void __sched interruptible_sleep_on(wait_queue_head_t *q)
5971	{
5972	sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
5973	}
5974	EXPORT_SYMBOL(interruptible_sleep_on);
5975
5976	long __sched
5977	interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
5978	{
5979	return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
5980	}
5981	EXPORT_SYMBOL(interruptible_sleep_on_timeout);
5982
5983	void __sched sleep_on(wait_queue_head_t *q)
5984	{
5985	sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
5986	}
5987	EXPORT_SYMBOL(sleep_on);
5988
5989	long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
5990	{
5991	return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
5992	}
5993	EXPORT_SYMBOL(sleep_on_timeout);
5994
5995	#ifdef CONFIG_RT_MUTEXES
5996
5997	/*
5998	* rt_mutex_setprio - set the current priority of a task
5999	* @p: task
6000	* @prio: prio value (kernel-internal form)
6001	*
6002	* This function changes the 'effective' priority of a task. It does
6003	* not touch ->normal_prio like __setscheduler().
6004	*
6005	* Used by the rt_mutex code to implement priority inheritance logic.
6006	*/
6007	void rt_mutex_setprio(struct task_struct *p, int prio)
6008	{
6009	unsigned long flags;
6010	int oldprio, on_rq, running;
6011	struct rq *rq;
6012	const struct sched_class *prev_class;
6013
6014	BUG_ON(prio < 0 \|\| prio > MAX_PRIO);
6015
6016	rq = task_rq_lock(p, &flags);
6017	update_rq_clock(rq);
6018
6019	oldprio = p->prio;
6020	prev_class = p->sched_class;
6021	on_rq = p->se.on_rq;
6022	running = task_current(rq, p);
6023	if (on_rq)
6024	dequeue_task(rq, p, 0);
6025	if (running)
6026	p->sched_class->put_prev_task(rq, p);
6027
6028	if (rt_prio(prio))
6029	p->sched_class = &rt_sched_class;
6030	else
6031	p->sched_class = &fair_sched_class;
6032
6033	p->prio = prio;
6034
6035	if (running)
6036	p->sched_class->set_curr_task(rq);
6037	if (on_rq) {
6038	enqueue_task(rq, p, 0);
6039
6040	check_class_changed(rq, p, prev_class, oldprio, running);
6041	}
6042	task_rq_unlock(rq, &flags);
6043	}
6044
6045	#endif
6046
6047	void set_user_nice(struct task_struct *p, long nice)
6048	{
6049	int old_prio, delta, on_rq;
6050	unsigned long flags;
6051	struct rq *rq;
6052
6053	if (TASK_NICE(p) == nice \|\| nice < -20 \|\| nice > 19)
6054	return;
6055	/*
6056	* We have to be careful, if called from sys_setpriority(),
6057	* the task might be in the middle of scheduling on another CPU.
6058	*/
6059	rq = task_rq_lock(p, &flags);
6060	update_rq_clock(rq);
6061	/*
6062	* The RT priorities are set via sched_setscheduler(), but we still
6063	* allow the 'normal' nice value to be set - but as expected
6064	* it wont have any effect on scheduling until the task is
6065	* SCHED_FIFO/SCHED_RR:
6066	*/
6067	if (task_has_rt_policy(p)) {
6068	p->static_prio = NICE_TO_PRIO(nice);
6069	goto out_unlock;
6070	}
6071	on_rq = p->se.on_rq;
6072	if (on_rq)
6073	dequeue_task(rq, p, 0);
6074
6075	p->static_prio = NICE_TO_PRIO(nice);
6076	set_load_weight(p);
6077	old_prio = p->prio;
6078	p->prio = effective_prio(p);
6079	delta = p->prio - old_prio;
6080
6081	if (on_rq) {
6082	enqueue_task(rq, p, 0);
6083	/*
6084	* If the task increased its priority or is running and
6085	* lowered its priority, then reschedule its CPU:
6086	*/
6087	if (delta < 0 \|\| (delta > 0 && task_running(rq, p)))
6088	resched_task(rq->curr);
6089	}
6090	out_unlock:
6091	task_rq_unlock(rq, &flags);
6092	}
6093	EXPORT_SYMBOL(set_user_nice);
6094
6095	/*
6096	* can_nice - check if a task can reduce its nice value
6097	* @p: task
6098	* @nice: nice value
6099	*/
6100	int can_nice(const struct task_struct *p, const int nice)
6101	{
6102	/* convert nice value [19,-20] to rlimit style value [1,40] */
6103	int nice_rlim = 20 - nice;
6104
6105	return (nice_rlim <= p->signal->rlim[RLIMIT_NICE].rlim_cur \|\|
6106	capable(CAP_SYS_NICE));
6107	}
6108	EXPORT_SYMBOL_GPL(can_nice);
6109
6110	#ifdef __ARCH_WANT_SYS_NICE
6111
6112	/*
6113	* sys_nice - change the priority of the current process.
6114	* @increment: priority increment
6115	*
6116	* sys_setpriority is a more generic, but much slower function that
6117	* does similar things.
6118	*/
6119	SYSCALL_DEFINE1(nice, int, increment)
6120	{
6121	long nice, retval;
6122
6123	/*
6124	* Setpriority might change our priority at the same moment.
6125	* We don't have to worry. Conceptually one call occurs first
6126	* and we have a single winner.
6127	*/
6128	if (increment < -40)
6129	increment = -40;
6130	if (increment > 40)
6131	increment = 40;
6132
6133	nice = TASK_NICE(current) + increment;
6134	if (nice < -20)
6135	nice = -20;
6136	if (nice > 19)
6137	nice = 19;
6138
6139	if (increment < 0 && !can_nice(current, nice))
6140	return -EPERM;
6141
6142	retval = security_task_setnice(current, nice);
6143	if (retval)
6144	return retval;
6145
6146	set_user_nice(current, nice);
6147	return 0;
6148	}
6149
6150	#endif
6151
6152	/**
6153	* task_prio - return the priority value of a given task.
6154	* @p: the task in question.
6155	*
6156	* This is the priority value as seen by users in /proc.
6157	* RT tasks are offset by -200. Normal tasks are centered
6158	* around 0, value goes from -16 to +15.
6159	*/
6160	int task_prio(const struct task_struct *p)
6161	{
6162	return p->prio - MAX_RT_PRIO;
6163	}
6164
6165	/**
6166	* task_nice - return the nice value of a given task.
6167	* @p: the task in question.
6168	*/
6169	int task_nice(const struct task_struct *p)
6170	{
6171	return TASK_NICE(p);
6172	}
6173	EXPORT_SYMBOL(task_nice);
6174
6175	/**
6176	* idle_cpu - is a given cpu idle currently?
6177	* @cpu: the processor in question.
6178	*/
6179	int idle_cpu(int cpu)
6180	{
6181	return cpu_curr(cpu) == cpu_rq(cpu)->idle;
6182	}
6183
6184	/**
6185	* idle_task - return the idle task for a given cpu.
6186	* @cpu: the processor in question.
6187	*/
6188	struct task_struct *idle_task(int cpu)
6189	{
6190	return cpu_rq(cpu)->idle;
6191	}
6192
6193	/**
6194	* find_process_by_pid - find a process with a matching PID value.
6195	* @pid: the pid in question.
6196	*/
6197	static struct task_struct *find_process_by_pid(pid_t pid)
6198	{
6199	return pid ? find_task_by_vpid(pid) : current;
6200	}
6201
6202	/* Actually do priority change: must hold rq lock. */
6203	static void
6204	__setscheduler(struct rq rq, struct task_struct p, int policy, int prio)
6205	{
6206	BUG_ON(p->se.on_rq);
6207
6208	p->policy = policy;
6209	switch (p->policy) {
6210	case SCHED_NORMAL:
6211	case SCHED_BATCH:
6212	case SCHED_IDLE:
6213	p->sched_class = &fair_sched_class;
6214	break;
6215	case SCHED_FIFO:
6216	case SCHED_RR:
6217	p->sched_class = &rt_sched_class;
6218	break;
6219	}
6220
6221	p->rt_priority = prio;
6222	p->normal_prio = normal_prio(p);
6223	/* we are holding p->pi_lock already */
6224	p->prio = rt_mutex_getprio(p);
6225	set_load_weight(p);
6226	}
6227
6228	/*
6229	* check the target process has a UID that matches the current process's
6230	*/
6231	static bool check_same_owner(struct task_struct *p)
6232	{
6233	const struct cred cred = current_cred(), pcred;
6234	bool match;
6235
6236	rcu_read_lock();
6237	pcred = __task_cred(p);
6238	match = (cred->euid == pcred->euid \|\|
6239	cred->euid == pcred->uid);
6240	rcu_read_unlock();
6241	return match;
6242	}
6243
6244	static int __sched_setscheduler(struct task_struct *p, int policy,
6245	struct sched_param *param, bool user)
6246	{
6247	int retval, oldprio, oldpolicy = -1, on_rq, running;
6248	unsigned long flags;
6249	const struct sched_class *prev_class;
6250	struct rq *rq;
6251	int reset_on_fork;
6252
6253	/* may grab non-irq protected spin_locks */
6254	BUG_ON(in_interrupt());
6255	recheck:
6256	/* double check policy once rq lock held */
6257	if (policy < 0) {
6258	reset_on_fork = p->sched_reset_on_fork;
6259	policy = oldpolicy = p->policy;
6260	} else {
6261	reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
6262	policy &= ~SCHED_RESET_ON_FORK;
6263
6264	if (policy != SCHED_FIFO && policy != SCHED_RR &&
6265	policy != SCHED_NORMAL && policy != SCHED_BATCH &&
6266	policy != SCHED_IDLE)
6267	return -EINVAL;
6268	}
6269
6270	/*
6271	* Valid priorities for SCHED_FIFO and SCHED_RR are
6272	* 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
6273	* SCHED_BATCH and SCHED_IDLE is 0.
6274	*/
6275	if (param->sched_priority < 0 \|\|
6276	(p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) \|\|
6277	(!p->mm && param->sched_priority > MAX_RT_PRIO-1))
6278	return -EINVAL;
6279	if (rt_policy(policy) != (param->sched_priority != 0))
6280	return -EINVAL;
6281
6282	/*
6283	* Allow unprivileged RT tasks to decrease priority:
6284	*/
6285	if (user && !capable(CAP_SYS_NICE)) {
6286	if (rt_policy(policy)) {
6287	unsigned long rlim_rtprio;
6288
6289	if (!lock_task_sighand(p, &flags))
6290	return -ESRCH;
6291	rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
6292	unlock_task_sighand(p, &flags);
6293
6294	/* can't set/change the rt policy */
6295	if (policy != p->policy && !rlim_rtprio)
6296	return -EPERM;
6297
6298	/* can't increase priority */
6299	if (param->sched_priority > p->rt_priority &&
6300	param->sched_priority > rlim_rtprio)
6301	return -EPERM;
6302	}
6303	/*
6304	* Like positive nice levels, dont allow tasks to
6305	* move out of SCHED_IDLE either:
6306	*/
6307	if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
6308	return -EPERM;
6309
6310	/* can't change other user's priorities */
6311	if (!check_same_owner(p))
6312	return -EPERM;
6313
6314	/* Normal users shall not reset the sched_reset_on_fork flag */
6315	if (p->sched_reset_on_fork && !reset_on_fork)
6316	return -EPERM;
6317	}
6318
6319	if (user) {
6320	#ifdef CONFIG_RT_GROUP_SCHED
6321	/*
6322	* Do not allow realtime tasks into groups that have no runtime
6323	* assigned.
6324	*/
6325	if (rt_bandwidth_enabled() && rt_policy(policy) &&
6326	task_group(p)->rt_bandwidth.rt_runtime == 0)
6327	return -EPERM;
6328	#endif
6329
6330	retval = security_task_setscheduler(p, policy, param);
6331	if (retval)
6332	return retval;
6333	}
6334
6335	/*
6336	* make sure no PI-waiters arrive (or leave) while we are
6337	* changing the priority of the task:
6338	*/
6339	spin_lock_irqsave(&p->pi_lock, flags);
6340	/*
6341	* To be able to change p->policy safely, the apropriate
6342	* runqueue lock must be held.
6343	*/
6344	rq = __task_rq_lock(p);
6345	/* recheck policy now with rq lock held */
6346	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
6347	policy = oldpolicy = -1;
6348	__task_rq_unlock(rq);
6349	spin_unlock_irqrestore(&p->pi_lock, flags);
6350	goto recheck;
6351	}
6352	update_rq_clock(rq);
6353	on_rq = p->se.on_rq;
6354	running = task_current(rq, p);
6355	if (on_rq)
6356	deactivate_task(rq, p, 0);
6357	if (running)
6358	p->sched_class->put_prev_task(rq, p);
6359
6360	p->sched_reset_on_fork = reset_on_fork;
6361
6362	oldprio = p->prio;
6363	prev_class = p->sched_class;
6364	__setscheduler(rq, p, policy, param->sched_priority);
6365
6366	if (running)
6367	p->sched_class->set_curr_task(rq);
6368	if (on_rq) {
6369	activate_task(rq, p, 0);
6370
6371	check_class_changed(rq, p, prev_class, oldprio, running);
6372	}
6373	__task_rq_unlock(rq);
6374	spin_unlock_irqrestore(&p->pi_lock, flags);
6375
6376	rt_mutex_adjust_pi(p);
6377
6378	return 0;
6379	}
6380
6381	/**
6382	* sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
6383	* @p: the task in question.
6384	* @policy: new policy.
6385	* @param: structure containing the new RT priority.
6386	*
6387	* NOTE that the task may be already dead.
6388	*/
6389	int sched_setscheduler(struct task_struct *p, int policy,
6390	struct sched_param *param)
6391	{
6392	return __sched_setscheduler(p, policy, param, true);
6393	}
6394	EXPORT_SYMBOL_GPL(sched_setscheduler);
6395
6396	/**
6397	* sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
6398	* @p: the task in question.
6399	* @policy: new policy.
6400	* @param: structure containing the new RT priority.
6401	*
6402	* Just like sched_setscheduler, only don't bother checking if the
6403	* current context has permission. For example, this is needed in
6404	* stop_machine(): we create temporary high priority worker threads,
6405	* but our caller might not have that capability.
6406	*/
6407	int sched_setscheduler_nocheck(struct task_struct *p, int policy,
6408	struct sched_param *param)
6409	{
6410	return __sched_setscheduler(p, policy, param, false);
6411	}
6412
6413	static int
6414	do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
6415	{
6416	struct sched_param lparam;
6417	struct task_struct *p;
6418	int retval;
6419
6420	if (!param \|\| pid < 0)
6421	return -EINVAL;
6422	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
6423	return -EFAULT;
6424
6425	rcu_read_lock();
6426	retval = -ESRCH;
6427	p = find_process_by_pid(pid);
6428	if (p != NULL)
6429	retval = sched_setscheduler(p, policy, &lparam);
6430	rcu_read_unlock();
6431
6432	return retval;
6433	}
6434
6435	/**
6436	* sys_sched_setscheduler - set/change the scheduler policy and RT priority
6437	* @pid: the pid in question.
6438	* @policy: new policy.
6439	* @param: structure containing the new RT priority.
6440	*/
6441	SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
6442	struct sched_param __user *, param)
6443	{
6444	/* negative values for policy are not valid */
6445	if (policy < 0)
6446	return -EINVAL;
6447
6448	return do_sched_setscheduler(pid, policy, param);
6449	}
6450
6451	/**
6452	* sys_sched_setparam - set/change the RT priority of a thread
6453	* @pid: the pid in question.
6454	* @param: structure containing the new RT priority.
6455	*/
6456	SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
6457	{
6458	return do_sched_setscheduler(pid, -1, param);
6459	}
6460
6461	/**
6462	* sys_sched_getscheduler - get the policy (scheduling class) of a thread
6463	* @pid: the pid in question.
6464	*/
6465	SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
6466	{
6467	struct task_struct *p;
6468	int retval;
6469
6470	if (pid < 0)
6471	return -EINVAL;
6472
6473	retval = -ESRCH;
6474	read_lock(&tasklist_lock);
6475	p = find_process_by_pid(pid);
6476	if (p) {
6477	retval = security_task_getscheduler(p);
6478	if (!retval)
6479	retval = p->policy
6480	\| (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
6481	}
6482	read_unlock(&tasklist_lock);
6483	return retval;
6484	}
6485
6486	/**
6487	* sys_sched_getparam - get the RT priority of a thread
6488	* @pid: the pid in question.
6489	* @param: structure containing the RT priority.
6490	*/
6491	SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
6492	{
6493	struct sched_param lp;
6494	struct task_struct *p;
6495	int retval;
6496
6497	if (!param \|\| pid < 0)
6498	return -EINVAL;
6499
6500	read_lock(&tasklist_lock);
6501	p = find_process_by_pid(pid);
6502	retval = -ESRCH;
6503	if (!p)
6504	goto out_unlock;
6505
6506	retval = security_task_getscheduler(p);
6507	if (retval)
6508	goto out_unlock;
6509
6510	lp.sched_priority = p->rt_priority;
6511	read_unlock(&tasklist_lock);
6512
6513	/*
6514	* This one might sleep, we cannot do it with a spinlock held ...
6515	*/
6516	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
6517
6518	return retval;
6519
6520	out_unlock:
6521	read_unlock(&tasklist_lock);
6522	return retval;
6523	}
6524
6525	long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
6526	{
6527	cpumask_var_t cpus_allowed, new_mask;
6528	struct task_struct *p;
6529	int retval;
6530
6531	get_online_cpus();
6532	read_lock(&tasklist_lock);
6533
6534	p = find_process_by_pid(pid);
6535	if (!p) {
6536	read_unlock(&tasklist_lock);
6537	put_online_cpus();
6538	return -ESRCH;
6539	}
6540
6541	/*
6542	* It is not safe to call set_cpus_allowed with the
6543	* tasklist_lock held. We will bump the task_struct's
6544	* usage count and then drop tasklist_lock.
6545	*/
6546	get_task_struct(p);
6547	read_unlock(&tasklist_lock);
6548
6549	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
6550	retval = -ENOMEM;
6551	goto out_put_task;
6552	}
6553	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
6554	retval = -ENOMEM;
6555	goto out_free_cpus_allowed;
6556	}
6557	retval = -EPERM;
6558	if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
6559	goto out_unlock;
6560
6561	retval = security_task_setscheduler(p, 0, NULL);
6562	if (retval)
6563	goto out_unlock;
6564
6565	cpuset_cpus_allowed(p, cpus_allowed);
6566	cpumask_and(new_mask, in_mask, cpus_allowed);
6567	again:
6568	retval = set_cpus_allowed_ptr(p, new_mask);
6569
6570	if (!retval) {
6571	cpuset_cpus_allowed(p, cpus_allowed);
6572	if (!cpumask_subset(new_mask, cpus_allowed)) {
6573	/*
6574	* We must have raced with a concurrent cpuset
6575	* update. Just reset the cpus_allowed to the
6576	* cpuset's cpus_allowed
6577	*/
6578	cpumask_copy(new_mask, cpus_allowed);
6579	goto again;
6580	}
6581	}
6582	out_unlock:
6583	free_cpumask_var(new_mask);
6584	out_free_cpus_allowed:
6585	free_cpumask_var(cpus_allowed);
6586	out_put_task:
6587	put_task_struct(p);
6588	put_online_cpus();
6589	return retval;
6590	}
6591
6592	static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
6593	struct cpumask *new_mask)
6594	{
6595	if (len < cpumask_size())
6596	cpumask_clear(new_mask);
6597	else if (len > cpumask_size())
6598	len = cpumask_size();
6599
6600	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
6601	}
6602
6603	/**
6604	* sys_sched_setaffinity - set the cpu affinity of a process
6605	* @pid: pid of the process
6606	* @len: length in bytes of the bitmask pointed to by user_mask_ptr
6607	* @user_mask_ptr: user-space pointer to the new cpu mask
6608	*/
6609	SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
6610	unsigned long __user *, user_mask_ptr)
6611	{
6612	cpumask_var_t new_mask;
6613	int retval;
6614
6615	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
6616	return -ENOMEM;
6617
6618	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
6619	if (retval == 0)
6620	retval = sched_setaffinity(pid, new_mask);
6621	free_cpumask_var(new_mask);
6622	return retval;
6623	}
6624
6625	long sched_getaffinity(pid_t pid, struct cpumask *mask)
6626	{
6627	struct task_struct *p;
6628	int retval;
6629
6630	get_online_cpus();
6631	read_lock(&tasklist_lock);
6632
6633	retval = -ESRCH;
6634	p = find_process_by_pid(pid);
6635	if (!p)
6636	goto out_unlock;
6637
6638	retval = security_task_getscheduler(p);
6639	if (retval)
6640	goto out_unlock;
6641
6642	cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
6643
6644	out_unlock:
6645	read_unlock(&tasklist_lock);
6646	put_online_cpus();
6647
6648	return retval;
6649	}
6650
6651	/**
6652	* sys_sched_getaffinity - get the cpu affinity of a process
6653	* @pid: pid of the process
6654	* @len: length in bytes of the bitmask pointed to by user_mask_ptr
6655	* @user_mask_ptr: user-space pointer to hold the current cpu mask
6656	*/
6657	SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
6658	unsigned long __user *, user_mask_ptr)
6659	{
6660	int ret;
6661	cpumask_var_t mask;
6662
6663	if (len < cpumask_size())
6664	return -EINVAL;
6665
6666	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
6667	return -ENOMEM;
6668
6669	ret = sched_getaffinity(pid, mask);
6670	if (ret == 0) {
6671	if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
6672	ret = -EFAULT;
6673	else
6674	ret = cpumask_size();
6675	}
6676	free_cpumask_var(mask);
6677
6678	return ret;
6679	}
6680
6681	/**
6682	* sys_sched_yield - yield the current processor to other threads.
6683	*
6684	* This function yields the current CPU to other tasks. If there are no
6685	* other threads running on this CPU then this function will return.
6686	*/
6687	SYSCALL_DEFINE0(sched_yield)
6688	{
6689	struct rq *rq = this_rq_lock();
6690
6691	schedstat_inc(rq, yld_count);
6692	current->sched_class->yield_task(rq);
6693
6694	/*
6695	* Since we are going to call schedule() anyway, there's
6696	* no need to preempt or enable interrupts:
6697	*/
6698	__release(rq->lock);
6699	spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
6700	_raw_spin_unlock(&rq->lock);
6701	preempt_enable_no_resched();
6702
6703	schedule();
6704
6705	return 0;
6706	}
6707
6708	static inline int should_resched(void)
6709	{
6710	return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
6711	}
6712
6713	static void __cond_resched(void)
6714	{
6715	add_preempt_count(PREEMPT_ACTIVE);
6716	schedule();
6717	sub_preempt_count(PREEMPT_ACTIVE);
6718	}
6719
6720	int __sched _cond_resched(void)
6721	{
6722	if (should_resched()) {
6723	__cond_resched();
6724	return 1;
6725	}
6726	return 0;
6727	}
6728	EXPORT_SYMBOL(_cond_resched);
6729
6730	/*
6731	* __cond_resched_lock() - if a reschedule is pending, drop the given lock,
6732	* call schedule, and on return reacquire the lock.
6733	*
6734	* This works OK both with and without CONFIG_PREEMPT. We do strange low-level
6735	* operations here to prevent schedule() from being called twice (once via
6736	* spin_unlock(), once by hand).
6737	*/
6738	int __cond_resched_lock(spinlock_t *lock)
6739	{
6740	int resched = should_resched();
6741	int ret = 0;
6742
6743	lockdep_assert_held(lock);
6744
6745	if (spin_needbreak(lock) \|\| resched) {
6746	spin_unlock(lock);
6747	if (resched)
6748	__cond_resched();
6749	else
6750	cpu_relax();
6751	ret = 1;
6752	spin_lock(lock);
6753	}
6754	return ret;
6755	}
6756	EXPORT_SYMBOL(__cond_resched_lock);
6757
6758	int __sched __cond_resched_softirq(void)
6759	{
6760	BUG_ON(!in_softirq());
6761
6762	if (should_resched()) {
6763	local_bh_enable();
6764	__cond_resched();
6765	local_bh_disable();
6766	return 1;
6767	}
6768	return 0;
6769	}
6770	EXPORT_SYMBOL(__cond_resched_softirq);
6771
6772	/**
6773	* yield - yield the current processor to other threads.
6774	*
6775	* This is a shortcut for kernel-space yielding - it marks the
6776	* thread runnable and calls sys_sched_yield().
6777	*/
6778	void __sched yield(void)
6779	{
6780	set_current_state(TASK_RUNNING);
6781	sys_sched_yield();
6782	}
6783	EXPORT_SYMBOL(yield);
6784
6785	/*
6786	* This task is about to go to sleep on IO. Increment rq->nr_iowait so
6787	* that process accounting knows that this is a task in IO wait state.
6788	*/
6789	void __sched io_schedule(void)
6790	{
6791	struct rq *rq = raw_rq();
6792
6793	delayacct_blkio_start();
6794	atomic_inc(&rq->nr_iowait);
6795	current->in_iowait = 1;
6796	schedule();
6797	current->in_iowait = 0;
6798	atomic_dec(&rq->nr_iowait);
6799	delayacct_blkio_end();
6800	}
6801	EXPORT_SYMBOL(io_schedule);
6802
6803	long __sched io_schedule_timeout(long timeout)
6804	{
6805	struct rq *rq = raw_rq();
6806	long ret;
6807
6808	delayacct_blkio_start();
6809	atomic_inc(&rq->nr_iowait);
6810	current->in_iowait = 1;
6811	ret = schedule_timeout(timeout);
6812	current->in_iowait = 0;
6813	atomic_dec(&rq->nr_iowait);
6814	delayacct_blkio_end();
6815	return ret;
6816	}
6817
6818	/**
6819	* sys_sched_get_priority_max - return maximum RT priority.
6820	* @policy: scheduling class.
6821	*
6822	* this syscall returns the maximum rt_priority that can be used
6823	* by a given scheduling class.
6824	*/
6825	SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
6826	{
6827	int ret = -EINVAL;
6828
6829	switch (policy) {
6830	case SCHED_FIFO:
6831	case SCHED_RR:
6832	ret = MAX_USER_RT_PRIO-1;
6833	break;
6834	case SCHED_NORMAL:
6835	case SCHED_BATCH:
6836	case SCHED_IDLE:
6837	ret = 0;
6838	break;
6839	}
6840	return ret;
6841	}
6842
6843	/**
6844	* sys_sched_get_priority_min - return minimum RT priority.
6845	* @policy: scheduling class.
6846	*
6847	* this syscall returns the minimum rt_priority that can be used
6848	* by a given scheduling class.
6849	*/
6850	SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
6851	{
6852	int ret = -EINVAL;
6853
6854	switch (policy) {
6855	case SCHED_FIFO:
6856	case SCHED_RR:
6857	ret = 1;
6858	break;
6859	case SCHED_NORMAL:
6860	case SCHED_BATCH:
6861	case SCHED_IDLE:
6862	ret = 0;
6863	}
6864	return ret;
6865	}
6866
6867	/**
6868	* sys_sched_rr_get_interval - return the default timeslice of a process.
6869	* @pid: pid of the process.
6870	* @interval: userspace pointer to the timeslice value.
6871	*
6872	* this syscall writes the default timeslice value of a given process
6873	* into the user-space timespec buffer. A value of '0' means infinity.
6874	*/
6875	SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
6876	struct timespec __user *, interval)
6877	{
6878	struct task_struct *p;
6879	unsigned int time_slice;
6880	int retval;
6881	struct timespec t;
6882
6883	if (pid < 0)
6884	return -EINVAL;
6885
6886	retval = -ESRCH;
6887	read_lock(&tasklist_lock);
6888	p = find_process_by_pid(pid);
6889	if (!p)
6890	goto out_unlock;
6891
6892	retval = security_task_getscheduler(p);
6893	if (retval)
6894	goto out_unlock;
6895
6896	time_slice = p->sched_class->get_rr_interval(p);
6897
6898	read_unlock(&tasklist_lock);
6899	jiffies_to_timespec(time_slice, &t);
6900	retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
6901	return retval;
6902
6903	out_unlock:
6904	read_unlock(&tasklist_lock);
6905	return retval;
6906	}
6907
6908	static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
6909
6910	void sched_show_task(struct task_struct *p)
6911	{
6912	unsigned long free = 0;
6913	unsigned state;
6914
6915	state = p->state ? __ffs(p->state) + 1 : 0;
6916	printk(KERN_INFO "%-13.13s %c", p->comm,
6917	state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
6918	#if BITS_PER_LONG == 32
6919	if (state == TASK_RUNNING)
6920	printk(KERN_CONT " running ");
6921	else
6922	printk(KERN_CONT " %08lx ", thread_saved_pc(p));
6923	#else
6924	if (state == TASK_RUNNING)
6925	printk(KERN_CONT " running task ");
6926	else
6927	printk(KERN_CONT " %016lx ", thread_saved_pc(p));
6928	#endif
6929	#ifdef CONFIG_DEBUG_STACK_USAGE
6930	free = stack_not_used(p);
6931	#endif
6932	printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
6933	task_pid_nr(p), task_pid_nr(p->real_parent),
6934	(unsigned long)task_thread_info(p)->flags);
6935
6936	show_stack(p, NULL);
6937	}
6938
6939	void show_state_filter(unsigned long state_filter)
6940	{
6941	struct task_struct g, p;
6942
6943	#if BITS_PER_LONG == 32
6944	printk(KERN_INFO
6945	" task PC stack pid father\n");
6946	#else
6947	printk(KERN_INFO
6948	" task PC stack pid father\n");
6949	#endif
6950	read_lock(&tasklist_lock);
6951	do_each_thread(g, p) {
6952	/*
6953	* reset the NMI-timeout, listing all files on a slow
6954	* console might take alot of time:
6955	*/
6956	touch_nmi_watchdog();
6957	if (!state_filter \|\| (p->state & state_filter))
6958	sched_show_task(p);
6959	} while_each_thread(g, p);
6960
6961	touch_all_softlockup_watchdogs();
6962
6963	#ifdef CONFIG_SCHED_DEBUG
6964	sysrq_sched_debug_show();
6965	#endif
6966	read_unlock(&tasklist_lock);
6967	/*
6968	* Only show locks if all tasks are dumped:
6969	*/
6970	if (state_filter == -1)
6971	debug_show_all_locks();
6972	}
6973
6974	void __cpuinit init_idle_bootup_task(struct task_struct *idle)
6975	{
6976	idle->sched_class = &idle_sched_class;
6977	}
6978
6979	/**
6980	* init_idle - set up an idle thread for a given CPU
6981	* @idle: task in question
6982	* @cpu: cpu the idle task belongs to
6983	*
6984	* NOTE: this function does not set the idle thread's NEED_RESCHED
6985	* flag, to make booting more robust.
6986	*/
6987	void __cpuinit init_idle(struct task_struct *idle, int cpu)
6988	{
6989	struct rq *rq = cpu_rq(cpu);
6990	unsigned long flags;
6991
6992	spin_lock_irqsave(&rq->lock, flags);
6993
6994	__sched_fork(idle);
6995	idle->se.exec_start = sched_clock();
6996
6997	cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
6998	__set_task_cpu(idle, cpu);
6999
7000	rq->curr = rq->idle = idle;
7001	#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
7002	idle->oncpu = 1;
7003	#endif
7004	spin_unlock_irqrestore(&rq->lock, flags);
7005
7006	/* Set the preempt count _outside_ the spinlocks! */
7007	#if defined(CONFIG_PREEMPT)
7008	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
7009	#else
7010	task_thread_info(idle)->preempt_count = 0;
7011	#endif
7012	/*
7013	* The idle tasks have their own, simple scheduling class:
7014	*/
7015	idle->sched_class = &idle_sched_class;
7016	ftrace_graph_init_task(idle);
7017	}
7018
7019	/*
7020	* In a system that switches off the HZ timer nohz_cpu_mask
7021	* indicates which cpus entered this state. This is used
7022	* in the rcu update to wait only for active cpus. For system
7023	* which do not switch off the HZ timer nohz_cpu_mask should
7024	* always be CPU_BITS_NONE.
7025	*/
7026	cpumask_var_t nohz_cpu_mask;
7027
7028	/*
7029	* Increase the granularity value when there are more CPUs,
7030	* because with more CPUs the 'effective latency' as visible
7031	* to users decreases. But the relationship is not linear,
7032	* so pick a second-best guess by going with the log2 of the
7033	* number of CPUs.
7034	*
7035	* This idea comes from the SD scheduler of Con Kolivas:
7036	*/
7037	static void update_sysctl(void)
7038	{
7039	unsigned int cpus = min(num_online_cpus(), 8U);
7040	unsigned int factor = 1 + ilog2(cpus);
7041
7042	#define SET_SYSCTL(name) \
7043	(sysctl_##name = (factor) * normalized_sysctl_##name)
7044	SET_SYSCTL(sched_min_granularity);
7045	SET_SYSCTL(sched_latency);
7046	SET_SYSCTL(sched_wakeup_granularity);
7047	SET_SYSCTL(sched_shares_ratelimit);
7048	#undef SET_SYSCTL
7049	}
7050
7051	static inline void sched_init_granularity(void)
7052	{
7053	update_sysctl();
7054	}
7055
7056	#ifdef CONFIG_SMP
7057	/*
7058	* This is how migration works:
7059	*
7060	* 1) we queue a struct migration_req structure in the source CPU's
7061	* runqueue and wake up that CPU's migration thread.
7062	* 2) we down() the locked semaphore => thread blocks.
7063	* 3) migration thread wakes up (implicitly it forces the migrated
7064	* thread off the CPU)
7065	* 4) it gets the migration request and checks whether the migrated
7066	* task is still in the wrong runqueue.
7067	* 5) if it's in the wrong runqueue then the migration thread removes
7068	* it and puts it into the right queue.
7069	* 6) migration thread up()s the semaphore.
7070	* 7) we wake up and the migration is done.
7071	*/
7072
7073	/*
7074	* Change a given task's CPU affinity. Migrate the thread to a
7075	* proper CPU and schedule it away if the CPU it's executing on
7076	* is removed from the allowed bitmask.
7077	*
7078	* NOTE: the caller must have a valid reference to the task, the
7079	* task must not exit() & deallocate itself prematurely. The
7080	* call is not atomic; no spinlocks may be held.
7081	*/
7082	int set_cpus_allowed_ptr(struct task_struct p, const struct cpumask new_mask)
7083	{
7084	struct migration_req req;
7085	unsigned long flags;
7086	struct rq *rq;
7087	int ret = 0;
7088
7089	rq = task_rq_lock(p, &flags);
7090	if (!cpumask_intersects(new_mask, cpu_active_mask)) {
7091	ret = -EINVAL;
7092	goto out;
7093	}
7094
7095	if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
7096	!cpumask_equal(&p->cpus_allowed, new_mask))) {
7097	ret = -EINVAL;
7098	goto out;
7099	}
7100
7101	if (p->sched_class->set_cpus_allowed)
7102	p->sched_class->set_cpus_allowed(p, new_mask);
7103	else {
7104	cpumask_copy(&p->cpus_allowed, new_mask);
7105	p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
7106	}
7107
7108	/* Can the task run on the task's current CPU? If so, we're done */
7109	if (cpumask_test_cpu(task_cpu(p), new_mask))
7110	goto out;
7111
7112	if (migrate_task(p, cpumask_any_and(cpu_active_mask, new_mask), &req)) {
7113	/* Need help from migration thread: drop lock and wait. */
7114	struct task_struct *mt = rq->migration_thread;
7115
7116	get_task_struct(mt);
7117	task_rq_unlock(rq, &flags);
7118	wake_up_process(rq->migration_thread);
7119	put_task_struct(mt);
7120	wait_for_completion(&req.done);
7121	tlb_migrate_finish(p->mm);
7122	return 0;
7123	}
7124	out:
7125	task_rq_unlock(rq, &flags);
7126
7127	return ret;
7128	}
7129	EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
7130
7131	/*
7132	* Move (not current) task off this cpu, onto dest cpu. We're doing
7133	* this because either it can't run here any more (set_cpus_allowed()
7134	* away from this CPU, or CPU going down), or because we're
7135	* attempting to rebalance this task on exec (sched_exec).
7136	*
7137	* So we race with normal scheduler movements, but that's OK, as long
7138	* as the task is no longer on this CPU.
7139	*
7140	* Returns non-zero if task was successfully migrated.
7141	*/
7142	static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
7143	{
7144	struct rq rq_dest, rq_src;
7145	int ret = 0, on_rq;
7146
7147	if (unlikely(!cpu_active(dest_cpu)))
7148	return ret;
7149
7150	rq_src = cpu_rq(src_cpu);
7151	rq_dest = cpu_rq(dest_cpu);
7152
7153	double_rq_lock(rq_src, rq_dest);
7154	/* Already moved. */
7155	if (task_cpu(p) != src_cpu)
7156	goto done;
7157	/* Affinity changed (again). */
7158	if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7159	goto fail;
7160
7161	on_rq = p->se.on_rq;
7162	if (on_rq)
7163	deactivate_task(rq_src, p, 0);
7164
7165	set_task_cpu(p, dest_cpu);
7166	if (on_rq) {
7167	activate_task(rq_dest, p, 0);
7168	check_preempt_curr(rq_dest, p, 0);
7169	}
7170	done:
7171	ret = 1;
7172	fail:
7173	double_rq_unlock(rq_src, rq_dest);
7174	return ret;
7175	}
7176
7177	#define RCU_MIGRATION_IDLE 0
7178	#define RCU_MIGRATION_NEED_QS 1
7179	#define RCU_MIGRATION_GOT_QS 2
7180	#define RCU_MIGRATION_MUST_SYNC 3
7181
7182	/*
7183	* migration_thread - this is a highprio system thread that performs
7184	* thread migration by bumping thread off CPU then 'pushing' onto
7185	* another runqueue.
7186	*/
7187	static int migration_thread(void *data)
7188	{
7189	int badcpu;
7190	int cpu = (long)data;
7191	struct rq *rq;
7192
7193	rq = cpu_rq(cpu);
7194	BUG_ON(rq->migration_thread != current);
7195
7196	set_current_state(TASK_INTERRUPTIBLE);
7197	while (!kthread_should_stop()) {
7198	struct migration_req *req;
7199	struct list_head *head;
7200
7201	spin_lock_irq(&rq->lock);
7202
7203	if (cpu_is_offline(cpu)) {
7204	spin_unlock_irq(&rq->lock);
7205	break;
7206	}
7207
7208	if (rq->active_balance) {
7209	active_load_balance(rq, cpu);
7210	rq->active_balance = 0;
7211	}
7212
7213	head = &rq->migration_queue;
7214
7215	if (list_empty(head)) {
7216	spin_unlock_irq(&rq->lock);
7217	schedule();
7218	set_current_state(TASK_INTERRUPTIBLE);
7219	continue;
7220	}
7221	req = list_entry(head->next, struct migration_req, list);
7222	list_del_init(head->next);
7223
7224	if (req->task != NULL) {
7225	spin_unlock(&rq->lock);
7226	__migrate_task(req->task, cpu, req->dest_cpu);
7227	} else if (likely(cpu == (badcpu = smp_processor_id()))) {
7228	req->dest_cpu = RCU_MIGRATION_GOT_QS;
7229	spin_unlock(&rq->lock);
7230	} else {
7231	req->dest_cpu = RCU_MIGRATION_MUST_SYNC;
7232	spin_unlock(&rq->lock);
7233	WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu);
7234	}
7235	local_irq_enable();
7236
7237	complete(&req->done);
7238	}
7239	__set_current_state(TASK_RUNNING);
7240
7241	return 0;
7242	}
7243
7244	#ifdef CONFIG_HOTPLUG_CPU
7245
7246	static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
7247	{
7248	int ret;
7249
7250	local_irq_disable();
7251	ret = __migrate_task(p, src_cpu, dest_cpu);
7252	local_irq_enable();
7253	return ret;
7254	}
7255
7256	/*
7257	* Figure out where task on dead CPU should go, use force if necessary.
7258	*/
7259	static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
7260	{
7261	int dest_cpu;
7262	const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
7263
7264	again:
7265	/* Look for allowed, online CPU in same node. */
7266	for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
7267	if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7268	goto move;
7269
7270	/* Any allowed, online CPU? */
7271	dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
7272	if (dest_cpu < nr_cpu_ids)
7273	goto move;
7274
7275	/* No more Mr. Nice Guy. */
7276	if (dest_cpu >= nr_cpu_ids) {
7277	cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
7278	dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
7279
7280	/*
7281	* Don't tell them about moving exiting tasks or
7282	* kernel threads (both mm NULL), since they never
7283	* leave kernel.
7284	*/
7285	if (p->mm && printk_ratelimit()) {
7286	printk(KERN_INFO "process %d (%s) no "
7287	"longer affine to cpu%d\n",
7288	task_pid_nr(p), p->comm, dead_cpu);
7289	}
7290	}
7291
7292	move:
7293	/* It can have affinity changed while we were choosing. */
7294	if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
7295	goto again;
7296	}
7297
7298	/*
7299	* While a dead CPU has no uninterruptible tasks queued at this point,
7300	* it might still have a nonzero ->nr_uninterruptible counter, because
7301	* for performance reasons the counter is not stricly tracking tasks to
7302	* their home CPUs. So we just add the counter to another CPU's counter,
7303	* to keep the global sum constant after CPU-down:
7304	*/
7305	static void migrate_nr_uninterruptible(struct rq *rq_src)
7306	{
7307	struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
7308	unsigned long flags;
7309
7310	local_irq_save(flags);
7311	double_rq_lock(rq_src, rq_dest);
7312	rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
7313	rq_src->nr_uninterruptible = 0;
7314	double_rq_unlock(rq_src, rq_dest);
7315	local_irq_restore(flags);
7316	}
7317
7318	/* Run through task list and migrate tasks from the dead cpu. */
7319	static void migrate_live_tasks(int src_cpu)
7320	{
7321	struct task_struct p, t;
7322
7323	read_lock(&tasklist_lock);
7324
7325	do_each_thread(t, p) {
7326	if (p == current)
7327	continue;
7328
7329	if (task_cpu(p) == src_cpu)
7330	move_task_off_dead_cpu(src_cpu, p);
7331	} while_each_thread(t, p);
7332
7333	read_unlock(&tasklist_lock);
7334	}
7335
7336	/*
7337	* Schedules idle task to be the next runnable task on current CPU.
7338	* It does so by boosting its priority to highest possible.
7339	* Used by CPU offline code.
7340	*/
7341	void sched_idle_next(void)
7342	{
7343	int this_cpu = smp_processor_id();
7344	struct rq *rq = cpu_rq(this_cpu);
7345	struct task_struct *p = rq->idle;
7346	unsigned long flags;
7347
7348	/* cpu has to be offline */
7349	BUG_ON(cpu_online(this_cpu));
7350
7351	/*
7352	* Strictly not necessary since rest of the CPUs are stopped by now
7353	* and interrupts disabled on the current cpu.
7354	*/
7355	spin_lock_irqsave(&rq->lock, flags);
7356
7357	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
7358
7359	update_rq_clock(rq);
7360	activate_task(rq, p, 0);
7361
7362	spin_unlock_irqrestore(&rq->lock, flags);
7363	}
7364
7365	/*
7366	* Ensures that the idle task is using init_mm right before its cpu goes
7367	* offline.
7368	*/
7369	void idle_task_exit(void)
7370	{
7371	struct mm_struct *mm = current->active_mm;
7372
7373	BUG_ON(cpu_online(smp_processor_id()));
7374
7375	if (mm != &init_mm)
7376	switch_mm(mm, &init_mm, current);
7377	mmdrop(mm);
7378	}
7379
7380	/* called under rq->lock with disabled interrupts */
7381	static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
7382	{
7383	struct rq *rq = cpu_rq(dead_cpu);
7384
7385	/* Must be exiting, otherwise would be on tasklist. */
7386	BUG_ON(!p->exit_state);
7387
7388	/* Cannot have done final schedule yet: would have vanished. */
7389	BUG_ON(p->state == TASK_DEAD);
7390
7391	get_task_struct(p);
7392
7393	/*
7394	* Drop lock around migration; if someone else moves it,
7395	* that's OK. No task can be added to this CPU, so iteration is
7396	* fine.
7397	*/
7398	spin_unlock_irq(&rq->lock);
7399	move_task_off_dead_cpu(dead_cpu, p);
7400	spin_lock_irq(&rq->lock);
7401
7402	put_task_struct(p);
7403	}
7404
7405	/* release_task() removes task from tasklist, so we won't find dead tasks. */
7406	static void migrate_dead_tasks(unsigned int dead_cpu)
7407	{
7408	struct rq *rq = cpu_rq(dead_cpu);
7409	struct task_struct *next;
7410
7411	for ( ; ; ) {
7412	if (!rq->nr_running)
7413	break;
7414	update_rq_clock(rq);
7415	next = pick_next_task(rq);
7416	if (!next)
7417	break;
7418	next->sched_class->put_prev_task(rq, next);
7419	migrate_dead(dead_cpu, next);
7420
7421	}
7422	}
7423
7424	/*
7425	* remove the tasks which were accounted by rq from calc_load_tasks.
7426	*/
7427	static void calc_global_load_remove(struct rq *rq)
7428	{
7429	atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
7430	rq->calc_load_active = 0;
7431	}
7432	#endif /* CONFIG_HOTPLUG_CPU */
7433
7434	#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
7435
7436	static struct ctl_table sd_ctl_dir[] = {
7437	{
7438	.procname = "sched_domain",
7439	.mode = 0555,
7440	},
7441	{0, },
7442	};
7443
7444	static struct ctl_table sd_ctl_root[] = {
7445	{
7446	.ctl_name = CTL_KERN,
7447	.procname = "kernel",
7448	.mode = 0555,
7449	.child = sd_ctl_dir,
7450	},
7451	{0, },
7452	};
7453
7454	static struct ctl_table *sd_alloc_ctl_entry(int n)
7455	{
7456	struct ctl_table *entry =
7457	kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
7458
7459	return entry;
7460	}
7461
7462	static void sd_free_ctl_entry(struct ctl_table **tablep)
7463	{
7464	struct ctl_table *entry;
7465
7466	/*
7467	* In the intermediate directories, both the child directory and
7468	* procname are dynamically allocated and could fail but the mode
7469	* will always be set. In the lowest directory the names are
7470	* static strings and all have proc handlers.
7471	*/
7472	for (entry = *tablep; entry->mode; entry++) {
7473	if (entry->child)
7474	sd_free_ctl_entry(&entry->child);
7475	if (entry->proc_handler == NULL)
7476	kfree(entry->procname);
7477	}
7478
7479	kfree(*tablep);
7480	*tablep = NULL;
7481	}
7482
7483	static void
7484	set_table_entry(struct ctl_table *entry,
7485	const char procname, void data, int maxlen,
7486	mode_t mode, proc_handler *proc_handler)
7487	{
7488	entry->procname = procname;
7489	entry->data = data;
7490	entry->maxlen = maxlen;
7491	entry->mode = mode;
7492	entry->proc_handler = proc_handler;
7493	}
7494
7495	static struct ctl_table *
7496	sd_alloc_ctl_domain_table(struct sched_domain *sd)
7497	{
7498	struct ctl_table *table = sd_alloc_ctl_entry(13);
7499
7500	if (table == NULL)
7501	return NULL;
7502
7503	set_table_entry(&table[0], "min_interval", &sd->min_interval,
7504	sizeof(long), 0644, proc_doulongvec_minmax);
7505	set_table_entry(&table[1], "max_interval", &sd->max_interval,
7506	sizeof(long), 0644, proc_doulongvec_minmax);
7507	set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
7508	sizeof(int), 0644, proc_dointvec_minmax);
7509	set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
7510	sizeof(int), 0644, proc_dointvec_minmax);
7511	set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
7512	sizeof(int), 0644, proc_dointvec_minmax);
7513	set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
7514	sizeof(int), 0644, proc_dointvec_minmax);
7515	set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
7516	sizeof(int), 0644, proc_dointvec_minmax);
7517	set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
7518	sizeof(int), 0644, proc_dointvec_minmax);
7519	set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
7520	sizeof(int), 0644, proc_dointvec_minmax);
7521	set_table_entry(&table[9], "cache_nice_tries",
7522	&sd->cache_nice_tries,
7523	sizeof(int), 0644, proc_dointvec_minmax);
7524	set_table_entry(&table[10], "flags", &sd->flags,
7525	sizeof(int), 0644, proc_dointvec_minmax);
7526	set_table_entry(&table[11], "name", sd->name,
7527	CORENAME_MAX_SIZE, 0444, proc_dostring);
7528	/* &table[12] is terminator */
7529
7530	return table;
7531	}
7532
7533	static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
7534	{
7535	struct ctl_table entry, table;
7536	struct sched_domain *sd;
7537	int domain_num = 0, i;
7538	char buf[32];
7539
7540	for_each_domain(cpu, sd)
7541	domain_num++;
7542	entry = table = sd_alloc_ctl_entry(domain_num + 1);
7543	if (table == NULL)
7544	return NULL;
7545
7546	i = 0;
7547	for_each_domain(cpu, sd) {
7548	snprintf(buf, 32, "domain%d", i);
7549	entry->procname = kstrdup(buf, GFP_KERNEL);
7550	entry->mode = 0555;
7551	entry->child = sd_alloc_ctl_domain_table(sd);
7552	entry++;
7553	i++;
7554	}
7555	return table;
7556	}
7557
7558	static struct ctl_table_header *sd_sysctl_header;
7559	static void register_sched_domain_sysctl(void)
7560	{
7561	int i, cpu_num = num_possible_cpus();
7562	struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
7563	char buf[32];
7564
7565	WARN_ON(sd_ctl_dir[0].child);
7566	sd_ctl_dir[0].child = entry;
7567
7568	if (entry == NULL)
7569	return;
7570
7571	for_each_possible_cpu(i) {
7572	snprintf(buf, 32, "cpu%d", i);
7573	entry->procname = kstrdup(buf, GFP_KERNEL);
7574	entry->mode = 0555;
7575	entry->child = sd_alloc_ctl_cpu_table(i);
7576	entry++;
7577	}
7578
7579	WARN_ON(sd_sysctl_header);
7580	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
7581	}
7582
7583	/* may be called multiple times per register */
7584	static void unregister_sched_domain_sysctl(void)
7585	{
7586	if (sd_sysctl_header)
7587	unregister_sysctl_table(sd_sysctl_header);
7588	sd_sysctl_header = NULL;
7589	if (sd_ctl_dir[0].child)
7590	sd_free_ctl_entry(&sd_ctl_dir[0].child);
7591	}
7592	#else
7593	static void register_sched_domain_sysctl(void)
7594	{
7595	}
7596	static void unregister_sched_domain_sysctl(void)
7597	{
7598	}
7599	#endif
7600
7601	static void set_rq_online(struct rq *rq)
7602	{
7603	if (!rq->online) {
7604	const struct sched_class *class;
7605
7606	cpumask_set_cpu(rq->cpu, rq->rd->online);
7607	rq->online = 1;
7608
7609	for_each_class(class) {
7610	if (class->rq_online)
7611	class->rq_online(rq);
7612	}
7613	}
7614	}
7615
7616	static void set_rq_offline(struct rq *rq)
7617	{
7618	if (rq->online) {
7619	const struct sched_class *class;
7620
7621	for_each_class(class) {
7622	if (class->rq_offline)
7623	class->rq_offline(rq);
7624	}
7625
7626	cpumask_clear_cpu(rq->cpu, rq->rd->online);
7627	rq->online = 0;
7628	}
7629	}
7630
7631	/*
7632	* migration_call - callback that gets triggered when a CPU is added.
7633	* Here we can start up the necessary migration thread for the new CPU.
7634	*/
7635	static int __cpuinit
7636	migration_call(struct notifier_block nfb, unsigned long action, void hcpu)
7637	{
7638	struct task_struct *p;
7639	int cpu = (long)hcpu;
7640	unsigned long flags;
7641	struct rq *rq;
7642
7643	switch (action) {
7644
7645	case CPU_UP_PREPARE:
7646	case CPU_UP_PREPARE_FROZEN:
7647	p = kthread_create(migration_thread, hcpu, "migration/%d", cpu);
7648	if (IS_ERR(p))
7649	return NOTIFY_BAD;
7650	kthread_bind(p, cpu);
7651	/* Must be high prio: stop_machine expects to yield to it. */
7652	rq = task_rq_lock(p, &flags);
7653	__setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
7654	task_rq_unlock(rq, &flags);
7655	get_task_struct(p);
7656	cpu_rq(cpu)->migration_thread = p;
7657	rq->calc_load_update = calc_load_update;
7658	break;
7659
7660	case CPU_ONLINE:
7661	case CPU_ONLINE_FROZEN:
7662	/* Strictly unnecessary, as first user will wake it. */
7663	wake_up_process(cpu_rq(cpu)->migration_thread);
7664
7665	/* Update our root-domain */
7666	rq = cpu_rq(cpu);
7667	spin_lock_irqsave(&rq->lock, flags);
7668	if (rq->rd) {
7669	BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7670
7671	set_rq_online(rq);
7672	}
7673	spin_unlock_irqrestore(&rq->lock, flags);
7674	break;
7675
7676	#ifdef CONFIG_HOTPLUG_CPU
7677	case CPU_UP_CANCELED:
7678	case CPU_UP_CANCELED_FROZEN:
7679	if (!cpu_rq(cpu)->migration_thread)
7680	break;
7681	/* Unbind it from offline cpu so it can run. Fall thru. */
7682	kthread_bind(cpu_rq(cpu)->migration_thread,
7683	cpumask_any(cpu_online_mask));
7684	kthread_stop(cpu_rq(cpu)->migration_thread);
7685	put_task_struct(cpu_rq(cpu)->migration_thread);
7686	cpu_rq(cpu)->migration_thread = NULL;
7687	break;
7688
7689	case CPU_DEAD:
7690	case CPU_DEAD_FROZEN:
7691	cpuset_lock(); /* around calls to cpuset_cpus_allowed_lock() */
7692	migrate_live_tasks(cpu);
7693	rq = cpu_rq(cpu);
7694	kthread_stop(rq->migration_thread);
7695	put_task_struct(rq->migration_thread);
7696	rq->migration_thread = NULL;
7697	/* Idle task back to normal (off runqueue, low prio) */
7698	spin_lock_irq(&rq->lock);
7699	update_rq_clock(rq);
7700	deactivate_task(rq, rq->idle, 0);
7701	__setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
7702	rq->idle->sched_class = &idle_sched_class;
7703	migrate_dead_tasks(cpu);
7704	spin_unlock_irq(&rq->lock);
7705	cpuset_unlock();
7706	migrate_nr_uninterruptible(rq);
7707	BUG_ON(rq->nr_running != 0);
7708	calc_global_load_remove(rq);
7709	/*
7710	* No need to migrate the tasks: it was best-effort if
7711	* they didn't take sched_hotcpu_mutex. Just wake up
7712	* the requestors.
7713	*/
7714	spin_lock_irq(&rq->lock);
7715	while (!list_empty(&rq->migration_queue)) {
7716	struct migration_req *req;
7717
7718	req = list_entry(rq->migration_queue.next,
7719	struct migration_req, list);
7720	list_del_init(&req->list);
7721	spin_unlock_irq(&rq->lock);
7722	complete(&req->done);
7723	spin_lock_irq(&rq->lock);
7724	}
7725	spin_unlock_irq(&rq->lock);
7726	break;
7727
7728	case CPU_DYING:
7729	case CPU_DYING_FROZEN:
7730	/* Update our root-domain */
7731	rq = cpu_rq(cpu);
7732	spin_lock_irqsave(&rq->lock, flags);
7733	if (rq->rd) {
7734	BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7735	set_rq_offline(rq);
7736	}
7737	spin_unlock_irqrestore(&rq->lock, flags);
7738	break;
7739	#endif
7740	}
7741	return NOTIFY_OK;
7742	}
7743
7744	/*
7745	* Register at high priority so that task migration (migrate_all_tasks)
7746	* happens before everything else. This has to be lower priority than
7747	* the notifier in the perf_event subsystem, though.
7748	*/
7749	static struct notifier_block __cpuinitdata migration_notifier = {
7750	.notifier_call = migration_call,
7751	.priority = 10
7752	};
7753
7754	static int __init migration_init(void)
7755	{
7756	void cpu = (void )(long)smp_processor_id();
7757	int err;
7758
7759	/* Start one for the boot CPU: */
7760	err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
7761	BUG_ON(err == NOTIFY_BAD);
7762	migration_call(&migration_notifier, CPU_ONLINE, cpu);
7763	register_cpu_notifier(&migration_notifier);
7764
7765	return 0;
7766	}
7767	early_initcall(migration_init);
7768	#endif
7769
7770	#ifdef CONFIG_SMP
7771
7772	#ifdef CONFIG_SCHED_DEBUG
7773
7774	static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
7775	struct cpumask *groupmask)
7776	{
7777	struct sched_group *group = sd->groups;
7778	char str[256];
7779
7780	cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
7781	cpumask_clear(groupmask);
7782
7783	printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
7784
7785	if (!(sd->flags & SD_LOAD_BALANCE)) {
7786	printk("does not load-balance\n");
7787	if (sd->parent)
7788	printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
7789	" has parent");
7790	return -1;
7791	}
7792
7793	printk(KERN_CONT "span %s level %s\n", str, sd->name);
7794
7795	if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
7796	printk(KERN_ERR "ERROR: domain->span does not contain "
7797	"CPU%d\n", cpu);
7798	}
7799	if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
7800	printk(KERN_ERR "ERROR: domain->groups does not contain"
7801	" CPU%d\n", cpu);
7802	}
7803
7804	printk(KERN_DEBUG "%*s groups:", level + 1, "");
7805	do {
7806	if (!group) {
7807	printk("\n");
7808	printk(KERN_ERR "ERROR: group is NULL\n");
7809	break;
7810	}
7811
7812	if (!group->cpu_power) {
7813	printk(KERN_CONT "\n");
7814	printk(KERN_ERR "ERROR: domain->cpu_power not "
7815	"set\n");
7816	break;
7817	}
7818
7819	if (!cpumask_weight(sched_group_cpus(group))) {
7820	printk(KERN_CONT "\n");
7821	printk(KERN_ERR "ERROR: empty group\n");
7822	break;
7823	}
7824
7825	if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
7826	printk(KERN_CONT "\n");
7827	printk(KERN_ERR "ERROR: repeated CPUs\n");
7828	break;
7829	}
7830
7831	cpumask_or(groupmask, groupmask, sched_group_cpus(group));
7832
7833	cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
7834
7835	printk(KERN_CONT " %s", str);
7836	if (group->cpu_power != SCHED_LOAD_SCALE) {
7837	printk(KERN_CONT " (cpu_power = %d)",
7838	group->cpu_power);
7839	}
7840
7841	group = group->next;
7842	} while (group != sd->groups);
7843	printk(KERN_CONT "\n");
7844
7845	if (!cpumask_equal(sched_domain_span(sd), groupmask))
7846	printk(KERN_ERR "ERROR: groups don't span domain->span\n");
7847
7848	if (sd->parent &&
7849	!cpumask_subset(groupmask, sched_domain_span(sd->parent)))
7850	printk(KERN_ERR "ERROR: parent span is not a superset "
7851	"of domain->span\n");
7852	return 0;
7853	}
7854
7855	static void sched_domain_debug(struct sched_domain *sd, int cpu)
7856	{
7857	cpumask_var_t groupmask;
7858	int level = 0;
7859
7860	if (!sd) {
7861	printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
7862	return;
7863	}
7864
7865	printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
7866
7867	if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
7868	printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
7869	return;
7870	}
7871
7872	for (;;) {
7873	if (sched_domain_debug_one(sd, cpu, level, groupmask))
7874	break;
7875	level++;
7876	sd = sd->parent;
7877	if (!sd)
7878	break;
7879	}
7880	free_cpumask_var(groupmask);
7881	}
7882	#else /* !CONFIG_SCHED_DEBUG */
7883	# define sched_domain_debug(sd, cpu) do { } while (0)
7884	#endif /* CONFIG_SCHED_DEBUG */
7885
7886	static int sd_degenerate(struct sched_domain *sd)
7887	{
7888	if (cpumask_weight(sched_domain_span(sd)) == 1)
7889	return 1;
7890
7891	/* Following flags need at least 2 groups */
7892	if (sd->flags & (SD_LOAD_BALANCE \|
7893	SD_BALANCE_NEWIDLE \|
7894	SD_BALANCE_FORK \|
7895	SD_BALANCE_EXEC \|
7896	SD_SHARE_CPUPOWER \|
7897	SD_SHARE_PKG_RESOURCES)) {
7898	if (sd->groups != sd->groups->next)
7899	return 0;
7900	}
7901
7902	/* Following flags don't use groups */
7903	if (sd->flags & (SD_WAKE_AFFINE))
7904	return 0;
7905
7906	return 1;
7907	}
7908
7909	static int
7910	sd_parent_degenerate(struct sched_domain sd, struct sched_domain parent)
7911	{
7912	unsigned long cflags = sd->flags, pflags = parent->flags;
7913
7914	if (sd_degenerate(parent))
7915	return 1;
7916
7917	if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
7918	return 0;
7919
7920	/* Flags needing groups don't count if only 1 group in parent */
7921	if (parent->groups == parent->groups->next) {
7922	pflags &= ~(SD_LOAD_BALANCE \|
7923	SD_BALANCE_NEWIDLE \|
7924	SD_BALANCE_FORK \|
7925	SD_BALANCE_EXEC \|
7926	SD_SHARE_CPUPOWER \|
7927	SD_SHARE_PKG_RESOURCES);
7928	if (nr_node_ids == 1)
7929	pflags &= ~SD_SERIALIZE;
7930	}
7931	if (~cflags & pflags)
7932	return 0;
7933
7934	return 1;
7935	}
7936
7937	static void free_rootdomain(struct root_domain *rd)
7938	{
7939	synchronize_sched();
7940
7941	cpupri_cleanup(&rd->cpupri);
7942
7943	free_cpumask_var(rd->rto_mask);
7944	free_cpumask_var(rd->online);
7945	free_cpumask_var(rd->span);
7946	kfree(rd);
7947	}
7948
7949	static void rq_attach_root(struct rq rq, struct root_domain rd)
7950	{
7951	struct root_domain *old_rd = NULL;
7952	unsigned long flags;
7953
7954	spin_lock_irqsave(&rq->lock, flags);
7955
7956	if (rq->rd) {
7957	old_rd = rq->rd;
7958
7959	if (cpumask_test_cpu(rq->cpu, old_rd->online))
7960	set_rq_offline(rq);
7961
7962	cpumask_clear_cpu(rq->cpu, old_rd->span);
7963
7964	/*
7965	* If we dont want to free the old_rt yet then
7966	* set old_rd to NULL to skip the freeing later
7967	* in this function:
7968	*/
7969	if (!atomic_dec_and_test(&old_rd->refcount))
7970	old_rd = NULL;
7971	}
7972
7973	atomic_inc(&rd->refcount);
7974	rq->rd = rd;
7975
7976	cpumask_set_cpu(rq->cpu, rd->span);
7977	if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
7978	set_rq_online(rq);
7979
7980	spin_unlock_irqrestore(&rq->lock, flags);
7981
7982	if (old_rd)
7983	free_rootdomain(old_rd);
7984	}
7985
7986	static int init_rootdomain(struct root_domain *rd, bool bootmem)
7987	{
7988	gfp_t gfp = GFP_KERNEL;
7989
7990	memset(rd, 0, sizeof(*rd));
7991
7992	if (bootmem)
7993	gfp = GFP_NOWAIT;
7994
7995	if (!alloc_cpumask_var(&rd->span, gfp))
7996	goto out;
7997	if (!alloc_cpumask_var(&rd->online, gfp))
7998	goto free_span;
7999	if (!alloc_cpumask_var(&rd->rto_mask, gfp))
8000	goto free_online;
8001
8002	if (cpupri_init(&rd->cpupri, bootmem) != 0)
8003	goto free_rto_mask;
8004	return 0;
8005
8006	free_rto_mask:
8007	free_cpumask_var(rd->rto_mask);
8008	free_online:
8009	free_cpumask_var(rd->online);
8010	free_span:
8011	free_cpumask_var(rd->span);
8012	out:
8013	return -ENOMEM;
8014	}
8015
8016	static void init_defrootdomain(void)
8017	{
8018	init_rootdomain(&def_root_domain, true);
8019
8020	atomic_set(&def_root_domain.refcount, 1);
8021	}
8022
8023	static struct root_domain *alloc_rootdomain(void)
8024	{
8025	struct root_domain *rd;
8026
8027	rd = kmalloc(sizeof(*rd), GFP_KERNEL);
8028	if (!rd)
8029	return NULL;
8030
8031	if (init_rootdomain(rd, false) != 0) {
8032	kfree(rd);
8033	return NULL;
8034	}
8035
8036	return rd;
8037	}
8038
8039	/*
8040	* Attach the domain 'sd' to 'cpu' as its base domain. Callers must
8041	* hold the hotplug lock.
8042	*/
8043	static void
8044	cpu_attach_domain(struct sched_domain sd, struct root_domain rd, int cpu)
8045	{
8046	struct rq *rq = cpu_rq(cpu);
8047	struct sched_domain *tmp;
8048
8049	/* Remove the sched domains which do not contribute to scheduling. */
8050	for (tmp = sd; tmp; ) {
8051	struct sched_domain *parent = tmp->parent;
8052	if (!parent)
8053	break;
8054
8055	if (sd_parent_degenerate(tmp, parent)) {
8056	tmp->parent = parent->parent;
8057	if (parent->parent)
8058	parent->parent->child = tmp;
8059	} else
8060	tmp = tmp->parent;
8061	}
8062
8063	if (sd && sd_degenerate(sd)) {
8064	sd = sd->parent;
8065	if (sd)
8066	sd->child = NULL;
8067	}
8068
8069	sched_domain_debug(sd, cpu);
8070
8071	rq_attach_root(rq, rd);
8072	rcu_assign_pointer(rq->sd, sd);
8073	}
8074
8075	/* cpus with isolated domains */
8076	static cpumask_var_t cpu_isolated_map;
8077
8078	/* Setup the mask of cpus configured for isolated domains */
8079	static int __init isolated_cpu_setup(char *str)
8080	{
8081	alloc_bootmem_cpumask_var(&cpu_isolated_map);
8082	cpulist_parse(str, cpu_isolated_map);
8083	return 1;
8084	}
8085
8086	__setup("isolcpus=", isolated_cpu_setup);
8087
8088	/*
8089	* init_sched_build_groups takes the cpumask we wish to span, and a pointer
8090	* to a function which identifies what group(along with sched group) a CPU
8091	* belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
8092	* (due to the fact that we keep track of groups covered with a struct cpumask).
8093	*
8094	* init_sched_build_groups will build a circular linked list of the groups
8095	* covered by the given span, and will set each group's ->cpumask correctly,
8096	* and ->cpu_power to 0.
8097	*/
8098	static void
8099	init_sched_build_groups(const struct cpumask *span,
8100	const struct cpumask *cpu_map,
8101	int (group_fn)(int cpu, const struct cpumask cpu_map,
8102	struct sched_group **sg,
8103	struct cpumask *tmpmask),
8104	struct cpumask covered, struct cpumask tmpmask)
8105	{
8106	struct sched_group first = NULL, last = NULL;
8107	int i;
8108
8109	cpumask_clear(covered);
8110
8111	for_each_cpu(i, span) {
8112	struct sched_group *sg;
8113	int group = group_fn(i, cpu_map, &sg, tmpmask);
8114	int j;
8115
8116	if (cpumask_test_cpu(i, covered))
8117	continue;
8118
8119	cpumask_clear(sched_group_cpus(sg));
8120	sg->cpu_power = 0;
8121
8122	for_each_cpu(j, span) {
8123	if (group_fn(j, cpu_map, NULL, tmpmask) != group)
8124	continue;
8125
8126	cpumask_set_cpu(j, covered);
8127	cpumask_set_cpu(j, sched_group_cpus(sg));
8128	}
8129	if (!first)
8130	first = sg;
8131	if (last)
8132	last->next = sg;
8133	last = sg;
8134	}
8135	last->next = first;
8136	}
8137
8138	#define SD_NODES_PER_DOMAIN 16
8139
8140	#ifdef CONFIG_NUMA
8141
8142	/**
8143	* find_next_best_node - find the next node to include in a sched_domain
8144	* @node: node whose sched_domain we're building
8145	* @used_nodes: nodes already in the sched_domain
8146	*
8147	* Find the next node to include in a given scheduling domain. Simply
8148	* finds the closest node not already in the @used_nodes map.
8149	*
8150	* Should use nodemask_t.
8151	*/
8152	static int find_next_best_node(int node, nodemask_t *used_nodes)
8153	{
8154	int i, n, val, min_val, best_node = 0;
8155
8156	min_val = INT_MAX;
8157
8158	for (i = 0; i < nr_node_ids; i++) {
8159	/* Start at @node */
8160	n = (node + i) % nr_node_ids;
8161
8162	if (!nr_cpus_node(n))
8163	continue;
8164
8165	/* Skip already used nodes */
8166	if (node_isset(n, *used_nodes))
8167	continue;
8168
8169	/* Simple min distance search */
8170	val = node_distance(node, n);
8171
8172	if (val < min_val) {
8173	min_val = val;
8174	best_node = n;
8175	}
8176	}
8177
8178	node_set(best_node, *used_nodes);
8179	return best_node;
8180	}
8181
8182	/**
8183	* sched_domain_node_span - get a cpumask for a node's sched_domain
8184	* @node: node whose cpumask we're constructing
8185	* @span: resulting cpumask
8186	*
8187	* Given a node, construct a good cpumask for its sched_domain to span. It
8188	* should be one that prevents unnecessary balancing, but also spreads tasks
8189	* out optimally.
8190	*/
8191	static void sched_domain_node_span(int node, struct cpumask *span)
8192	{
8193	nodemask_t used_nodes;
8194	int i;
8195
8196	cpumask_clear(span);
8197	nodes_clear(used_nodes);
8198
8199	cpumask_or(span, span, cpumask_of_node(node));
8200	node_set(node, used_nodes);
8201
8202	for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
8203	int next_node = find_next_best_node(node, &used_nodes);
8204
8205	cpumask_or(span, span, cpumask_of_node(next_node));
8206	}
8207	}
8208	#endif /* CONFIG_NUMA */
8209
8210	int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
8211
8212	/*
8213	* The cpus mask in sched_group and sched_domain hangs off the end.
8214	*
8215	* ( See the the comments in include/linux/sched.h:struct sched_group
8216	* and struct sched_domain. )
8217	*/
8218	struct static_sched_group {
8219	struct sched_group sg;
8220	DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
8221	};
8222
8223	struct static_sched_domain {
8224	struct sched_domain sd;
8225	DECLARE_BITMAP(span, CONFIG_NR_CPUS);
8226	};
8227
8228	struct s_data {
8229	#ifdef CONFIG_NUMA
8230	int sd_allnodes;
8231	cpumask_var_t domainspan;
8232	cpumask_var_t covered;
8233	cpumask_var_t notcovered;
8234	#endif
8235	cpumask_var_t nodemask;
8236	cpumask_var_t this_sibling_map;
8237	cpumask_var_t this_core_map;
8238	cpumask_var_t send_covered;
8239	cpumask_var_t tmpmask;
8240	struct sched_group **sched_group_nodes;
8241	struct root_domain *rd;
8242	};
8243
8244	enum s_alloc {
8245	sa_sched_groups = 0,
8246	sa_rootdomain,
8247	sa_tmpmask,
8248	sa_send_covered,
8249	sa_this_core_map,
8250	sa_this_sibling_map,
8251	sa_nodemask,
8252	sa_sched_group_nodes,
8253	#ifdef CONFIG_NUMA
8254	sa_notcovered,
8255	sa_covered,
8256	sa_domainspan,
8257	#endif
8258	sa_none,
8259	};
8260
8261	/*
8262	* SMT sched-domains:
8263	*/
8264	#ifdef CONFIG_SCHED_SMT
8265	static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
8266	static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
8267
8268	static int
8269	cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
8270	struct sched_group *sg, struct cpumask unused)
8271	{
8272	if (sg)
8273	*sg = &per_cpu(sched_group_cpus, cpu).sg;
8274	return cpu;
8275	}
8276	#endif /* CONFIG_SCHED_SMT */
8277
8278	/*
8279	* multi-core sched-domains:
8280	*/
8281	#ifdef CONFIG_SCHED_MC
8282	static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
8283	static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
8284	#endif /* CONFIG_SCHED_MC */
8285
8286	#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
8287	static int
8288	cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
8289	struct sched_group *sg, struct cpumask mask)
8290	{
8291	int group;
8292
8293	cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
8294	group = cpumask_first(mask);
8295	if (sg)
8296	*sg = &per_cpu(sched_group_core, group).sg;
8297	return group;
8298	}
8299	#elif defined(CONFIG_SCHED_MC)
8300	static int
8301	cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
8302	struct sched_group *sg, struct cpumask unused)
8303	{
8304	if (sg)
8305	*sg = &per_cpu(sched_group_core, cpu).sg;
8306	return cpu;
8307	}
8308	#endif
8309
8310	static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
8311	static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
8312
8313	static int
8314	cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
8315	struct sched_group *sg, struct cpumask mask)
8316	{
8317	int group;
8318	#ifdef CONFIG_SCHED_MC
8319	cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
8320	group = cpumask_first(mask);
8321	#elif defined(CONFIG_SCHED_SMT)
8322	cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
8323	group = cpumask_first(mask);
8324	#else
8325	group = cpu;
8326	#endif
8327	if (sg)
8328	*sg = &per_cpu(sched_group_phys, group).sg;
8329	return group;
8330	}
8331
8332	#ifdef CONFIG_NUMA
8333	/*
8334	* The init_sched_build_groups can't handle what we want to do with node
8335	* groups, so roll our own. Now each node has its own list of groups which
8336	* gets dynamically allocated.
8337	*/
8338	static DEFINE_PER_CPU(struct static_sched_domain, node_domains);
8339	static struct sched_group ***sched_group_nodes_bycpu;
8340
8341	static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains);
8342	static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
8343
8344	static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
8345	struct sched_group **sg,
8346	struct cpumask *nodemask)
8347	{
8348	int group;
8349
8350	cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
8351	group = cpumask_first(nodemask);
8352
8353	if (sg)
8354	*sg = &per_cpu(sched_group_allnodes, group).sg;
8355	return group;
8356	}
8357
8358	static void init_numa_sched_groups_power(struct sched_group *group_head)
8359	{
8360	struct sched_group *sg = group_head;
8361	int j;
8362
8363	if (!sg)
8364	return;
8365	do {
8366	for_each_cpu(j, sched_group_cpus(sg)) {
8367	struct sched_domain *sd;
8368
8369	sd = &per_cpu(phys_domains, j).sd;
8370	if (j != group_first_cpu(sd->groups)) {
8371	/*
8372	* Only add "power" once for each
8373	* physical package.
8374	*/
8375	continue;
8376	}
8377
8378	sg->cpu_power += sd->groups->cpu_power;
8379	}
8380	sg = sg->next;
8381	} while (sg != group_head);
8382	}
8383
8384	static int build_numa_sched_groups(struct s_data *d,
8385	const struct cpumask *cpu_map, int num)
8386	{
8387	struct sched_domain *sd;
8388	struct sched_group sg, prev;
8389	int n, j;
8390
8391	cpumask_clear(d->covered);
8392	cpumask_and(d->nodemask, cpumask_of_node(num), cpu_map);
8393	if (cpumask_empty(d->nodemask)) {
8394	d->sched_group_nodes[num] = NULL;
8395	goto out;
8396	}
8397
8398	sched_domain_node_span(num, d->domainspan);
8399	cpumask_and(d->domainspan, d->domainspan, cpu_map);
8400
8401	sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
8402	GFP_KERNEL, num);
8403	if (!sg) {
8404	printk(KERN_WARNING "Can not alloc domain group for node %d\n",
8405	num);
8406	return -ENOMEM;
8407	}
8408	d->sched_group_nodes[num] = sg;
8409
8410	for_each_cpu(j, d->nodemask) {
8411	sd = &per_cpu(node_domains, j).sd;
8412	sd->groups = sg;
8413	}
8414
8415	sg->cpu_power = 0;
8416	cpumask_copy(sched_group_cpus(sg), d->nodemask);
8417	sg->next = sg;
8418	cpumask_or(d->covered, d->covered, d->nodemask);
8419
8420	prev = sg;
8421	for (j = 0; j < nr_node_ids; j++) {
8422	n = (num + j) % nr_node_ids;
8423	cpumask_complement(d->notcovered, d->covered);
8424	cpumask_and(d->tmpmask, d->notcovered, cpu_map);
8425	cpumask_and(d->tmpmask, d->tmpmask, d->domainspan);
8426	if (cpumask_empty(d->tmpmask))
8427	break;
8428	cpumask_and(d->tmpmask, d->tmpmask, cpumask_of_node(n));
8429	if (cpumask_empty(d->tmpmask))
8430	continue;
8431	sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
8432	GFP_KERNEL, num);
8433	if (!sg) {
8434	printk(KERN_WARNING
8435	"Can not alloc domain group for node %d\n", j);
8436	return -ENOMEM;
8437	}
8438	sg->cpu_power = 0;
8439	cpumask_copy(sched_group_cpus(sg), d->tmpmask);
8440	sg->next = prev->next;
8441	cpumask_or(d->covered, d->covered, d->tmpmask);
8442	prev->next = sg;
8443	prev = sg;
8444	}
8445	out:
8446	return 0;
8447	}
8448	#endif /* CONFIG_NUMA */
8449
8450	#ifdef CONFIG_NUMA
8451	/* Free memory allocated for various sched_group structures */
8452	static void free_sched_groups(const struct cpumask *cpu_map,
8453	struct cpumask *nodemask)
8454	{
8455	int cpu, i;
8456
8457	for_each_cpu(cpu, cpu_map) {
8458	struct sched_group **sched_group_nodes
8459	= sched_group_nodes_bycpu[cpu];
8460
8461	if (!sched_group_nodes)
8462	continue;
8463
8464	for (i = 0; i < nr_node_ids; i++) {
8465	struct sched_group oldsg, sg = sched_group_nodes[i];
8466
8467	cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
8468	if (cpumask_empty(nodemask))
8469	continue;
8470
8471	if (sg == NULL)
8472	continue;
8473	sg = sg->next;
8474	next_sg:
8475	oldsg = sg;
8476	sg = sg->next;
8477	kfree(oldsg);
8478	if (oldsg != sched_group_nodes[i])
8479	goto next_sg;
8480	}
8481	kfree(sched_group_nodes);
8482	sched_group_nodes_bycpu[cpu] = NULL;
8483	}
8484	}
8485	#else /* !CONFIG_NUMA */
8486	static void free_sched_groups(const struct cpumask *cpu_map,
8487	struct cpumask *nodemask)
8488	{
8489	}
8490	#endif /* CONFIG_NUMA */
8491
8492	/*
8493	* Initialize sched groups cpu_power.
8494	*
8495	* cpu_power indicates the capacity of sched group, which is used while
8496	* distributing the load between different sched groups in a sched domain.
8497	* Typically cpu_power for all the groups in a sched domain will be same unless
8498	* there are asymmetries in the topology. If there are asymmetries, group
8499	* having more cpu_power will pickup more load compared to the group having
8500	* less cpu_power.
8501	*/
8502	static void init_sched_groups_power(int cpu, struct sched_domain *sd)
8503	{
8504	struct sched_domain *child;
8505	struct sched_group *group;
8506	long power;
8507	int weight;
8508
8509	WARN_ON(!sd \|\| !sd->groups);
8510
8511	if (cpu != group_first_cpu(sd->groups))
8512	return;
8513
8514	child = sd->child;
8515
8516	sd->groups->cpu_power = 0;
8517
8518	if (!child) {
8519	power = SCHED_LOAD_SCALE;
8520	weight = cpumask_weight(sched_domain_span(sd));
8521	/*
8522	* SMT siblings share the power of a single core.
8523	* Usually multiple threads get a better yield out of
8524	* that one core than a single thread would have,
8525	* reflect that in sd->smt_gain.
8526	*/
8527	if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
8528	power *= sd->smt_gain;
8529	power /= weight;
8530	power >>= SCHED_LOAD_SHIFT;
8531	}
8532	sd->groups->cpu_power += power;
8533	return;
8534	}
8535
8536	/*
8537	* Add cpu_power of each child group to this groups cpu_power.
8538	*/
8539	group = child->groups;
8540	do {
8541	sd->groups->cpu_power += group->cpu_power;
8542	group = group->next;
8543	} while (group != child->groups);
8544	}
8545
8546	/*
8547	* Initializers for schedule domains
8548	* Non-inlined to reduce accumulated stack pressure in build_sched_domains()
8549	*/
8550
8551	#ifdef CONFIG_SCHED_DEBUG
8552	# define SD_INIT_NAME(sd, type) sd->name = #type
8553	#else
8554	# define SD_INIT_NAME(sd, type) do { } while (0)
8555	#endif
8556
8557	#define SD_INIT(sd, type) sd_init_##type(sd)
8558
8559	#define SD_INIT_FUNC(type) \
8560	static noinline void sd_init_##type(struct sched_domain *sd) \
8561	{ \
8562	memset(sd, 0, sizeof(*sd)); \
8563	*sd = SD_##type##_INIT; \
8564	sd->level = SD_LV_##type; \
8565	SD_INIT_NAME(sd, type); \
8566	}
8567
8568	SD_INIT_FUNC(CPU)
8569	#ifdef CONFIG_NUMA
8570	SD_INIT_FUNC(ALLNODES)
8571	SD_INIT_FUNC(NODE)
8572	#endif
8573	#ifdef CONFIG_SCHED_SMT
8574	SD_INIT_FUNC(SIBLING)
8575	#endif
8576	#ifdef CONFIG_SCHED_MC
8577	SD_INIT_FUNC(MC)
8578	#endif
8579
8580	static int default_relax_domain_level = -1;
8581
8582	static int __init setup_relax_domain_level(char *str)
8583	{
8584	unsigned long val;
8585
8586	val = simple_strtoul(str, NULL, 0);
8587	if (val < SD_LV_MAX)
8588	default_relax_domain_level = val;
8589
8590	return 1;
8591	}
8592	__setup("relax_domain_level=", setup_relax_domain_level);
8593
8594	static void set_domain_attribute(struct sched_domain *sd,
8595	struct sched_domain_attr *attr)
8596	{
8597	int request;
8598
8599	if (!attr \|\| attr->relax_domain_level < 0) {
8600	if (default_relax_domain_level < 0)
8601	return;
8602	else
8603	request = default_relax_domain_level;
8604	} else
8605	request = attr->relax_domain_level;
8606	if (request < sd->level) {
8607	/* turn off idle balance on this domain */
8608	sd->flags &= ~(SD_BALANCE_WAKE\|SD_BALANCE_NEWIDLE);
8609	} else {
8610	/* turn on idle balance on this domain */
8611	sd->flags \|= (SD_BALANCE_WAKE\|SD_BALANCE_NEWIDLE);
8612	}
8613	}
8614
8615	static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
8616	const struct cpumask *cpu_map)
8617	{
8618	switch (what) {
8619	case sa_sched_groups:
8620	free_sched_groups(cpu_map, d->tmpmask); /* fall through */
8621	d->sched_group_nodes = NULL;
8622	case sa_rootdomain:
8623	free_rootdomain(d->rd); /* fall through */
8624	case sa_tmpmask:
8625	free_cpumask_var(d->tmpmask); /* fall through */
8626	case sa_send_covered:
8627	free_cpumask_var(d->send_covered); /* fall through */
8628	case sa_this_core_map:
8629	free_cpumask_var(d->this_core_map); /* fall through */
8630	case sa_this_sibling_map:
8631	free_cpumask_var(d->this_sibling_map); /* fall through */
8632	case sa_nodemask:
8633	free_cpumask_var(d->nodemask); /* fall through */
8634	case sa_sched_group_nodes:
8635	#ifdef CONFIG_NUMA
8636	kfree(d->sched_group_nodes); /* fall through */
8637	case sa_notcovered:
8638	free_cpumask_var(d->notcovered); /* fall through */
8639	case sa_covered:
8640	free_cpumask_var(d->covered); /* fall through */
8641	case sa_domainspan:
8642	free_cpumask_var(d->domainspan); /* fall through */
8643	#endif
8644	case sa_none:
8645	break;
8646	}
8647	}
8648
8649	static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
8650	const struct cpumask *cpu_map)
8651	{
8652	#ifdef CONFIG_NUMA
8653	if (!alloc_cpumask_var(&d->domainspan, GFP_KERNEL))
8654	return sa_none;
8655	if (!alloc_cpumask_var(&d->covered, GFP_KERNEL))
8656	return sa_domainspan;
8657	if (!alloc_cpumask_var(&d->notcovered, GFP_KERNEL))
8658	return sa_covered;
8659	/* Allocate the per-node list of sched groups */
8660	d->sched_group_nodes = kcalloc(nr_node_ids,
8661	sizeof(struct sched_group *), GFP_KERNEL);
8662	if (!d->sched_group_nodes) {
8663	printk(KERN_WARNING "Can not alloc sched group node list\n");
8664	return sa_notcovered;
8665	}
8666	sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
8667	#endif
8668	if (!alloc_cpumask_var(&d->nodemask, GFP_KERNEL))
8669	return sa_sched_group_nodes;
8670	if (!alloc_cpumask_var(&d->this_sibling_map, GFP_KERNEL))
8671	return sa_nodemask;
8672	if (!alloc_cpumask_var(&d->this_core_map, GFP_KERNEL))
8673	return sa_this_sibling_map;
8674	if (!alloc_cpumask_var(&d->send_covered, GFP_KERNEL))
8675	return sa_this_core_map;
8676	if (!alloc_cpumask_var(&d->tmpmask, GFP_KERNEL))
8677	return sa_send_covered;
8678	d->rd = alloc_rootdomain();
8679	if (!d->rd) {
8680	printk(KERN_WARNING "Cannot alloc root domain\n");
8681	return sa_tmpmask;
8682	}
8683	return sa_rootdomain;
8684	}
8685
8686	static struct sched_domain __build_numa_sched_domains(struct s_data d,
8687	const struct cpumask cpu_map, struct sched_domain_attr attr, int i)
8688	{
8689	struct sched_domain *sd = NULL;
8690	#ifdef CONFIG_NUMA
8691	struct sched_domain *parent;
8692
8693	d->sd_allnodes = 0;
8694	if (cpumask_weight(cpu_map) >
8695	SD_NODES_PER_DOMAIN * cpumask_weight(d->nodemask)) {
8696	sd = &per_cpu(allnodes_domains, i).sd;
8697	SD_INIT(sd, ALLNODES);
8698	set_domain_attribute(sd, attr);
8699	cpumask_copy(sched_domain_span(sd), cpu_map);
8700	cpu_to_allnodes_group(i, cpu_map, &sd->groups, d->tmpmask);
8701	d->sd_allnodes = 1;
8702	}
8703	parent = sd;
8704
8705	sd = &per_cpu(node_domains, i).sd;
8706	SD_INIT(sd, NODE);
8707	set_domain_attribute(sd, attr);
8708	sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
8709	sd->parent = parent;
8710	if (parent)
8711	parent->child = sd;
8712	cpumask_and(sched_domain_span(sd), sched_domain_span(sd), cpu_map);
8713	#endif
8714	return sd;
8715	}
8716
8717	static struct sched_domain __build_cpu_sched_domain(struct s_data d,
8718	const struct cpumask cpu_map, struct sched_domain_attr attr,
8719	struct sched_domain *parent, int i)
8720	{
8721	struct sched_domain *sd;
8722	sd = &per_cpu(phys_domains, i).sd;
8723	SD_INIT(sd, CPU);
8724	set_domain_attribute(sd, attr);
8725	cpumask_copy(sched_domain_span(sd), d->nodemask);
8726	sd->parent = parent;
8727	if (parent)
8728	parent->child = sd;
8729	cpu_to_phys_group(i, cpu_map, &sd->groups, d->tmpmask);
8730	return sd;
8731	}
8732
8733	static struct sched_domain __build_mc_sched_domain(struct s_data d,
8734	const struct cpumask cpu_map, struct sched_domain_attr attr,
8735	struct sched_domain *parent, int i)
8736	{
8737	struct sched_domain *sd = parent;
8738	#ifdef CONFIG_SCHED_MC
8739	sd = &per_cpu(core_domains, i).sd;
8740	SD_INIT(sd, MC);
8741	set_domain_attribute(sd, attr);
8742	cpumask_and(sched_domain_span(sd), cpu_map, cpu_coregroup_mask(i));
8743	sd->parent = parent;
8744	parent->child = sd;
8745	cpu_to_core_group(i, cpu_map, &sd->groups, d->tmpmask);
8746	#endif
8747	return sd;
8748	}
8749
8750	static struct sched_domain __build_smt_sched_domain(struct s_data d,
8751	const struct cpumask cpu_map, struct sched_domain_attr attr,
8752	struct sched_domain *parent, int i)
8753	{
8754	struct sched_domain *sd = parent;
8755	#ifdef CONFIG_SCHED_SMT
8756	sd = &per_cpu(cpu_domains, i).sd;
8757	SD_INIT(sd, SIBLING);
8758	set_domain_attribute(sd, attr);
8759	cpumask_and(sched_domain_span(sd), cpu_map, topology_thread_cpumask(i));
8760	sd->parent = parent;
8761	parent->child = sd;
8762	cpu_to_cpu_group(i, cpu_map, &sd->groups, d->tmpmask);
8763	#endif
8764	return sd;
8765	}
8766
8767	static void build_sched_groups(struct s_data *d, enum sched_domain_level l,
8768	const struct cpumask *cpu_map, int cpu)
8769	{
8770	switch (l) {
8771	#ifdef CONFIG_SCHED_SMT
8772	case SD_LV_SIBLING: /* set up CPU (sibling) groups */
8773	cpumask_and(d->this_sibling_map, cpu_map,
8774	topology_thread_cpumask(cpu));
8775	if (cpu == cpumask_first(d->this_sibling_map))
8776	init_sched_build_groups(d->this_sibling_map, cpu_map,
8777	&cpu_to_cpu_group,
8778	d->send_covered, d->tmpmask);
8779	break;
8780	#endif
8781	#ifdef CONFIG_SCHED_MC
8782	case SD_LV_MC: /* set up multi-core groups */
8783	cpumask_and(d->this_core_map, cpu_map, cpu_coregroup_mask(cpu));
8784	if (cpu == cpumask_first(d->this_core_map))
8785	init_sched_build_groups(d->this_core_map, cpu_map,
8786	&cpu_to_core_group,
8787	d->send_covered, d->tmpmask);
8788	break;
8789	#endif
8790	case SD_LV_CPU: /* set up physical groups */
8791	cpumask_and(d->nodemask, cpumask_of_node(cpu), cpu_map);
8792	if (!cpumask_empty(d->nodemask))
8793	init_sched_build_groups(d->nodemask, cpu_map,
8794	&cpu_to_phys_group,
8795	d->send_covered, d->tmpmask);
8796	break;
8797	#ifdef CONFIG_NUMA
8798	case SD_LV_ALLNODES:
8799	init_sched_build_groups(cpu_map, cpu_map, &cpu_to_allnodes_group,
8800	d->send_covered, d->tmpmask);
8801	break;
8802	#endif
8803	default:
8804	break;
8805	}
8806	}
8807
8808	/*
8809	* Build sched domains for a given set of cpus and attach the sched domains
8810	* to the individual cpus
8811	*/
8812	static int __build_sched_domains(const struct cpumask *cpu_map,
8813	struct sched_domain_attr *attr)
8814	{
8815	enum s_alloc alloc_state = sa_none;
8816	struct s_data d;
8817	struct sched_domain *sd;
8818	int i;
8819	#ifdef CONFIG_NUMA
8820	d.sd_allnodes = 0;
8821	#endif
8822
8823	alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
8824	if (alloc_state != sa_rootdomain)
8825	goto error;
8826	alloc_state = sa_sched_groups;
8827
8828	/*
8829	* Set up domains for cpus specified by the cpu_map.
8830	*/
8831	for_each_cpu(i, cpu_map) {
8832	cpumask_and(d.nodemask, cpumask_of_node(cpu_to_node(i)),
8833	cpu_map);
8834
8835	sd = __build_numa_sched_domains(&d, cpu_map, attr, i);
8836	sd = __build_cpu_sched_domain(&d, cpu_map, attr, sd, i);
8837	sd = __build_mc_sched_domain(&d, cpu_map, attr, sd, i);
8838	sd = __build_smt_sched_domain(&d, cpu_map, attr, sd, i);
8839	}
8840
8841	for_each_cpu(i, cpu_map) {
8842	build_sched_groups(&d, SD_LV_SIBLING, cpu_map, i);
8843	build_sched_groups(&d, SD_LV_MC, cpu_map, i);
8844	}
8845
8846	/* Set up physical groups */
8847	for (i = 0; i < nr_node_ids; i++)
8848	build_sched_groups(&d, SD_LV_CPU, cpu_map, i);
8849
8850	#ifdef CONFIG_NUMA
8851	/* Set up node groups */
8852	if (d.sd_allnodes)
8853	build_sched_groups(&d, SD_LV_ALLNODES, cpu_map, 0);
8854
8855	for (i = 0; i < nr_node_ids; i++)
8856	if (build_numa_sched_groups(&d, cpu_map, i))
8857	goto error;
8858	#endif
8859
8860	/* Calculate CPU power for physical packages and nodes */
8861	#ifdef CONFIG_SCHED_SMT
8862	for_each_cpu(i, cpu_map) {
8863	sd = &per_cpu(cpu_domains, i).sd;
8864	init_sched_groups_power(i, sd);
8865	}
8866	#endif
8867	#ifdef CONFIG_SCHED_MC
8868	for_each_cpu(i, cpu_map) {
8869	sd = &per_cpu(core_domains, i).sd;
8870	init_sched_groups_power(i, sd);
8871	}
8872	#endif
8873
8874	for_each_cpu(i, cpu_map) {
8875	sd = &per_cpu(phys_domains, i).sd;
8876	init_sched_groups_power(i, sd);
8877	}
8878
8879	#ifdef CONFIG_NUMA
8880	for (i = 0; i < nr_node_ids; i++)
8881	init_numa_sched_groups_power(d.sched_group_nodes[i]);
8882
8883	if (d.sd_allnodes) {
8884	struct sched_group *sg;
8885
8886	cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
8887	d.tmpmask);
8888	init_numa_sched_groups_power(sg);
8889	}
8890	#endif
8891
8892	/* Attach the domains */
8893	for_each_cpu(i, cpu_map) {
8894	#ifdef CONFIG_SCHED_SMT
8895	sd = &per_cpu(cpu_domains, i).sd;
8896	#elif defined(CONFIG_SCHED_MC)
8897	sd = &per_cpu(core_domains, i).sd;
8898	#else
8899	sd = &per_cpu(phys_domains, i).sd;
8900	#endif
8901	cpu_attach_domain(sd, d.rd, i);
8902	}
8903
8904	d.sched_group_nodes = NULL; /* don't free this we still need it */
8905	__free_domain_allocs(&d, sa_tmpmask, cpu_map);
8906	return 0;
8907
8908	error:
8909	__free_domain_allocs(&d, alloc_state, cpu_map);
8910	return -ENOMEM;
8911	}
8912
8913	static int build_sched_domains(const struct cpumask *cpu_map)
8914	{
8915	return __build_sched_domains(cpu_map, NULL);
8916	}
8917
8918	static struct cpumask doms_cur; / current sched domains */
8919	static int ndoms_cur; /* number of sched domains in 'doms_cur' */
8920	static struct sched_domain_attr *dattr_cur;
8921	/* attribues of custom domains in 'doms_cur' */
8922
8923	/*
8924	* Special case: If a kmalloc of a doms_cur partition (array of
8925	* cpumask) fails, then fallback to a single sched domain,
8926	* as determined by the single cpumask fallback_doms.
8927	*/
8928	static cpumask_var_t fallback_doms;
8929
8930	/*
8931	* arch_update_cpu_topology lets virtualized architectures update the
8932	* cpu core maps. It is supposed to return 1 if the topology changed
8933	* or 0 if it stayed the same.
8934	*/
8935	int __attribute__((weak)) arch_update_cpu_topology(void)
8936	{
8937	return 0;
8938	}
8939
8940	/*
8941	* Set up scheduler domains and groups. Callers must hold the hotplug lock.
8942	* For now this just excludes isolated cpus, but could be used to
8943	* exclude other special cases in the future.
8944	*/
8945	static int arch_init_sched_domains(const struct cpumask *cpu_map)
8946	{
8947	int err;
8948
8949	arch_update_cpu_topology();
8950	ndoms_cur = 1;
8951	doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
8952	if (!doms_cur)
8953	doms_cur = fallback_doms;
8954	cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
8955	dattr_cur = NULL;
8956	err = build_sched_domains(doms_cur);
8957	register_sched_domain_sysctl();
8958
8959	return err;
8960	}
8961
8962	static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
8963	struct cpumask *tmpmask)
8964	{
8965	free_sched_groups(cpu_map, tmpmask);
8966	}
8967
8968	/*
8969	* Detach sched domains from a group of cpus specified in cpu_map
8970	* These cpus will now be attached to the NULL domain
8971	*/
8972	static void detach_destroy_domains(const struct cpumask *cpu_map)
8973	{
8974	/* Save because hotplug lock held. */
8975	static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
8976	int i;
8977
8978	for_each_cpu(i, cpu_map)
8979	cpu_attach_domain(NULL, &def_root_domain, i);
8980	synchronize_sched();
8981	arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
8982	}
8983
8984	/* handle null as "default" */
8985	static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
8986	struct sched_domain_attr *new, int idx_new)
8987	{
8988	struct sched_domain_attr tmp;
8989
8990	/* fast path */
8991	if (!new && !cur)
8992	return 1;
8993
8994	tmp = SD_ATTR_INIT;
8995	return !memcmp(cur ? (cur + idx_cur) : &tmp,
8996	new ? (new + idx_new) : &tmp,
8997	sizeof(struct sched_domain_attr));
8998	}
8999
9000	/*
9001	* Partition sched domains as specified by the 'ndoms_new'
9002	* cpumasks in the array doms_new[] of cpumasks. This compares
9003	* doms_new[] to the current sched domain partitioning, doms_cur[].
9004	* It destroys each deleted domain and builds each new domain.
9005	*
9006	* 'doms_new' is an array of cpumask's of length 'ndoms_new'.
9007	* The masks don't intersect (don't overlap.) We should setup one
9008	* sched domain for each mask. CPUs not in any of the cpumasks will
9009	* not be load balanced. If the same cpumask appears both in the
9010	* current 'doms_cur' domains and in the new 'doms_new', we can leave
9011	* it as it is.
9012	*
9013	* The passed in 'doms_new' should be kmalloc'd. This routine takes
9014	* ownership of it and will kfree it when done with it. If the caller
9015	* failed the kmalloc call, then it can pass in doms_new == NULL &&
9016	* ndoms_new == 1, and partition_sched_domains() will fallback to
9017	* the single partition 'fallback_doms', it also forces the domains
9018	* to be rebuilt.
9019	*
9020	* If doms_new == NULL it will be replaced with cpu_online_mask.
9021	* ndoms_new == 0 is a special case for destroying existing domains,
9022	* and it will not create the default domain.
9023	*
9024	* Call with hotplug lock held
9025	*/
9026	/* FIXME: Change to struct cpumask doms_new[] /
9027	void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
9028	struct sched_domain_attr *dattr_new)
9029	{
9030	int i, j, n;
9031	int new_topology;
9032
9033	mutex_lock(&sched_domains_mutex);
9034
9035	/* always unregister in case we don't destroy any domains */
9036	unregister_sched_domain_sysctl();
9037
9038	/* Let architecture update cpu core mappings. */
9039	new_topology = arch_update_cpu_topology();
9040
9041	n = doms_new ? ndoms_new : 0;
9042
9043	/* Destroy deleted domains */
9044	for (i = 0; i < ndoms_cur; i++) {
9045	for (j = 0; j < n && !new_topology; j++) {
9046	if (cpumask_equal(&doms_cur[i], &doms_new[j])
9047	&& dattrs_equal(dattr_cur, i, dattr_new, j))
9048	goto match1;
9049	}
9050	/* no match - a current sched domain not in new doms_new[] */
9051	detach_destroy_domains(doms_cur + i);
9052	match1:
9053	;
9054	}
9055
9056	if (doms_new == NULL) {
9057	ndoms_cur = 0;
9058	doms_new = fallback_doms;
9059	cpumask_andnot(&doms_new[0], cpu_active_mask, cpu_isolated_map);
9060	WARN_ON_ONCE(dattr_new);
9061	}
9062
9063	/* Build new domains */
9064	for (i = 0; i < ndoms_new; i++) {
9065	for (j = 0; j < ndoms_cur && !new_topology; j++) {
9066	if (cpumask_equal(&doms_new[i], &doms_cur[j])
9067	&& dattrs_equal(dattr_new, i, dattr_cur, j))
9068	goto match2;
9069	}
9070	/* no match - add a new doms_new */
9071	__build_sched_domains(doms_new + i,
9072	dattr_new ? dattr_new + i : NULL);
9073	match2:
9074	;
9075	}
9076
9077	/* Remember the new sched domains */
9078	if (doms_cur != fallback_doms)
9079	kfree(doms_cur);
9080	kfree(dattr_cur); /* kfree(NULL) is safe */
9081	doms_cur = doms_new;
9082	dattr_cur = dattr_new;
9083	ndoms_cur = ndoms_new;
9084
9085	register_sched_domain_sysctl();
9086
9087	mutex_unlock(&sched_domains_mutex);
9088	}
9089
9090	#if defined(CONFIG_SCHED_MC) \|\| defined(CONFIG_SCHED_SMT)
9091	static void arch_reinit_sched_domains(void)
9092	{
9093	get_online_cpus();
9094
9095	/* Destroy domains first to force the rebuild */
9096	partition_sched_domains(0, NULL, NULL);
9097
9098	rebuild_sched_domains();
9099	put_online_cpus();
9100	}
9101
9102	static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
9103	{
9104	unsigned int level = 0;
9105
9106	if (sscanf(buf, "%u", &level) != 1)
9107	return -EINVAL;
9108
9109	/*
9110	* level is always be positive so don't check for
9111	* level < POWERSAVINGS_BALANCE_NONE which is 0
9112	* What happens on 0 or 1 byte write,
9113	* need to check for count as well?
9114	*/
9115
9116	if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
9117	return -EINVAL;
9118
9119	if (smt)
9120	sched_smt_power_savings = level;
9121	else
9122	sched_mc_power_savings = level;
9123
9124	arch_reinit_sched_domains();
9125
9126	return count;
9127	}
9128
9129	#ifdef CONFIG_SCHED_MC
9130	static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
9131	char *page)
9132	{
9133	return sprintf(page, "%u\n", sched_mc_power_savings);
9134	}
9135	static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
9136	const char *buf, size_t count)
9137	{
9138	return sched_power_savings_store(buf, count, 0);
9139	}
9140	static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
9141	sched_mc_power_savings_show,
9142	sched_mc_power_savings_store);
9143	#endif
9144
9145	#ifdef CONFIG_SCHED_SMT
9146	static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
9147	char *page)
9148	{
9149	return sprintf(page, "%u\n", sched_smt_power_savings);
9150	}
9151	static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
9152	const char *buf, size_t count)
9153	{
9154	return sched_power_savings_store(buf, count, 1);
9155	}
9156	static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
9157	sched_smt_power_savings_show,
9158	sched_smt_power_savings_store);
9159	#endif
9160
9161	int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
9162	{
9163	int err = 0;
9164
9165	#ifdef CONFIG_SCHED_SMT
9166	if (smt_capable())
9167	err = sysfs_create_file(&cls->kset.kobj,
9168	&attr_sched_smt_power_savings.attr);
9169	#endif
9170	#ifdef CONFIG_SCHED_MC
9171	if (!err && mc_capable())
9172	err = sysfs_create_file(&cls->kset.kobj,
9173	&attr_sched_mc_power_savings.attr);
9174	#endif
9175	return err;
9176	}
9177	#endif /* CONFIG_SCHED_MC \|\| CONFIG_SCHED_SMT */
9178
9179	#ifndef CONFIG_CPUSETS
9180	/*
9181	* Add online and remove offline CPUs from the scheduler domains.
9182	* When cpusets are enabled they take over this function.
9183	*/
9184	static int update_sched_domains(struct notifier_block *nfb,
9185	unsigned long action, void *hcpu)
9186	{
9187	switch (action) {
9188	case CPU_ONLINE:
9189	case CPU_ONLINE_FROZEN:
9190	case CPU_DOWN_PREPARE:
9191	case CPU_DOWN_PREPARE_FROZEN:
9192	case CPU_DOWN_FAILED:
9193	case CPU_DOWN_FAILED_FROZEN:
9194	partition_sched_domains(1, NULL, NULL);
9195	return NOTIFY_OK;
9196
9197	default:
9198	return NOTIFY_DONE;
9199	}
9200	}
9201	#endif
9202
9203	static int update_runtime(struct notifier_block *nfb,
9204	unsigned long action, void *hcpu)
9205	{
9206	int cpu = (int)(long)hcpu;
9207
9208	switch (action) {
9209	case CPU_DOWN_PREPARE:
9210	case CPU_DOWN_PREPARE_FROZEN:
9211	disable_runtime(cpu_rq(cpu));
9212	return NOTIFY_OK;
9213
9214	case CPU_DOWN_FAILED:
9215	case CPU_DOWN_FAILED_FROZEN:
9216	case CPU_ONLINE:
9217	case CPU_ONLINE_FROZEN:
9218	enable_runtime(cpu_rq(cpu));
9219	return NOTIFY_OK;
9220
9221	default:
9222	return NOTIFY_DONE;
9223	}
9224	}
9225
9226	void __init sched_init_smp(void)
9227	{
9228	cpumask_var_t non_isolated_cpus;
9229
9230	alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
9231	alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
9232
9233	#if defined(CONFIG_NUMA)
9234	sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
9235	GFP_KERNEL);
9236	BUG_ON(sched_group_nodes_bycpu == NULL);
9237	#endif
9238	get_online_cpus();
9239	mutex_lock(&sched_domains_mutex);
9240	arch_init_sched_domains(cpu_active_mask);
9241	cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
9242	if (cpumask_empty(non_isolated_cpus))
9243	cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
9244	mutex_unlock(&sched_domains_mutex);
9245	put_online_cpus();
9246
9247	#ifndef CONFIG_CPUSETS
9248	/* XXX: Theoretical race here - CPU may be hotplugged now */
9249	hotcpu_notifier(update_sched_domains, 0);
9250	#endif
9251
9252	/* RT runtime code needs to handle some hotplug events */
9253	hotcpu_notifier(update_runtime, 0);
9254
9255	init_hrtick();
9256
9257	/* Move init over to a non-isolated CPU */
9258	if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
9259	BUG();
9260	sched_init_granularity();
9261	free_cpumask_var(non_isolated_cpus);
9262
9263	init_sched_rt_class();
9264	}
9265	#else
9266	void __init sched_init_smp(void)
9267	{
9268	sched_init_granularity();
9269	}
9270	#endif /* CONFIG_SMP */
9271
9272	const_debug unsigned int sysctl_timer_migration = 1;
9273
9274	int in_sched_functions(unsigned long addr)
9275	{
9276	return in_lock_functions(addr) \|\|
9277	(addr >= (unsigned long)__sched_text_start
9278	&& addr < (unsigned long)__sched_text_end);
9279	}
9280
9281	static void init_cfs_rq(struct cfs_rq cfs_rq, struct rq rq)
9282	{
9283	cfs_rq->tasks_timeline = RB_ROOT;
9284	INIT_LIST_HEAD(&cfs_rq->tasks);
9285	#ifdef CONFIG_FAIR_GROUP_SCHED
9286	cfs_rq->rq = rq;
9287	#endif
9288	cfs_rq->min_vruntime = (u64)(-(1LL << 20));
9289	}
9290
9291	static void init_rt_rq(struct rt_rq rt_rq, struct rq rq)
9292	{
9293	struct rt_prio_array *array;
9294	int i;
9295
9296	array = &rt_rq->active;
9297	for (i = 0; i < MAX_RT_PRIO; i++) {
9298	INIT_LIST_HEAD(array->queue + i);
9299	__clear_bit(i, array->bitmap);
9300	}
9301	/* delimiter for bitsearch: */
9302	__set_bit(MAX_RT_PRIO, array->bitmap);
9303
9304	#if defined CONFIG_SMP \|\| defined CONFIG_RT_GROUP_SCHED
9305	rt_rq->highest_prio.curr = MAX_RT_PRIO;
9306	#ifdef CONFIG_SMP
9307	rt_rq->highest_prio.next = MAX_RT_PRIO;
9308	#endif
9309	#endif
9310	#ifdef CONFIG_SMP
9311	rt_rq->rt_nr_migratory = 0;
9312	rt_rq->overloaded = 0;
9313	plist_head_init(&rt_rq->pushable_tasks, &rq->lock);
9314	#endif
9315
9316	rt_rq->rt_time = 0;
9317	rt_rq->rt_throttled = 0;
9318	rt_rq->rt_runtime = 0;
9319	spin_lock_init(&rt_rq->rt_runtime_lock);
9320
9321	#ifdef CONFIG_RT_GROUP_SCHED
9322	rt_rq->rt_nr_boosted = 0;
9323	rt_rq->rq = rq;
9324	#endif
9325	}
9326
9327	#ifdef CONFIG_FAIR_GROUP_SCHED
9328	static void init_tg_cfs_entry(struct task_group tg, struct cfs_rq cfs_rq,
9329	struct sched_entity *se, int cpu, int add,
9330	struct sched_entity *parent)
9331	{
9332	struct rq *rq = cpu_rq(cpu);
9333	tg->cfs_rq[cpu] = cfs_rq;
9334	init_cfs_rq(cfs_rq, rq);
9335	cfs_rq->tg = tg;
9336	if (add)
9337	list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
9338
9339	tg->se[cpu] = se;
9340	/* se could be NULL for init_task_group */
9341	if (!se)
9342	return;
9343
9344	if (!parent)
9345	se->cfs_rq = &rq->cfs;
9346	else
9347	se->cfs_rq = parent->my_q;
9348
9349	se->my_q = cfs_rq;
9350	se->load.weight = tg->shares;
9351	se->load.inv_weight = 0;
9352	se->parent = parent;
9353	}
9354	#endif
9355
9356	#ifdef CONFIG_RT_GROUP_SCHED
9357	static void init_tg_rt_entry(struct task_group tg, struct rt_rq rt_rq,
9358	struct sched_rt_entity *rt_se, int cpu, int add,
9359	struct sched_rt_entity *parent)
9360	{
9361	struct rq *rq = cpu_rq(cpu);
9362
9363	tg->rt_rq[cpu] = rt_rq;
9364	init_rt_rq(rt_rq, rq);
9365	rt_rq->tg = tg;
9366	rt_rq->rt_se = rt_se;
9367	rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
9368	if (add)
9369	list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
9370
9371	tg->rt_se[cpu] = rt_se;
9372	if (!rt_se)
9373	return;
9374
9375	if (!parent)
9376	rt_se->rt_rq = &rq->rt;
9377	else
9378	rt_se->rt_rq = parent->my_q;
9379
9380	rt_se->my_q = rt_rq;
9381	rt_se->parent = parent;
9382	INIT_LIST_HEAD(&rt_se->run_list);
9383	}
9384	#endif
9385
9386	void __init sched_init(void)
9387	{
9388	int i, j;
9389	unsigned long alloc_size = 0, ptr;
9390
9391	#ifdef CONFIG_FAIR_GROUP_SCHED
9392	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
9393	#endif
9394	#ifdef CONFIG_RT_GROUP_SCHED
9395	alloc_size += 2 * nr_cpu_ids * sizeof(void **);
9396	#endif
9397	#ifdef CONFIG_USER_SCHED
9398	alloc_size *= 2;
9399	#endif
9400	#ifdef CONFIG_CPUMASK_OFFSTACK
9401	alloc_size += num_possible_cpus() * cpumask_size();
9402	#endif
9403	/*
9404	* As sched_init() is called before page_alloc is setup,
9405	* we use alloc_bootmem().
9406	*/
9407	if (alloc_size) {
9408	ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
9409
9410	#ifdef CONFIG_FAIR_GROUP_SCHED
9411	init_task_group.se = (struct sched_entity **)ptr;
9412	ptr += nr_cpu_ids * sizeof(void **);
9413
9414	init_task_group.cfs_rq = (struct cfs_rq **)ptr;
9415	ptr += nr_cpu_ids * sizeof(void **);
9416
9417	#ifdef CONFIG_USER_SCHED
9418	root_task_group.se = (struct sched_entity **)ptr;
9419	ptr += nr_cpu_ids * sizeof(void **);
9420
9421	root_task_group.cfs_rq = (struct cfs_rq **)ptr;
9422	ptr += nr_cpu_ids * sizeof(void **);
9423	#endif /* CONFIG_USER_SCHED */
9424	#endif /* CONFIG_FAIR_GROUP_SCHED */
9425	#ifdef CONFIG_RT_GROUP_SCHED
9426	init_task_group.rt_se = (struct sched_rt_entity **)ptr;
9427	ptr += nr_cpu_ids * sizeof(void **);
9428
9429	init_task_group.rt_rq = (struct rt_rq **)ptr;
9430	ptr += nr_cpu_ids * sizeof(void **);
9431
9432	#ifdef CONFIG_USER_SCHED
9433	root_task_group.rt_se = (struct sched_rt_entity **)ptr;
9434	ptr += nr_cpu_ids * sizeof(void **);
9435
9436	root_task_group.rt_rq = (struct rt_rq **)ptr;
9437	ptr += nr_cpu_ids * sizeof(void **);
9438	#endif /* CONFIG_USER_SCHED */
9439	#endif /* CONFIG_RT_GROUP_SCHED */
9440	#ifdef CONFIG_CPUMASK_OFFSTACK
9441	for_each_possible_cpu(i) {
9442	per_cpu(load_balance_tmpmask, i) = (void *)ptr;
9443	ptr += cpumask_size();
9444	}
9445	#endif /* CONFIG_CPUMASK_OFFSTACK */
9446	}
9447
9448	#ifdef CONFIG_SMP
9449	init_defrootdomain();
9450	#endif
9451
9452	init_rt_bandwidth(&def_rt_bandwidth,
9453	global_rt_period(), global_rt_runtime());
9454
9455	#ifdef CONFIG_RT_GROUP_SCHED
9456	init_rt_bandwidth(&init_task_group.rt_bandwidth,
9457	global_rt_period(), global_rt_runtime());
9458	#ifdef CONFIG_USER_SCHED
9459	init_rt_bandwidth(&root_task_group.rt_bandwidth,
9460	global_rt_period(), RUNTIME_INF);
9461	#endif /* CONFIG_USER_SCHED */
9462	#endif /* CONFIG_RT_GROUP_SCHED */
9463
9464	#ifdef CONFIG_GROUP_SCHED
9465	list_add(&init_task_group.list, &task_groups);
9466	INIT_LIST_HEAD(&init_task_group.children);
9467
9468	#ifdef CONFIG_USER_SCHED
9469	INIT_LIST_HEAD(&root_task_group.children);
9470	init_task_group.parent = &root_task_group;
9471	list_add(&init_task_group.siblings, &root_task_group.children);
9472	#endif /* CONFIG_USER_SCHED */
9473	#endif /* CONFIG_GROUP_SCHED */
9474
9475	#if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP
9476	update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long),
9477	__alignof__(unsigned long));
9478	#endif
9479	for_each_possible_cpu(i) {
9480	struct rq *rq;
9481
9482	rq = cpu_rq(i);
9483	spin_lock_init(&rq->lock);
9484	rq->nr_running = 0;
9485	rq->calc_load_active = 0;
9486	rq->calc_load_update = jiffies + LOAD_FREQ;
9487	init_cfs_rq(&rq->cfs, rq);
9488	init_rt_rq(&rq->rt, rq);
9489	#ifdef CONFIG_FAIR_GROUP_SCHED
9490	init_task_group.shares = init_task_group_load;
9491	INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
9492	#ifdef CONFIG_CGROUP_SCHED
9493	/*
9494	* How much cpu bandwidth does init_task_group get?
9495	*
9496	* In case of task-groups formed thr' the cgroup filesystem, it
9497	* gets 100% of the cpu resources in the system. This overall
9498	* system cpu resource is divided among the tasks of
9499	* init_task_group and its child task-groups in a fair manner,
9500	* based on each entity's (task or task-group's) weight
9501	* (se->load.weight).
9502	*
9503	* In other words, if init_task_group has 10 tasks of weight
9504	* 1024) and two child groups A0 and A1 (of weight 1024 each),
9505	* then A0's share of the cpu resource is:
9506	*
9507	* A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
9508	*
9509	* We achieve this by letting init_task_group's tasks sit
9510	* directly in rq->cfs (i.e init_task_group->se[] = NULL).
9511	*/
9512	init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
9513	#elif defined CONFIG_USER_SCHED
9514	root_task_group.shares = NICE_0_LOAD;
9515	init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
9516	/*
9517	* In case of task-groups formed thr' the user id of tasks,
9518	* init_task_group represents tasks belonging to root user.
9519	* Hence it forms a sibling of all subsequent groups formed.
9520	* In this case, init_task_group gets only a fraction of overall
9521	* system cpu resource, based on the weight assigned to root
9522	* user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
9523	* by letting tasks of init_task_group sit in a separate cfs_rq
9524	* (init_tg_cfs_rq) and having one entity represent this group of
9525	* tasks in rq->cfs (i.e init_task_group->se[] != NULL).
9526	*/
9527	init_tg_cfs_entry(&init_task_group,
9528	&per_cpu(init_tg_cfs_rq, i),
9529	&per_cpu(init_sched_entity, i), i, 1,
9530	root_task_group.se[i]);
9531
9532	#endif
9533	#endif /* CONFIG_FAIR_GROUP_SCHED */
9534
9535	rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
9536	#ifdef CONFIG_RT_GROUP_SCHED
9537	INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
9538	#ifdef CONFIG_CGROUP_SCHED
9539	init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
9540	#elif defined CONFIG_USER_SCHED
9541	init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
9542	init_tg_rt_entry(&init_task_group,
9543	&per_cpu(init_rt_rq, i),
9544	&per_cpu(init_sched_rt_entity, i), i, 1,
9545	root_task_group.rt_se[i]);
9546	#endif
9547	#endif
9548
9549	for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
9550	rq->cpu_load[j] = 0;
9551	#ifdef CONFIG_SMP
9552	rq->sd = NULL;
9553	rq->rd = NULL;
9554	rq->post_schedule = 0;
9555	rq->active_balance = 0;
9556	rq->next_balance = jiffies;
9557	rq->push_cpu = 0;
9558	rq->cpu = i;
9559	rq->online = 0;
9560	rq->migration_thread = NULL;
9561	rq->idle_stamp = 0;
9562	rq->avg_idle = 2*sysctl_sched_migration_cost;
9563	INIT_LIST_HEAD(&rq->migration_queue);
9564	rq_attach_root(rq, &def_root_domain);
9565	#endif
9566	init_rq_hrtick(rq);
9567	atomic_set(&rq->nr_iowait, 0);
9568	}
9569
9570	set_load_weight(&init_task);
9571
9572	#ifdef CONFIG_PREEMPT_NOTIFIERS
9573	INIT_HLIST_HEAD(&init_task.preempt_notifiers);
9574	#endif
9575
9576	#ifdef CONFIG_SMP
9577	open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
9578	#endif
9579
9580	#ifdef CONFIG_RT_MUTEXES
9581	plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
9582	#endif
9583
9584	/*
9585	* The boot idle thread does lazy MMU switching as well:
9586	*/
9587	atomic_inc(&init_mm.mm_count);
9588	enter_lazy_tlb(&init_mm, current);
9589
9590	/*
9591	* Make us the idle thread. Technically, schedule() should not be
9592	* called from this thread, however somewhere below it might be,
9593	* but because we are the idle thread, we just pick up running again
9594	* when this runqueue becomes "idle".
9595	*/
9596	init_idle(current, smp_processor_id());
9597
9598	calc_load_update = jiffies + LOAD_FREQ;
9599
9600	/*
9601	* During early bootup we pretend to be a normal task:
9602	*/
9603	current->sched_class = &fair_sched_class;
9604
9605	/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
9606	zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
9607	#ifdef CONFIG_SMP
9608	#ifdef CONFIG_NO_HZ
9609	zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT);
9610	alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT);
9611	#endif
9612	/* May be allocated at isolcpus cmdline parse time */
9613	if (cpu_isolated_map == NULL)
9614	zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
9615	#endif /* SMP */
9616
9617	perf_event_init();
9618
9619	scheduler_running = 1;
9620	}
9621
9622	#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
9623	static inline int preempt_count_equals(int preempt_offset)
9624	{
9625	int nested = preempt_count() & ~PREEMPT_ACTIVE;
9626
9627	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
9628	}
9629
9630	void __might_sleep(char *file, int line, int preempt_offset)
9631	{
9632	#ifdef in_atomic
9633	static unsigned long prev_jiffy; /* ratelimiting */
9634
9635	if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) \|\|
9636	system_state != SYSTEM_RUNNING \|\| oops_in_progress)
9637	return;
9638	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
9639	return;
9640	prev_jiffy = jiffies;
9641
9642	printk(KERN_ERR
9643	"BUG: sleeping function called from invalid context at %s:%d\n",
9644	file, line);
9645	printk(KERN_ERR
9646	"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
9647	in_atomic(), irqs_disabled(),
9648	current->pid, current->comm);
9649
9650	debug_show_held_locks(current);
9651	if (irqs_disabled())
9652	print_irqtrace_events(current);
9653	dump_stack();
9654	#endif
9655	}
9656	EXPORT_SYMBOL(__might_sleep);
9657	#endif
9658
9659	#ifdef CONFIG_MAGIC_SYSRQ
9660	static void normalize_task(struct rq rq, struct task_struct p)
9661	{
9662	int on_rq;
9663
9664	update_rq_clock(rq);
9665	on_rq = p->se.on_rq;
9666	if (on_rq)
9667	deactivate_task(rq, p, 0);
9668	__setscheduler(rq, p, SCHED_NORMAL, 0);
9669	if (on_rq) {
9670	activate_task(rq, p, 0);
9671	resched_task(rq->curr);
9672	}
9673	}
9674
9675	void normalize_rt_tasks(void)
9676	{
9677	struct task_struct g, p;
9678	unsigned long flags;
9679	struct rq *rq;
9680
9681	read_lock_irqsave(&tasklist_lock, flags);
9682	do_each_thread(g, p) {
9683	/*
9684	* Only normalize user tasks:
9685	*/
9686	if (!p->mm)
9687	continue;
9688
9689	p->se.exec_start = 0;
9690	#ifdef CONFIG_SCHEDSTATS
9691	p->se.wait_start = 0;
9692	p->se.sleep_start = 0;
9693	p->se.block_start = 0;
9694	#endif
9695
9696	if (!rt_task(p)) {
9697	/*
9698	* Renice negative nice level userspace
9699	* tasks back to 0:
9700	*/
9701	if (TASK_NICE(p) < 0 && p->mm)
9702	set_user_nice(p, 0);
9703	continue;
9704	}
9705
9706	spin_lock(&p->pi_lock);
9707	rq = __task_rq_lock(p);
9708
9709	normalize_task(rq, p);
9710
9711	__task_rq_unlock(rq);
9712	spin_unlock(&p->pi_lock);
9713	} while_each_thread(g, p);
9714
9715	read_unlock_irqrestore(&tasklist_lock, flags);
9716	}
9717
9718	#endif /* CONFIG_MAGIC_SYSRQ */
9719
9720	#ifdef CONFIG_IA64
9721	/*
9722	* These functions are only useful for the IA64 MCA handling.
9723	*
9724	* They can only be called when the whole system has been
9725	* stopped - every CPU needs to be quiescent, and no scheduling
9726	* activity can take place. Using them for anything else would
9727	* be a serious bug, and as a result, they aren't even visible
9728	* under any other configuration.
9729	*/
9730
9731	/**
9732	* curr_task - return the current task for a given cpu.
9733	* @cpu: the processor in question.
9734	*
9735	* ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
9736	*/
9737	struct task_struct *curr_task(int cpu)
9738	{
9739	return cpu_curr(cpu);
9740	}
9741
9742	/**
9743	* set_curr_task - set the current task for a given cpu.
9744	* @cpu: the processor in question.
9745	* @p: the task pointer to set.
9746	*
9747	* Description: This function must only be used when non-maskable interrupts
9748	* are serviced on a separate stack. It allows the architecture to switch the
9749	* notion of the current task on a cpu in a non-blocking manner. This function
9750	* must be called with all CPU's synchronized, and interrupts disabled, the
9751	* and caller must save the original value of the current task (see
9752	* curr_task() above) and restore that value before reenabling interrupts and
9753	* re-starting the system.
9754	*
9755	* ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
9756	*/
9757	void set_curr_task(int cpu, struct task_struct *p)
9758	{
9759	cpu_curr(cpu) = p;
9760	}
9761
9762	#endif
9763
9764	#ifdef CONFIG_FAIR_GROUP_SCHED
9765	static void free_fair_sched_group(struct task_group *tg)
9766	{
9767	int i;
9768
9769	for_each_possible_cpu(i) {
9770	if (tg->cfs_rq)
9771	kfree(tg->cfs_rq[i]);
9772	if (tg->se)
9773	kfree(tg->se[i]);
9774	}
9775
9776	kfree(tg->cfs_rq);
9777	kfree(tg->se);
9778	}
9779
9780	static
9781	int alloc_fair_sched_group(struct task_group tg, struct task_group parent)
9782	{
9783	struct cfs_rq *cfs_rq;
9784	struct sched_entity *se;
9785	struct rq *rq;
9786	int i;
9787
9788	tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
9789	if (!tg->cfs_rq)
9790	goto err;
9791	tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
9792	if (!tg->se)
9793	goto err;
9794
9795	tg->shares = NICE_0_LOAD;
9796
9797	for_each_possible_cpu(i) {
9798	rq = cpu_rq(i);
9799
9800	cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
9801	GFP_KERNEL, cpu_to_node(i));
9802	if (!cfs_rq)
9803	goto err;
9804
9805	se = kzalloc_node(sizeof(struct sched_entity),
9806	GFP_KERNEL, cpu_to_node(i));
9807	if (!se)
9808	goto err;
9809
9810	init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
9811	}
9812
9813	return 1;
9814
9815	err:
9816	return 0;
9817	}
9818
9819	static inline void register_fair_sched_group(struct task_group *tg, int cpu)
9820	{
9821	list_add_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list,
9822	&cpu_rq(cpu)->leaf_cfs_rq_list);
9823	}
9824
9825	static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
9826	{
9827	list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
9828	}
9829	#else /* !CONFG_FAIR_GROUP_SCHED */
9830	static inline void free_fair_sched_group(struct task_group *tg)
9831	{
9832	}
9833
9834	static inline
9835	int alloc_fair_sched_group(struct task_group tg, struct task_group parent)
9836	{
9837	return 1;
9838	}
9839
9840	static inline void register_fair_sched_group(struct task_group *tg, int cpu)
9841	{
9842	}
9843
9844	static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
9845	{
9846	}
9847	#endif /* CONFIG_FAIR_GROUP_SCHED */
9848
9849	#ifdef CONFIG_RT_GROUP_SCHED
9850	static void free_rt_sched_group(struct task_group *tg)
9851	{
9852	int i;
9853
9854	destroy_rt_bandwidth(&tg->rt_bandwidth);
9855
9856	for_each_possible_cpu(i) {
9857	if (tg->rt_rq)
9858	kfree(tg->rt_rq[i]);
9859	if (tg->rt_se)
9860	kfree(tg->rt_se[i]);
9861	}
9862
9863	kfree(tg->rt_rq);
9864	kfree(tg->rt_se);
9865	}
9866
9867	static
9868	int alloc_rt_sched_group(struct task_group tg, struct task_group parent)
9869	{
9870	struct rt_rq *rt_rq;
9871	struct sched_rt_entity *rt_se;
9872	struct rq *rq;
9873	int i;
9874
9875	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
9876	if (!tg->rt_rq)
9877	goto err;
9878	tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
9879	if (!tg->rt_se)
9880	goto err;
9881
9882	init_rt_bandwidth(&tg->rt_bandwidth,
9883	ktime_to_ns(def_rt_bandwidth.rt_period), 0);
9884
9885	for_each_possible_cpu(i) {
9886	rq = cpu_rq(i);
9887
9888	rt_rq = kzalloc_node(sizeof(struct rt_rq),
9889	GFP_KERNEL, cpu_to_node(i));
9890	if (!rt_rq)
9891	goto err;
9892
9893	rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
9894	GFP_KERNEL, cpu_to_node(i));
9895	if (!rt_se)
9896	goto err;
9897
9898	init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
9899	}
9900
9901	return 1;
9902
9903	err:
9904	return 0;
9905	}
9906
9907	static inline void register_rt_sched_group(struct task_group *tg, int cpu)
9908	{
9909	list_add_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list,
9910	&cpu_rq(cpu)->leaf_rt_rq_list);
9911	}
9912
9913	static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
9914	{
9915	list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
9916	}
9917	#else /* !CONFIG_RT_GROUP_SCHED */
9918	static inline void free_rt_sched_group(struct task_group *tg)
9919	{
9920	}
9921
9922	static inline
9923	int alloc_rt_sched_group(struct task_group tg, struct task_group parent)
9924	{
9925	return 1;
9926	}
9927
9928	static inline void register_rt_sched_group(struct task_group *tg, int cpu)
9929	{
9930	}
9931
9932	static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
9933	{
9934	}
9935	#endif /* CONFIG_RT_GROUP_SCHED */
9936
9937	#ifdef CONFIG_GROUP_SCHED
9938	static void free_sched_group(struct task_group *tg)
9939	{
9940	free_fair_sched_group(tg);
9941	free_rt_sched_group(tg);
9942	kfree(tg);
9943	}
9944
9945	/* allocate runqueue etc for a new task group */
9946	struct task_group sched_create_group(struct task_group parent)
9947	{
9948	struct task_group *tg;
9949	unsigned long flags;
9950	int i;
9951
9952	tg = kzalloc(sizeof(*tg), GFP_KERNEL);
9953	if (!tg)
9954	return ERR_PTR(-ENOMEM);
9955
9956	if (!alloc_fair_sched_group(tg, parent))
9957	goto err;
9958
9959	if (!alloc_rt_sched_group(tg, parent))
9960	goto err;
9961
9962	spin_lock_irqsave(&task_group_lock, flags);
9963	for_each_possible_cpu(i) {
9964	register_fair_sched_group(tg, i);
9965	register_rt_sched_group(tg, i);
9966	}
9967	list_add_rcu(&tg->list, &task_groups);
9968
9969	WARN_ON(!parent); /* root should already exist */
9970
9971	tg->parent = parent;
9972	INIT_LIST_HEAD(&tg->children);
9973	list_add_rcu(&tg->siblings, &parent->children);
9974	spin_unlock_irqrestore(&task_group_lock, flags);
9975
9976	return tg;
9977
9978	err:
9979	free_sched_group(tg);
9980	return ERR_PTR(-ENOMEM);
9981	}
9982
9983	/* rcu callback to free various structures associated with a task group */
9984	static void free_sched_group_rcu(struct rcu_head *rhp)
9985	{
9986	/* now it should be safe to free those cfs_rqs */
9987	free_sched_group(container_of(rhp, struct task_group, rcu));
9988	}
9989
9990	/* Destroy runqueue etc associated with a task group */
9991	void sched_destroy_group(struct task_group *tg)
9992	{
9993	unsigned long flags;
9994	int i;
9995
9996	spin_lock_irqsave(&task_group_lock, flags);
9997	for_each_possible_cpu(i) {
9998	unregister_fair_sched_group(tg, i);
9999	unregister_rt_sched_group(tg, i);
10000	}
10001	list_del_rcu(&tg->list);
10002	list_del_rcu(&tg->siblings);
10003	spin_unlock_irqrestore(&task_group_lock, flags);
10004
10005	/* wait for possible concurrent references to cfs_rqs complete */
10006	call_rcu(&tg->rcu, free_sched_group_rcu);
10007	}
10008
10009	/* change task's runqueue when it moves between groups.
10010	* The caller of this function should have put the task in its new group
10011	* by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
10012	* reflect its new group.
10013	*/
10014	void sched_move_task(struct task_struct *tsk)
10015	{
10016	int on_rq, running;
10017	unsigned long flags;
10018	struct rq *rq;
10019
10020	rq = task_rq_lock(tsk, &flags);
10021
10022	update_rq_clock(rq);
10023
10024	running = task_current(rq, tsk);
10025	on_rq = tsk->se.on_rq;
10026
10027	if (on_rq)
10028	dequeue_task(rq, tsk, 0);
10029	if (unlikely(running))
10030	tsk->sched_class->put_prev_task(rq, tsk);
10031
10032	set_task_rq(tsk, task_cpu(tsk));
10033
10034	#ifdef CONFIG_FAIR_GROUP_SCHED
10035	if (tsk->sched_class->moved_group)
10036	tsk->sched_class->moved_group(tsk);
10037	#endif
10038
10039	if (unlikely(running))
10040	tsk->sched_class->set_curr_task(rq);
10041	if (on_rq)
10042	enqueue_task(rq, tsk, 0);
10043
10044	task_rq_unlock(rq, &flags);
10045	}
10046	#endif /* CONFIG_GROUP_SCHED */
10047
10048	#ifdef CONFIG_FAIR_GROUP_SCHED
10049	static void __set_se_shares(struct sched_entity *se, unsigned long shares)
10050	{
10051	struct cfs_rq *cfs_rq = se->cfs_rq;
10052	int on_rq;
10053
10054	on_rq = se->on_rq;
10055	if (on_rq)
10056	dequeue_entity(cfs_rq, se, 0);
10057
10058	se->load.weight = shares;
10059	se->load.inv_weight = 0;
10060
10061	if (on_rq)
10062	enqueue_entity(cfs_rq, se, 0);
10063	}
10064
10065	static void set_se_shares(struct sched_entity *se, unsigned long shares)
10066	{
10067	struct cfs_rq *cfs_rq = se->cfs_rq;
10068	struct rq *rq = cfs_rq->rq;
10069	unsigned long flags;
10070
10071	spin_lock_irqsave(&rq->lock, flags);
10072	__set_se_shares(se, shares);
10073	spin_unlock_irqrestore(&rq->lock, flags);
10074	}
10075
10076	static DEFINE_MUTEX(shares_mutex);
10077
10078	int sched_group_set_shares(struct task_group *tg, unsigned long shares)
10079	{
10080	int i;
10081	unsigned long flags;
10082
10083	/*
10084	* We can't change the weight of the root cgroup.
10085	*/
10086	if (!tg->se[0])
10087	return -EINVAL;
10088
10089	if (shares < MIN_SHARES)
10090	shares = MIN_SHARES;
10091	else if (shares > MAX_SHARES)
10092	shares = MAX_SHARES;
10093
10094	mutex_lock(&shares_mutex);
10095	if (tg->shares == shares)
10096	goto done;
10097
10098	spin_lock_irqsave(&task_group_lock, flags);
10099	for_each_possible_cpu(i)
10100	unregister_fair_sched_group(tg, i);
10101	list_del_rcu(&tg->siblings);
10102	spin_unlock_irqrestore(&task_group_lock, flags);
10103
10104	/* wait for any ongoing reference to this group to finish */
10105	synchronize_sched();
10106
10107	/*
10108	* Now we are free to modify the group's share on each cpu
10109	* w/o tripping rebalance_share or load_balance_fair.
10110	*/
10111	tg->shares = shares;
10112	for_each_possible_cpu(i) {
10113	/*
10114	* force a rebalance
10115	*/
10116	cfs_rq_set_shares(tg->cfs_rq[i], 0);
10117	set_se_shares(tg->se[i], shares);
10118	}
10119
10120	/*
10121	* Enable load balance activity on this group, by inserting it back on
10122	* each cpu's rq->leaf_cfs_rq_list.
10123	*/
10124	spin_lock_irqsave(&task_group_lock, flags);
10125	for_each_possible_cpu(i)
10126	register_fair_sched_group(tg, i);
10127	list_add_rcu(&tg->siblings, &tg->parent->children);
10128	spin_unlock_irqrestore(&task_group_lock, flags);
10129	done:
10130	mutex_unlock(&shares_mutex);
10131	return 0;
10132	}
10133
10134	unsigned long sched_group_shares(struct task_group *tg)
10135	{
10136	return tg->shares;
10137	}
10138	#endif
10139
10140	#ifdef CONFIG_RT_GROUP_SCHED
10141	/*
10142	* Ensure that the real time constraints are schedulable.
10143	*/
10144	static DEFINE_MUTEX(rt_constraints_mutex);
10145
10146	static unsigned long to_ratio(u64 period, u64 runtime)
10147	{
10148	if (runtime == RUNTIME_INF)
10149	return 1ULL << 20;
10150
10151	return div64_u64(runtime << 20, period);
10152	}
10153
10154	/* Must be called with tasklist_lock held */
10155	static inline int tg_has_rt_tasks(struct task_group *tg)
10156	{
10157	struct task_struct g, p;
10158
10159	do_each_thread(g, p) {
10160	if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
10161	return 1;
10162	} while_each_thread(g, p);
10163
10164	return 0;
10165	}
10166
10167	struct rt_schedulable_data {
10168	struct task_group *tg;
10169	u64 rt_period;
10170	u64 rt_runtime;
10171	};
10172
10173	static int tg_schedulable(struct task_group tg, void data)
10174	{
10175	struct rt_schedulable_data *d = data;
10176	struct task_group *child;
10177	unsigned long total, sum = 0;
10178	u64 period, runtime;
10179
10180	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
10181	runtime = tg->rt_bandwidth.rt_runtime;
10182
10183	if (tg == d->tg) {
10184	period = d->rt_period;
10185	runtime = d->rt_runtime;
10186	}
10187
10188	#ifdef CONFIG_USER_SCHED
10189	if (tg == &root_task_group) {
10190	period = global_rt_period();
10191	runtime = global_rt_runtime();
10192	}
10193	#endif
10194
10195	/*
10196	* Cannot have more runtime than the period.
10197	*/
10198	if (runtime > period && runtime != RUNTIME_INF)
10199	return -EINVAL;
10200
10201	/*
10202	* Ensure we don't starve existing RT tasks.
10203	*/
10204	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
10205	return -EBUSY;
10206
10207	total = to_ratio(period, runtime);
10208
10209	/*
10210	* Nobody can have more than the global setting allows.
10211	*/
10212	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
10213	return -EINVAL;
10214
10215	/*
10216	* The sum of our children's runtime should not exceed our own.
10217	*/
10218	list_for_each_entry_rcu(child, &tg->children, siblings) {
10219	period = ktime_to_ns(child->rt_bandwidth.rt_period);
10220	runtime = child->rt_bandwidth.rt_runtime;
10221
10222	if (child == d->tg) {
10223	period = d->rt_period;
10224	runtime = d->rt_runtime;
10225	}
10226
10227	sum += to_ratio(period, runtime);
10228	}
10229
10230	if (sum > total)
10231	return -EINVAL;
10232
10233	return 0;
10234	}
10235
10236	static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
10237	{
10238	struct rt_schedulable_data data = {
10239	.tg = tg,
10240	.rt_period = period,
10241	.rt_runtime = runtime,
10242	};
10243
10244	return walk_tg_tree(tg_schedulable, tg_nop, &data);
10245	}
10246
10247	static int tg_set_bandwidth(struct task_group *tg,
10248	u64 rt_period, u64 rt_runtime)
10249	{
10250	int i, err = 0;
10251
10252	mutex_lock(&rt_constraints_mutex);
10253	read_lock(&tasklist_lock);
10254	err = __rt_schedulable(tg, rt_period, rt_runtime);
10255	if (err)
10256	goto unlock;
10257
10258	spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
10259	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
10260	tg->rt_bandwidth.rt_runtime = rt_runtime;
10261
10262	for_each_possible_cpu(i) {
10263	struct rt_rq *rt_rq = tg->rt_rq[i];
10264
10265	spin_lock(&rt_rq->rt_runtime_lock);
10266	rt_rq->rt_runtime = rt_runtime;
10267	spin_unlock(&rt_rq->rt_runtime_lock);
10268	}
10269	spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
10270	unlock:
10271	read_unlock(&tasklist_lock);
10272	mutex_unlock(&rt_constraints_mutex);
10273
10274	return err;
10275	}
10276
10277	int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
10278	{
10279	u64 rt_runtime, rt_period;
10280
10281	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
10282	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
10283	if (rt_runtime_us < 0)
10284	rt_runtime = RUNTIME_INF;
10285
10286	return tg_set_bandwidth(tg, rt_period, rt_runtime);
10287	}
10288
10289	long sched_group_rt_runtime(struct task_group *tg)
10290	{
10291	u64 rt_runtime_us;
10292
10293	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
10294	return -1;
10295
10296	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
10297	do_div(rt_runtime_us, NSEC_PER_USEC);
10298	return rt_runtime_us;
10299	}
10300
10301	int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
10302	{
10303	u64 rt_runtime, rt_period;
10304
10305	rt_period = (u64)rt_period_us * NSEC_PER_USEC;
10306	rt_runtime = tg->rt_bandwidth.rt_runtime;
10307
10308	if (rt_period == 0)
10309	return -EINVAL;
10310
10311	return tg_set_bandwidth(tg, rt_period, rt_runtime);
10312	}
10313
10314	long sched_group_rt_period(struct task_group *tg)
10315	{
10316	u64 rt_period_us;
10317
10318	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
10319	do_div(rt_period_us, NSEC_PER_USEC);
10320	return rt_period_us;
10321	}
10322
10323	static int sched_rt_global_constraints(void)
10324	{
10325	u64 runtime, period;
10326	int ret = 0;
10327
10328	if (sysctl_sched_rt_period <= 0)
10329	return -EINVAL;
10330
10331	runtime = global_rt_runtime();
10332	period = global_rt_period();
10333
10334	/*
10335	* Sanity check on the sysctl variables.
10336	*/
10337	if (runtime > period && runtime != RUNTIME_INF)
10338	return -EINVAL;
10339
10340	mutex_lock(&rt_constraints_mutex);
10341	read_lock(&tasklist_lock);
10342	ret = __rt_schedulable(NULL, 0, 0);
10343	read_unlock(&tasklist_lock);
10344	mutex_unlock(&rt_constraints_mutex);
10345
10346	return ret;
10347	}
10348
10349	int sched_rt_can_attach(struct task_group tg, struct task_struct tsk)
10350	{
10351	/* Don't accept realtime tasks when there is no way for them to run */
10352	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
10353	return 0;
10354
10355	return 1;
10356	}
10357
10358	#else /* !CONFIG_RT_GROUP_SCHED */
10359	static int sched_rt_global_constraints(void)
10360	{
10361	unsigned long flags;
10362	int i;
10363
10364	if (sysctl_sched_rt_period <= 0)
10365	return -EINVAL;
10366
10367	/*
10368	* There's always some RT tasks in the root group
10369	* -- migration, kstopmachine etc..
10370	*/
10371	if (sysctl_sched_rt_runtime == 0)
10372	return -EBUSY;
10373
10374	spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
10375	for_each_possible_cpu(i) {
10376	struct rt_rq *rt_rq = &cpu_rq(i)->rt;
10377
10378	spin_lock(&rt_rq->rt_runtime_lock);
10379	rt_rq->rt_runtime = global_rt_runtime();
10380	spin_unlock(&rt_rq->rt_runtime_lock);
10381	}
10382	spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
10383
10384	return 0;
10385	}
10386	#endif /* CONFIG_RT_GROUP_SCHED */
10387
10388	int sched_rt_handler(struct ctl_table *table, int write,
10389	void __user buffer, size_t lenp,
10390	loff_t *ppos)
10391	{
10392	int ret;
10393	int old_period, old_runtime;
10394	static DEFINE_MUTEX(mutex);
10395
10396	mutex_lock(&mutex);
10397	old_period = sysctl_sched_rt_period;
10398	old_runtime = sysctl_sched_rt_runtime;
10399
10400	ret = proc_dointvec(table, write, buffer, lenp, ppos);
10401
10402	if (!ret && write) {
10403	ret = sched_rt_global_constraints();
10404	if (ret) {
10405	sysctl_sched_rt_period = old_period;
10406	sysctl_sched_rt_runtime = old_runtime;
10407	} else {
10408	def_rt_bandwidth.rt_runtime = global_rt_runtime();
10409	def_rt_bandwidth.rt_period =
10410	ns_to_ktime(global_rt_period());
10411	}
10412	}
10413	mutex_unlock(&mutex);
10414
10415	return ret;
10416	}
10417
10418	#ifdef CONFIG_CGROUP_SCHED
10419
10420	/* return corresponding task_group object of a cgroup */
10421	static inline struct task_group cgroup_tg(struct cgroup cgrp)
10422	{
10423	return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
10424	struct task_group, css);
10425	}
10426
10427	static struct cgroup_subsys_state *
10428	cpu_cgroup_create(struct cgroup_subsys ss, struct cgroup cgrp)
10429	{
10430	struct task_group tg, parent;
10431
10432	if (!cgrp->parent) {
10433	/* This is early initialization for the top cgroup */
10434	return &init_task_group.css;
10435	}
10436
10437	parent = cgroup_tg(cgrp->parent);
10438	tg = sched_create_group(parent);
10439	if (IS_ERR(tg))
10440	return ERR_PTR(-ENOMEM);
10441
10442	return &tg->css;
10443	}
10444
10445	static void
10446	cpu_cgroup_destroy(struct cgroup_subsys ss, struct cgroup cgrp)
10447	{
10448	struct task_group *tg = cgroup_tg(cgrp);
10449
10450	sched_destroy_group(tg);
10451	}
10452
10453	static int
10454	cpu_cgroup_can_attach_task(struct cgroup cgrp, struct task_struct tsk)
10455	{
10456	#ifdef CONFIG_RT_GROUP_SCHED
10457	if (!sched_rt_can_attach(cgroup_tg(cgrp), tsk))
10458	return -EINVAL;
10459	#else
10460	/* We don't support RT-tasks being in separate groups */
10461	if (tsk->sched_class != &fair_sched_class)
10462	return -EINVAL;
10463	#endif
10464	return 0;
10465	}
10466
10467	static int
10468	cpu_cgroup_can_attach(struct cgroup_subsys ss, struct cgroup cgrp,
10469	struct task_struct *tsk, bool threadgroup)
10470	{
10471	int retval = cpu_cgroup_can_attach_task(cgrp, tsk);
10472	if (retval)
10473	return retval;
10474	if (threadgroup) {
10475	struct task_struct *c;
10476	rcu_read_lock();
10477	list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
10478	retval = cpu_cgroup_can_attach_task(cgrp, c);
10479	if (retval) {
10480	rcu_read_unlock();
10481	return retval;
10482	}
10483	}
10484	rcu_read_unlock();
10485	}
10486	return 0;
10487	}
10488
10489	static void
10490	cpu_cgroup_attach(struct cgroup_subsys ss, struct cgroup cgrp,
10491	struct cgroup old_cont, struct task_struct tsk,
10492	bool threadgroup)
10493	{
10494	sched_move_task(tsk);
10495	if (threadgroup) {
10496	struct task_struct *c;
10497	rcu_read_lock();
10498	list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
10499	sched_move_task(c);
10500	}
10501	rcu_read_unlock();
10502	}
10503	}
10504
10505	#ifdef CONFIG_FAIR_GROUP_SCHED
10506	static int cpu_shares_write_u64(struct cgroup cgrp, struct cftype cftype,
10507	u64 shareval)
10508	{
10509	return sched_group_set_shares(cgroup_tg(cgrp), shareval);
10510	}
10511
10512	static u64 cpu_shares_read_u64(struct cgroup cgrp, struct cftype cft)
10513	{
10514	struct task_group *tg = cgroup_tg(cgrp);
10515
10516	return (u64) tg->shares;
10517	}
10518	#endif /* CONFIG_FAIR_GROUP_SCHED */
10519
10520	#ifdef CONFIG_RT_GROUP_SCHED
10521	static int cpu_rt_runtime_write(struct cgroup cgrp, struct cftype cft,
10522	s64 val)
10523	{
10524	return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
10525	}
10526
10527	static s64 cpu_rt_runtime_read(struct cgroup cgrp, struct cftype cft)
10528	{
10529	return sched_group_rt_runtime(cgroup_tg(cgrp));
10530	}
10531
10532	static int cpu_rt_period_write_uint(struct cgroup cgrp, struct cftype cftype,
10533	u64 rt_period_us)
10534	{
10535	return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
10536	}
10537
10538	static u64 cpu_rt_period_read_uint(struct cgroup cgrp, struct cftype cft)
10539	{
10540	return sched_group_rt_period(cgroup_tg(cgrp));
10541	}
10542	#endif /* CONFIG_RT_GROUP_SCHED */
10543
10544	static struct cftype cpu_files[] = {
10545	#ifdef CONFIG_FAIR_GROUP_SCHED
10546	{
10547	.name = "shares",
10548	.read_u64 = cpu_shares_read_u64,
10549	.write_u64 = cpu_shares_write_u64,
10550	},
10551	#endif
10552	#ifdef CONFIG_RT_GROUP_SCHED
10553	{
10554	.name = "rt_runtime_us",
10555	.read_s64 = cpu_rt_runtime_read,
10556	.write_s64 = cpu_rt_runtime_write,
10557	},
10558	{
10559	.name = "rt_period_us",
10560	.read_u64 = cpu_rt_period_read_uint,
10561	.write_u64 = cpu_rt_period_write_uint,
10562	},
10563	#endif
10564	};
10565
10566	static int cpu_cgroup_populate(struct cgroup_subsys ss, struct cgroup cont)
10567	{
10568	return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
10569	}
10570
10571	struct cgroup_subsys cpu_cgroup_subsys = {
10572	.name = "cpu",
10573	.create = cpu_cgroup_create,
10574	.destroy = cpu_cgroup_destroy,
10575	.can_attach = cpu_cgroup_can_attach,
10576	.attach = cpu_cgroup_attach,
10577	.populate = cpu_cgroup_populate,
10578	.subsys_id = cpu_cgroup_subsys_id,
10579	.early_init = 1,
10580	};
10581
10582	#endif /* CONFIG_CGROUP_SCHED */
10583
10584	#ifdef CONFIG_CGROUP_CPUACCT
10585
10586	/*
10587	* CPU accounting code for task groups.
10588	*
10589	* Based on the work by Paul Menage (menage@google.com) and Balbir Singh
10590	* (balbir@in.ibm.com).
10591	*/
10592
10593	/* track cpu usage of a group of tasks and its child groups */
10594	struct cpuacct {
10595	struct cgroup_subsys_state css;
10596	/* cpuusage holds pointer to a u64-type object on every cpu */
10597	u64 *cpuusage;
10598	struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
10599	struct cpuacct *parent;
10600	};
10601
10602	struct cgroup_subsys cpuacct_subsys;
10603
10604	/* return cpu accounting group corresponding to this container */
10605	static inline struct cpuacct cgroup_ca(struct cgroup cgrp)
10606	{
10607	return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
10608	struct cpuacct, css);
10609	}
10610
10611	/* return cpu accounting group to which this task belongs */
10612	static inline struct cpuacct task_ca(struct task_struct tsk)
10613	{
10614	return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
10615	struct cpuacct, css);
10616	}
10617
10618	/* create a new cpu accounting group */
10619	static struct cgroup_subsys_state *cpuacct_create(
10620	struct cgroup_subsys ss, struct cgroup cgrp)
10621	{
10622	struct cpuacct ca = kzalloc(sizeof(ca), GFP_KERNEL);
10623	int i;
10624
10625	if (!ca)
10626	goto out;
10627
10628	ca->cpuusage = alloc_percpu(u64);
10629	if (!ca->cpuusage)
10630	goto out_free_ca;
10631
10632	for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
10633	if (percpu_counter_init(&ca->cpustat[i], 0))
10634	goto out_free_counters;
10635
10636	if (cgrp->parent)
10637	ca->parent = cgroup_ca(cgrp->parent);
10638
10639	return &ca->css;
10640
10641	out_free_counters:
10642	while (--i >= 0)
10643	percpu_counter_destroy(&ca->cpustat[i]);
10644	free_percpu(ca->cpuusage);
10645	out_free_ca:
10646	kfree(ca);
10647	out:
10648	return ERR_PTR(-ENOMEM);
10649	}
10650
10651	/* destroy an existing cpu accounting group */
10652	static void
10653	cpuacct_destroy(struct cgroup_subsys ss, struct cgroup cgrp)
10654	{
10655	struct cpuacct *ca = cgroup_ca(cgrp);
10656	int i;
10657
10658	for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
10659	percpu_counter_destroy(&ca->cpustat[i]);
10660	free_percpu(ca->cpuusage);
10661	kfree(ca);
10662	}
10663
10664	static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
10665	{
10666	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
10667	u64 data;
10668
10669	#ifndef CONFIG_64BIT
10670	/*
10671	* Take rq->lock to make 64-bit read safe on 32-bit platforms.
10672	*/
10673	spin_lock_irq(&cpu_rq(cpu)->lock);
10674	data = *cpuusage;
10675	spin_unlock_irq(&cpu_rq(cpu)->lock);
10676	#else
10677	data = *cpuusage;
10678	#endif
10679
10680	return data;
10681	}
10682
10683	static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
10684	{
10685	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
10686
10687	#ifndef CONFIG_64BIT
10688	/*
10689	* Take rq->lock to make 64-bit write safe on 32-bit platforms.
10690	*/
10691	spin_lock_irq(&cpu_rq(cpu)->lock);
10692	*cpuusage = val;
10693	spin_unlock_irq(&cpu_rq(cpu)->lock);
10694	#else
10695	*cpuusage = val;
10696	#endif
10697	}
10698
10699	/* return total cpu usage (in nanoseconds) of a group */
10700	static u64 cpuusage_read(struct cgroup cgrp, struct cftype cft)
10701	{
10702	struct cpuacct *ca = cgroup_ca(cgrp);
10703	u64 totalcpuusage = 0;
10704	int i;
10705
10706	for_each_present_cpu(i)
10707	totalcpuusage += cpuacct_cpuusage_read(ca, i);
10708
10709	return totalcpuusage;
10710	}
10711
10712	static int cpuusage_write(struct cgroup cgrp, struct cftype cftype,
10713	u64 reset)
10714	{
10715	struct cpuacct *ca = cgroup_ca(cgrp);
10716	int err = 0;
10717	int i;
10718
10719	if (reset) {
10720	err = -EINVAL;
10721	goto out;
10722	}
10723
10724	for_each_present_cpu(i)
10725	cpuacct_cpuusage_write(ca, i, 0);
10726
10727	out:
10728	return err;
10729	}
10730
10731	static int cpuacct_percpu_seq_read(struct cgroup cgroup, struct cftype cft,
10732	struct seq_file *m)
10733	{
10734	struct cpuacct *ca = cgroup_ca(cgroup);
10735	u64 percpu;
10736	int i;
10737
10738	for_each_present_cpu(i) {
10739	percpu = cpuacct_cpuusage_read(ca, i);
10740	seq_printf(m, "%llu ", (unsigned long long) percpu);
10741	}
10742	seq_printf(m, "\n");
10743	return 0;
10744	}
10745
10746	static const char *cpuacct_stat_desc[] = {
10747	[CPUACCT_STAT_USER] = "user",
10748	[CPUACCT_STAT_SYSTEM] = "system",
10749	};
10750
10751	static int cpuacct_stats_show(struct cgroup cgrp, struct cftype cft,
10752	struct cgroup_map_cb *cb)
10753	{
10754	struct cpuacct *ca = cgroup_ca(cgrp);
10755	int i;
10756
10757	for (i = 0; i < CPUACCT_STAT_NSTATS; i++) {
10758	s64 val = percpu_counter_read(&ca->cpustat[i]);
10759	val = cputime64_to_clock_t(val);
10760	cb->fill(cb, cpuacct_stat_desc[i], val);
10761	}
10762	return 0;
10763	}
10764
10765	static struct cftype files[] = {
10766	{
10767	.name = "usage",
10768	.read_u64 = cpuusage_read,
10769	.write_u64 = cpuusage_write,
10770	},
10771	{
10772	.name = "usage_percpu",
10773	.read_seq_string = cpuacct_percpu_seq_read,
10774	},
10775	{
10776	.name = "stat",
10777	.read_map = cpuacct_stats_show,
10778	},
10779	};
10780
10781	static int cpuacct_populate(struct cgroup_subsys ss, struct cgroup cgrp)
10782	{
10783	return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
10784	}
10785
10786	/*
10787	* charge this task's execution time to its accounting group.
10788	*
10789	* called with rq->lock held.
10790	*/
10791	static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
10792	{
10793	struct cpuacct *ca;
10794	int cpu;
10795
10796	if (unlikely(!cpuacct_subsys.active))
10797	return;
10798
10799	cpu = task_cpu(tsk);
10800
10801	rcu_read_lock();
10802
10803	ca = task_ca(tsk);
10804
10805	for (; ca; ca = ca->parent) {
10806	u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
10807	*cpuusage += cputime;
10808	}
10809
10810	rcu_read_unlock();
10811	}
10812
10813	/*
10814	* Charge the system/user time to the task's accounting group.
10815	*/
10816	static void cpuacct_update_stats(struct task_struct *tsk,
10817	enum cpuacct_stat_index idx, cputime_t val)
10818	{
10819	struct cpuacct *ca;
10820
10821	if (unlikely(!cpuacct_subsys.active))
10822	return;
10823
10824	rcu_read_lock();
10825	ca = task_ca(tsk);
10826
10827	do {
10828	percpu_counter_add(&ca->cpustat[idx], val);
10829	ca = ca->parent;
10830	} while (ca);
10831	rcu_read_unlock();
10832	}
10833
10834	struct cgroup_subsys cpuacct_subsys = {
10835	.name = "cpuacct",
10836	.create = cpuacct_create,
10837	.destroy = cpuacct_destroy,
10838	.populate = cpuacct_populate,
10839	.subsys_id = cpuacct_subsys_id,
10840	};
10841	#endif /* CONFIG_CGROUP_CPUACCT */
10842
10843	#ifndef CONFIG_SMP
10844
10845	int rcu_expedited_torture_stats(char *page)
10846	{
10847	return 0;
10848	}
10849	EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
10850
10851	void synchronize_sched_expedited(void)
10852	{
10853	}
10854	EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
10855
10856	#else /* #ifndef CONFIG_SMP */
10857
10858	static DEFINE_PER_CPU(struct migration_req, rcu_migration_req);
10859	static DEFINE_MUTEX(rcu_sched_expedited_mutex);
10860
10861	#define RCU_EXPEDITED_STATE_POST -2
10862	#define RCU_EXPEDITED_STATE_IDLE -1
10863
10864	static int rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
10865
10866	int rcu_expedited_torture_stats(char *page)
10867	{
10868	int cnt = 0;
10869	int cpu;
10870
10871	cnt += sprintf(&page[cnt], "state: %d /", rcu_expedited_state);
10872	for_each_online_cpu(cpu) {
10873	cnt += sprintf(&page[cnt], " %d:%d",
10874	cpu, per_cpu(rcu_migration_req, cpu).dest_cpu);
10875	}
10876	cnt += sprintf(&page[cnt], "\n");
10877	return cnt;
10878	}
10879	EXPORT_SYMBOL_GPL(rcu_expedited_torture_stats);
10880
10881	static long synchronize_sched_expedited_count;
10882
10883	/*
10884	* Wait for an rcu-sched grace period to elapse, but use "big hammer"
10885	* approach to force grace period to end quickly. This consumes
10886	* significant time on all CPUs, and is thus not recommended for
10887	* any sort of common-case code.
10888	*
10889	* Note that it is illegal to call this function while holding any
10890	* lock that is acquired by a CPU-hotplug notifier. Failing to
10891	* observe this restriction will result in deadlock.
10892	*/
10893	void synchronize_sched_expedited(void)
10894	{
10895	int cpu;
10896	unsigned long flags;
10897	bool need_full_sync = 0;
10898	struct rq *rq;
10899	struct migration_req *req;
10900	long snap;
10901	int trycount = 0;
10902
10903	smp_mb(); /* ensure prior mod happens before capturing snap. */
10904	snap = ACCESS_ONCE(synchronize_sched_expedited_count) + 1;
10905	get_online_cpus();
10906	while (!mutex_trylock(&rcu_sched_expedited_mutex)) {
10907	put_online_cpus();
10908	if (trycount++ < 10)
10909	udelay(trycount * num_online_cpus());
10910	else {
10911	synchronize_sched();
10912	return;
10913	}
10914	if (ACCESS_ONCE(synchronize_sched_expedited_count) - snap > 0) {
10915	smp_mb(); /* ensure test happens before caller kfree */
10916	return;
10917	}
10918	get_online_cpus();
10919	}
10920	rcu_expedited_state = RCU_EXPEDITED_STATE_POST;
10921	for_each_online_cpu(cpu) {
10922	rq = cpu_rq(cpu);
10923	req = &per_cpu(rcu_migration_req, cpu);
10924	init_completion(&req->done);
10925	req->task = NULL;
10926	req->dest_cpu = RCU_MIGRATION_NEED_QS;
10927	spin_lock_irqsave(&rq->lock, flags);
10928	list_add(&req->list, &rq->migration_queue);
10929	spin_unlock_irqrestore(&rq->lock, flags);
10930	wake_up_process(rq->migration_thread);
10931	}
10932	for_each_online_cpu(cpu) {
10933	rcu_expedited_state = cpu;
10934	req = &per_cpu(rcu_migration_req, cpu);
10935	rq = cpu_rq(cpu);
10936	wait_for_completion(&req->done);
10937	spin_lock_irqsave(&rq->lock, flags);
10938	if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC))
10939	need_full_sync = 1;
10940	req->dest_cpu = RCU_MIGRATION_IDLE;
10941	spin_unlock_irqrestore(&rq->lock, flags);
10942	}
10943	rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
10944	mutex_unlock(&rcu_sched_expedited_mutex);
10945	put_online_cpus();
10946	if (need_full_sync)
10947	synchronize_sched();
10948	}
10949	EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
10950
10951	#endif /* #else #ifndef CONFIG_SMP */
10952

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/kernel/sched.c

interactive