Root/
1 | /* CPU control. |
2 | * (C) 2001, 2002, 2003, 2004 Rusty Russell |
3 | * |
4 | * This code is licenced under the GPL. |
5 | */ |
6 | #include <linux/proc_fs.h> |
7 | #include <linux/smp.h> |
8 | #include <linux/init.h> |
9 | #include <linux/notifier.h> |
10 | #include <linux/sched.h> |
11 | #include <linux/unistd.h> |
12 | #include <linux/cpu.h> |
13 | #include <linux/module.h> |
14 | #include <linux/kthread.h> |
15 | #include <linux/stop_machine.h> |
16 | #include <linux/mutex.h> |
17 | #include <linux/gfp.h> |
18 | |
19 | #ifdef CONFIG_SMP |
20 | /* Serializes the updates to cpu_online_mask, cpu_present_mask */ |
21 | static DEFINE_MUTEX(cpu_add_remove_lock); |
22 | |
23 | static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain); |
24 | |
25 | /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. |
26 | * Should always be manipulated under cpu_add_remove_lock |
27 | */ |
28 | static int cpu_hotplug_disabled; |
29 | |
30 | static struct { |
31 | struct task_struct *active_writer; |
32 | struct mutex lock; /* Synchronizes accesses to refcount, */ |
33 | /* |
34 | * Also blocks the new readers during |
35 | * an ongoing cpu hotplug operation. |
36 | */ |
37 | int refcount; |
38 | } cpu_hotplug = { |
39 | .active_writer = NULL, |
40 | .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), |
41 | .refcount = 0, |
42 | }; |
43 | |
44 | #ifdef CONFIG_HOTPLUG_CPU |
45 | |
46 | void get_online_cpus(void) |
47 | { |
48 | might_sleep(); |
49 | if (cpu_hotplug.active_writer == current) |
50 | return; |
51 | mutex_lock(&cpu_hotplug.lock); |
52 | cpu_hotplug.refcount++; |
53 | mutex_unlock(&cpu_hotplug.lock); |
54 | |
55 | } |
56 | EXPORT_SYMBOL_GPL(get_online_cpus); |
57 | |
58 | void put_online_cpus(void) |
59 | { |
60 | if (cpu_hotplug.active_writer == current) |
61 | return; |
62 | mutex_lock(&cpu_hotplug.lock); |
63 | if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer)) |
64 | wake_up_process(cpu_hotplug.active_writer); |
65 | mutex_unlock(&cpu_hotplug.lock); |
66 | |
67 | } |
68 | EXPORT_SYMBOL_GPL(put_online_cpus); |
69 | |
70 | #endif /* CONFIG_HOTPLUG_CPU */ |
71 | |
72 | /* |
73 | * The following two API's must be used when attempting |
74 | * to serialize the updates to cpu_online_mask, cpu_present_mask. |
75 | */ |
76 | void cpu_maps_update_begin(void) |
77 | { |
78 | mutex_lock(&cpu_add_remove_lock); |
79 | } |
80 | |
81 | void cpu_maps_update_done(void) |
82 | { |
83 | mutex_unlock(&cpu_add_remove_lock); |
84 | } |
85 | |
86 | /* |
87 | * This ensures that the hotplug operation can begin only when the |
88 | * refcount goes to zero. |
89 | * |
90 | * Note that during a cpu-hotplug operation, the new readers, if any, |
91 | * will be blocked by the cpu_hotplug.lock |
92 | * |
93 | * Since cpu_hotplug_begin() is always called after invoking |
94 | * cpu_maps_update_begin(), we can be sure that only one writer is active. |
95 | * |
96 | * Note that theoretically, there is a possibility of a livelock: |
97 | * - Refcount goes to zero, last reader wakes up the sleeping |
98 | * writer. |
99 | * - Last reader unlocks the cpu_hotplug.lock. |
100 | * - A new reader arrives at this moment, bumps up the refcount. |
101 | * - The writer acquires the cpu_hotplug.lock finds the refcount |
102 | * non zero and goes to sleep again. |
103 | * |
104 | * However, this is very difficult to achieve in practice since |
105 | * get_online_cpus() not an api which is called all that often. |
106 | * |
107 | */ |
108 | static void cpu_hotplug_begin(void) |
109 | { |
110 | cpu_hotplug.active_writer = current; |
111 | |
112 | for (;;) { |
113 | mutex_lock(&cpu_hotplug.lock); |
114 | if (likely(!cpu_hotplug.refcount)) |
115 | break; |
116 | __set_current_state(TASK_UNINTERRUPTIBLE); |
117 | mutex_unlock(&cpu_hotplug.lock); |
118 | schedule(); |
119 | } |
120 | } |
121 | |
122 | static void cpu_hotplug_done(void) |
123 | { |
124 | cpu_hotplug.active_writer = NULL; |
125 | mutex_unlock(&cpu_hotplug.lock); |
126 | } |
127 | /* Need to know about CPUs going up/down? */ |
128 | int __ref register_cpu_notifier(struct notifier_block *nb) |
129 | { |
130 | int ret; |
131 | cpu_maps_update_begin(); |
132 | ret = raw_notifier_chain_register(&cpu_chain, nb); |
133 | cpu_maps_update_done(); |
134 | return ret; |
135 | } |
136 | |
137 | #ifdef CONFIG_HOTPLUG_CPU |
138 | |
139 | EXPORT_SYMBOL(register_cpu_notifier); |
140 | |
141 | void __ref unregister_cpu_notifier(struct notifier_block *nb) |
142 | { |
143 | cpu_maps_update_begin(); |
144 | raw_notifier_chain_unregister(&cpu_chain, nb); |
145 | cpu_maps_update_done(); |
146 | } |
147 | EXPORT_SYMBOL(unregister_cpu_notifier); |
148 | |
149 | static inline void check_for_tasks(int cpu) |
150 | { |
151 | struct task_struct *p; |
152 | |
153 | write_lock_irq(&tasklist_lock); |
154 | for_each_process(p) { |
155 | if (task_cpu(p) == cpu && p->state == TASK_RUNNING && |
156 | (!cputime_eq(p->utime, cputime_zero) || |
157 | !cputime_eq(p->stime, cputime_zero))) |
158 | printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d " |
159 | "(state = %ld, flags = %x)\n", |
160 | p->comm, task_pid_nr(p), cpu, |
161 | p->state, p->flags); |
162 | } |
163 | write_unlock_irq(&tasklist_lock); |
164 | } |
165 | |
166 | struct take_cpu_down_param { |
167 | unsigned long mod; |
168 | void *hcpu; |
169 | }; |
170 | |
171 | /* Take this CPU down. */ |
172 | static int __ref take_cpu_down(void *_param) |
173 | { |
174 | struct take_cpu_down_param *param = _param; |
175 | int err; |
176 | |
177 | /* Ensure this CPU doesn't handle any more interrupts. */ |
178 | err = __cpu_disable(); |
179 | if (err < 0) |
180 | return err; |
181 | |
182 | raw_notifier_call_chain(&cpu_chain, CPU_DYING | param->mod, |
183 | param->hcpu); |
184 | |
185 | /* Force idle task to run as soon as we yield: it should |
186 | immediately notice cpu is offline and die quickly. */ |
187 | sched_idle_next(); |
188 | return 0; |
189 | } |
190 | |
191 | /* Requires cpu_add_remove_lock to be held */ |
192 | static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) |
193 | { |
194 | int err, nr_calls = 0; |
195 | cpumask_var_t old_allowed; |
196 | void *hcpu = (void *)(long)cpu; |
197 | unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; |
198 | struct take_cpu_down_param tcd_param = { |
199 | .mod = mod, |
200 | .hcpu = hcpu, |
201 | }; |
202 | |
203 | if (num_online_cpus() == 1) |
204 | return -EBUSY; |
205 | |
206 | if (!cpu_online(cpu)) |
207 | return -EINVAL; |
208 | |
209 | if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL)) |
210 | return -ENOMEM; |
211 | |
212 | cpu_hotplug_begin(); |
213 | set_cpu_active(cpu, false); |
214 | err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod, |
215 | hcpu, -1, &nr_calls); |
216 | if (err == NOTIFY_BAD) { |
217 | set_cpu_active(cpu, true); |
218 | |
219 | nr_calls--; |
220 | __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod, |
221 | hcpu, nr_calls, NULL); |
222 | printk("%s: attempt to take down CPU %u failed\n", |
223 | __func__, cpu); |
224 | err = -EINVAL; |
225 | goto out_release; |
226 | } |
227 | |
228 | /* Ensure that we are not runnable on dying cpu */ |
229 | cpumask_copy(old_allowed, ¤t->cpus_allowed); |
230 | set_cpus_allowed_ptr(current, cpu_active_mask); |
231 | |
232 | err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu)); |
233 | if (err) { |
234 | set_cpu_active(cpu, true); |
235 | /* CPU didn't die: tell everyone. Can't complain. */ |
236 | if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod, |
237 | hcpu) == NOTIFY_BAD) |
238 | BUG(); |
239 | |
240 | goto out_allowed; |
241 | } |
242 | BUG_ON(cpu_online(cpu)); |
243 | |
244 | /* Wait for it to sleep (leaving idle task). */ |
245 | while (!idle_cpu(cpu)) |
246 | yield(); |
247 | |
248 | /* This actually kills the CPU. */ |
249 | __cpu_die(cpu); |
250 | |
251 | /* CPU is completely dead: tell everyone. Too late to complain. */ |
252 | if (raw_notifier_call_chain(&cpu_chain, CPU_DEAD | mod, |
253 | hcpu) == NOTIFY_BAD) |
254 | BUG(); |
255 | |
256 | check_for_tasks(cpu); |
257 | |
258 | out_allowed: |
259 | set_cpus_allowed_ptr(current, old_allowed); |
260 | out_release: |
261 | cpu_hotplug_done(); |
262 | if (!err) { |
263 | if (raw_notifier_call_chain(&cpu_chain, CPU_POST_DEAD | mod, |
264 | hcpu) == NOTIFY_BAD) |
265 | BUG(); |
266 | } |
267 | free_cpumask_var(old_allowed); |
268 | return err; |
269 | } |
270 | |
271 | int __ref cpu_down(unsigned int cpu) |
272 | { |
273 | int err; |
274 | |
275 | err = stop_machine_create(); |
276 | if (err) |
277 | return err; |
278 | cpu_maps_update_begin(); |
279 | |
280 | if (cpu_hotplug_disabled) { |
281 | err = -EBUSY; |
282 | goto out; |
283 | } |
284 | |
285 | err = _cpu_down(cpu, 0); |
286 | |
287 | out: |
288 | cpu_maps_update_done(); |
289 | stop_machine_destroy(); |
290 | return err; |
291 | } |
292 | EXPORT_SYMBOL(cpu_down); |
293 | #endif /*CONFIG_HOTPLUG_CPU*/ |
294 | |
295 | /* Requires cpu_add_remove_lock to be held */ |
296 | static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) |
297 | { |
298 | int ret, nr_calls = 0; |
299 | void *hcpu = (void *)(long)cpu; |
300 | unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; |
301 | |
302 | if (cpu_online(cpu) || !cpu_present(cpu)) |
303 | return -EINVAL; |
304 | |
305 | cpu_hotplug_begin(); |
306 | ret = __raw_notifier_call_chain(&cpu_chain, CPU_UP_PREPARE | mod, hcpu, |
307 | -1, &nr_calls); |
308 | if (ret == NOTIFY_BAD) { |
309 | nr_calls--; |
310 | printk("%s: attempt to bring up CPU %u failed\n", |
311 | __func__, cpu); |
312 | ret = -EINVAL; |
313 | goto out_notify; |
314 | } |
315 | |
316 | /* Arch-specific enabling code. */ |
317 | ret = __cpu_up(cpu); |
318 | if (ret != 0) |
319 | goto out_notify; |
320 | BUG_ON(!cpu_online(cpu)); |
321 | |
322 | set_cpu_active(cpu, true); |
323 | |
324 | /* Now call notifier in preparation. */ |
325 | raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu); |
326 | |
327 | out_notify: |
328 | if (ret != 0) |
329 | __raw_notifier_call_chain(&cpu_chain, |
330 | CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL); |
331 | cpu_hotplug_done(); |
332 | |
333 | return ret; |
334 | } |
335 | |
336 | int __cpuinit cpu_up(unsigned int cpu) |
337 | { |
338 | int err = 0; |
339 | if (!cpu_possible(cpu)) { |
340 | printk(KERN_ERR "can't online cpu %d because it is not " |
341 | "configured as may-hotadd at boot time\n", cpu); |
342 | #if defined(CONFIG_IA64) |
343 | printk(KERN_ERR "please check additional_cpus= boot " |
344 | "parameter\n"); |
345 | #endif |
346 | return -EINVAL; |
347 | } |
348 | |
349 | cpu_maps_update_begin(); |
350 | |
351 | if (cpu_hotplug_disabled) { |
352 | err = -EBUSY; |
353 | goto out; |
354 | } |
355 | |
356 | err = _cpu_up(cpu, 0); |
357 | |
358 | out: |
359 | cpu_maps_update_done(); |
360 | return err; |
361 | } |
362 | |
363 | #ifdef CONFIG_PM_SLEEP_SMP |
364 | static cpumask_var_t frozen_cpus; |
365 | |
366 | int disable_nonboot_cpus(void) |
367 | { |
368 | int cpu, first_cpu, error; |
369 | |
370 | error = stop_machine_create(); |
371 | if (error) |
372 | return error; |
373 | cpu_maps_update_begin(); |
374 | first_cpu = cpumask_first(cpu_online_mask); |
375 | /* |
376 | * We take down all of the non-boot CPUs in one shot to avoid races |
377 | * with the userspace trying to use the CPU hotplug at the same time |
378 | */ |
379 | cpumask_clear(frozen_cpus); |
380 | |
381 | printk("Disabling non-boot CPUs ...\n"); |
382 | for_each_online_cpu(cpu) { |
383 | if (cpu == first_cpu) |
384 | continue; |
385 | error = _cpu_down(cpu, 1); |
386 | if (!error) |
387 | cpumask_set_cpu(cpu, frozen_cpus); |
388 | else { |
389 | printk(KERN_ERR "Error taking CPU%d down: %d\n", |
390 | cpu, error); |
391 | break; |
392 | } |
393 | } |
394 | |
395 | if (!error) { |
396 | BUG_ON(num_online_cpus() > 1); |
397 | /* Make sure the CPUs won't be enabled by someone else */ |
398 | cpu_hotplug_disabled = 1; |
399 | } else { |
400 | printk(KERN_ERR "Non-boot CPUs are not disabled\n"); |
401 | } |
402 | cpu_maps_update_done(); |
403 | stop_machine_destroy(); |
404 | return error; |
405 | } |
406 | |
407 | void __weak arch_enable_nonboot_cpus_begin(void) |
408 | { |
409 | } |
410 | |
411 | void __weak arch_enable_nonboot_cpus_end(void) |
412 | { |
413 | } |
414 | |
415 | void __ref enable_nonboot_cpus(void) |
416 | { |
417 | int cpu, error; |
418 | |
419 | /* Allow everyone to use the CPU hotplug again */ |
420 | cpu_maps_update_begin(); |
421 | cpu_hotplug_disabled = 0; |
422 | if (cpumask_empty(frozen_cpus)) |
423 | goto out; |
424 | |
425 | printk("Enabling non-boot CPUs ...\n"); |
426 | |
427 | arch_enable_nonboot_cpus_begin(); |
428 | |
429 | for_each_cpu(cpu, frozen_cpus) { |
430 | error = _cpu_up(cpu, 1); |
431 | if (!error) { |
432 | printk("CPU%d is up\n", cpu); |
433 | continue; |
434 | } |
435 | printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error); |
436 | } |
437 | |
438 | arch_enable_nonboot_cpus_end(); |
439 | |
440 | cpumask_clear(frozen_cpus); |
441 | out: |
442 | cpu_maps_update_done(); |
443 | } |
444 | |
445 | static int alloc_frozen_cpus(void) |
446 | { |
447 | if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) |
448 | return -ENOMEM; |
449 | return 0; |
450 | } |
451 | core_initcall(alloc_frozen_cpus); |
452 | #endif /* CONFIG_PM_SLEEP_SMP */ |
453 | |
454 | /** |
455 | * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers |
456 | * @cpu: cpu that just started |
457 | * |
458 | * This function calls the cpu_chain notifiers with CPU_STARTING. |
459 | * It must be called by the arch code on the new cpu, before the new cpu |
460 | * enables interrupts and before the "boot" cpu returns from __cpu_up(). |
461 | */ |
462 | void __cpuinit notify_cpu_starting(unsigned int cpu) |
463 | { |
464 | unsigned long val = CPU_STARTING; |
465 | |
466 | #ifdef CONFIG_PM_SLEEP_SMP |
467 | if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus)) |
468 | val = CPU_STARTING_FROZEN; |
469 | #endif /* CONFIG_PM_SLEEP_SMP */ |
470 | raw_notifier_call_chain(&cpu_chain, val, (void *)(long)cpu); |
471 | } |
472 | |
473 | #endif /* CONFIG_SMP */ |
474 | |
475 | /* |
476 | * cpu_bit_bitmap[] is a special, "compressed" data structure that |
477 | * represents all NR_CPUS bits binary values of 1<<nr. |
478 | * |
479 | * It is used by cpumask_of() to get a constant address to a CPU |
480 | * mask value that has a single bit set only. |
481 | */ |
482 | |
483 | /* cpu_bit_bitmap[0] is empty - so we can back into it */ |
484 | #define MASK_DECLARE_1(x) [x+1][0] = 1UL << (x) |
485 | #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) |
486 | #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) |
487 | #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) |
488 | |
489 | const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { |
490 | |
491 | MASK_DECLARE_8(0), MASK_DECLARE_8(8), |
492 | MASK_DECLARE_8(16), MASK_DECLARE_8(24), |
493 | #if BITS_PER_LONG > 32 |
494 | MASK_DECLARE_8(32), MASK_DECLARE_8(40), |
495 | MASK_DECLARE_8(48), MASK_DECLARE_8(56), |
496 | #endif |
497 | }; |
498 | EXPORT_SYMBOL_GPL(cpu_bit_bitmap); |
499 | |
500 | const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; |
501 | EXPORT_SYMBOL(cpu_all_bits); |
502 | |
503 | #ifdef CONFIG_INIT_ALL_POSSIBLE |
504 | static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly |
505 | = CPU_BITS_ALL; |
506 | #else |
507 | static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly; |
508 | #endif |
509 | const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits); |
510 | EXPORT_SYMBOL(cpu_possible_mask); |
511 | |
512 | static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly; |
513 | const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits); |
514 | EXPORT_SYMBOL(cpu_online_mask); |
515 | |
516 | static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly; |
517 | const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits); |
518 | EXPORT_SYMBOL(cpu_present_mask); |
519 | |
520 | static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly; |
521 | const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits); |
522 | EXPORT_SYMBOL(cpu_active_mask); |
523 | |
524 | void set_cpu_possible(unsigned int cpu, bool possible) |
525 | { |
526 | if (possible) |
527 | cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits)); |
528 | else |
529 | cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits)); |
530 | } |
531 | |
532 | void set_cpu_present(unsigned int cpu, bool present) |
533 | { |
534 | if (present) |
535 | cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits)); |
536 | else |
537 | cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits)); |
538 | } |
539 | |
540 | void set_cpu_online(unsigned int cpu, bool online) |
541 | { |
542 | if (online) |
543 | cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits)); |
544 | else |
545 | cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits)); |
546 | } |
547 | |
548 | void set_cpu_active(unsigned int cpu, bool active) |
549 | { |
550 | if (active) |
551 | cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits)); |
552 | else |
553 | cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits)); |
554 | } |
555 | |
556 | void init_cpu_present(const struct cpumask *src) |
557 | { |
558 | cpumask_copy(to_cpumask(cpu_present_bits), src); |
559 | } |
560 | |
561 | void init_cpu_possible(const struct cpumask *src) |
562 | { |
563 | cpumask_copy(to_cpumask(cpu_possible_bits), src); |
564 | } |
565 | |
566 | void init_cpu_online(const struct cpumask *src) |
567 | { |
568 | cpumask_copy(to_cpumask(cpu_online_bits), src); |
569 | } |
570 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9