Root/
1 | /* |
2 | * intel_idle.c - native hardware idle loop for modern Intel processors |
3 | * |
4 | * Copyright (c) 2010, Intel Corporation. |
5 | * Len Brown <len.brown@intel.com> |
6 | * |
7 | * This program is free software; you can redistribute it and/or modify it |
8 | * under the terms and conditions of the GNU General Public License, |
9 | * version 2, as published by the Free Software Foundation. |
10 | * |
11 | * This program is distributed in the hope it will be useful, but WITHOUT |
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
14 | * more details. |
15 | * |
16 | * You should have received a copy of the GNU General Public License along with |
17 | * this program; if not, write to the Free Software Foundation, Inc., |
18 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
19 | */ |
20 | |
21 | /* |
22 | * intel_idle is a cpuidle driver that loads on specific Intel processors |
23 | * in lieu of the legacy ACPI processor_idle driver. The intent is to |
24 | * make Linux more efficient on these processors, as intel_idle knows |
25 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. |
26 | */ |
27 | |
28 | /* |
29 | * Design Assumptions |
30 | * |
31 | * All CPUs have same idle states as boot CPU |
32 | * |
33 | * Chipset BM_STS (bus master status) bit is a NOP |
34 | * for preventing entry into deep C-stats |
35 | */ |
36 | |
37 | /* |
38 | * Known limitations |
39 | * |
40 | * The driver currently initializes for_each_online_cpu() upon modprobe. |
41 | * It it unaware of subsequent processors hot-added to the system. |
42 | * This means that if you boot with maxcpus=n and later online |
43 | * processors above n, those processors will use C1 only. |
44 | * |
45 | * ACPI has a .suspend hack to turn off deep c-statees during suspend |
46 | * to avoid complications with the lapic timer workaround. |
47 | * Have not seen issues with suspend, but may need same workaround here. |
48 | * |
49 | * There is currently no kernel-based automatic probing/loading mechanism |
50 | * if the driver is built as a module. |
51 | */ |
52 | |
53 | /* un-comment DEBUG to enable pr_debug() statements */ |
54 | #define DEBUG |
55 | |
56 | #include <linux/kernel.h> |
57 | #include <linux/cpuidle.h> |
58 | #include <linux/clockchips.h> |
59 | #include <linux/hrtimer.h> /* ktime_get_real() */ |
60 | #include <trace/events/power.h> |
61 | #include <linux/sched.h> |
62 | #include <linux/notifier.h> |
63 | #include <linux/cpu.h> |
64 | #include <asm/mwait.h> |
65 | #include <asm/msr.h> |
66 | |
67 | #define INTEL_IDLE_VERSION "0.4" |
68 | #define PREFIX "intel_idle: " |
69 | |
70 | static struct cpuidle_driver intel_idle_driver = { |
71 | .name = "intel_idle", |
72 | .owner = THIS_MODULE, |
73 | }; |
74 | /* intel_idle.max_cstate=0 disables driver */ |
75 | static int max_cstate = MWAIT_MAX_NUM_CSTATES - 1; |
76 | |
77 | static unsigned int mwait_substates; |
78 | |
79 | #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF |
80 | /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ |
81 | static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ |
82 | |
83 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
84 | static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state); |
85 | |
86 | static struct cpuidle_state *cpuidle_state_table; |
87 | |
88 | /* |
89 | * Hardware C-state auto-demotion may not always be optimal. |
90 | * Indicate which enable bits to clear here. |
91 | */ |
92 | static unsigned long long auto_demotion_disable_flags; |
93 | |
94 | /* |
95 | * Set this flag for states where the HW flushes the TLB for us |
96 | * and so we don't need cross-calls to keep it consistent. |
97 | * If this flag is set, SW flushes the TLB, so even if the |
98 | * HW doesn't do the flushing, this flag is safe to use. |
99 | */ |
100 | #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 |
101 | |
102 | /* |
103 | * States are indexed by the cstate number, |
104 | * which is also the index into the MWAIT hint array. |
105 | * Thus C0 is a dummy. |
106 | */ |
107 | static struct cpuidle_state nehalem_cstates[MWAIT_MAX_NUM_CSTATES] = { |
108 | { /* MWAIT C0 */ }, |
109 | { /* MWAIT C1 */ |
110 | .name = "NHM-C1", |
111 | .desc = "MWAIT 0x00", |
112 | .driver_data = (void *) 0x00, |
113 | .flags = CPUIDLE_FLAG_TIME_VALID, |
114 | .exit_latency = 3, |
115 | .target_residency = 6, |
116 | .enter = &intel_idle }, |
117 | { /* MWAIT C2 */ |
118 | .name = "NHM-C3", |
119 | .desc = "MWAIT 0x10", |
120 | .driver_data = (void *) 0x10, |
121 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
122 | .exit_latency = 20, |
123 | .target_residency = 80, |
124 | .enter = &intel_idle }, |
125 | { /* MWAIT C3 */ |
126 | .name = "NHM-C6", |
127 | .desc = "MWAIT 0x20", |
128 | .driver_data = (void *) 0x20, |
129 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
130 | .exit_latency = 200, |
131 | .target_residency = 800, |
132 | .enter = &intel_idle }, |
133 | }; |
134 | |
135 | static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = { |
136 | { /* MWAIT C0 */ }, |
137 | { /* MWAIT C1 */ |
138 | .name = "SNB-C1", |
139 | .desc = "MWAIT 0x00", |
140 | .driver_data = (void *) 0x00, |
141 | .flags = CPUIDLE_FLAG_TIME_VALID, |
142 | .exit_latency = 1, |
143 | .target_residency = 1, |
144 | .enter = &intel_idle }, |
145 | { /* MWAIT C2 */ |
146 | .name = "SNB-C3", |
147 | .desc = "MWAIT 0x10", |
148 | .driver_data = (void *) 0x10, |
149 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
150 | .exit_latency = 80, |
151 | .target_residency = 211, |
152 | .enter = &intel_idle }, |
153 | { /* MWAIT C3 */ |
154 | .name = "SNB-C6", |
155 | .desc = "MWAIT 0x20", |
156 | .driver_data = (void *) 0x20, |
157 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
158 | .exit_latency = 104, |
159 | .target_residency = 345, |
160 | .enter = &intel_idle }, |
161 | { /* MWAIT C4 */ |
162 | .name = "SNB-C7", |
163 | .desc = "MWAIT 0x30", |
164 | .driver_data = (void *) 0x30, |
165 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
166 | .exit_latency = 109, |
167 | .target_residency = 345, |
168 | .enter = &intel_idle }, |
169 | }; |
170 | |
171 | static struct cpuidle_state atom_cstates[MWAIT_MAX_NUM_CSTATES] = { |
172 | { /* MWAIT C0 */ }, |
173 | { /* MWAIT C1 */ |
174 | .name = "ATM-C1", |
175 | .desc = "MWAIT 0x00", |
176 | .driver_data = (void *) 0x00, |
177 | .flags = CPUIDLE_FLAG_TIME_VALID, |
178 | .exit_latency = 1, |
179 | .target_residency = 4, |
180 | .enter = &intel_idle }, |
181 | { /* MWAIT C2 */ |
182 | .name = "ATM-C2", |
183 | .desc = "MWAIT 0x10", |
184 | .driver_data = (void *) 0x10, |
185 | .flags = CPUIDLE_FLAG_TIME_VALID, |
186 | .exit_latency = 20, |
187 | .target_residency = 80, |
188 | .enter = &intel_idle }, |
189 | { /* MWAIT C3 */ }, |
190 | { /* MWAIT C4 */ |
191 | .name = "ATM-C4", |
192 | .desc = "MWAIT 0x30", |
193 | .driver_data = (void *) 0x30, |
194 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
195 | .exit_latency = 100, |
196 | .target_residency = 400, |
197 | .enter = &intel_idle }, |
198 | { /* MWAIT C5 */ }, |
199 | { /* MWAIT C6 */ |
200 | .name = "ATM-C6", |
201 | .desc = "MWAIT 0x52", |
202 | .driver_data = (void *) 0x52, |
203 | .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
204 | .exit_latency = 140, |
205 | .target_residency = 560, |
206 | .enter = &intel_idle }, |
207 | }; |
208 | |
209 | /** |
210 | * intel_idle |
211 | * @dev: cpuidle_device |
212 | * @state: cpuidle state |
213 | * |
214 | */ |
215 | static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state) |
216 | { |
217 | unsigned long ecx = 1; /* break on interrupt flag */ |
218 | unsigned long eax = (unsigned long)cpuidle_get_statedata(state); |
219 | unsigned int cstate; |
220 | ktime_t kt_before, kt_after; |
221 | s64 usec_delta; |
222 | int cpu = smp_processor_id(); |
223 | |
224 | cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; |
225 | |
226 | local_irq_disable(); |
227 | |
228 | /* |
229 | * leave_mm() to avoid costly and often unnecessary wakeups |
230 | * for flushing the user TLB's associated with the active mm. |
231 | */ |
232 | if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
233 | leave_mm(cpu); |
234 | |
235 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
236 | clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu); |
237 | |
238 | kt_before = ktime_get_real(); |
239 | |
240 | stop_critical_timings(); |
241 | if (!need_resched()) { |
242 | |
243 | __monitor((void *)¤t_thread_info()->flags, 0, 0); |
244 | smp_mb(); |
245 | if (!need_resched()) |
246 | __mwait(eax, ecx); |
247 | } |
248 | |
249 | start_critical_timings(); |
250 | |
251 | kt_after = ktime_get_real(); |
252 | usec_delta = ktime_to_us(ktime_sub(kt_after, kt_before)); |
253 | |
254 | local_irq_enable(); |
255 | |
256 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
257 | clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu); |
258 | |
259 | return usec_delta; |
260 | } |
261 | |
262 | static void __setup_broadcast_timer(void *arg) |
263 | { |
264 | unsigned long reason = (unsigned long)arg; |
265 | int cpu = smp_processor_id(); |
266 | |
267 | reason = reason ? |
268 | CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF; |
269 | |
270 | clockevents_notify(reason, &cpu); |
271 | } |
272 | |
273 | static int setup_broadcast_cpuhp_notify(struct notifier_block *n, |
274 | unsigned long action, void *hcpu) |
275 | { |
276 | int hotcpu = (unsigned long)hcpu; |
277 | |
278 | switch (action & 0xf) { |
279 | case CPU_ONLINE: |
280 | smp_call_function_single(hotcpu, __setup_broadcast_timer, |
281 | (void *)true, 1); |
282 | break; |
283 | } |
284 | return NOTIFY_OK; |
285 | } |
286 | |
287 | static struct notifier_block setup_broadcast_notifier = { |
288 | .notifier_call = setup_broadcast_cpuhp_notify, |
289 | }; |
290 | |
291 | static void auto_demotion_disable(void *dummy) |
292 | { |
293 | unsigned long long msr_bits; |
294 | |
295 | rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); |
296 | msr_bits &= ~auto_demotion_disable_flags; |
297 | wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); |
298 | } |
299 | |
300 | /* |
301 | * intel_idle_probe() |
302 | */ |
303 | static int intel_idle_probe(void) |
304 | { |
305 | unsigned int eax, ebx, ecx; |
306 | |
307 | if (max_cstate == 0) { |
308 | pr_debug(PREFIX "disabled\n"); |
309 | return -EPERM; |
310 | } |
311 | |
312 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) |
313 | return -ENODEV; |
314 | |
315 | if (!boot_cpu_has(X86_FEATURE_MWAIT)) |
316 | return -ENODEV; |
317 | |
318 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) |
319 | return -ENODEV; |
320 | |
321 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); |
322 | |
323 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || |
324 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK)) |
325 | return -ENODEV; |
326 | |
327 | pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); |
328 | |
329 | |
330 | if (boot_cpu_data.x86 != 6) /* family 6 */ |
331 | return -ENODEV; |
332 | |
333 | switch (boot_cpu_data.x86_model) { |
334 | |
335 | case 0x1A: /* Core i7, Xeon 5500 series */ |
336 | case 0x1E: /* Core i7 and i5 Processor - Lynnfield Jasper Forest */ |
337 | case 0x1F: /* Core i7 and i5 Processor - Nehalem */ |
338 | case 0x2E: /* Nehalem-EX Xeon */ |
339 | case 0x2F: /* Westmere-EX Xeon */ |
340 | case 0x25: /* Westmere */ |
341 | case 0x2C: /* Westmere */ |
342 | cpuidle_state_table = nehalem_cstates; |
343 | auto_demotion_disable_flags = |
344 | (NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE); |
345 | break; |
346 | |
347 | case 0x1C: /* 28 - Atom Processor */ |
348 | cpuidle_state_table = atom_cstates; |
349 | break; |
350 | |
351 | case 0x26: /* 38 - Lincroft Atom Processor */ |
352 | cpuidle_state_table = atom_cstates; |
353 | auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE; |
354 | break; |
355 | |
356 | case 0x2A: /* SNB */ |
357 | case 0x2D: /* SNB Xeon */ |
358 | cpuidle_state_table = snb_cstates; |
359 | break; |
360 | |
361 | default: |
362 | pr_debug(PREFIX "does not run on family %d model %d\n", |
363 | boot_cpu_data.x86, boot_cpu_data.x86_model); |
364 | return -ENODEV; |
365 | } |
366 | |
367 | if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ |
368 | lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; |
369 | else { |
370 | smp_call_function(__setup_broadcast_timer, (void *)true, 1); |
371 | register_cpu_notifier(&setup_broadcast_notifier); |
372 | } |
373 | |
374 | pr_debug(PREFIX "v" INTEL_IDLE_VERSION |
375 | " model 0x%X\n", boot_cpu_data.x86_model); |
376 | |
377 | pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", |
378 | lapic_timer_reliable_states); |
379 | return 0; |
380 | } |
381 | |
382 | /* |
383 | * intel_idle_cpuidle_devices_uninit() |
384 | * unregister, free cpuidle_devices |
385 | */ |
386 | static void intel_idle_cpuidle_devices_uninit(void) |
387 | { |
388 | int i; |
389 | struct cpuidle_device *dev; |
390 | |
391 | for_each_online_cpu(i) { |
392 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); |
393 | cpuidle_unregister_device(dev); |
394 | } |
395 | |
396 | free_percpu(intel_idle_cpuidle_devices); |
397 | return; |
398 | } |
399 | /* |
400 | * intel_idle_cpuidle_devices_init() |
401 | * allocate, initialize, register cpuidle_devices |
402 | */ |
403 | static int intel_idle_cpuidle_devices_init(void) |
404 | { |
405 | int i, cstate; |
406 | struct cpuidle_device *dev; |
407 | |
408 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
409 | if (intel_idle_cpuidle_devices == NULL) |
410 | return -ENOMEM; |
411 | |
412 | for_each_online_cpu(i) { |
413 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); |
414 | |
415 | dev->state_count = 1; |
416 | |
417 | for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) { |
418 | int num_substates; |
419 | |
420 | if (cstate > max_cstate) { |
421 | printk(PREFIX "max_cstate %d reached\n", |
422 | max_cstate); |
423 | break; |
424 | } |
425 | |
426 | /* does the state exist in CPUID.MWAIT? */ |
427 | num_substates = (mwait_substates >> ((cstate) * 4)) |
428 | & MWAIT_SUBSTATE_MASK; |
429 | if (num_substates == 0) |
430 | continue; |
431 | /* is the state not enabled? */ |
432 | if (cpuidle_state_table[cstate].enter == NULL) { |
433 | /* does the driver not know about the state? */ |
434 | if (*cpuidle_state_table[cstate].name == '\0') |
435 | pr_debug(PREFIX "unaware of model 0x%x" |
436 | " MWAIT %d please" |
437 | " contact lenb@kernel.org", |
438 | boot_cpu_data.x86_model, cstate); |
439 | continue; |
440 | } |
441 | |
442 | if ((cstate > 2) && |
443 | !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) |
444 | mark_tsc_unstable("TSC halts in idle" |
445 | " states deeper than C2"); |
446 | |
447 | dev->states[dev->state_count] = /* structure copy */ |
448 | cpuidle_state_table[cstate]; |
449 | |
450 | dev->state_count += 1; |
451 | } |
452 | |
453 | dev->cpu = i; |
454 | if (cpuidle_register_device(dev)) { |
455 | pr_debug(PREFIX "cpuidle_register_device %d failed!\n", |
456 | i); |
457 | intel_idle_cpuidle_devices_uninit(); |
458 | return -EIO; |
459 | } |
460 | } |
461 | if (auto_demotion_disable_flags) |
462 | smp_call_function(auto_demotion_disable, NULL, 1); |
463 | |
464 | return 0; |
465 | } |
466 | |
467 | |
468 | static int __init intel_idle_init(void) |
469 | { |
470 | int retval; |
471 | |
472 | /* Do not load intel_idle at all for now if idle= is passed */ |
473 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) |
474 | return -ENODEV; |
475 | |
476 | retval = intel_idle_probe(); |
477 | if (retval) |
478 | return retval; |
479 | |
480 | retval = cpuidle_register_driver(&intel_idle_driver); |
481 | if (retval) { |
482 | printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", |
483 | cpuidle_get_driver()->name); |
484 | return retval; |
485 | } |
486 | |
487 | retval = intel_idle_cpuidle_devices_init(); |
488 | if (retval) { |
489 | cpuidle_unregister_driver(&intel_idle_driver); |
490 | return retval; |
491 | } |
492 | |
493 | return 0; |
494 | } |
495 | |
496 | static void __exit intel_idle_exit(void) |
497 | { |
498 | intel_idle_cpuidle_devices_uninit(); |
499 | cpuidle_unregister_driver(&intel_idle_driver); |
500 | |
501 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) { |
502 | smp_call_function(__setup_broadcast_timer, (void *)false, 1); |
503 | unregister_cpu_notifier(&setup_broadcast_notifier); |
504 | } |
505 | |
506 | return; |
507 | } |
508 | |
509 | module_init(intel_idle_init); |
510 | module_exit(intel_idle_exit); |
511 | |
512 | module_param(max_cstate, int, 0444); |
513 | |
514 | MODULE_AUTHOR("Len Brown <len.brown@intel.com>"); |
515 | MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION); |
516 | MODULE_LICENSE("GPL"); |
517 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9