Kernel

Kernel Git Source Tree

Root/mm/memory-failure.c

1	/*
2	* Copyright (C) 2008, 2009 Intel Corporation
3	* Authors: Andi Kleen, Fengguang Wu
4	*
5	* This software may be redistributed and/or modified under the terms of
6	* the GNU General Public License ("GPL") version 2 only as published by the
7	* Free Software Foundation.
8	*
9	* High level machine check handler. Handles pages reported by the
10	* hardware as being corrupted usually due to a multi-bit ECC memory or cache
11	* failure.
12	*
13	* In addition there is a "soft offline" entry point that allows stop using
14	* not-yet-corrupted-by-suspicious pages without killing anything.
15	*
16	* Handles page cache pages in various states. The tricky part
17	* here is that we can access any page asynchronously in respect to
18	* other VM users, because memory failures could happen anytime and
19	* anywhere. This could violate some of their assumptions. This is why
20	* this code has to be extremely careful. Generally it tries to use
21	* normal locking rules, as in get the standard locks, even if that means
22	* the error handling takes potentially a long time.
23	*
24	* There are several operations here with exponential complexity because
25	* of unsuitable VM data structures. For example the operation to map back
26	* from RMAP chains to processes has to walk the complete process list and
27	* has non linear complexity with the number. But since memory corruptions
28	* are rare we hope to get away with this. This avoids impacting the core
29	* VM.
30	*/
31
32	/*
33	* Notebook:
34	* - hugetlb needs more code
35	* - kcore/oldmem/vmcore/mem/kmem check for hwpoison pages
36	* - pass bad pages to kdump next kernel
37	*/
38	#include <linux/kernel.h>
39	#include <linux/mm.h>
40	#include <linux/page-flags.h>
41	#include <linux/kernel-page-flags.h>
42	#include <linux/sched.h>
43	#include <linux/ksm.h>
44	#include <linux/rmap.h>
45	#include <linux/export.h>
46	#include <linux/pagemap.h>
47	#include <linux/swap.h>
48	#include <linux/backing-dev.h>
49	#include <linux/migrate.h>
50	#include <linux/page-isolation.h>
51	#include <linux/suspend.h>
52	#include <linux/slab.h>
53	#include <linux/swapops.h>
54	#include <linux/hugetlb.h>
55	#include <linux/memory_hotplug.h>
56	#include <linux/mm_inline.h>
57	#include <linux/kfifo.h>
58	#include "internal.h"
59
60	int sysctl_memory_failure_early_kill __read_mostly = 0;
61
62	int sysctl_memory_failure_recovery __read_mostly = 1;
63
64	atomic_long_t mce_bad_pages __read_mostly = ATOMIC_LONG_INIT(0);
65
66	#if defined(CONFIG_HWPOISON_INJECT) \|\| defined(CONFIG_HWPOISON_INJECT_MODULE)
67
68	u32 hwpoison_filter_enable = 0;
69	u32 hwpoison_filter_dev_major = ~0U;
70	u32 hwpoison_filter_dev_minor = ~0U;
71	u64 hwpoison_filter_flags_mask;
72	u64 hwpoison_filter_flags_value;
73	EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
74	EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
75	EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
76	EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
77	EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
78
79	static int hwpoison_filter_dev(struct page *p)
80	{
81	struct address_space *mapping;
82	dev_t dev;
83
84	if (hwpoison_filter_dev_major == ~0U &&
85	hwpoison_filter_dev_minor == ~0U)
86	return 0;
87
88	/*
89	* page_mapping() does not accept slab pages.
90	*/
91	if (PageSlab(p))
92	return -EINVAL;
93
94	mapping = page_mapping(p);
95	if (mapping == NULL \|\| mapping->host == NULL)
96	return -EINVAL;
97
98	dev = mapping->host->i_sb->s_dev;
99	if (hwpoison_filter_dev_major != ~0U &&
100	hwpoison_filter_dev_major != MAJOR(dev))
101	return -EINVAL;
102	if (hwpoison_filter_dev_minor != ~0U &&
103	hwpoison_filter_dev_minor != MINOR(dev))
104	return -EINVAL;
105
106	return 0;
107	}
108
109	static int hwpoison_filter_flags(struct page *p)
110	{
111	if (!hwpoison_filter_flags_mask)
112	return 0;
113
114	if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
115	hwpoison_filter_flags_value)
116	return 0;
117	else
118	return -EINVAL;
119	}
120
121	/*
122	* This allows stress tests to limit test scope to a collection of tasks
123	* by putting them under some memcg. This prevents killing unrelated/important
124	* processes such as /sbin/init. Note that the target task may share clean
125	* pages with init (eg. libc text), which is harmless. If the target task
126	* share _dirty_ pages with another task B, the test scheme must make sure B
127	* is also included in the memcg. At last, due to race conditions this filter
128	* can only guarantee that the page either belongs to the memcg tasks, or is
129	* a freed page.
130	*/
131	#ifdef CONFIG_MEMCG_SWAP
132	u64 hwpoison_filter_memcg;
133	EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
134	static int hwpoison_filter_task(struct page *p)
135	{
136	struct mem_cgroup *mem;
137	struct cgroup_subsys_state *css;
138	unsigned long ino;
139
140	if (!hwpoison_filter_memcg)
141	return 0;
142
143	mem = try_get_mem_cgroup_from_page(p);
144	if (!mem)
145	return -EINVAL;
146
147	css = mem_cgroup_css(mem);
148	/* root_mem_cgroup has NULL dentries */
149	if (!css->cgroup->dentry)
150	return -EINVAL;
151
152	ino = css->cgroup->dentry->d_inode->i_ino;
153	css_put(css);
154
155	if (ino != hwpoison_filter_memcg)
156	return -EINVAL;
157
158	return 0;
159	}
160	#else
161	static int hwpoison_filter_task(struct page *p) { return 0; }
162	#endif
163
164	int hwpoison_filter(struct page *p)
165	{
166	if (!hwpoison_filter_enable)
167	return 0;
168
169	if (hwpoison_filter_dev(p))
170	return -EINVAL;
171
172	if (hwpoison_filter_flags(p))
173	return -EINVAL;
174
175	if (hwpoison_filter_task(p))
176	return -EINVAL;
177
178	return 0;
179	}
180	#else
181	int hwpoison_filter(struct page *p)
182	{
183	return 0;
184	}
185	#endif
186
187	EXPORT_SYMBOL_GPL(hwpoison_filter);
188
189	/*
190	* Send all the processes who have the page mapped a signal.
191	* ``action optional'' if they are not immediately affected by the error
192	* ``action required'' if error happened in current execution context
193	*/
194	static int kill_proc(struct task_struct *t, unsigned long addr, int trapno,
195	unsigned long pfn, struct page *page, int flags)
196	{
197	struct siginfo si;
198	int ret;
199
200	printk(KERN_ERR
201	"MCE %#lx: Killing %s:%d due to hardware memory corruption\n",
202	pfn, t->comm, t->pid);
203	si.si_signo = SIGBUS;
204	si.si_errno = 0;
205	si.si_addr = (void *)addr;
206	#ifdef __ARCH_SI_TRAPNO
207	si.si_trapno = trapno;
208	#endif
209	si.si_addr_lsb = compound_trans_order(compound_head(page)) + PAGE_SHIFT;
210
211	if ((flags & MF_ACTION_REQUIRED) && t == current) {
212	si.si_code = BUS_MCEERR_AR;
213	ret = force_sig_info(SIGBUS, &si, t);
214	} else {
215	/*
216	* Don't use force here, it's convenient if the signal
217	* can be temporarily blocked.
218	* This could cause a loop when the user sets SIGBUS
219	* to SIG_IGN, but hopefully no one will do that?
220	*/
221	si.si_code = BUS_MCEERR_AO;
222	ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
223	}
224	if (ret < 0)
225	printk(KERN_INFO "MCE: Error sending signal to %s:%d: %d\n",
226	t->comm, t->pid, ret);
227	return ret;
228	}
229
230	/*
231	* When a unknown page type is encountered drain as many buffers as possible
232	* in the hope to turn the page into a LRU or free page, which we can handle.
233	*/
234	void shake_page(struct page *p, int access)
235	{
236	if (!PageSlab(p)) {
237	lru_add_drain_all();
238	if (PageLRU(p))
239	return;
240	drain_all_pages();
241	if (PageLRU(p) \|\| is_free_buddy_page(p))
242	return;
243	}
244
245	/*
246	* Only call shrink_slab here (which would also shrink other caches) if
247	* access is not potentially fatal.
248	*/
249	if (access) {
250	int nr;
251	do {
252	struct shrink_control shrink = {
253	.gfp_mask = GFP_KERNEL,
254	};
255
256	nr = shrink_slab(&shrink, 1000, 1000);
257	if (page_count(p) == 1)
258	break;
259	} while (nr > 10);
260	}
261	}
262	EXPORT_SYMBOL_GPL(shake_page);
263
264	/*
265	* Kill all processes that have a poisoned page mapped and then isolate
266	* the page.
267	*
268	* General strategy:
269	* Find all processes having the page mapped and kill them.
270	* But we keep a page reference around so that the page is not
271	* actually freed yet.
272	* Then stash the page away
273	*
274	* There's no convenient way to get back to mapped processes
275	* from the VMAs. So do a brute-force search over all
276	* running processes.
277	*
278	* Remember that machine checks are not common (or rather
279	* if they are common you have other problems), so this shouldn't
280	* be a performance issue.
281	*
282	* Also there are some races possible while we get from the
283	* error detection to actually handle it.
284	*/
285
286	struct to_kill {
287	struct list_head nd;
288	struct task_struct *tsk;
289	unsigned long addr;
290	char addr_valid;
291	};
292
293	/*
294	* Failure handling: if we can't find or can't kill a process there's
295	* not much we can do. We just print a message and ignore otherwise.
296	*/
297
298	/*
299	* Schedule a process for later kill.
300	* Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
301	* TBD would GFP_NOIO be enough?
302	*/
303	static void add_to_kill(struct task_struct tsk, struct page p,
304	struct vm_area_struct *vma,
305	struct list_head *to_kill,
306	struct to_kill **tkc)
307	{
308	struct to_kill *tk;
309
310	if (*tkc) {
311	tk = *tkc;
312	*tkc = NULL;
313	} else {
314	tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
315	if (!tk) {
316	printk(KERN_ERR
317	"MCE: Out of memory while machine check handling\n");
318	return;
319	}
320	}
321	tk->addr = page_address_in_vma(p, vma);
322	tk->addr_valid = 1;
323
324	/*
325	* In theory we don't have to kill when the page was
326	* munmaped. But it could be also a mremap. Since that's
327	* likely very rare kill anyways just out of paranoia, but use
328	* a SIGKILL because the error is not contained anymore.
329	*/
330	if (tk->addr == -EFAULT) {
331	pr_info("MCE: Unable to find user space address %lx in %s\n",
332	page_to_pfn(p), tsk->comm);
333	tk->addr_valid = 0;
334	}
335	get_task_struct(tsk);
336	tk->tsk = tsk;
337	list_add_tail(&tk->nd, to_kill);
338	}
339
340	/*
341	* Kill the processes that have been collected earlier.
342	*
343	* Only do anything when DOIT is set, otherwise just free the list
344	* (this is used for clean pages which do not need killing)
345	* Also when FAIL is set do a force kill because something went
346	* wrong earlier.
347	*/
348	static void kill_procs(struct list_head *to_kill, int forcekill, int trapno,
349	int fail, struct page *page, unsigned long pfn,
350	int flags)
351	{
352	struct to_kill tk, next;
353
354	list_for_each_entry_safe (tk, next, to_kill, nd) {
355	if (forcekill) {
356	/*
357	* In case something went wrong with munmapping
358	* make sure the process doesn't catch the
359	* signal and then access the memory. Just kill it.
360	*/
361	if (fail \|\| tk->addr_valid == 0) {
362	printk(KERN_ERR
363	"MCE %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
364	pfn, tk->tsk->comm, tk->tsk->pid);
365	force_sig(SIGKILL, tk->tsk);
366	}
367
368	/*
369	* In theory the process could have mapped
370	* something else on the address in-between. We could
371	* check for that, but we need to tell the
372	* process anyways.
373	*/
374	else if (kill_proc(tk->tsk, tk->addr, trapno,
375	pfn, page, flags) < 0)
376	printk(KERN_ERR
377	"MCE %#lx: Cannot send advisory machine check signal to %s:%d\n",
378	pfn, tk->tsk->comm, tk->tsk->pid);
379	}
380	put_task_struct(tk->tsk);
381	kfree(tk);
382	}
383	}
384
385	static int task_early_kill(struct task_struct *tsk)
386	{
387	if (!tsk->mm)
388	return 0;
389	if (tsk->flags & PF_MCE_PROCESS)
390	return !!(tsk->flags & PF_MCE_EARLY);
391	return sysctl_memory_failure_early_kill;
392	}
393
394	/*
395	* Collect processes when the error hit an anonymous page.
396	*/
397	static void collect_procs_anon(struct page page, struct list_head to_kill,
398	struct to_kill **tkc)
399	{
400	struct vm_area_struct *vma;
401	struct task_struct *tsk;
402	struct anon_vma *av;
403
404	av = page_lock_anon_vma(page);
405	if (av == NULL) /* Not actually mapped anymore */
406	return;
407
408	read_lock(&tasklist_lock);
409	for_each_process (tsk) {
410	struct anon_vma_chain *vmac;
411
412	if (!task_early_kill(tsk))
413	continue;
414	list_for_each_entry(vmac, &av->head, same_anon_vma) {
415	vma = vmac->vma;
416	if (!page_mapped_in_vma(page, vma))
417	continue;
418	if (vma->vm_mm == tsk->mm)
419	add_to_kill(tsk, page, vma, to_kill, tkc);
420	}
421	}
422	read_unlock(&tasklist_lock);
423	page_unlock_anon_vma(av);
424	}
425
426	/*
427	* Collect processes when the error hit a file mapped page.
428	*/
429	static void collect_procs_file(struct page page, struct list_head to_kill,
430	struct to_kill **tkc)
431	{
432	struct vm_area_struct *vma;
433	struct task_struct *tsk;
434	struct prio_tree_iter iter;
435	struct address_space *mapping = page->mapping;
436
437	mutex_lock(&mapping->i_mmap_mutex);
438	read_lock(&tasklist_lock);
439	for_each_process(tsk) {
440	pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
441
442	if (!task_early_kill(tsk))
443	continue;
444
445	vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff,
446	pgoff) {
447	/*
448	* Send early kill signal to tasks where a vma covers
449	* the page but the corrupted page is not necessarily
450	* mapped it in its pte.
451	* Assume applications who requested early kill want
452	* to be informed of all such data corruptions.
453	*/
454	if (vma->vm_mm == tsk->mm)
455	add_to_kill(tsk, page, vma, to_kill, tkc);
456	}
457	}
458	read_unlock(&tasklist_lock);
459	mutex_unlock(&mapping->i_mmap_mutex);
460	}
461
462	/*
463	* Collect the processes who have the corrupted page mapped to kill.
464	* This is done in two steps for locking reasons.
465	* First preallocate one tokill structure outside the spin locks,
466	* so that we can kill at least one process reasonably reliable.
467	*/
468	static void collect_procs(struct page page, struct list_head tokill)
469	{
470	struct to_kill *tk;
471
472	if (!page->mapping)
473	return;
474
475	tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
476	if (!tk)
477	return;
478	if (PageAnon(page))
479	collect_procs_anon(page, tokill, &tk);
480	else
481	collect_procs_file(page, tokill, &tk);
482	kfree(tk);
483	}
484
485	/*
486	* Error handlers for various types of pages.
487	*/
488
489	enum outcome {
490	IGNORED, /* Error: cannot be handled */
491	FAILED, /* Error: handling failed */
492	DELAYED, /* Will be handled later */
493	RECOVERED, /* Successfully recovered */
494	};
495
496	static const char *action_name[] = {
497	[IGNORED] = "Ignored",
498	[FAILED] = "Failed",
499	[DELAYED] = "Delayed",
500	[RECOVERED] = "Recovered",
501	};
502
503	/*
504	* XXX: It is possible that a page is isolated from LRU cache,
505	* and then kept in swap cache or failed to remove from page cache.
506	* The page count will stop it from being freed by unpoison.
507	* Stress tests should be aware of this memory leak problem.
508	*/
509	static int delete_from_lru_cache(struct page *p)
510	{
511	if (!isolate_lru_page(p)) {
512	/*
513	* Clear sensible page flags, so that the buddy system won't
514	* complain when the page is unpoison-and-freed.
515	*/
516	ClearPageActive(p);
517	ClearPageUnevictable(p);
518	/*
519	* drop the page count elevated by isolate_lru_page()
520	*/
521	page_cache_release(p);
522	return 0;
523	}
524	return -EIO;
525	}
526
527	/*
528	* Error hit kernel page.
529	* Do nothing, try to be lucky and not touch this instead. For a few cases we
530	* could be more sophisticated.
531	*/
532	static int me_kernel(struct page *p, unsigned long pfn)
533	{
534	return IGNORED;
535	}
536
537	/*
538	* Page in unknown state. Do nothing.
539	*/
540	static int me_unknown(struct page *p, unsigned long pfn)
541	{
542	printk(KERN_ERR "MCE %#lx: Unknown page state\n", pfn);
543	return FAILED;
544	}
545
546	/*
547	* Clean (or cleaned) page cache page.
548	*/
549	static int me_pagecache_clean(struct page *p, unsigned long pfn)
550	{
551	int err;
552	int ret = FAILED;
553	struct address_space *mapping;
554
555	delete_from_lru_cache(p);
556
557	/*
558	* For anonymous pages we're done the only reference left
559	* should be the one m_f() holds.
560	*/
561	if (PageAnon(p))
562	return RECOVERED;
563
564	/*
565	* Now truncate the page in the page cache. This is really
566	* more like a "temporary hole punch"
567	* Don't do this for block devices when someone else
568	* has a reference, because it could be file system metadata
569	* and that's not safe to truncate.
570	*/
571	mapping = page_mapping(p);
572	if (!mapping) {
573	/*
574	* Page has been teared down in the meanwhile
575	*/
576	return FAILED;
577	}
578
579	/*
580	* Truncation is a bit tricky. Enable it per file system for now.
581	*
582	* Open: to take i_mutex or not for this? Right now we don't.
583	*/
584	if (mapping->a_ops->error_remove_page) {
585	err = mapping->a_ops->error_remove_page(mapping, p);
586	if (err != 0) {
587	printk(KERN_INFO "MCE %#lx: Failed to punch page: %d\n",
588	pfn, err);
589	} else if (page_has_private(p) &&
590	!try_to_release_page(p, GFP_NOIO)) {
591	pr_info("MCE %#lx: failed to release buffers\n", pfn);
592	} else {
593	ret = RECOVERED;
594	}
595	} else {
596	/*
597	* If the file system doesn't support it just invalidate
598	* This fails on dirty or anything with private pages
599	*/
600	if (invalidate_inode_page(p))
601	ret = RECOVERED;
602	else
603	printk(KERN_INFO "MCE %#lx: Failed to invalidate\n",
604	pfn);
605	}
606	return ret;
607	}
608
609	/*
610	* Dirty cache page page
611	* Issues: when the error hit a hole page the error is not properly
612	* propagated.
613	*/
614	static int me_pagecache_dirty(struct page *p, unsigned long pfn)
615	{
616	struct address_space *mapping = page_mapping(p);
617
618	SetPageError(p);
619	/* TBD: print more information about the file. */
620	if (mapping) {
621	/*
622	* IO error will be reported by write(), fsync(), etc.
623	* who check the mapping.
624	* This way the application knows that something went
625	* wrong with its dirty file data.
626	*
627	* There's one open issue:
628	*
629	* The EIO will be only reported on the next IO
630	* operation and then cleared through the IO map.
631	* Normally Linux has two mechanisms to pass IO error
632	* first through the AS_EIO flag in the address space
633	* and then through the PageError flag in the page.
634	* Since we drop pages on memory failure handling the
635	* only mechanism open to use is through AS_AIO.
636	*
637	* This has the disadvantage that it gets cleared on
638	* the first operation that returns an error, while
639	* the PageError bit is more sticky and only cleared
640	* when the page is reread or dropped. If an
641	* application assumes it will always get error on
642	* fsync, but does other operations on the fd before
643	* and the page is dropped between then the error
644	* will not be properly reported.
645	*
646	* This can already happen even without hwpoisoned
647	* pages: first on metadata IO errors (which only
648	* report through AS_EIO) or when the page is dropped
649	* at the wrong time.
650	*
651	* So right now we assume that the application DTRT on
652	* the first EIO, but we're not worse than other parts
653	* of the kernel.
654	*/
655	mapping_set_error(mapping, EIO);
656	}
657
658	return me_pagecache_clean(p, pfn);
659	}
660
661	/*
662	* Clean and dirty swap cache.
663	*
664	* Dirty swap cache page is tricky to handle. The page could live both in page
665	* cache and swap cache(ie. page is freshly swapped in). So it could be
666	* referenced concurrently by 2 types of PTEs:
667	* normal PTEs and swap PTEs. We try to handle them consistently by calling
668	* try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
669	* and then
670	* - clear dirty bit to prevent IO
671	* - remove from LRU
672	* - but keep in the swap cache, so that when we return to it on
673	* a later page fault, we know the application is accessing
674	* corrupted data and shall be killed (we installed simple
675	* interception code in do_swap_page to catch it).
676	*
677	* Clean swap cache pages can be directly isolated. A later page fault will
678	* bring in the known good data from disk.
679	*/
680	static int me_swapcache_dirty(struct page *p, unsigned long pfn)
681	{
682	ClearPageDirty(p);
683	/* Trigger EIO in shmem: */
684	ClearPageUptodate(p);
685
686	if (!delete_from_lru_cache(p))
687	return DELAYED;
688	else
689	return FAILED;
690	}
691
692	static int me_swapcache_clean(struct page *p, unsigned long pfn)
693	{
694	delete_from_swap_cache(p);
695
696	if (!delete_from_lru_cache(p))
697	return RECOVERED;
698	else
699	return FAILED;
700	}
701
702	/*
703	* Huge pages. Needs work.
704	* Issues:
705	* - Error on hugepage is contained in hugepage unit (not in raw page unit.)
706	* To narrow down kill region to one page, we need to break up pmd.
707	*/
708	static int me_huge_page(struct page *p, unsigned long pfn)
709	{
710	int res = 0;
711	struct page *hpage = compound_head(p);
712	/*
713	* We can safely recover from error on free or reserved (i.e.
714	* not in-use) hugepage by dequeuing it from freelist.
715	* To check whether a hugepage is in-use or not, we can't use
716	* page->lru because it can be used in other hugepage operations,
717	* such as __unmap_hugepage_range() and gather_surplus_pages().
718	* So instead we use page_mapping() and PageAnon().
719	* We assume that this function is called with page lock held,
720	* so there is no race between isolation and mapping/unmapping.
721	*/
722	if (!(page_mapping(hpage) \|\| PageAnon(hpage))) {
723	res = dequeue_hwpoisoned_huge_page(hpage);
724	if (!res)
725	return RECOVERED;
726	}
727	return DELAYED;
728	}
729
730	/*
731	* Various page states we can handle.
732	*
733	* A page state is defined by its current page->flags bits.
734	* The table matches them in order and calls the right handler.
735	*
736	* This is quite tricky because we can access page at any time
737	* in its live cycle, so all accesses have to be extremely careful.
738	*
739	* This is not complete. More states could be added.
740	* For any missing state don't attempt recovery.
741	*/
742
743	#define dirty (1UL << PG_dirty)
744	#define sc (1UL << PG_swapcache)
745	#define unevict (1UL << PG_unevictable)
746	#define mlock (1UL << PG_mlocked)
747	#define writeback (1UL << PG_writeback)
748	#define lru (1UL << PG_lru)
749	#define swapbacked (1UL << PG_swapbacked)
750	#define head (1UL << PG_head)
751	#define tail (1UL << PG_tail)
752	#define compound (1UL << PG_compound)
753	#define slab (1UL << PG_slab)
754	#define reserved (1UL << PG_reserved)
755
756	static struct page_state {
757	unsigned long mask;
758	unsigned long res;
759	char *msg;
760	int (action)(struct page p, unsigned long pfn);
761	} error_states[] = {
762	{ reserved, reserved, "reserved kernel", me_kernel },
763	/*
764	* free pages are specially detected outside this table:
765	* PG_buddy pages only make a small fraction of all free pages.
766	*/
767
768	/*
769	* Could in theory check if slab page is free or if we can drop
770	* currently unused objects without touching them. But just
771	* treat it as standard kernel for now.
772	*/
773	{ slab, slab, "kernel slab", me_kernel },
774
775	#ifdef CONFIG_PAGEFLAGS_EXTENDED
776	{ head, head, "huge", me_huge_page },
777	{ tail, tail, "huge", me_huge_page },
778	#else
779	{ compound, compound, "huge", me_huge_page },
780	#endif
781
782	{ sc\|dirty, sc\|dirty, "swapcache", me_swapcache_dirty },
783	{ sc\|dirty, sc, "swapcache", me_swapcache_clean },
784
785	{ unevict\|dirty, unevict\|dirty, "unevictable LRU", me_pagecache_dirty},
786	{ unevict, unevict, "unevictable LRU", me_pagecache_clean},
787
788	{ mlock\|dirty, mlock\|dirty, "mlocked LRU", me_pagecache_dirty },
789	{ mlock, mlock, "mlocked LRU", me_pagecache_clean },
790
791	{ lru\|dirty, lru\|dirty, "LRU", me_pagecache_dirty },
792	{ lru\|dirty, lru, "clean LRU", me_pagecache_clean },
793
794	/*
795	* Catchall entry: must be at end.
796	*/
797	{ 0, 0, "unknown page state", me_unknown },
798	};
799
800	#undef dirty
801	#undef sc
802	#undef unevict
803	#undef mlock
804	#undef writeback
805	#undef lru
806	#undef swapbacked
807	#undef head
808	#undef tail
809	#undef compound
810	#undef slab
811	#undef reserved
812
813	static void action_result(unsigned long pfn, char *msg, int result)
814	{
815	struct page *page = pfn_to_page(pfn);
816
817	printk(KERN_ERR "MCE %#lx: %s%s page recovery: %s\n",
818	pfn,
819	PageDirty(page) ? "dirty " : "",
820	msg, action_name[result]);
821	}
822
823	static int page_action(struct page_state ps, struct page p,
824	unsigned long pfn)
825	{
826	int result;
827	int count;
828
829	result = ps->action(p, pfn);
830	action_result(pfn, ps->msg, result);
831
832	count = page_count(p) - 1;
833	if (ps->action == me_swapcache_dirty && result == DELAYED)
834	count--;
835	if (count != 0) {
836	printk(KERN_ERR
837	"MCE %#lx: %s page still referenced by %d users\n",
838	pfn, ps->msg, count);
839	result = FAILED;
840	}
841
842	/* Could do more checks here if page looks ok */
843	/*
844	* Could adjust zone counters here to correct for the missing page.
845	*/
846
847	return (result == RECOVERED \|\| result == DELAYED) ? 0 : -EBUSY;
848	}
849
850	/*
851	* Do all that is necessary to remove user space mappings. Unmap
852	* the pages and send SIGBUS to the processes if the data was dirty.
853	*/
854	static int hwpoison_user_mappings(struct page *p, unsigned long pfn,
855	int trapno, int flags)
856	{
857	enum ttu_flags ttu = TTU_UNMAP \| TTU_IGNORE_MLOCK \| TTU_IGNORE_ACCESS;
858	struct address_space *mapping;
859	LIST_HEAD(tokill);
860	int ret;
861	int kill = 1, forcekill;
862	struct page *hpage = compound_head(p);
863	struct page *ppage;
864
865	if (PageReserved(p) \|\| PageSlab(p))
866	return SWAP_SUCCESS;
867
868	/*
869	* This check implies we don't kill processes if their pages
870	* are in the swap cache early. Those are always late kills.
871	*/
872	if (!page_mapped(hpage))
873	return SWAP_SUCCESS;
874
875	if (PageKsm(p))
876	return SWAP_FAIL;
877
878	if (PageSwapCache(p)) {
879	printk(KERN_ERR
880	"MCE %#lx: keeping poisoned page in swap cache\n", pfn);
881	ttu \|= TTU_IGNORE_HWPOISON;
882	}
883
884	/*
885	* Propagate the dirty bit from PTEs to struct page first, because we
886	* need this to decide if we should kill or just drop the page.
887	* XXX: the dirty test could be racy: set_page_dirty() may not always
888	* be called inside page lock (it's recommended but not enforced).
889	*/
890	mapping = page_mapping(hpage);
891	if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
892	mapping_cap_writeback_dirty(mapping)) {
893	if (page_mkclean(hpage)) {
894	SetPageDirty(hpage);
895	} else {
896	kill = 0;
897	ttu \|= TTU_IGNORE_HWPOISON;
898	printk(KERN_INFO
899	"MCE %#lx: corrupted page was clean: dropped without side effects\n",
900	pfn);
901	}
902	}
903
904	/*
905	* ppage: poisoned page
906	* if p is regular page(4k page)
907	* ppage == real poisoned page;
908	* else p is hugetlb or THP, ppage == head page.
909	*/
910	ppage = hpage;
911
912	if (PageTransHuge(hpage)) {
913	/*
914	* Verify that this isn't a hugetlbfs head page, the check for
915	* PageAnon is just for avoid tripping a split_huge_page
916	* internal debug check, as split_huge_page refuses to deal with
917	* anything that isn't an anon page. PageAnon can't go away fro
918	* under us because we hold a refcount on the hpage, without a
919	* refcount on the hpage. split_huge_page can't be safely called
920	* in the first place, having a refcount on the tail isn't
921	* enough * to be safe.
922	*/
923	if (!PageHuge(hpage) && PageAnon(hpage)) {
924	if (unlikely(split_huge_page(hpage))) {
925	/*
926	* FIXME: if splitting THP is failed, it is
927	* better to stop the following operation rather
928	* than causing panic by unmapping. System might
929	* survive if the page is freed later.
930	*/
931	printk(KERN_INFO
932	"MCE %#lx: failed to split THP\n", pfn);
933
934	BUG_ON(!PageHWPoison(p));
935	return SWAP_FAIL;
936	}
937	/* THP is split, so ppage should be the real poisoned page. */
938	ppage = p;
939	}
940	}
941
942	/*
943	* First collect all the processes that have the page
944	* mapped in dirty form. This has to be done before try_to_unmap,
945	* because ttu takes the rmap data structures down.
946	*
947	* Error handling: We ignore errors here because
948	* there's nothing that can be done.
949	*/
950	if (kill)
951	collect_procs(ppage, &tokill);
952
953	if (hpage != ppage)
954	lock_page(ppage);
955
956	ret = try_to_unmap(ppage, ttu);
957	if (ret != SWAP_SUCCESS)
958	printk(KERN_ERR "MCE %#lx: failed to unmap page (mapcount=%d)\n",
959	pfn, page_mapcount(ppage));
960
961	if (hpage != ppage)
962	unlock_page(ppage);
963
964	/*
965	* Now that the dirty bit has been propagated to the
966	* struct page and all unmaps done we can decide if
967	* killing is needed or not. Only kill when the page
968	* was dirty or the process is not restartable,
969	* otherwise the tokill list is merely
970	* freed. When there was a problem unmapping earlier
971	* use a more force-full uncatchable kill to prevent
972	* any accesses to the poisoned memory.
973	*/
974	forcekill = PageDirty(ppage) \|\| (flags & MF_MUST_KILL);
975	kill_procs(&tokill, forcekill, trapno,
976	ret != SWAP_SUCCESS, p, pfn, flags);
977
978	return ret;
979	}
980
981	static void set_page_hwpoison_huge_page(struct page *hpage)
982	{
983	int i;
984	int nr_pages = 1 << compound_trans_order(hpage);
985	for (i = 0; i < nr_pages; i++)
986	SetPageHWPoison(hpage + i);
987	}
988
989	static void clear_page_hwpoison_huge_page(struct page *hpage)
990	{
991	int i;
992	int nr_pages = 1 << compound_trans_order(hpage);
993	for (i = 0; i < nr_pages; i++)
994	ClearPageHWPoison(hpage + i);
995	}
996
997	/**
998	* memory_failure - Handle memory failure of a page.
999	* @pfn: Page Number of the corrupted page
1000	* @trapno: Trap number reported in the signal to user space.
1001	* @flags: fine tune action taken
1002	*
1003	* This function is called by the low level machine check code
1004	* of an architecture when it detects hardware memory corruption
1005	* of a page. It tries its best to recover, which includes
1006	* dropping pages, killing processes etc.
1007	*
1008	* The function is primarily of use for corruptions that
1009	* happen outside the current execution context (e.g. when
1010	* detected by a background scrubber)
1011	*
1012	* Must run in process context (e.g. a work queue) with interrupts
1013	* enabled and no spinlocks hold.
1014	*/
1015	int memory_failure(unsigned long pfn, int trapno, int flags)
1016	{
1017	struct page_state *ps;
1018	struct page *p;
1019	struct page *hpage;
1020	int res;
1021	unsigned int nr_pages;
1022
1023	if (!sysctl_memory_failure_recovery)
1024	panic("Memory failure from trap %d on page %lx", trapno, pfn);
1025
1026	if (!pfn_valid(pfn)) {
1027	printk(KERN_ERR
1028	"MCE %#lx: memory outside kernel control\n",
1029	pfn);
1030	return -ENXIO;
1031	}
1032
1033	p = pfn_to_page(pfn);
1034	hpage = compound_head(p);
1035	if (TestSetPageHWPoison(p)) {
1036	printk(KERN_ERR "MCE %#lx: already hardware poisoned\n", pfn);
1037	return 0;
1038	}
1039
1040	nr_pages = 1 << compound_trans_order(hpage);
1041	atomic_long_add(nr_pages, &mce_bad_pages);
1042
1043	/*
1044	* We need/can do nothing about count=0 pages.
1045	* 1) it's a free page, and therefore in safe hand:
1046	* prep_new_page() will be the gate keeper.
1047	* 2) it's a free hugepage, which is also safe:
1048	* an affected hugepage will be dequeued from hugepage freelist,
1049	* so there's no concern about reusing it ever after.
1050	* 3) it's part of a non-compound high order page.
1051	* Implies some kernel user: cannot stop them from
1052	* R/W the page; let's pray that the page has been
1053	* used and will be freed some time later.
1054	* In fact it's dangerous to directly bump up page count from 0,
1055	* that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
1056	*/
1057	if (!(flags & MF_COUNT_INCREASED) &&
1058	!get_page_unless_zero(hpage)) {
1059	if (is_free_buddy_page(p)) {
1060	action_result(pfn, "free buddy", DELAYED);
1061	return 0;
1062	} else if (PageHuge(hpage)) {
1063	/*
1064	* Check "just unpoisoned", "filter hit", and
1065	* "race with other subpage."
1066	*/
1067	lock_page(hpage);
1068	if (!PageHWPoison(hpage)
1069	\|\| (hwpoison_filter(p) && TestClearPageHWPoison(p))
1070	\|\| (p != hpage && TestSetPageHWPoison(hpage))) {
1071	atomic_long_sub(nr_pages, &mce_bad_pages);
1072	return 0;
1073	}
1074	set_page_hwpoison_huge_page(hpage);
1075	res = dequeue_hwpoisoned_huge_page(hpage);
1076	action_result(pfn, "free huge",
1077	res ? IGNORED : DELAYED);
1078	unlock_page(hpage);
1079	return res;
1080	} else {
1081	action_result(pfn, "high order kernel", IGNORED);
1082	return -EBUSY;
1083	}
1084	}
1085
1086	/*
1087	* We ignore non-LRU pages for good reasons.
1088	* - PG_locked is only well defined for LRU pages and a few others
1089	* - to avoid races with __set_page_locked()
1090	* - to avoid races with __SetPageSlab*() (and more non-atomic ops)
1091	* The check (unnecessarily) ignores LRU pages being isolated and
1092	* walked by the page reclaim code, however that's not a big loss.
1093	*/
1094	if (!PageHuge(p) && !PageTransTail(p)) {
1095	if (!PageLRU(p))
1096	shake_page(p, 0);
1097	if (!PageLRU(p)) {
1098	/*
1099	* shake_page could have turned it free.
1100	*/
1101	if (is_free_buddy_page(p)) {
1102	action_result(pfn, "free buddy, 2nd try",
1103	DELAYED);
1104	return 0;
1105	}
1106	action_result(pfn, "non LRU", IGNORED);
1107	put_page(p);
1108	return -EBUSY;
1109	}
1110	}
1111
1112	/*
1113	* Lock the page and wait for writeback to finish.
1114	* It's very difficult to mess with pages currently under IO
1115	* and in many cases impossible, so we just avoid it here.
1116	*/
1117	lock_page(hpage);
1118
1119	/*
1120	* unpoison always clear PG_hwpoison inside page lock
1121	*/
1122	if (!PageHWPoison(p)) {
1123	printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn);
1124	res = 0;
1125	goto out;
1126	}
1127	if (hwpoison_filter(p)) {
1128	if (TestClearPageHWPoison(p))
1129	atomic_long_sub(nr_pages, &mce_bad_pages);
1130	unlock_page(hpage);
1131	put_page(hpage);
1132	return 0;
1133	}
1134
1135	/*
1136	* For error on the tail page, we should set PG_hwpoison
1137	* on the head page to show that the hugepage is hwpoisoned
1138	*/
1139	if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
1140	action_result(pfn, "hugepage already hardware poisoned",
1141	IGNORED);
1142	unlock_page(hpage);
1143	put_page(hpage);
1144	return 0;
1145	}
1146	/*
1147	* Set PG_hwpoison on all pages in an error hugepage,
1148	* because containment is done in hugepage unit for now.
1149	* Since we have done TestSetPageHWPoison() for the head page with
1150	* page lock held, we can safely set PG_hwpoison bits on tail pages.
1151	*/
1152	if (PageHuge(p))
1153	set_page_hwpoison_huge_page(hpage);
1154
1155	wait_on_page_writeback(p);
1156
1157	/*
1158	* Now take care of user space mappings.
1159	* Abort on fail: __delete_from_page_cache() assumes unmapped page.
1160	*/
1161	if (hwpoison_user_mappings(p, pfn, trapno, flags) != SWAP_SUCCESS) {
1162	printk(KERN_ERR "MCE %#lx: cannot unmap page, give up\n", pfn);
1163	res = -EBUSY;
1164	goto out;
1165	}
1166
1167	/*
1168	* Torn down by someone else?
1169	*/
1170	if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
1171	action_result(pfn, "already truncated LRU", IGNORED);
1172	res = -EBUSY;
1173	goto out;
1174	}
1175
1176	res = -EBUSY;
1177	for (ps = error_states;; ps++) {
1178	if ((p->flags & ps->mask) == ps->res) {
1179	res = page_action(ps, p, pfn);
1180	break;
1181	}
1182	}
1183	out:
1184	unlock_page(hpage);
1185	return res;
1186	}
1187	EXPORT_SYMBOL_GPL(memory_failure);
1188
1189	#define MEMORY_FAILURE_FIFO_ORDER 4
1190	#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER)
1191
1192	struct memory_failure_entry {
1193	unsigned long pfn;
1194	int trapno;
1195	int flags;
1196	};
1197
1198	struct memory_failure_cpu {
1199	DECLARE_KFIFO(fifo, struct memory_failure_entry,
1200	MEMORY_FAILURE_FIFO_SIZE);
1201	spinlock_t lock;
1202	struct work_struct work;
1203	};
1204
1205	static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
1206
1207	/**
1208	* memory_failure_queue - Schedule handling memory failure of a page.
1209	* @pfn: Page Number of the corrupted page
1210	* @trapno: Trap number reported in the signal to user space.
1211	* @flags: Flags for memory failure handling
1212	*
1213	* This function is called by the low level hardware error handler
1214	* when it detects hardware memory corruption of a page. It schedules
1215	* the recovering of error page, including dropping pages, killing
1216	* processes etc.
1217	*
1218	* The function is primarily of use for corruptions that
1219	* happen outside the current execution context (e.g. when
1220	* detected by a background scrubber)
1221	*
1222	* Can run in IRQ context.
1223	*/
1224	void memory_failure_queue(unsigned long pfn, int trapno, int flags)
1225	{
1226	struct memory_failure_cpu *mf_cpu;
1227	unsigned long proc_flags;
1228	struct memory_failure_entry entry = {
1229	.pfn = pfn,
1230	.trapno = trapno,
1231	.flags = flags,
1232	};
1233
1234	mf_cpu = &get_cpu_var(memory_failure_cpu);
1235	spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1236	if (kfifo_put(&mf_cpu->fifo, &entry))
1237	schedule_work_on(smp_processor_id(), &mf_cpu->work);
1238	else
1239	pr_err("Memory failure: buffer overflow when queuing memory failure at 0x%#lx\n",
1240	pfn);
1241	spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1242	put_cpu_var(memory_failure_cpu);
1243	}
1244	EXPORT_SYMBOL_GPL(memory_failure_queue);
1245
1246	static void memory_failure_work_func(struct work_struct *work)
1247	{
1248	struct memory_failure_cpu *mf_cpu;
1249	struct memory_failure_entry entry = { 0, };
1250	unsigned long proc_flags;
1251	int gotten;
1252
1253	mf_cpu = &__get_cpu_var(memory_failure_cpu);
1254	for (;;) {
1255	spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1256	gotten = kfifo_get(&mf_cpu->fifo, &entry);
1257	spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1258	if (!gotten)
1259	break;
1260	memory_failure(entry.pfn, entry.trapno, entry.flags);
1261	}
1262	}
1263
1264	static int __init memory_failure_init(void)
1265	{
1266	struct memory_failure_cpu *mf_cpu;
1267	int cpu;
1268
1269	for_each_possible_cpu(cpu) {
1270	mf_cpu = &per_cpu(memory_failure_cpu, cpu);
1271	spin_lock_init(&mf_cpu->lock);
1272	INIT_KFIFO(mf_cpu->fifo);
1273	INIT_WORK(&mf_cpu->work, memory_failure_work_func);
1274	}
1275
1276	return 0;
1277	}
1278	core_initcall(memory_failure_init);
1279
1280	/**
1281	* unpoison_memory - Unpoison a previously poisoned page
1282	* @pfn: Page number of the to be unpoisoned page
1283	*
1284	* Software-unpoison a page that has been poisoned by
1285	* memory_failure() earlier.
1286	*
1287	* This is only done on the software-level, so it only works
1288	* for linux injected failures, not real hardware failures
1289	*
1290	* Returns 0 for success, otherwise -errno.
1291	*/
1292	int unpoison_memory(unsigned long pfn)
1293	{
1294	struct page *page;
1295	struct page *p;
1296	int freeit = 0;
1297	unsigned int nr_pages;
1298
1299	if (!pfn_valid(pfn))
1300	return -ENXIO;
1301
1302	p = pfn_to_page(pfn);
1303	page = compound_head(p);
1304
1305	if (!PageHWPoison(p)) {
1306	pr_info("MCE: Page was already unpoisoned %#lx\n", pfn);
1307	return 0;
1308	}
1309
1310	nr_pages = 1 << compound_trans_order(page);
1311
1312	if (!get_page_unless_zero(page)) {
1313	/*
1314	* Since HWPoisoned hugepage should have non-zero refcount,
1315	* race between memory failure and unpoison seems to happen.
1316	* In such case unpoison fails and memory failure runs
1317	* to the end.
1318	*/
1319	if (PageHuge(page)) {
1320	pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn);
1321	return 0;
1322	}
1323	if (TestClearPageHWPoison(p))
1324	atomic_long_sub(nr_pages, &mce_bad_pages);
1325	pr_info("MCE: Software-unpoisoned free page %#lx\n", pfn);
1326	return 0;
1327	}
1328
1329	lock_page(page);
1330	/*
1331	* This test is racy because PG_hwpoison is set outside of page lock.
1332	* That's acceptable because that won't trigger kernel panic. Instead,
1333	* the PG_hwpoison page will be caught and isolated on the entrance to
1334	* the free buddy page pool.
1335	*/
1336	if (TestClearPageHWPoison(page)) {
1337	pr_info("MCE: Software-unpoisoned page %#lx\n", pfn);
1338	atomic_long_sub(nr_pages, &mce_bad_pages);
1339	freeit = 1;
1340	if (PageHuge(page))
1341	clear_page_hwpoison_huge_page(page);
1342	}
1343	unlock_page(page);
1344
1345	put_page(page);
1346	if (freeit)
1347	put_page(page);
1348
1349	return 0;
1350	}
1351	EXPORT_SYMBOL(unpoison_memory);
1352
1353	static struct page new_page(struct page p, unsigned long private, int **x)
1354	{
1355	int nid = page_to_nid(p);
1356	if (PageHuge(p))
1357	return alloc_huge_page_node(page_hstate(compound_head(p)),
1358	nid);
1359	else
1360	return alloc_pages_exact_node(nid, GFP_HIGHUSER_MOVABLE, 0);
1361	}
1362
1363	/*
1364	* Safely get reference count of an arbitrary page.
1365	* Returns 0 for a free page, -EIO for a zero refcount page
1366	* that is not free, and 1 for any other page type.
1367	* For 1 the page is returned with increased page count, otherwise not.
1368	*/
1369	static int get_any_page(struct page *p, unsigned long pfn, int flags)
1370	{
1371	int ret;
1372
1373	if (flags & MF_COUNT_INCREASED)
1374	return 1;
1375
1376	/*
1377	* The lock_memory_hotplug prevents a race with memory hotplug.
1378	* This is a big hammer, a better would be nicer.
1379	*/
1380	lock_memory_hotplug();
1381
1382	/*
1383	* Isolate the page, so that it doesn't get reallocated if it
1384	* was free.
1385	*/
1386	set_migratetype_isolate(p);
1387	/*
1388	* When the target page is a free hugepage, just remove it
1389	* from free hugepage list.
1390	*/
1391	if (!get_page_unless_zero(compound_head(p))) {
1392	if (PageHuge(p)) {
1393	pr_info("%s: %#lx free huge page\n", __func__, pfn);
1394	ret = dequeue_hwpoisoned_huge_page(compound_head(p));
1395	} else if (is_free_buddy_page(p)) {
1396	pr_info("%s: %#lx free buddy page\n", __func__, pfn);
1397	/* Set hwpoison bit while page is still isolated */
1398	SetPageHWPoison(p);
1399	ret = 0;
1400	} else {
1401	pr_info("%s: %#lx: unknown zero refcount page type %lx\n",
1402	__func__, pfn, p->flags);
1403	ret = -EIO;
1404	}
1405	} else {
1406	/* Not a free page */
1407	ret = 1;
1408	}
1409	unset_migratetype_isolate(p, MIGRATE_MOVABLE);
1410	unlock_memory_hotplug();
1411	return ret;
1412	}
1413
1414	static int soft_offline_huge_page(struct page *page, int flags)
1415	{
1416	int ret;
1417	unsigned long pfn = page_to_pfn(page);
1418	struct page *hpage = compound_head(page);
1419
1420	ret = get_any_page(page, pfn, flags);
1421	if (ret < 0)
1422	return ret;
1423	if (ret == 0)
1424	goto done;
1425
1426	if (PageHWPoison(hpage)) {
1427	put_page(hpage);
1428	pr_info("soft offline: %#lx hugepage already poisoned\n", pfn);
1429	return -EBUSY;
1430	}
1431
1432	/* Keep page count to indicate a given hugepage is isolated. */
1433	ret = migrate_huge_page(hpage, new_page, MPOL_MF_MOVE_ALL, false,
1434	MIGRATE_SYNC);
1435	put_page(hpage);
1436	if (ret) {
1437	pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
1438	pfn, ret, page->flags);
1439	return ret;
1440	}
1441	done:
1442	if (!PageHWPoison(hpage))
1443	atomic_long_add(1 << compound_trans_order(hpage),
1444	&mce_bad_pages);
1445	set_page_hwpoison_huge_page(hpage);
1446	dequeue_hwpoisoned_huge_page(hpage);
1447	/* keep elevated page count for bad page */
1448	return ret;
1449	}
1450
1451	/**
1452	* soft_offline_page - Soft offline a page.
1453	* @page: page to offline
1454	* @flags: flags. Same as memory_failure().
1455	*
1456	* Returns 0 on success, otherwise negated errno.
1457	*
1458	* Soft offline a page, by migration or invalidation,
1459	* without killing anything. This is for the case when
1460	* a page is not corrupted yet (so it's still valid to access),
1461	* but has had a number of corrected errors and is better taken
1462	* out.
1463	*
1464	* The actual policy on when to do that is maintained by
1465	* user space.
1466	*
1467	* This should never impact any application or cause data loss,
1468	* however it might take some time.
1469	*
1470	* This is not a 100% solution for all memory, but tries to be
1471	* ``good enough'' for the majority of memory.
1472	*/
1473	int soft_offline_page(struct page *page, int flags)
1474	{
1475	int ret;
1476	unsigned long pfn = page_to_pfn(page);
1477
1478	if (PageHuge(page))
1479	return soft_offline_huge_page(page, flags);
1480
1481	ret = get_any_page(page, pfn, flags);
1482	if (ret < 0)
1483	return ret;
1484	if (ret == 0)
1485	goto done;
1486
1487	/*
1488	* Page cache page we can handle?
1489	*/
1490	if (!PageLRU(page)) {
1491	/*
1492	* Try to free it.
1493	*/
1494	put_page(page);
1495	shake_page(page, 1);
1496
1497	/*
1498	* Did it turn free?
1499	*/
1500	ret = get_any_page(page, pfn, 0);
1501	if (ret < 0)
1502	return ret;
1503	if (ret == 0)
1504	goto done;
1505	}
1506	if (!PageLRU(page)) {
1507	pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n",
1508	pfn, page->flags);
1509	return -EIO;
1510	}
1511
1512	lock_page(page);
1513	wait_on_page_writeback(page);
1514
1515	/*
1516	* Synchronized using the page lock with memory_failure()
1517	*/
1518	if (PageHWPoison(page)) {
1519	unlock_page(page);
1520	put_page(page);
1521	pr_info("soft offline: %#lx page already poisoned\n", pfn);
1522	return -EBUSY;
1523	}
1524
1525	/*
1526	* Try to invalidate first. This should work for
1527	* non dirty unmapped page cache pages.
1528	*/
1529	ret = invalidate_inode_page(page);
1530	unlock_page(page);
1531	/*
1532	* RED-PEN would be better to keep it isolated here, but we
1533	* would need to fix isolation locking first.
1534	*/
1535	if (ret == 1) {
1536	put_page(page);
1537	ret = 0;
1538	pr_info("soft_offline: %#lx: invalidated\n", pfn);
1539	goto done;
1540	}
1541
1542	/*
1543	* Simple invalidation didn't work.
1544	* Try to migrate to a new page instead. migrate.c
1545	* handles a large number of cases for us.
1546	*/
1547	ret = isolate_lru_page(page);
1548	/*
1549	* Drop page reference which is came from get_any_page()
1550	* successful isolate_lru_page() already took another one.
1551	*/
1552	put_page(page);
1553	if (!ret) {
1554	LIST_HEAD(pagelist);
1555	inc_zone_page_state(page, NR_ISOLATED_ANON +
1556	page_is_file_cache(page));
1557	list_add(&page->lru, &pagelist);
1558	ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
1559	false, MIGRATE_SYNC);
1560	if (ret) {
1561	putback_lru_pages(&pagelist);
1562	pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
1563	pfn, ret, page->flags);
1564	if (ret > 0)
1565	ret = -EIO;
1566	}
1567	} else {
1568	pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n",
1569	pfn, ret, page_count(page), page->flags);
1570	}
1571	if (ret)
1572	return ret;
1573
1574	done:
1575	atomic_long_add(1, &mce_bad_pages);
1576	SetPageHWPoison(page);
1577	/* keep elevated page count for bad page */
1578	return ret;
1579	}
1580

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/memory-failure.c

interactive