Root/
1 | /* |
2 | * Memory Migration functionality - linux/mm/migration.c |
3 | * |
4 | * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter |
5 | * |
6 | * Page migration was first developed in the context of the memory hotplug |
7 | * project. The main authors of the migration code are: |
8 | * |
9 | * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> |
10 | * Hirokazu Takahashi <taka@valinux.co.jp> |
11 | * Dave Hansen <haveblue@us.ibm.com> |
12 | * Christoph Lameter |
13 | */ |
14 | |
15 | #include <linux/migrate.h> |
16 | #include <linux/module.h> |
17 | #include <linux/swap.h> |
18 | #include <linux/swapops.h> |
19 | #include <linux/pagemap.h> |
20 | #include <linux/buffer_head.h> |
21 | #include <linux/mm_inline.h> |
22 | #include <linux/nsproxy.h> |
23 | #include <linux/pagevec.h> |
24 | #include <linux/ksm.h> |
25 | #include <linux/rmap.h> |
26 | #include <linux/topology.h> |
27 | #include <linux/cpu.h> |
28 | #include <linux/cpuset.h> |
29 | #include <linux/writeback.h> |
30 | #include <linux/mempolicy.h> |
31 | #include <linux/vmalloc.h> |
32 | #include <linux/security.h> |
33 | #include <linux/memcontrol.h> |
34 | #include <linux/syscalls.h> |
35 | #include <linux/gfp.h> |
36 | |
37 | #include "internal.h" |
38 | |
39 | #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) |
40 | |
41 | /* |
42 | * migrate_prep() needs to be called before we start compiling a list of pages |
43 | * to be migrated using isolate_lru_page(). |
44 | */ |
45 | int migrate_prep(void) |
46 | { |
47 | /* |
48 | * Clear the LRU lists so pages can be isolated. |
49 | * Note that pages may be moved off the LRU after we have |
50 | * drained them. Those pages will fail to migrate like other |
51 | * pages that may be busy. |
52 | */ |
53 | lru_add_drain_all(); |
54 | |
55 | return 0; |
56 | } |
57 | |
58 | /* |
59 | * Add isolated pages on the list back to the LRU under page lock |
60 | * to avoid leaking evictable pages back onto unevictable list. |
61 | * |
62 | * returns the number of pages put back. |
63 | */ |
64 | int putback_lru_pages(struct list_head *l) |
65 | { |
66 | struct page *page; |
67 | struct page *page2; |
68 | int count = 0; |
69 | |
70 | list_for_each_entry_safe(page, page2, l, lru) { |
71 | list_del(&page->lru); |
72 | dec_zone_page_state(page, NR_ISOLATED_ANON + |
73 | page_is_file_cache(page)); |
74 | putback_lru_page(page); |
75 | count++; |
76 | } |
77 | return count; |
78 | } |
79 | |
80 | /* |
81 | * Restore a potential migration pte to a working pte entry |
82 | */ |
83 | static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, |
84 | unsigned long addr, void *old) |
85 | { |
86 | struct mm_struct *mm = vma->vm_mm; |
87 | swp_entry_t entry; |
88 | pgd_t *pgd; |
89 | pud_t *pud; |
90 | pmd_t *pmd; |
91 | pte_t *ptep, pte; |
92 | spinlock_t *ptl; |
93 | |
94 | pgd = pgd_offset(mm, addr); |
95 | if (!pgd_present(*pgd)) |
96 | goto out; |
97 | |
98 | pud = pud_offset(pgd, addr); |
99 | if (!pud_present(*pud)) |
100 | goto out; |
101 | |
102 | pmd = pmd_offset(pud, addr); |
103 | if (!pmd_present(*pmd)) |
104 | goto out; |
105 | |
106 | ptep = pte_offset_map(pmd, addr); |
107 | |
108 | if (!is_swap_pte(*ptep)) { |
109 | pte_unmap(ptep); |
110 | goto out; |
111 | } |
112 | |
113 | ptl = pte_lockptr(mm, pmd); |
114 | spin_lock(ptl); |
115 | pte = *ptep; |
116 | if (!is_swap_pte(pte)) |
117 | goto unlock; |
118 | |
119 | entry = pte_to_swp_entry(pte); |
120 | |
121 | if (!is_migration_entry(entry) || |
122 | migration_entry_to_page(entry) != old) |
123 | goto unlock; |
124 | |
125 | get_page(new); |
126 | pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); |
127 | if (is_write_migration_entry(entry)) |
128 | pte = pte_mkwrite(pte); |
129 | flush_cache_page(vma, addr, pte_pfn(pte)); |
130 | set_pte_at(mm, addr, ptep, pte); |
131 | |
132 | if (PageAnon(new)) |
133 | page_add_anon_rmap(new, vma, addr); |
134 | else |
135 | page_add_file_rmap(new); |
136 | |
137 | /* No need to invalidate - it was non-present before */ |
138 | update_mmu_cache(vma, addr, ptep); |
139 | unlock: |
140 | pte_unmap_unlock(ptep, ptl); |
141 | out: |
142 | return SWAP_AGAIN; |
143 | } |
144 | |
145 | /* |
146 | * Get rid of all migration entries and replace them by |
147 | * references to the indicated page. |
148 | */ |
149 | static void remove_migration_ptes(struct page *old, struct page *new) |
150 | { |
151 | rmap_walk(new, remove_migration_pte, old); |
152 | } |
153 | |
154 | /* |
155 | * Something used the pte of a page under migration. We need to |
156 | * get to the page and wait until migration is finished. |
157 | * When we return from this function the fault will be retried. |
158 | * |
159 | * This function is called from do_swap_page(). |
160 | */ |
161 | void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, |
162 | unsigned long address) |
163 | { |
164 | pte_t *ptep, pte; |
165 | spinlock_t *ptl; |
166 | swp_entry_t entry; |
167 | struct page *page; |
168 | |
169 | ptep = pte_offset_map_lock(mm, pmd, address, &ptl); |
170 | pte = *ptep; |
171 | if (!is_swap_pte(pte)) |
172 | goto out; |
173 | |
174 | entry = pte_to_swp_entry(pte); |
175 | if (!is_migration_entry(entry)) |
176 | goto out; |
177 | |
178 | page = migration_entry_to_page(entry); |
179 | |
180 | /* |
181 | * Once radix-tree replacement of page migration started, page_count |
182 | * *must* be zero. And, we don't want to call wait_on_page_locked() |
183 | * against a page without get_page(). |
184 | * So, we use get_page_unless_zero(), here. Even failed, page fault |
185 | * will occur again. |
186 | */ |
187 | if (!get_page_unless_zero(page)) |
188 | goto out; |
189 | pte_unmap_unlock(ptep, ptl); |
190 | wait_on_page_locked(page); |
191 | put_page(page); |
192 | return; |
193 | out: |
194 | pte_unmap_unlock(ptep, ptl); |
195 | } |
196 | |
197 | /* |
198 | * Replace the page in the mapping. |
199 | * |
200 | * The number of remaining references must be: |
201 | * 1 for anonymous pages without a mapping |
202 | * 2 for pages with a mapping |
203 | * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. |
204 | */ |
205 | static int migrate_page_move_mapping(struct address_space *mapping, |
206 | struct page *newpage, struct page *page) |
207 | { |
208 | int expected_count; |
209 | void **pslot; |
210 | |
211 | if (!mapping) { |
212 | /* Anonymous page without mapping */ |
213 | if (page_count(page) != 1) |
214 | return -EAGAIN; |
215 | return 0; |
216 | } |
217 | |
218 | spin_lock_irq(&mapping->tree_lock); |
219 | |
220 | pslot = radix_tree_lookup_slot(&mapping->page_tree, |
221 | page_index(page)); |
222 | |
223 | expected_count = 2 + page_has_private(page); |
224 | if (page_count(page) != expected_count || |
225 | (struct page *)radix_tree_deref_slot(pslot) != page) { |
226 | spin_unlock_irq(&mapping->tree_lock); |
227 | return -EAGAIN; |
228 | } |
229 | |
230 | if (!page_freeze_refs(page, expected_count)) { |
231 | spin_unlock_irq(&mapping->tree_lock); |
232 | return -EAGAIN; |
233 | } |
234 | |
235 | /* |
236 | * Now we know that no one else is looking at the page. |
237 | */ |
238 | get_page(newpage); /* add cache reference */ |
239 | if (PageSwapCache(page)) { |
240 | SetPageSwapCache(newpage); |
241 | set_page_private(newpage, page_private(page)); |
242 | } |
243 | |
244 | radix_tree_replace_slot(pslot, newpage); |
245 | |
246 | page_unfreeze_refs(page, expected_count); |
247 | /* |
248 | * Drop cache reference from old page. |
249 | * We know this isn't the last reference. |
250 | */ |
251 | __put_page(page); |
252 | |
253 | /* |
254 | * If moved to a different zone then also account |
255 | * the page for that zone. Other VM counters will be |
256 | * taken care of when we establish references to the |
257 | * new page and drop references to the old page. |
258 | * |
259 | * Note that anonymous pages are accounted for |
260 | * via NR_FILE_PAGES and NR_ANON_PAGES if they |
261 | * are mapped to swap space. |
262 | */ |
263 | __dec_zone_page_state(page, NR_FILE_PAGES); |
264 | __inc_zone_page_state(newpage, NR_FILE_PAGES); |
265 | if (PageSwapBacked(page)) { |
266 | __dec_zone_page_state(page, NR_SHMEM); |
267 | __inc_zone_page_state(newpage, NR_SHMEM); |
268 | } |
269 | spin_unlock_irq(&mapping->tree_lock); |
270 | |
271 | return 0; |
272 | } |
273 | |
274 | /* |
275 | * Copy the page to its new location |
276 | */ |
277 | static void migrate_page_copy(struct page *newpage, struct page *page) |
278 | { |
279 | copy_highpage(newpage, page); |
280 | |
281 | if (PageError(page)) |
282 | SetPageError(newpage); |
283 | if (PageReferenced(page)) |
284 | SetPageReferenced(newpage); |
285 | if (PageUptodate(page)) |
286 | SetPageUptodate(newpage); |
287 | if (TestClearPageActive(page)) { |
288 | VM_BUG_ON(PageUnevictable(page)); |
289 | SetPageActive(newpage); |
290 | } else if (TestClearPageUnevictable(page)) |
291 | SetPageUnevictable(newpage); |
292 | if (PageChecked(page)) |
293 | SetPageChecked(newpage); |
294 | if (PageMappedToDisk(page)) |
295 | SetPageMappedToDisk(newpage); |
296 | |
297 | if (PageDirty(page)) { |
298 | clear_page_dirty_for_io(page); |
299 | /* |
300 | * Want to mark the page and the radix tree as dirty, and |
301 | * redo the accounting that clear_page_dirty_for_io undid, |
302 | * but we can't use set_page_dirty because that function |
303 | * is actually a signal that all of the page has become dirty. |
304 | * Wheras only part of our page may be dirty. |
305 | */ |
306 | __set_page_dirty_nobuffers(newpage); |
307 | } |
308 | |
309 | mlock_migrate_page(newpage, page); |
310 | ksm_migrate_page(newpage, page); |
311 | |
312 | ClearPageSwapCache(page); |
313 | ClearPagePrivate(page); |
314 | set_page_private(page, 0); |
315 | page->mapping = NULL; |
316 | |
317 | /* |
318 | * If any waiters have accumulated on the new page then |
319 | * wake them up. |
320 | */ |
321 | if (PageWriteback(newpage)) |
322 | end_page_writeback(newpage); |
323 | } |
324 | |
325 | /************************************************************ |
326 | * Migration functions |
327 | ***********************************************************/ |
328 | |
329 | /* Always fail migration. Used for mappings that are not movable */ |
330 | int fail_migrate_page(struct address_space *mapping, |
331 | struct page *newpage, struct page *page) |
332 | { |
333 | return -EIO; |
334 | } |
335 | EXPORT_SYMBOL(fail_migrate_page); |
336 | |
337 | /* |
338 | * Common logic to directly migrate a single page suitable for |
339 | * pages that do not use PagePrivate/PagePrivate2. |
340 | * |
341 | * Pages are locked upon entry and exit. |
342 | */ |
343 | int migrate_page(struct address_space *mapping, |
344 | struct page *newpage, struct page *page) |
345 | { |
346 | int rc; |
347 | |
348 | BUG_ON(PageWriteback(page)); /* Writeback must be complete */ |
349 | |
350 | rc = migrate_page_move_mapping(mapping, newpage, page); |
351 | |
352 | if (rc) |
353 | return rc; |
354 | |
355 | migrate_page_copy(newpage, page); |
356 | return 0; |
357 | } |
358 | EXPORT_SYMBOL(migrate_page); |
359 | |
360 | #ifdef CONFIG_BLOCK |
361 | /* |
362 | * Migration function for pages with buffers. This function can only be used |
363 | * if the underlying filesystem guarantees that no other references to "page" |
364 | * exist. |
365 | */ |
366 | int buffer_migrate_page(struct address_space *mapping, |
367 | struct page *newpage, struct page *page) |
368 | { |
369 | struct buffer_head *bh, *head; |
370 | int rc; |
371 | |
372 | if (!page_has_buffers(page)) |
373 | return migrate_page(mapping, newpage, page); |
374 | |
375 | head = page_buffers(page); |
376 | |
377 | rc = migrate_page_move_mapping(mapping, newpage, page); |
378 | |
379 | if (rc) |
380 | return rc; |
381 | |
382 | bh = head; |
383 | do { |
384 | get_bh(bh); |
385 | lock_buffer(bh); |
386 | bh = bh->b_this_page; |
387 | |
388 | } while (bh != head); |
389 | |
390 | ClearPagePrivate(page); |
391 | set_page_private(newpage, page_private(page)); |
392 | set_page_private(page, 0); |
393 | put_page(page); |
394 | get_page(newpage); |
395 | |
396 | bh = head; |
397 | do { |
398 | set_bh_page(bh, newpage, bh_offset(bh)); |
399 | bh = bh->b_this_page; |
400 | |
401 | } while (bh != head); |
402 | |
403 | SetPagePrivate(newpage); |
404 | |
405 | migrate_page_copy(newpage, page); |
406 | |
407 | bh = head; |
408 | do { |
409 | unlock_buffer(bh); |
410 | put_bh(bh); |
411 | bh = bh->b_this_page; |
412 | |
413 | } while (bh != head); |
414 | |
415 | return 0; |
416 | } |
417 | EXPORT_SYMBOL(buffer_migrate_page); |
418 | #endif |
419 | |
420 | /* |
421 | * Writeback a page to clean the dirty state |
422 | */ |
423 | static int writeout(struct address_space *mapping, struct page *page) |
424 | { |
425 | struct writeback_control wbc = { |
426 | .sync_mode = WB_SYNC_NONE, |
427 | .nr_to_write = 1, |
428 | .range_start = 0, |
429 | .range_end = LLONG_MAX, |
430 | .nonblocking = 1, |
431 | .for_reclaim = 1 |
432 | }; |
433 | int rc; |
434 | |
435 | if (!mapping->a_ops->writepage) |
436 | /* No write method for the address space */ |
437 | return -EINVAL; |
438 | |
439 | if (!clear_page_dirty_for_io(page)) |
440 | /* Someone else already triggered a write */ |
441 | return -EAGAIN; |
442 | |
443 | /* |
444 | * A dirty page may imply that the underlying filesystem has |
445 | * the page on some queue. So the page must be clean for |
446 | * migration. Writeout may mean we loose the lock and the |
447 | * page state is no longer what we checked for earlier. |
448 | * At this point we know that the migration attempt cannot |
449 | * be successful. |
450 | */ |
451 | remove_migration_ptes(page, page); |
452 | |
453 | rc = mapping->a_ops->writepage(page, &wbc); |
454 | |
455 | if (rc != AOP_WRITEPAGE_ACTIVATE) |
456 | /* unlocked. Relock */ |
457 | lock_page(page); |
458 | |
459 | return (rc < 0) ? -EIO : -EAGAIN; |
460 | } |
461 | |
462 | /* |
463 | * Default handling if a filesystem does not provide a migration function. |
464 | */ |
465 | static int fallback_migrate_page(struct address_space *mapping, |
466 | struct page *newpage, struct page *page) |
467 | { |
468 | if (PageDirty(page)) |
469 | return writeout(mapping, page); |
470 | |
471 | /* |
472 | * Buffers may be managed in a filesystem specific way. |
473 | * We must have no buffers or drop them. |
474 | */ |
475 | if (page_has_private(page) && |
476 | !try_to_release_page(page, GFP_KERNEL)) |
477 | return -EAGAIN; |
478 | |
479 | return migrate_page(mapping, newpage, page); |
480 | } |
481 | |
482 | /* |
483 | * Move a page to a newly allocated page |
484 | * The page is locked and all ptes have been successfully removed. |
485 | * |
486 | * The new page will have replaced the old page if this function |
487 | * is successful. |
488 | * |
489 | * Return value: |
490 | * < 0 - error code |
491 | * == 0 - success |
492 | */ |
493 | static int move_to_new_page(struct page *newpage, struct page *page) |
494 | { |
495 | struct address_space *mapping; |
496 | int rc; |
497 | |
498 | /* |
499 | * Block others from accessing the page when we get around to |
500 | * establishing additional references. We are the only one |
501 | * holding a reference to the new page at this point. |
502 | */ |
503 | if (!trylock_page(newpage)) |
504 | BUG(); |
505 | |
506 | /* Prepare mapping for the new page.*/ |
507 | newpage->index = page->index; |
508 | newpage->mapping = page->mapping; |
509 | if (PageSwapBacked(page)) |
510 | SetPageSwapBacked(newpage); |
511 | |
512 | mapping = page_mapping(page); |
513 | if (!mapping) |
514 | rc = migrate_page(mapping, newpage, page); |
515 | else if (mapping->a_ops->migratepage) |
516 | /* |
517 | * Most pages have a mapping and most filesystems |
518 | * should provide a migration function. Anonymous |
519 | * pages are part of swap space which also has its |
520 | * own migration function. This is the most common |
521 | * path for page migration. |
522 | */ |
523 | rc = mapping->a_ops->migratepage(mapping, |
524 | newpage, page); |
525 | else |
526 | rc = fallback_migrate_page(mapping, newpage, page); |
527 | |
528 | if (!rc) |
529 | remove_migration_ptes(page, newpage); |
530 | else |
531 | newpage->mapping = NULL; |
532 | |
533 | unlock_page(newpage); |
534 | |
535 | return rc; |
536 | } |
537 | |
538 | /* |
539 | * Obtain the lock on page, remove all ptes and migrate the page |
540 | * to the newly allocated page in newpage. |
541 | */ |
542 | static int unmap_and_move(new_page_t get_new_page, unsigned long private, |
543 | struct page *page, int force, int offlining) |
544 | { |
545 | int rc = 0; |
546 | int *result = NULL; |
547 | struct page *newpage = get_new_page(page, private, &result); |
548 | int rcu_locked = 0; |
549 | int charge = 0; |
550 | struct mem_cgroup *mem = NULL; |
551 | |
552 | if (!newpage) |
553 | return -ENOMEM; |
554 | |
555 | if (page_count(page) == 1) { |
556 | /* page was freed from under us. So we are done. */ |
557 | goto move_newpage; |
558 | } |
559 | |
560 | /* prepare cgroup just returns 0 or -ENOMEM */ |
561 | rc = -EAGAIN; |
562 | |
563 | if (!trylock_page(page)) { |
564 | if (!force) |
565 | goto move_newpage; |
566 | lock_page(page); |
567 | } |
568 | |
569 | /* |
570 | * Only memory hotplug's offline_pages() caller has locked out KSM, |
571 | * and can safely migrate a KSM page. The other cases have skipped |
572 | * PageKsm along with PageReserved - but it is only now when we have |
573 | * the page lock that we can be certain it will not go KSM beneath us |
574 | * (KSM will not upgrade a page from PageAnon to PageKsm when it sees |
575 | * its pagecount raised, but only here do we take the page lock which |
576 | * serializes that). |
577 | */ |
578 | if (PageKsm(page) && !offlining) { |
579 | rc = -EBUSY; |
580 | goto unlock; |
581 | } |
582 | |
583 | /* charge against new page */ |
584 | charge = mem_cgroup_prepare_migration(page, &mem); |
585 | if (charge == -ENOMEM) { |
586 | rc = -ENOMEM; |
587 | goto unlock; |
588 | } |
589 | BUG_ON(charge); |
590 | |
591 | if (PageWriteback(page)) { |
592 | if (!force) |
593 | goto uncharge; |
594 | wait_on_page_writeback(page); |
595 | } |
596 | /* |
597 | * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, |
598 | * we cannot notice that anon_vma is freed while we migrates a page. |
599 | * This rcu_read_lock() delays freeing anon_vma pointer until the end |
600 | * of migration. File cache pages are no problem because of page_lock() |
601 | * File Caches may use write_page() or lock_page() in migration, then, |
602 | * just care Anon page here. |
603 | */ |
604 | if (PageAnon(page)) { |
605 | rcu_read_lock(); |
606 | rcu_locked = 1; |
607 | } |
608 | |
609 | /* |
610 | * Corner case handling: |
611 | * 1. When a new swap-cache page is read into, it is added to the LRU |
612 | * and treated as swapcache but it has no rmap yet. |
613 | * Calling try_to_unmap() against a page->mapping==NULL page will |
614 | * trigger a BUG. So handle it here. |
615 | * 2. An orphaned page (see truncate_complete_page) might have |
616 | * fs-private metadata. The page can be picked up due to memory |
617 | * offlining. Everywhere else except page reclaim, the page is |
618 | * invisible to the vm, so the page can not be migrated. So try to |
619 | * free the metadata, so the page can be freed. |
620 | */ |
621 | if (!page->mapping) { |
622 | if (!PageAnon(page) && page_has_private(page)) { |
623 | /* |
624 | * Go direct to try_to_free_buffers() here because |
625 | * a) that's what try_to_release_page() would do anyway |
626 | * b) we may be under rcu_read_lock() here, so we can't |
627 | * use GFP_KERNEL which is what try_to_release_page() |
628 | * needs to be effective. |
629 | */ |
630 | try_to_free_buffers(page); |
631 | goto rcu_unlock; |
632 | } |
633 | goto skip_unmap; |
634 | } |
635 | |
636 | /* Establish migration ptes or remove ptes */ |
637 | try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); |
638 | |
639 | skip_unmap: |
640 | if (!page_mapped(page)) |
641 | rc = move_to_new_page(newpage, page); |
642 | |
643 | if (rc) |
644 | remove_migration_ptes(page, page); |
645 | rcu_unlock: |
646 | if (rcu_locked) |
647 | rcu_read_unlock(); |
648 | uncharge: |
649 | if (!charge) |
650 | mem_cgroup_end_migration(mem, page, newpage); |
651 | unlock: |
652 | unlock_page(page); |
653 | |
654 | if (rc != -EAGAIN) { |
655 | /* |
656 | * A page that has been migrated has all references |
657 | * removed and will be freed. A page that has not been |
658 | * migrated will have kepts its references and be |
659 | * restored. |
660 | */ |
661 | list_del(&page->lru); |
662 | dec_zone_page_state(page, NR_ISOLATED_ANON + |
663 | page_is_file_cache(page)); |
664 | putback_lru_page(page); |
665 | } |
666 | |
667 | move_newpage: |
668 | |
669 | /* |
670 | * Move the new page to the LRU. If migration was not successful |
671 | * then this will free the page. |
672 | */ |
673 | putback_lru_page(newpage); |
674 | |
675 | if (result) { |
676 | if (rc) |
677 | *result = rc; |
678 | else |
679 | *result = page_to_nid(newpage); |
680 | } |
681 | return rc; |
682 | } |
683 | |
684 | /* |
685 | * migrate_pages |
686 | * |
687 | * The function takes one list of pages to migrate and a function |
688 | * that determines from the page to be migrated and the private data |
689 | * the target of the move and allocates the page. |
690 | * |
691 | * The function returns after 10 attempts or if no pages |
692 | * are movable anymore because to has become empty |
693 | * or no retryable pages exist anymore. All pages will be |
694 | * returned to the LRU or freed. |
695 | * |
696 | * Return: Number of pages not migrated or error code. |
697 | */ |
698 | int migrate_pages(struct list_head *from, |
699 | new_page_t get_new_page, unsigned long private, int offlining) |
700 | { |
701 | int retry = 1; |
702 | int nr_failed = 0; |
703 | int pass = 0; |
704 | struct page *page; |
705 | struct page *page2; |
706 | int swapwrite = current->flags & PF_SWAPWRITE; |
707 | int rc; |
708 | |
709 | if (!swapwrite) |
710 | current->flags |= PF_SWAPWRITE; |
711 | |
712 | for(pass = 0; pass < 10 && retry; pass++) { |
713 | retry = 0; |
714 | |
715 | list_for_each_entry_safe(page, page2, from, lru) { |
716 | cond_resched(); |
717 | |
718 | rc = unmap_and_move(get_new_page, private, |
719 | page, pass > 2, offlining); |
720 | |
721 | switch(rc) { |
722 | case -ENOMEM: |
723 | goto out; |
724 | case -EAGAIN: |
725 | retry++; |
726 | break; |
727 | case 0: |
728 | break; |
729 | default: |
730 | /* Permanent failure */ |
731 | nr_failed++; |
732 | break; |
733 | } |
734 | } |
735 | } |
736 | rc = 0; |
737 | out: |
738 | if (!swapwrite) |
739 | current->flags &= ~PF_SWAPWRITE; |
740 | |
741 | putback_lru_pages(from); |
742 | |
743 | if (rc) |
744 | return rc; |
745 | |
746 | return nr_failed + retry; |
747 | } |
748 | |
749 | #ifdef CONFIG_NUMA |
750 | /* |
751 | * Move a list of individual pages |
752 | */ |
753 | struct page_to_node { |
754 | unsigned long addr; |
755 | struct page *page; |
756 | int node; |
757 | int status; |
758 | }; |
759 | |
760 | static struct page *new_page_node(struct page *p, unsigned long private, |
761 | int **result) |
762 | { |
763 | struct page_to_node *pm = (struct page_to_node *)private; |
764 | |
765 | while (pm->node != MAX_NUMNODES && pm->page != p) |
766 | pm++; |
767 | |
768 | if (pm->node == MAX_NUMNODES) |
769 | return NULL; |
770 | |
771 | *result = &pm->status; |
772 | |
773 | return alloc_pages_exact_node(pm->node, |
774 | GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0); |
775 | } |
776 | |
777 | /* |
778 | * Move a set of pages as indicated in the pm array. The addr |
779 | * field must be set to the virtual address of the page to be moved |
780 | * and the node number must contain a valid target node. |
781 | * The pm array ends with node = MAX_NUMNODES. |
782 | */ |
783 | static int do_move_page_to_node_array(struct mm_struct *mm, |
784 | struct page_to_node *pm, |
785 | int migrate_all) |
786 | { |
787 | int err; |
788 | struct page_to_node *pp; |
789 | LIST_HEAD(pagelist); |
790 | |
791 | down_read(&mm->mmap_sem); |
792 | |
793 | /* |
794 | * Build a list of pages to migrate |
795 | */ |
796 | for (pp = pm; pp->node != MAX_NUMNODES; pp++) { |
797 | struct vm_area_struct *vma; |
798 | struct page *page; |
799 | |
800 | err = -EFAULT; |
801 | vma = find_vma(mm, pp->addr); |
802 | if (!vma || !vma_migratable(vma)) |
803 | goto set_status; |
804 | |
805 | page = follow_page(vma, pp->addr, FOLL_GET); |
806 | |
807 | err = PTR_ERR(page); |
808 | if (IS_ERR(page)) |
809 | goto set_status; |
810 | |
811 | err = -ENOENT; |
812 | if (!page) |
813 | goto set_status; |
814 | |
815 | /* Use PageReserved to check for zero page */ |
816 | if (PageReserved(page) || PageKsm(page)) |
817 | goto put_and_set; |
818 | |
819 | pp->page = page; |
820 | err = page_to_nid(page); |
821 | |
822 | if (err == pp->node) |
823 | /* |
824 | * Node already in the right place |
825 | */ |
826 | goto put_and_set; |
827 | |
828 | err = -EACCES; |
829 | if (page_mapcount(page) > 1 && |
830 | !migrate_all) |
831 | goto put_and_set; |
832 | |
833 | err = isolate_lru_page(page); |
834 | if (!err) { |
835 | list_add_tail(&page->lru, &pagelist); |
836 | inc_zone_page_state(page, NR_ISOLATED_ANON + |
837 | page_is_file_cache(page)); |
838 | } |
839 | put_and_set: |
840 | /* |
841 | * Either remove the duplicate refcount from |
842 | * isolate_lru_page() or drop the page ref if it was |
843 | * not isolated. |
844 | */ |
845 | put_page(page); |
846 | set_status: |
847 | pp->status = err; |
848 | } |
849 | |
850 | err = 0; |
851 | if (!list_empty(&pagelist)) |
852 | err = migrate_pages(&pagelist, new_page_node, |
853 | (unsigned long)pm, 0); |
854 | |
855 | up_read(&mm->mmap_sem); |
856 | return err; |
857 | } |
858 | |
859 | /* |
860 | * Migrate an array of page address onto an array of nodes and fill |
861 | * the corresponding array of status. |
862 | */ |
863 | static int do_pages_move(struct mm_struct *mm, struct task_struct *task, |
864 | unsigned long nr_pages, |
865 | const void __user * __user *pages, |
866 | const int __user *nodes, |
867 | int __user *status, int flags) |
868 | { |
869 | struct page_to_node *pm; |
870 | nodemask_t task_nodes; |
871 | unsigned long chunk_nr_pages; |
872 | unsigned long chunk_start; |
873 | int err; |
874 | |
875 | task_nodes = cpuset_mems_allowed(task); |
876 | |
877 | err = -ENOMEM; |
878 | pm = (struct page_to_node *)__get_free_page(GFP_KERNEL); |
879 | if (!pm) |
880 | goto out; |
881 | |
882 | migrate_prep(); |
883 | |
884 | /* |
885 | * Store a chunk of page_to_node array in a page, |
886 | * but keep the last one as a marker |
887 | */ |
888 | chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1; |
889 | |
890 | for (chunk_start = 0; |
891 | chunk_start < nr_pages; |
892 | chunk_start += chunk_nr_pages) { |
893 | int j; |
894 | |
895 | if (chunk_start + chunk_nr_pages > nr_pages) |
896 | chunk_nr_pages = nr_pages - chunk_start; |
897 | |
898 | /* fill the chunk pm with addrs and nodes from user-space */ |
899 | for (j = 0; j < chunk_nr_pages; j++) { |
900 | const void __user *p; |
901 | int node; |
902 | |
903 | err = -EFAULT; |
904 | if (get_user(p, pages + j + chunk_start)) |
905 | goto out_pm; |
906 | pm[j].addr = (unsigned long) p; |
907 | |
908 | if (get_user(node, nodes + j + chunk_start)) |
909 | goto out_pm; |
910 | |
911 | err = -ENODEV; |
912 | if (node < 0 || node >= MAX_NUMNODES) |
913 | goto out_pm; |
914 | |
915 | if (!node_state(node, N_HIGH_MEMORY)) |
916 | goto out_pm; |
917 | |
918 | err = -EACCES; |
919 | if (!node_isset(node, task_nodes)) |
920 | goto out_pm; |
921 | |
922 | pm[j].node = node; |
923 | } |
924 | |
925 | /* End marker for this chunk */ |
926 | pm[chunk_nr_pages].node = MAX_NUMNODES; |
927 | |
928 | /* Migrate this chunk */ |
929 | err = do_move_page_to_node_array(mm, pm, |
930 | flags & MPOL_MF_MOVE_ALL); |
931 | if (err < 0) |
932 | goto out_pm; |
933 | |
934 | /* Return status information */ |
935 | for (j = 0; j < chunk_nr_pages; j++) |
936 | if (put_user(pm[j].status, status + j + chunk_start)) { |
937 | err = -EFAULT; |
938 | goto out_pm; |
939 | } |
940 | } |
941 | err = 0; |
942 | |
943 | out_pm: |
944 | free_page((unsigned long)pm); |
945 | out: |
946 | return err; |
947 | } |
948 | |
949 | /* |
950 | * Determine the nodes of an array of pages and store it in an array of status. |
951 | */ |
952 | static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, |
953 | const void __user **pages, int *status) |
954 | { |
955 | unsigned long i; |
956 | |
957 | down_read(&mm->mmap_sem); |
958 | |
959 | for (i = 0; i < nr_pages; i++) { |
960 | unsigned long addr = (unsigned long)(*pages); |
961 | struct vm_area_struct *vma; |
962 | struct page *page; |
963 | int err = -EFAULT; |
964 | |
965 | vma = find_vma(mm, addr); |
966 | if (!vma) |
967 | goto set_status; |
968 | |
969 | page = follow_page(vma, addr, 0); |
970 | |
971 | err = PTR_ERR(page); |
972 | if (IS_ERR(page)) |
973 | goto set_status; |
974 | |
975 | err = -ENOENT; |
976 | /* Use PageReserved to check for zero page */ |
977 | if (!page || PageReserved(page) || PageKsm(page)) |
978 | goto set_status; |
979 | |
980 | err = page_to_nid(page); |
981 | set_status: |
982 | *status = err; |
983 | |
984 | pages++; |
985 | status++; |
986 | } |
987 | |
988 | up_read(&mm->mmap_sem); |
989 | } |
990 | |
991 | /* |
992 | * Determine the nodes of a user array of pages and store it in |
993 | * a user array of status. |
994 | */ |
995 | static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, |
996 | const void __user * __user *pages, |
997 | int __user *status) |
998 | { |
999 | #define DO_PAGES_STAT_CHUNK_NR 16 |
1000 | const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; |
1001 | int chunk_status[DO_PAGES_STAT_CHUNK_NR]; |
1002 | |
1003 | while (nr_pages) { |
1004 | unsigned long chunk_nr; |
1005 | |
1006 | chunk_nr = nr_pages; |
1007 | if (chunk_nr > DO_PAGES_STAT_CHUNK_NR) |
1008 | chunk_nr = DO_PAGES_STAT_CHUNK_NR; |
1009 | |
1010 | if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages))) |
1011 | break; |
1012 | |
1013 | do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); |
1014 | |
1015 | if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) |
1016 | break; |
1017 | |
1018 | pages += chunk_nr; |
1019 | status += chunk_nr; |
1020 | nr_pages -= chunk_nr; |
1021 | } |
1022 | return nr_pages ? -EFAULT : 0; |
1023 | } |
1024 | |
1025 | /* |
1026 | * Move a list of pages in the address space of the currently executing |
1027 | * process. |
1028 | */ |
1029 | SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, |
1030 | const void __user * __user *, pages, |
1031 | const int __user *, nodes, |
1032 | int __user *, status, int, flags) |
1033 | { |
1034 | const struct cred *cred = current_cred(), *tcred; |
1035 | struct task_struct *task; |
1036 | struct mm_struct *mm; |
1037 | int err; |
1038 | |
1039 | /* Check flags */ |
1040 | if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) |
1041 | return -EINVAL; |
1042 | |
1043 | if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) |
1044 | return -EPERM; |
1045 | |
1046 | /* Find the mm_struct */ |
1047 | read_lock(&tasklist_lock); |
1048 | task = pid ? find_task_by_vpid(pid) : current; |
1049 | if (!task) { |
1050 | read_unlock(&tasklist_lock); |
1051 | return -ESRCH; |
1052 | } |
1053 | mm = get_task_mm(task); |
1054 | read_unlock(&tasklist_lock); |
1055 | |
1056 | if (!mm) |
1057 | return -EINVAL; |
1058 | |
1059 | /* |
1060 | * Check if this process has the right to modify the specified |
1061 | * process. The right exists if the process has administrative |
1062 | * capabilities, superuser privileges or the same |
1063 | * userid as the target process. |
1064 | */ |
1065 | rcu_read_lock(); |
1066 | tcred = __task_cred(task); |
1067 | if (cred->euid != tcred->suid && cred->euid != tcred->uid && |
1068 | cred->uid != tcred->suid && cred->uid != tcred->uid && |
1069 | !capable(CAP_SYS_NICE)) { |
1070 | rcu_read_unlock(); |
1071 | err = -EPERM; |
1072 | goto out; |
1073 | } |
1074 | rcu_read_unlock(); |
1075 | |
1076 | err = security_task_movememory(task); |
1077 | if (err) |
1078 | goto out; |
1079 | |
1080 | if (nodes) { |
1081 | err = do_pages_move(mm, task, nr_pages, pages, nodes, status, |
1082 | flags); |
1083 | } else { |
1084 | err = do_pages_stat(mm, nr_pages, pages, status); |
1085 | } |
1086 | |
1087 | out: |
1088 | mmput(mm); |
1089 | return err; |
1090 | } |
1091 | |
1092 | /* |
1093 | * Call migration functions in the vma_ops that may prepare |
1094 | * memory in a vm for migration. migration functions may perform |
1095 | * the migration for vmas that do not have an underlying page struct. |
1096 | */ |
1097 | int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, |
1098 | const nodemask_t *from, unsigned long flags) |
1099 | { |
1100 | struct vm_area_struct *vma; |
1101 | int err = 0; |
1102 | |
1103 | for (vma = mm->mmap; vma && !err; vma = vma->vm_next) { |
1104 | if (vma->vm_ops && vma->vm_ops->migrate) { |
1105 | err = vma->vm_ops->migrate(vma, to, from, flags); |
1106 | if (err) |
1107 | break; |
1108 | } |
1109 | } |
1110 | return err; |
1111 | } |
1112 | #endif |
1113 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9