Root/kernel/power/snapshot.c

1/*
2 * linux/kernel/power/snapshot.c
3 *
4 * This file provides system snapshot/restore functionality for swsusp.
5 *
6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8 *
9 * This file is released under the GPLv2.
10 *
11 */
12
13#include <linux/version.h>
14#include <linux/module.h>
15#include <linux/mm.h>
16#include <linux/suspend.h>
17#include <linux/delay.h>
18#include <linux/bitops.h>
19#include <linux/spinlock.h>
20#include <linux/kernel.h>
21#include <linux/pm.h>
22#include <linux/device.h>
23#include <linux/init.h>
24#include <linux/bootmem.h>
25#include <linux/syscalls.h>
26#include <linux/console.h>
27#include <linux/highmem.h>
28#include <linux/list.h>
29#include <linux/slab.h>
30
31#include <asm/uaccess.h>
32#include <asm/mmu_context.h>
33#include <asm/pgtable.h>
34#include <asm/tlbflush.h>
35#include <asm/io.h>
36
37#include "power.h"
38
39static int swsusp_page_is_free(struct page *);
40static void swsusp_set_page_forbidden(struct page *);
41static void swsusp_unset_page_forbidden(struct page *);
42
43/*
44 * Number of bytes to reserve for memory allocations made by device drivers
45 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't
46 * cause image creation to fail (tunable via /sys/power/reserved_size).
47 */
48unsigned long reserved_size;
49
50void __init hibernate_reserved_size_init(void)
51{
52    reserved_size = SPARE_PAGES * PAGE_SIZE;
53}
54
55/*
56 * Preferred image size in bytes (tunable via /sys/power/image_size).
57 * When it is set to N, swsusp will do its best to ensure the image
58 * size will not exceed N bytes, but if that is impossible, it will
59 * try to create the smallest image possible.
60 */
61unsigned long image_size;
62
63void __init hibernate_image_size_init(void)
64{
65    image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE;
66}
67
68/* List of PBEs needed for restoring the pages that were allocated before
69 * the suspend and included in the suspend image, but have also been
70 * allocated by the "resume" kernel, so their contents cannot be written
71 * directly to their "original" page frames.
72 */
73struct pbe *restore_pblist;
74
75/* Pointer to an auxiliary buffer (1 page) */
76static void *buffer;
77
78/**
79 * @safe_needed - on resume, for storing the PBE list and the image,
80 * we can only use memory pages that do not conflict with the pages
81 * used before suspend. The unsafe pages have PageNosaveFree set
82 * and we count them using unsafe_pages.
83 *
84 * Each allocated image page is marked as PageNosave and PageNosaveFree
85 * so that swsusp_free() can release it.
86 */
87
88#define PG_ANY 0
89#define PG_SAFE 1
90#define PG_UNSAFE_CLEAR 1
91#define PG_UNSAFE_KEEP 0
92
93static unsigned int allocated_unsafe_pages;
94
95static void *get_image_page(gfp_t gfp_mask, int safe_needed)
96{
97    void *res;
98
99    res = (void *)get_zeroed_page(gfp_mask);
100    if (safe_needed)
101        while (res && swsusp_page_is_free(virt_to_page(res))) {
102            /* The page is unsafe, mark it for swsusp_free() */
103            swsusp_set_page_forbidden(virt_to_page(res));
104            allocated_unsafe_pages++;
105            res = (void *)get_zeroed_page(gfp_mask);
106        }
107    if (res) {
108        swsusp_set_page_forbidden(virt_to_page(res));
109        swsusp_set_page_free(virt_to_page(res));
110    }
111    return res;
112}
113
114unsigned long get_safe_page(gfp_t gfp_mask)
115{
116    return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
117}
118
119static struct page *alloc_image_page(gfp_t gfp_mask)
120{
121    struct page *page;
122
123    page = alloc_page(gfp_mask);
124    if (page) {
125        swsusp_set_page_forbidden(page);
126        swsusp_set_page_free(page);
127    }
128    return page;
129}
130
131/**
132 * free_image_page - free page represented by @addr, allocated with
133 * get_image_page (page flags set by it must be cleared)
134 */
135
136static inline void free_image_page(void *addr, int clear_nosave_free)
137{
138    struct page *page;
139
140    BUG_ON(!virt_addr_valid(addr));
141
142    page = virt_to_page(addr);
143
144    swsusp_unset_page_forbidden(page);
145    if (clear_nosave_free)
146        swsusp_unset_page_free(page);
147
148    __free_page(page);
149}
150
151/* struct linked_page is used to build chains of pages */
152
153#define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
154
155struct linked_page {
156    struct linked_page *next;
157    char data[LINKED_PAGE_DATA_SIZE];
158} __attribute__((packed));
159
160static inline void
161free_list_of_pages(struct linked_page *list, int clear_page_nosave)
162{
163    while (list) {
164        struct linked_page *lp = list->next;
165
166        free_image_page(list, clear_page_nosave);
167        list = lp;
168    }
169}
170
171/**
172  * struct chain_allocator is used for allocating small objects out of
173  * a linked list of pages called 'the chain'.
174  *
175  * The chain grows each time when there is no room for a new object in
176  * the current page. The allocated objects cannot be freed individually.
177  * It is only possible to free them all at once, by freeing the entire
178  * chain.
179  *
180  * NOTE: The chain allocator may be inefficient if the allocated objects
181  * are not much smaller than PAGE_SIZE.
182  */
183
184struct chain_allocator {
185    struct linked_page *chain; /* the chain */
186    unsigned int used_space; /* total size of objects allocated out
187                     * of the current page
188                     */
189    gfp_t gfp_mask; /* mask for allocating pages */
190    int safe_needed; /* if set, only "safe" pages are allocated */
191};
192
193static void
194chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
195{
196    ca->chain = NULL;
197    ca->used_space = LINKED_PAGE_DATA_SIZE;
198    ca->gfp_mask = gfp_mask;
199    ca->safe_needed = safe_needed;
200}
201
202static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
203{
204    void *ret;
205
206    if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
207        struct linked_page *lp;
208
209        lp = get_image_page(ca->gfp_mask, ca->safe_needed);
210        if (!lp)
211            return NULL;
212
213        lp->next = ca->chain;
214        ca->chain = lp;
215        ca->used_space = 0;
216    }
217    ret = ca->chain->data + ca->used_space;
218    ca->used_space += size;
219    return ret;
220}
221
222/**
223 * Data types related to memory bitmaps.
224 *
225 * Memory bitmap is a structure consiting of many linked lists of
226 * objects. The main list's elements are of type struct zone_bitmap
227 * and each of them corresonds to one zone. For each zone bitmap
228 * object there is a list of objects of type struct bm_block that
229 * represent each blocks of bitmap in which information is stored.
230 *
231 * struct memory_bitmap contains a pointer to the main list of zone
232 * bitmap objects, a struct bm_position used for browsing the bitmap,
233 * and a pointer to the list of pages used for allocating all of the
234 * zone bitmap objects and bitmap block objects.
235 *
236 * NOTE: It has to be possible to lay out the bitmap in memory
237 * using only allocations of order 0. Additionally, the bitmap is
238 * designed to work with arbitrary number of zones (this is over the
239 * top for now, but let's avoid making unnecessary assumptions ;-).
240 *
241 * struct zone_bitmap contains a pointer to a list of bitmap block
242 * objects and a pointer to the bitmap block object that has been
243 * most recently used for setting bits. Additionally, it contains the
244 * pfns that correspond to the start and end of the represented zone.
245 *
246 * struct bm_block contains a pointer to the memory page in which
247 * information is stored (in the form of a block of bitmap)
248 * It also contains the pfns that correspond to the start and end of
249 * the represented memory area.
250 */
251
252#define BM_END_OF_MAP (~0UL)
253
254#define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE)
255
256struct bm_block {
257    struct list_head hook; /* hook into a list of bitmap blocks */
258    unsigned long start_pfn; /* pfn represented by the first bit */
259    unsigned long end_pfn; /* pfn represented by the last bit plus 1 */
260    unsigned long *data; /* bitmap representing pages */
261};
262
263static inline unsigned long bm_block_bits(struct bm_block *bb)
264{
265    return bb->end_pfn - bb->start_pfn;
266}
267
268/* strcut bm_position is used for browsing memory bitmaps */
269
270struct bm_position {
271    struct bm_block *block;
272    int bit;
273};
274
275struct memory_bitmap {
276    struct list_head blocks; /* list of bitmap blocks */
277    struct linked_page *p_list; /* list of pages used to store zone
278                     * bitmap objects and bitmap block
279                     * objects
280                     */
281    struct bm_position cur; /* most recently used bit position */
282};
283
284/* Functions that operate on memory bitmaps */
285
286static void memory_bm_position_reset(struct memory_bitmap *bm)
287{
288    bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook);
289    bm->cur.bit = 0;
290}
291
292static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
293
294/**
295 * create_bm_block_list - create a list of block bitmap objects
296 * @pages - number of pages to track
297 * @list - list to put the allocated blocks into
298 * @ca - chain allocator to be used for allocating memory
299 */
300static int create_bm_block_list(unsigned long pages,
301                struct list_head *list,
302                struct chain_allocator *ca)
303{
304    unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
305
306    while (nr_blocks-- > 0) {
307        struct bm_block *bb;
308
309        bb = chain_alloc(ca, sizeof(struct bm_block));
310        if (!bb)
311            return -ENOMEM;
312        list_add(&bb->hook, list);
313    }
314
315    return 0;
316}
317
318struct mem_extent {
319    struct list_head hook;
320    unsigned long start;
321    unsigned long end;
322};
323
324/**
325 * free_mem_extents - free a list of memory extents
326 * @list - list of extents to empty
327 */
328static void free_mem_extents(struct list_head *list)
329{
330    struct mem_extent *ext, *aux;
331
332    list_for_each_entry_safe(ext, aux, list, hook) {
333        list_del(&ext->hook);
334        kfree(ext);
335    }
336}
337
338/**
339 * create_mem_extents - create a list of memory extents representing
340 * contiguous ranges of PFNs
341 * @list - list to put the extents into
342 * @gfp_mask - mask to use for memory allocations
343 */
344static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
345{
346    struct zone *zone;
347
348    INIT_LIST_HEAD(list);
349
350    for_each_populated_zone(zone) {
351        unsigned long zone_start, zone_end;
352        struct mem_extent *ext, *cur, *aux;
353
354        zone_start = zone->zone_start_pfn;
355        zone_end = zone->zone_start_pfn + zone->spanned_pages;
356
357        list_for_each_entry(ext, list, hook)
358            if (zone_start <= ext->end)
359                break;
360
361        if (&ext->hook == list || zone_end < ext->start) {
362            /* New extent is necessary */
363            struct mem_extent *new_ext;
364
365            new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask);
366            if (!new_ext) {
367                free_mem_extents(list);
368                return -ENOMEM;
369            }
370            new_ext->start = zone_start;
371            new_ext->end = zone_end;
372            list_add_tail(&new_ext->hook, &ext->hook);
373            continue;
374        }
375
376        /* Merge this zone's range of PFNs with the existing one */
377        if (zone_start < ext->start)
378            ext->start = zone_start;
379        if (zone_end > ext->end)
380            ext->end = zone_end;
381
382        /* More merging may be possible */
383        cur = ext;
384        list_for_each_entry_safe_continue(cur, aux, list, hook) {
385            if (zone_end < cur->start)
386                break;
387            if (zone_end < cur->end)
388                ext->end = cur->end;
389            list_del(&cur->hook);
390            kfree(cur);
391        }
392    }
393
394    return 0;
395}
396
397/**
398  * memory_bm_create - allocate memory for a memory bitmap
399  */
400static int
401memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
402{
403    struct chain_allocator ca;
404    struct list_head mem_extents;
405    struct mem_extent *ext;
406    int error;
407
408    chain_init(&ca, gfp_mask, safe_needed);
409    INIT_LIST_HEAD(&bm->blocks);
410
411    error = create_mem_extents(&mem_extents, gfp_mask);
412    if (error)
413        return error;
414
415    list_for_each_entry(ext, &mem_extents, hook) {
416        struct bm_block *bb;
417        unsigned long pfn = ext->start;
418        unsigned long pages = ext->end - ext->start;
419
420        bb = list_entry(bm->blocks.prev, struct bm_block, hook);
421
422        error = create_bm_block_list(pages, bm->blocks.prev, &ca);
423        if (error)
424            goto Error;
425
426        list_for_each_entry_continue(bb, &bm->blocks, hook) {
427            bb->data = get_image_page(gfp_mask, safe_needed);
428            if (!bb->data) {
429                error = -ENOMEM;
430                goto Error;
431            }
432
433            bb->start_pfn = pfn;
434            if (pages >= BM_BITS_PER_BLOCK) {
435                pfn += BM_BITS_PER_BLOCK;
436                pages -= BM_BITS_PER_BLOCK;
437            } else {
438                /* This is executed only once in the loop */
439                pfn += pages;
440            }
441            bb->end_pfn = pfn;
442        }
443    }
444
445    bm->p_list = ca.chain;
446    memory_bm_position_reset(bm);
447 Exit:
448    free_mem_extents(&mem_extents);
449    return error;
450
451 Error:
452    bm->p_list = ca.chain;
453    memory_bm_free(bm, PG_UNSAFE_CLEAR);
454    goto Exit;
455}
456
457/**
458  * memory_bm_free - free memory occupied by the memory bitmap @bm
459  */
460static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
461{
462    struct bm_block *bb;
463
464    list_for_each_entry(bb, &bm->blocks, hook)
465        if (bb->data)
466            free_image_page(bb->data, clear_nosave_free);
467
468    free_list_of_pages(bm->p_list, clear_nosave_free);
469
470    INIT_LIST_HEAD(&bm->blocks);
471}
472
473/**
474 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
475 * to given pfn. The cur_zone_bm member of @bm and the cur_block member
476 * of @bm->cur_zone_bm are updated.
477 */
478static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
479                void **addr, unsigned int *bit_nr)
480{
481    struct bm_block *bb;
482
483    /*
484     * Check if the pfn corresponds to the current bitmap block and find
485     * the block where it fits if this is not the case.
486     */
487    bb = bm->cur.block;
488    if (pfn < bb->start_pfn)
489        list_for_each_entry_continue_reverse(bb, &bm->blocks, hook)
490            if (pfn >= bb->start_pfn)
491                break;
492
493    if (pfn >= bb->end_pfn)
494        list_for_each_entry_continue(bb, &bm->blocks, hook)
495            if (pfn >= bb->start_pfn && pfn < bb->end_pfn)
496                break;
497
498    if (&bb->hook == &bm->blocks)
499        return -EFAULT;
500
501    /* The block has been found */
502    bm->cur.block = bb;
503    pfn -= bb->start_pfn;
504    bm->cur.bit = pfn + 1;
505    *bit_nr = pfn;
506    *addr = bb->data;
507    return 0;
508}
509
510static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
511{
512    void *addr;
513    unsigned int bit;
514    int error;
515
516    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
517    BUG_ON(error);
518    set_bit(bit, addr);
519}
520
521static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
522{
523    void *addr;
524    unsigned int bit;
525    int error;
526
527    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
528    if (!error)
529        set_bit(bit, addr);
530    return error;
531}
532
533static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
534{
535    void *addr;
536    unsigned int bit;
537    int error;
538
539    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
540    BUG_ON(error);
541    clear_bit(bit, addr);
542}
543
544static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
545{
546    void *addr;
547    unsigned int bit;
548    int error;
549
550    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
551    BUG_ON(error);
552    return test_bit(bit, addr);
553}
554
555static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
556{
557    void *addr;
558    unsigned int bit;
559
560    return !memory_bm_find_bit(bm, pfn, &addr, &bit);
561}
562
563/**
564 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
565 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
566 * returned.
567 *
568 * It is required to run memory_bm_position_reset() before the first call to
569 * this function.
570 */
571
572static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
573{
574    struct bm_block *bb;
575    int bit;
576
577    bb = bm->cur.block;
578    do {
579        bit = bm->cur.bit;
580        bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
581        if (bit < bm_block_bits(bb))
582            goto Return_pfn;
583
584        bb = list_entry(bb->hook.next, struct bm_block, hook);
585        bm->cur.block = bb;
586        bm->cur.bit = 0;
587    } while (&bb->hook != &bm->blocks);
588
589    memory_bm_position_reset(bm);
590    return BM_END_OF_MAP;
591
592 Return_pfn:
593    bm->cur.bit = bit + 1;
594    return bb->start_pfn + bit;
595}
596
597/**
598 * This structure represents a range of page frames the contents of which
599 * should not be saved during the suspend.
600 */
601
602struct nosave_region {
603    struct list_head list;
604    unsigned long start_pfn;
605    unsigned long end_pfn;
606};
607
608static LIST_HEAD(nosave_regions);
609
610/**
611 * register_nosave_region - register a range of page frames the contents
612 * of which should not be saved during the suspend (to be used in the early
613 * initialization code)
614 */
615
616void __init
617__register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
618             int use_kmalloc)
619{
620    struct nosave_region *region;
621
622    if (start_pfn >= end_pfn)
623        return;
624
625    if (!list_empty(&nosave_regions)) {
626        /* Try to extend the previous region (they should be sorted) */
627        region = list_entry(nosave_regions.prev,
628                    struct nosave_region, list);
629        if (region->end_pfn == start_pfn) {
630            region->end_pfn = end_pfn;
631            goto Report;
632        }
633    }
634    if (use_kmalloc) {
635        /* during init, this shouldn't fail */
636        region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
637        BUG_ON(!region);
638    } else
639        /* This allocation cannot fail */
640        region = alloc_bootmem(sizeof(struct nosave_region));
641    region->start_pfn = start_pfn;
642    region->end_pfn = end_pfn;
643    list_add_tail(&region->list, &nosave_regions);
644 Report:
645    printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
646        start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
647}
648
649/*
650 * Set bits in this map correspond to the page frames the contents of which
651 * should not be saved during the suspend.
652 */
653static struct memory_bitmap *forbidden_pages_map;
654
655/* Set bits in this map correspond to free page frames. */
656static struct memory_bitmap *free_pages_map;
657
658/*
659 * Each page frame allocated for creating the image is marked by setting the
660 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
661 */
662
663void swsusp_set_page_free(struct page *page)
664{
665    if (free_pages_map)
666        memory_bm_set_bit(free_pages_map, page_to_pfn(page));
667}
668
669static int swsusp_page_is_free(struct page *page)
670{
671    return free_pages_map ?
672        memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
673}
674
675void swsusp_unset_page_free(struct page *page)
676{
677    if (free_pages_map)
678        memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
679}
680
681static void swsusp_set_page_forbidden(struct page *page)
682{
683    if (forbidden_pages_map)
684        memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
685}
686
687int swsusp_page_is_forbidden(struct page *page)
688{
689    return forbidden_pages_map ?
690        memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
691}
692
693static void swsusp_unset_page_forbidden(struct page *page)
694{
695    if (forbidden_pages_map)
696        memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
697}
698
699/**
700 * mark_nosave_pages - set bits corresponding to the page frames the
701 * contents of which should not be saved in a given bitmap.
702 */
703
704static void mark_nosave_pages(struct memory_bitmap *bm)
705{
706    struct nosave_region *region;
707
708    if (list_empty(&nosave_regions))
709        return;
710
711    list_for_each_entry(region, &nosave_regions, list) {
712        unsigned long pfn;
713
714        pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
715                region->start_pfn << PAGE_SHIFT,
716                region->end_pfn << PAGE_SHIFT);
717
718        for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
719            if (pfn_valid(pfn)) {
720                /*
721                 * It is safe to ignore the result of
722                 * mem_bm_set_bit_check() here, since we won't
723                 * touch the PFNs for which the error is
724                 * returned anyway.
725                 */
726                mem_bm_set_bit_check(bm, pfn);
727            }
728    }
729}
730
731/**
732 * create_basic_memory_bitmaps - create bitmaps needed for marking page
733 * frames that should not be saved and free page frames. The pointers
734 * forbidden_pages_map and free_pages_map are only modified if everything
735 * goes well, because we don't want the bits to be used before both bitmaps
736 * are set up.
737 */
738
739int create_basic_memory_bitmaps(void)
740{
741    struct memory_bitmap *bm1, *bm2;
742    int error = 0;
743
744    BUG_ON(forbidden_pages_map || free_pages_map);
745
746    bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
747    if (!bm1)
748        return -ENOMEM;
749
750    error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
751    if (error)
752        goto Free_first_object;
753
754    bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
755    if (!bm2)
756        goto Free_first_bitmap;
757
758    error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
759    if (error)
760        goto Free_second_object;
761
762    forbidden_pages_map = bm1;
763    free_pages_map = bm2;
764    mark_nosave_pages(forbidden_pages_map);
765
766    pr_debug("PM: Basic memory bitmaps created\n");
767
768    return 0;
769
770 Free_second_object:
771    kfree(bm2);
772 Free_first_bitmap:
773     memory_bm_free(bm1, PG_UNSAFE_CLEAR);
774 Free_first_object:
775    kfree(bm1);
776    return -ENOMEM;
777}
778
779/**
780 * free_basic_memory_bitmaps - free memory bitmaps allocated by
781 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary
782 * so that the bitmaps themselves are not referred to while they are being
783 * freed.
784 */
785
786void free_basic_memory_bitmaps(void)
787{
788    struct memory_bitmap *bm1, *bm2;
789
790    BUG_ON(!(forbidden_pages_map && free_pages_map));
791
792    bm1 = forbidden_pages_map;
793    bm2 = free_pages_map;
794    forbidden_pages_map = NULL;
795    free_pages_map = NULL;
796    memory_bm_free(bm1, PG_UNSAFE_CLEAR);
797    kfree(bm1);
798    memory_bm_free(bm2, PG_UNSAFE_CLEAR);
799    kfree(bm2);
800
801    pr_debug("PM: Basic memory bitmaps freed\n");
802}
803
804/**
805 * snapshot_additional_pages - estimate the number of additional pages
806 * be needed for setting up the suspend image data structures for given
807 * zone (usually the returned value is greater than the exact number)
808 */
809
810unsigned int snapshot_additional_pages(struct zone *zone)
811{
812    unsigned int res;
813
814    res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
815    res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
816    return 2 * res;
817}
818
819#ifdef CONFIG_HIGHMEM
820/**
821 * count_free_highmem_pages - compute the total number of free highmem
822 * pages, system-wide.
823 */
824
825static unsigned int count_free_highmem_pages(void)
826{
827    struct zone *zone;
828    unsigned int cnt = 0;
829
830    for_each_populated_zone(zone)
831        if (is_highmem(zone))
832            cnt += zone_page_state(zone, NR_FREE_PAGES);
833
834    return cnt;
835}
836
837/**
838 * saveable_highmem_page - Determine whether a highmem page should be
839 * included in the suspend image.
840 *
841 * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
842 * and it isn't a part of a free chunk of pages.
843 */
844static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)
845{
846    struct page *page;
847
848    if (!pfn_valid(pfn))
849        return NULL;
850
851    page = pfn_to_page(pfn);
852    if (page_zone(page) != zone)
853        return NULL;
854
855    BUG_ON(!PageHighMem(page));
856
857    if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) ||
858        PageReserved(page))
859        return NULL;
860
861    return page;
862}
863
864/**
865 * count_highmem_pages - compute the total number of saveable highmem
866 * pages.
867 */
868
869static unsigned int count_highmem_pages(void)
870{
871    struct zone *zone;
872    unsigned int n = 0;
873
874    for_each_populated_zone(zone) {
875        unsigned long pfn, max_zone_pfn;
876
877        if (!is_highmem(zone))
878            continue;
879
880        mark_free_pages(zone);
881        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
882        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
883            if (saveable_highmem_page(zone, pfn))
884                n++;
885    }
886    return n;
887}
888#else
889static inline void *saveable_highmem_page(struct zone *z, unsigned long p)
890{
891    return NULL;
892}
893#endif /* CONFIG_HIGHMEM */
894
895/**
896 * saveable_page - Determine whether a non-highmem page should be included
897 * in the suspend image.
898 *
899 * We should save the page if it isn't Nosave, and is not in the range
900 * of pages statically defined as 'unsaveable', and it isn't a part of
901 * a free chunk of pages.
902 */
903static struct page *saveable_page(struct zone *zone, unsigned long pfn)
904{
905    struct page *page;
906
907    if (!pfn_valid(pfn))
908        return NULL;
909
910    page = pfn_to_page(pfn);
911    if (page_zone(page) != zone)
912        return NULL;
913
914    BUG_ON(PageHighMem(page));
915
916    if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
917        return NULL;
918
919    if (PageReserved(page)
920        && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
921        return NULL;
922
923    return page;
924}
925
926/**
927 * count_data_pages - compute the total number of saveable non-highmem
928 * pages.
929 */
930
931static unsigned int count_data_pages(void)
932{
933    struct zone *zone;
934    unsigned long pfn, max_zone_pfn;
935    unsigned int n = 0;
936
937    for_each_populated_zone(zone) {
938        if (is_highmem(zone))
939            continue;
940
941        mark_free_pages(zone);
942        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
943        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
944            if (saveable_page(zone, pfn))
945                n++;
946    }
947    return n;
948}
949
950/* This is needed, because copy_page and memcpy are not usable for copying
951 * task structs.
952 */
953static inline void do_copy_page(long *dst, long *src)
954{
955    int n;
956
957    for (n = PAGE_SIZE / sizeof(long); n; n--)
958        *dst++ = *src++;
959}
960
961
962/**
963 * safe_copy_page - check if the page we are going to copy is marked as
964 * present in the kernel page tables (this always is the case if
965 * CONFIG_DEBUG_PAGEALLOC is not set and in that case
966 * kernel_page_present() always returns 'true').
967 */
968static void safe_copy_page(void *dst, struct page *s_page)
969{
970    if (kernel_page_present(s_page)) {
971        do_copy_page(dst, page_address(s_page));
972    } else {
973        kernel_map_pages(s_page, 1, 1);
974        do_copy_page(dst, page_address(s_page));
975        kernel_map_pages(s_page, 1, 0);
976    }
977}
978
979
980#ifdef CONFIG_HIGHMEM
981static inline struct page *
982page_is_saveable(struct zone *zone, unsigned long pfn)
983{
984    return is_highmem(zone) ?
985        saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
986}
987
988static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
989{
990    struct page *s_page, *d_page;
991    void *src, *dst;
992
993    s_page = pfn_to_page(src_pfn);
994    d_page = pfn_to_page(dst_pfn);
995    if (PageHighMem(s_page)) {
996        src = kmap_atomic(s_page, KM_USER0);
997        dst = kmap_atomic(d_page, KM_USER1);
998        do_copy_page(dst, src);
999        kunmap_atomic(dst, KM_USER1);
1000        kunmap_atomic(src, KM_USER0);
1001    } else {
1002        if (PageHighMem(d_page)) {
1003            /* Page pointed to by src may contain some kernel
1004             * data modified by kmap_atomic()
1005             */
1006            safe_copy_page(buffer, s_page);
1007            dst = kmap_atomic(d_page, KM_USER0);
1008            copy_page(dst, buffer);
1009            kunmap_atomic(dst, KM_USER0);
1010        } else {
1011            safe_copy_page(page_address(d_page), s_page);
1012        }
1013    }
1014}
1015#else
1016#define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
1017
1018static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1019{
1020    safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1021                pfn_to_page(src_pfn));
1022}
1023#endif /* CONFIG_HIGHMEM */
1024
1025static void
1026copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
1027{
1028    struct zone *zone;
1029    unsigned long pfn;
1030
1031    for_each_populated_zone(zone) {
1032        unsigned long max_zone_pfn;
1033
1034        mark_free_pages(zone);
1035        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1036        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1037            if (page_is_saveable(zone, pfn))
1038                memory_bm_set_bit(orig_bm, pfn);
1039    }
1040    memory_bm_position_reset(orig_bm);
1041    memory_bm_position_reset(copy_bm);
1042    for(;;) {
1043        pfn = memory_bm_next_pfn(orig_bm);
1044        if (unlikely(pfn == BM_END_OF_MAP))
1045            break;
1046        copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1047    }
1048}
1049
1050/* Total number of image pages */
1051static unsigned int nr_copy_pages;
1052/* Number of pages needed for saving the original pfns of the image pages */
1053static unsigned int nr_meta_pages;
1054/*
1055 * Numbers of normal and highmem page frames allocated for hibernation image
1056 * before suspending devices.
1057 */
1058unsigned int alloc_normal, alloc_highmem;
1059/*
1060 * Memory bitmap used for marking saveable pages (during hibernation) or
1061 * hibernation image pages (during restore)
1062 */
1063static struct memory_bitmap orig_bm;
1064/*
1065 * Memory bitmap used during hibernation for marking allocated page frames that
1066 * will contain copies of saveable pages. During restore it is initially used
1067 * for marking hibernation image pages, but then the set bits from it are
1068 * duplicated in @orig_bm and it is released. On highmem systems it is next
1069 * used for marking "safe" highmem pages, but it has to be reinitialized for
1070 * this purpose.
1071 */
1072static struct memory_bitmap copy_bm;
1073
1074/**
1075 * swsusp_free - free pages allocated for the suspend.
1076 *
1077 * Suspend pages are alocated before the atomic copy is made, so we
1078 * need to release them after the resume.
1079 */
1080
1081void swsusp_free(void)
1082{
1083    struct zone *zone;
1084    unsigned long pfn, max_zone_pfn;
1085
1086    for_each_populated_zone(zone) {
1087        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1088        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1089            if (pfn_valid(pfn)) {
1090                struct page *page = pfn_to_page(pfn);
1091
1092                if (swsusp_page_is_forbidden(page) &&
1093                    swsusp_page_is_free(page)) {
1094                    swsusp_unset_page_forbidden(page);
1095                    swsusp_unset_page_free(page);
1096                    __free_page(page);
1097                }
1098            }
1099    }
1100    nr_copy_pages = 0;
1101    nr_meta_pages = 0;
1102    restore_pblist = NULL;
1103    buffer = NULL;
1104    alloc_normal = 0;
1105    alloc_highmem = 0;
1106}
1107
1108/* Helper functions used for the shrinking of memory. */
1109
1110#define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN)
1111
1112/**
1113 * preallocate_image_pages - Allocate a number of pages for hibernation image
1114 * @nr_pages: Number of page frames to allocate.
1115 * @mask: GFP flags to use for the allocation.
1116 *
1117 * Return value: Number of page frames actually allocated
1118 */
1119static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1120{
1121    unsigned long nr_alloc = 0;
1122
1123    while (nr_pages > 0) {
1124        struct page *page;
1125
1126        page = alloc_image_page(mask);
1127        if (!page)
1128            break;
1129        memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1130        if (PageHighMem(page))
1131            alloc_highmem++;
1132        else
1133            alloc_normal++;
1134        nr_pages--;
1135        nr_alloc++;
1136    }
1137
1138    return nr_alloc;
1139}
1140
1141static unsigned long preallocate_image_memory(unsigned long nr_pages,
1142                          unsigned long avail_normal)
1143{
1144    unsigned long alloc;
1145
1146    if (avail_normal <= alloc_normal)
1147        return 0;
1148
1149    alloc = avail_normal - alloc_normal;
1150    if (nr_pages < alloc)
1151        alloc = nr_pages;
1152
1153    return preallocate_image_pages(alloc, GFP_IMAGE);
1154}
1155
1156#ifdef CONFIG_HIGHMEM
1157static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1158{
1159    return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
1160}
1161
1162/**
1163 * __fraction - Compute (an approximation of) x * (multiplier / base)
1164 */
1165static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1166{
1167    x *= multiplier;
1168    do_div(x, base);
1169    return (unsigned long)x;
1170}
1171
1172static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1173                        unsigned long highmem,
1174                        unsigned long total)
1175{
1176    unsigned long alloc = __fraction(nr_pages, highmem, total);
1177
1178    return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
1179}
1180#else /* CONFIG_HIGHMEM */
1181static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1182{
1183    return 0;
1184}
1185
1186static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1187                        unsigned long highmem,
1188                        unsigned long total)
1189{
1190    return 0;
1191}
1192#endif /* CONFIG_HIGHMEM */
1193
1194/**
1195 * free_unnecessary_pages - Release preallocated pages not needed for the image
1196 */
1197static void free_unnecessary_pages(void)
1198{
1199    unsigned long save, to_free_normal, to_free_highmem;
1200
1201    save = count_data_pages();
1202    if (alloc_normal >= save) {
1203        to_free_normal = alloc_normal - save;
1204        save = 0;
1205    } else {
1206        to_free_normal = 0;
1207        save -= alloc_normal;
1208    }
1209    save += count_highmem_pages();
1210    if (alloc_highmem >= save) {
1211        to_free_highmem = alloc_highmem - save;
1212    } else {
1213        to_free_highmem = 0;
1214        save -= alloc_highmem;
1215        if (to_free_normal > save)
1216            to_free_normal -= save;
1217        else
1218            to_free_normal = 0;
1219    }
1220
1221    memory_bm_position_reset(&copy_bm);
1222
1223    while (to_free_normal > 0 || to_free_highmem > 0) {
1224        unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1225        struct page *page = pfn_to_page(pfn);
1226
1227        if (PageHighMem(page)) {
1228            if (!to_free_highmem)
1229                continue;
1230            to_free_highmem--;
1231            alloc_highmem--;
1232        } else {
1233            if (!to_free_normal)
1234                continue;
1235            to_free_normal--;
1236            alloc_normal--;
1237        }
1238        memory_bm_clear_bit(&copy_bm, pfn);
1239        swsusp_unset_page_forbidden(page);
1240        swsusp_unset_page_free(page);
1241        __free_page(page);
1242    }
1243}
1244
1245/**
1246 * minimum_image_size - Estimate the minimum acceptable size of an image
1247 * @saveable: Number of saveable pages in the system.
1248 *
1249 * We want to avoid attempting to free too much memory too hard, so estimate the
1250 * minimum acceptable size of a hibernation image to use as the lower limit for
1251 * preallocating memory.
1252 *
1253 * We assume that the minimum image size should be proportional to
1254 *
1255 * [number of saveable pages] - [number of pages that can be freed in theory]
1256 *
1257 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1258 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
1259 * minus mapped file pages.
1260 */
1261static unsigned long minimum_image_size(unsigned long saveable)
1262{
1263    unsigned long size;
1264
1265    size = global_page_state(NR_SLAB_RECLAIMABLE)
1266        + global_page_state(NR_ACTIVE_ANON)
1267        + global_page_state(NR_INACTIVE_ANON)
1268        + global_page_state(NR_ACTIVE_FILE)
1269        + global_page_state(NR_INACTIVE_FILE)
1270        - global_page_state(NR_FILE_MAPPED);
1271
1272    return saveable <= size ? 0 : saveable - size;
1273}
1274
1275/**
1276 * hibernate_preallocate_memory - Preallocate memory for hibernation image
1277 *
1278 * To create a hibernation image it is necessary to make a copy of every page
1279 * frame in use. We also need a number of page frames to be free during
1280 * hibernation for allocations made while saving the image and for device
1281 * drivers, in case they need to allocate memory from their hibernation
1282 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1283 * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through
1284 * /sys/power/reserved_size, respectively). To make this happen, we compute the
1285 * total number of available page frames and allocate at least
1286 *
1287 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2
1288 * + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1289 *
1290 * of them, which corresponds to the maximum size of a hibernation image.
1291 *
1292 * If image_size is set below the number following from the above formula,
1293 * the preallocation of memory is continued until the total number of saveable
1294 * pages in the system is below the requested image size or the minimum
1295 * acceptable image size returned by minimum_image_size(), whichever is greater.
1296 */
1297int hibernate_preallocate_memory(void)
1298{
1299    struct zone *zone;
1300    unsigned long saveable, size, max_size, count, highmem, pages = 0;
1301    unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1302    struct timeval start, stop;
1303    int error;
1304
1305    printk(KERN_INFO "PM: Preallocating image memory... ");
1306    do_gettimeofday(&start);
1307
1308    error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1309    if (error)
1310        goto err_out;
1311
1312    error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1313    if (error)
1314        goto err_out;
1315
1316    alloc_normal = 0;
1317    alloc_highmem = 0;
1318
1319    /* Count the number of saveable data pages. */
1320    save_highmem = count_highmem_pages();
1321    saveable = count_data_pages();
1322
1323    /*
1324     * Compute the total number of page frames we can use (count) and the
1325     * number of pages needed for image metadata (size).
1326     */
1327    count = saveable;
1328    saveable += save_highmem;
1329    highmem = save_highmem;
1330    size = 0;
1331    for_each_populated_zone(zone) {
1332        size += snapshot_additional_pages(zone);
1333        if (is_highmem(zone))
1334            highmem += zone_page_state(zone, NR_FREE_PAGES);
1335        else
1336            count += zone_page_state(zone, NR_FREE_PAGES);
1337    }
1338    avail_normal = count;
1339    count += highmem;
1340    count -= totalreserve_pages;
1341
1342    /* Compute the maximum number of saveable pages to leave in memory. */
1343    max_size = (count - (size + PAGES_FOR_IO)) / 2
1344            - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
1345    /* Compute the desired number of image pages specified by image_size. */
1346    size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1347    if (size > max_size)
1348        size = max_size;
1349    /*
1350     * If the desired number of image pages is at least as large as the
1351     * current number of saveable pages in memory, allocate page frames for
1352     * the image and we're done.
1353     */
1354    if (size >= saveable) {
1355        pages = preallocate_image_highmem(save_highmem);
1356        pages += preallocate_image_memory(saveable - pages, avail_normal);
1357        goto out;
1358    }
1359
1360    /* Estimate the minimum size of the image. */
1361    pages = minimum_image_size(saveable);
1362    /*
1363     * To avoid excessive pressure on the normal zone, leave room in it to
1364     * accommodate an image of the minimum size (unless it's already too
1365     * small, in which case don't preallocate pages from it at all).
1366     */
1367    if (avail_normal > pages)
1368        avail_normal -= pages;
1369    else
1370        avail_normal = 0;
1371    if (size < pages)
1372        size = min_t(unsigned long, pages, max_size);
1373
1374    /*
1375     * Let the memory management subsystem know that we're going to need a
1376     * large number of page frames to allocate and make it free some memory.
1377     * NOTE: If this is not done, performance will be hurt badly in some
1378     * test cases.
1379     */
1380    shrink_all_memory(saveable - size);
1381
1382    /*
1383     * The number of saveable pages in memory was too high, so apply some
1384     * pressure to decrease it. First, make room for the largest possible
1385     * image and fail if that doesn't work. Next, try to decrease the size
1386     * of the image as much as indicated by 'size' using allocations from
1387     * highmem and non-highmem zones separately.
1388     */
1389    pages_highmem = preallocate_image_highmem(highmem / 2);
1390    alloc = (count - max_size) - pages_highmem;
1391    pages = preallocate_image_memory(alloc, avail_normal);
1392    if (pages < alloc) {
1393        /* We have exhausted non-highmem pages, try highmem. */
1394        alloc -= pages;
1395        pages += pages_highmem;
1396        pages_highmem = preallocate_image_highmem(alloc);
1397        if (pages_highmem < alloc)
1398            goto err_out;
1399        pages += pages_highmem;
1400        /*
1401         * size is the desired number of saveable pages to leave in
1402         * memory, so try to preallocate (all memory - size) pages.
1403         */
1404        alloc = (count - pages) - size;
1405        pages += preallocate_image_highmem(alloc);
1406    } else {
1407        /*
1408         * There are approximately max_size saveable pages at this point
1409         * and we want to reduce this number down to size.
1410         */
1411        alloc = max_size - size;
1412        size = preallocate_highmem_fraction(alloc, highmem, count);
1413        pages_highmem += size;
1414        alloc -= size;
1415        size = preallocate_image_memory(alloc, avail_normal);
1416        pages_highmem += preallocate_image_highmem(alloc - size);
1417        pages += pages_highmem + size;
1418    }
1419
1420    /*
1421     * We only need as many page frames for the image as there are saveable
1422     * pages in memory, but we have allocated more. Release the excessive
1423     * ones now.
1424     */
1425    free_unnecessary_pages();
1426
1427 out:
1428    do_gettimeofday(&stop);
1429    printk(KERN_CONT "done (allocated %lu pages)\n", pages);
1430    swsusp_show_speed(&start, &stop, pages, "Allocated");
1431
1432    return 0;
1433
1434 err_out:
1435    printk(KERN_CONT "\n");
1436    swsusp_free();
1437    return -ENOMEM;
1438}
1439
1440#ifdef CONFIG_HIGHMEM
1441/**
1442  * count_pages_for_highmem - compute the number of non-highmem pages
1443  * that will be necessary for creating copies of highmem pages.
1444  */
1445
1446static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1447{
1448    unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1449
1450    if (free_highmem >= nr_highmem)
1451        nr_highmem = 0;
1452    else
1453        nr_highmem -= free_highmem;
1454
1455    return nr_highmem;
1456}
1457#else
1458static unsigned int
1459count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1460#endif /* CONFIG_HIGHMEM */
1461
1462/**
1463 * enough_free_mem - Make sure we have enough free memory for the
1464 * snapshot image.
1465 */
1466
1467static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1468{
1469    struct zone *zone;
1470    unsigned int free = alloc_normal;
1471
1472    for_each_populated_zone(zone)
1473        if (!is_highmem(zone))
1474            free += zone_page_state(zone, NR_FREE_PAGES);
1475
1476    nr_pages += count_pages_for_highmem(nr_highmem);
1477    pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1478        nr_pages, PAGES_FOR_IO, free);
1479
1480    return free > nr_pages + PAGES_FOR_IO;
1481}
1482
1483#ifdef CONFIG_HIGHMEM
1484/**
1485 * get_highmem_buffer - if there are some highmem pages in the suspend
1486 * image, we may need the buffer to copy them and/or load their data.
1487 */
1488
1489static inline int get_highmem_buffer(int safe_needed)
1490{
1491    buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1492    return buffer ? 0 : -ENOMEM;
1493}
1494
1495/**
1496 * alloc_highmem_image_pages - allocate some highmem pages for the image.
1497 * Try to allocate as many pages as needed, but if the number of free
1498 * highmem pages is lesser than that, allocate them all.
1499 */
1500
1501static inline unsigned int
1502alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1503{
1504    unsigned int to_alloc = count_free_highmem_pages();
1505
1506    if (to_alloc > nr_highmem)
1507        to_alloc = nr_highmem;
1508
1509    nr_highmem -= to_alloc;
1510    while (to_alloc-- > 0) {
1511        struct page *page;
1512
1513        page = alloc_image_page(__GFP_HIGHMEM);
1514        memory_bm_set_bit(bm, page_to_pfn(page));
1515    }
1516    return nr_highmem;
1517}
1518#else
1519static inline int get_highmem_buffer(int safe_needed) { return 0; }
1520
1521static inline unsigned int
1522alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1523#endif /* CONFIG_HIGHMEM */
1524
1525/**
1526 * swsusp_alloc - allocate memory for the suspend image
1527 *
1528 * We first try to allocate as many highmem pages as there are
1529 * saveable highmem pages in the system. If that fails, we allocate
1530 * non-highmem pages for the copies of the remaining highmem ones.
1531 *
1532 * In this approach it is likely that the copies of highmem pages will
1533 * also be located in the high memory, because of the way in which
1534 * copy_data_pages() works.
1535 */
1536
1537static int
1538swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1539        unsigned int nr_pages, unsigned int nr_highmem)
1540{
1541    if (nr_highmem > 0) {
1542        if (get_highmem_buffer(PG_ANY))
1543            goto err_out;
1544        if (nr_highmem > alloc_highmem) {
1545            nr_highmem -= alloc_highmem;
1546            nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
1547        }
1548    }
1549    if (nr_pages > alloc_normal) {
1550        nr_pages -= alloc_normal;
1551        while (nr_pages-- > 0) {
1552            struct page *page;
1553
1554            page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1555            if (!page)
1556                goto err_out;
1557            memory_bm_set_bit(copy_bm, page_to_pfn(page));
1558        }
1559    }
1560
1561    return 0;
1562
1563 err_out:
1564    swsusp_free();
1565    return -ENOMEM;
1566}
1567
1568asmlinkage int swsusp_save(void)
1569{
1570    unsigned int nr_pages, nr_highmem;
1571
1572    printk(KERN_INFO "PM: Creating hibernation image:\n");
1573
1574    drain_local_pages(NULL);
1575    nr_pages = count_data_pages();
1576    nr_highmem = count_highmem_pages();
1577    printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1578
1579    if (!enough_free_mem(nr_pages, nr_highmem)) {
1580        printk(KERN_ERR "PM: Not enough free memory\n");
1581        return -ENOMEM;
1582    }
1583
1584    if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1585        printk(KERN_ERR "PM: Memory allocation failed\n");
1586        return -ENOMEM;
1587    }
1588
1589    /* During allocating of suspend pagedir, new cold pages may appear.
1590     * Kill them.
1591     */
1592    drain_local_pages(NULL);
1593    copy_data_pages(&copy_bm, &orig_bm);
1594
1595    /*
1596     * End of critical section. From now on, we can write to memory,
1597     * but we should not touch disk. This specially means we must _not_
1598     * touch swap space! Except we must write out our image of course.
1599     */
1600
1601    nr_pages += nr_highmem;
1602    nr_copy_pages = nr_pages;
1603    nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1604
1605    printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1606        nr_pages);
1607
1608    return 0;
1609}
1610
1611#ifndef CONFIG_ARCH_HIBERNATION_HEADER
1612static int init_header_complete(struct swsusp_info *info)
1613{
1614    memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1615    info->version_code = LINUX_VERSION_CODE;
1616    return 0;
1617}
1618
1619static char *check_image_kernel(struct swsusp_info *info)
1620{
1621    if (info->version_code != LINUX_VERSION_CODE)
1622        return "kernel version";
1623    if (strcmp(info->uts.sysname,init_utsname()->sysname))
1624        return "system type";
1625    if (strcmp(info->uts.release,init_utsname()->release))
1626        return "kernel release";
1627    if (strcmp(info->uts.version,init_utsname()->version))
1628        return "version";
1629    if (strcmp(info->uts.machine,init_utsname()->machine))
1630        return "machine";
1631    return NULL;
1632}
1633#endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1634
1635unsigned long snapshot_get_image_size(void)
1636{
1637    return nr_copy_pages + nr_meta_pages + 1;
1638}
1639
1640static int init_header(struct swsusp_info *info)
1641{
1642    memset(info, 0, sizeof(struct swsusp_info));
1643    info->num_physpages = num_physpages;
1644    info->image_pages = nr_copy_pages;
1645    info->pages = snapshot_get_image_size();
1646    info->size = info->pages;
1647    info->size <<= PAGE_SHIFT;
1648    return init_header_complete(info);
1649}
1650
1651/**
1652 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1653 * are stored in the array @buf[] (1 page at a time)
1654 */
1655
1656static inline void
1657pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1658{
1659    int j;
1660
1661    for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1662        buf[j] = memory_bm_next_pfn(bm);
1663        if (unlikely(buf[j] == BM_END_OF_MAP))
1664            break;
1665    }
1666}
1667
1668/**
1669 * snapshot_read_next - used for reading the system memory snapshot.
1670 *
1671 * On the first call to it @handle should point to a zeroed
1672 * snapshot_handle structure. The structure gets updated and a pointer
1673 * to it should be passed to this function every next time.
1674 *
1675 * On success the function returns a positive number. Then, the caller
1676 * is allowed to read up to the returned number of bytes from the memory
1677 * location computed by the data_of() macro.
1678 *
1679 * The function returns 0 to indicate the end of data stream condition,
1680 * and a negative number is returned on error. In such cases the
1681 * structure pointed to by @handle is not updated and should not be used
1682 * any more.
1683 */
1684
1685int snapshot_read_next(struct snapshot_handle *handle)
1686{
1687    if (handle->cur > nr_meta_pages + nr_copy_pages)
1688        return 0;
1689
1690    if (!buffer) {
1691        /* This makes the buffer be freed by swsusp_free() */
1692        buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1693        if (!buffer)
1694            return -ENOMEM;
1695    }
1696    if (!handle->cur) {
1697        int error;
1698
1699        error = init_header((struct swsusp_info *)buffer);
1700        if (error)
1701            return error;
1702        handle->buffer = buffer;
1703        memory_bm_position_reset(&orig_bm);
1704        memory_bm_position_reset(&copy_bm);
1705    } else if (handle->cur <= nr_meta_pages) {
1706        clear_page(buffer);
1707        pack_pfns(buffer, &orig_bm);
1708    } else {
1709        struct page *page;
1710
1711        page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1712        if (PageHighMem(page)) {
1713            /* Highmem pages are copied to the buffer,
1714             * because we can't return with a kmapped
1715             * highmem page (we may not be called again).
1716             */
1717            void *kaddr;
1718
1719            kaddr = kmap_atomic(page, KM_USER0);
1720            copy_page(buffer, kaddr);
1721            kunmap_atomic(kaddr, KM_USER0);
1722            handle->buffer = buffer;
1723        } else {
1724            handle->buffer = page_address(page);
1725        }
1726    }
1727    handle->cur++;
1728    return PAGE_SIZE;
1729}
1730
1731/**
1732 * mark_unsafe_pages - mark the pages that cannot be used for storing
1733 * the image during resume, because they conflict with the pages that
1734 * had been used before suspend
1735 */
1736
1737static int mark_unsafe_pages(struct memory_bitmap *bm)
1738{
1739    struct zone *zone;
1740    unsigned long pfn, max_zone_pfn;
1741
1742    /* Clear page flags */
1743    for_each_populated_zone(zone) {
1744        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1745        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1746            if (pfn_valid(pfn))
1747                swsusp_unset_page_free(pfn_to_page(pfn));
1748    }
1749
1750    /* Mark pages that correspond to the "original" pfns as "unsafe" */
1751    memory_bm_position_reset(bm);
1752    do {
1753        pfn = memory_bm_next_pfn(bm);
1754        if (likely(pfn != BM_END_OF_MAP)) {
1755            if (likely(pfn_valid(pfn)))
1756                swsusp_set_page_free(pfn_to_page(pfn));
1757            else
1758                return -EFAULT;
1759        }
1760    } while (pfn != BM_END_OF_MAP);
1761
1762    allocated_unsafe_pages = 0;
1763
1764    return 0;
1765}
1766
1767static void
1768duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1769{
1770    unsigned long pfn;
1771
1772    memory_bm_position_reset(src);
1773    pfn = memory_bm_next_pfn(src);
1774    while (pfn != BM_END_OF_MAP) {
1775        memory_bm_set_bit(dst, pfn);
1776        pfn = memory_bm_next_pfn(src);
1777    }
1778}
1779
1780static int check_header(struct swsusp_info *info)
1781{
1782    char *reason;
1783
1784    reason = check_image_kernel(info);
1785    if (!reason && info->num_physpages != num_physpages)
1786        reason = "memory size";
1787    if (reason) {
1788        printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1789        return -EPERM;
1790    }
1791    return 0;
1792}
1793
1794/**
1795 * load header - check the image header and copy data from it
1796 */
1797
1798static int
1799load_header(struct swsusp_info *info)
1800{
1801    int error;
1802
1803    restore_pblist = NULL;
1804    error = check_header(info);
1805    if (!error) {
1806        nr_copy_pages = info->image_pages;
1807        nr_meta_pages = info->pages - info->image_pages - 1;
1808    }
1809    return error;
1810}
1811
1812/**
1813 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1814 * the corresponding bit in the memory bitmap @bm
1815 */
1816static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1817{
1818    int j;
1819
1820    for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1821        if (unlikely(buf[j] == BM_END_OF_MAP))
1822            break;
1823
1824        if (memory_bm_pfn_present(bm, buf[j]))
1825            memory_bm_set_bit(bm, buf[j]);
1826        else
1827            return -EFAULT;
1828    }
1829
1830    return 0;
1831}
1832
1833/* List of "safe" pages that may be used to store data loaded from the suspend
1834 * image
1835 */
1836static struct linked_page *safe_pages_list;
1837
1838#ifdef CONFIG_HIGHMEM
1839/* struct highmem_pbe is used for creating the list of highmem pages that
1840 * should be restored atomically during the resume from disk, because the page
1841 * frames they have occupied before the suspend are in use.
1842 */
1843struct highmem_pbe {
1844    struct page *copy_page; /* data is here now */
1845    struct page *orig_page; /* data was here before the suspend */
1846    struct highmem_pbe *next;
1847};
1848
1849/* List of highmem PBEs needed for restoring the highmem pages that were
1850 * allocated before the suspend and included in the suspend image, but have
1851 * also been allocated by the "resume" kernel, so their contents cannot be
1852 * written directly to their "original" page frames.
1853 */
1854static struct highmem_pbe *highmem_pblist;
1855
1856/**
1857 * count_highmem_image_pages - compute the number of highmem pages in the
1858 * suspend image. The bits in the memory bitmap @bm that correspond to the
1859 * image pages are assumed to be set.
1860 */
1861
1862static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1863{
1864    unsigned long pfn;
1865    unsigned int cnt = 0;
1866
1867    memory_bm_position_reset(bm);
1868    pfn = memory_bm_next_pfn(bm);
1869    while (pfn != BM_END_OF_MAP) {
1870        if (PageHighMem(pfn_to_page(pfn)))
1871            cnt++;
1872
1873        pfn = memory_bm_next_pfn(bm);
1874    }
1875    return cnt;
1876}
1877
1878/**
1879 * prepare_highmem_image - try to allocate as many highmem pages as
1880 * there are highmem image pages (@nr_highmem_p points to the variable
1881 * containing the number of highmem image pages). The pages that are
1882 * "safe" (ie. will not be overwritten when the suspend image is
1883 * restored) have the corresponding bits set in @bm (it must be
1884 * unitialized).
1885 *
1886 * NOTE: This function should not be called if there are no highmem
1887 * image pages.
1888 */
1889
1890static unsigned int safe_highmem_pages;
1891
1892static struct memory_bitmap *safe_highmem_bm;
1893
1894static int
1895prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1896{
1897    unsigned int to_alloc;
1898
1899    if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1900        return -ENOMEM;
1901
1902    if (get_highmem_buffer(PG_SAFE))
1903        return -ENOMEM;
1904
1905    to_alloc = count_free_highmem_pages();
1906    if (to_alloc > *nr_highmem_p)
1907        to_alloc = *nr_highmem_p;
1908    else
1909        *nr_highmem_p = to_alloc;
1910
1911    safe_highmem_pages = 0;
1912    while (to_alloc-- > 0) {
1913        struct page *page;
1914
1915        page = alloc_page(__GFP_HIGHMEM);
1916        if (!swsusp_page_is_free(page)) {
1917            /* The page is "safe", set its bit the bitmap */
1918            memory_bm_set_bit(bm, page_to_pfn(page));
1919            safe_highmem_pages++;
1920        }
1921        /* Mark the page as allocated */
1922        swsusp_set_page_forbidden(page);
1923        swsusp_set_page_free(page);
1924    }
1925    memory_bm_position_reset(bm);
1926    safe_highmem_bm = bm;
1927    return 0;
1928}
1929
1930/**
1931 * get_highmem_page_buffer - for given highmem image page find the buffer
1932 * that suspend_write_next() should set for its caller to write to.
1933 *
1934 * If the page is to be saved to its "original" page frame or a copy of
1935 * the page is to be made in the highmem, @buffer is returned. Otherwise,
1936 * the copy of the page is to be made in normal memory, so the address of
1937 * the copy is returned.
1938 *
1939 * If @buffer is returned, the caller of suspend_write_next() will write
1940 * the page's contents to @buffer, so they will have to be copied to the
1941 * right location on the next call to suspend_write_next() and it is done
1942 * with the help of copy_last_highmem_page(). For this purpose, if
1943 * @buffer is returned, @last_highmem page is set to the page to which
1944 * the data will have to be copied from @buffer.
1945 */
1946
1947static struct page *last_highmem_page;
1948
1949static void *
1950get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1951{
1952    struct highmem_pbe *pbe;
1953    void *kaddr;
1954
1955    if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1956        /* We have allocated the "original" page frame and we can
1957         * use it directly to store the loaded page.
1958         */
1959        last_highmem_page = page;
1960        return buffer;
1961    }
1962    /* The "original" page frame has not been allocated and we have to
1963     * use a "safe" page frame to store the loaded page.
1964     */
1965    pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1966    if (!pbe) {
1967        swsusp_free();
1968        return ERR_PTR(-ENOMEM);
1969    }
1970    pbe->orig_page = page;
1971    if (safe_highmem_pages > 0) {
1972        struct page *tmp;
1973
1974        /* Copy of the page will be stored in high memory */
1975        kaddr = buffer;
1976        tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1977        safe_highmem_pages--;
1978        last_highmem_page = tmp;
1979        pbe->copy_page = tmp;
1980    } else {
1981        /* Copy of the page will be stored in normal memory */
1982        kaddr = safe_pages_list;
1983        safe_pages_list = safe_pages_list->next;
1984        pbe->copy_page = virt_to_page(kaddr);
1985    }
1986    pbe->next = highmem_pblist;
1987    highmem_pblist = pbe;
1988    return kaddr;
1989}
1990
1991/**
1992 * copy_last_highmem_page - copy the contents of a highmem image from
1993 * @buffer, where the caller of snapshot_write_next() has place them,
1994 * to the right location represented by @last_highmem_page .
1995 */
1996
1997static void copy_last_highmem_page(void)
1998{
1999    if (last_highmem_page) {
2000        void *dst;
2001
2002        dst = kmap_atomic(last_highmem_page, KM_USER0);
2003        copy_page(dst, buffer);
2004        kunmap_atomic(dst, KM_USER0);
2005        last_highmem_page = NULL;
2006    }
2007}
2008
2009static inline int last_highmem_page_copied(void)
2010{
2011    return !last_highmem_page;
2012}
2013
2014static inline void free_highmem_data(void)
2015{
2016    if (safe_highmem_bm)
2017        memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
2018
2019    if (buffer)
2020        free_image_page(buffer, PG_UNSAFE_CLEAR);
2021}
2022#else
2023static inline int get_safe_write_buffer(void) { return 0; }
2024
2025static unsigned int
2026count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2027
2028static inline int
2029prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
2030{
2031    return 0;
2032}
2033
2034static inline void *
2035get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
2036{
2037    return ERR_PTR(-EINVAL);
2038}
2039
2040static inline void copy_last_highmem_page(void) {}
2041static inline int last_highmem_page_copied(void) { return 1; }
2042static inline void free_highmem_data(void) {}
2043#endif /* CONFIG_HIGHMEM */
2044
2045/**
2046 * prepare_image - use the memory bitmap @bm to mark the pages that will
2047 * be overwritten in the process of restoring the system memory state
2048 * from the suspend image ("unsafe" pages) and allocate memory for the
2049 * image.
2050 *
2051 * The idea is to allocate a new memory bitmap first and then allocate
2052 * as many pages as needed for the image data, but not to assign these
2053 * pages to specific tasks initially. Instead, we just mark them as
2054 * allocated and create a lists of "safe" pages that will be used
2055 * later. On systems with high memory a list of "safe" highmem pages is
2056 * also created.
2057 */
2058
2059#define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2060
2061static int
2062prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
2063{
2064    unsigned int nr_pages, nr_highmem;
2065    struct linked_page *sp_list, *lp;
2066    int error;
2067
2068    /* If there is no highmem, the buffer will not be necessary */
2069    free_image_page(buffer, PG_UNSAFE_CLEAR);
2070    buffer = NULL;
2071
2072    nr_highmem = count_highmem_image_pages(bm);
2073    error = mark_unsafe_pages(bm);
2074    if (error)
2075        goto Free;
2076
2077    error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2078    if (error)
2079        goto Free;
2080
2081    duplicate_memory_bitmap(new_bm, bm);
2082    memory_bm_free(bm, PG_UNSAFE_KEEP);
2083    if (nr_highmem > 0) {
2084        error = prepare_highmem_image(bm, &nr_highmem);
2085        if (error)
2086            goto Free;
2087    }
2088    /* Reserve some safe pages for potential later use.
2089     *
2090     * NOTE: This way we make sure there will be enough safe pages for the
2091     * chain_alloc() in get_buffer(). It is a bit wasteful, but
2092     * nr_copy_pages cannot be greater than 50% of the memory anyway.
2093     */
2094    sp_list = NULL;
2095    /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2096    nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2097    nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2098    while (nr_pages > 0) {
2099        lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2100        if (!lp) {
2101            error = -ENOMEM;
2102            goto Free;
2103        }
2104        lp->next = sp_list;
2105        sp_list = lp;
2106        nr_pages--;
2107    }
2108    /* Preallocate memory for the image */
2109    safe_pages_list = NULL;
2110    nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2111    while (nr_pages > 0) {
2112        lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2113        if (!lp) {
2114            error = -ENOMEM;
2115            goto Free;
2116        }
2117        if (!swsusp_page_is_free(virt_to_page(lp))) {
2118            /* The page is "safe", add it to the list */
2119            lp->next = safe_pages_list;
2120            safe_pages_list = lp;
2121        }
2122        /* Mark the page as allocated */
2123        swsusp_set_page_forbidden(virt_to_page(lp));
2124        swsusp_set_page_free(virt_to_page(lp));
2125        nr_pages--;
2126    }
2127    /* Free the reserved safe pages so that chain_alloc() can use them */
2128    while (sp_list) {
2129        lp = sp_list->next;
2130        free_image_page(sp_list, PG_UNSAFE_CLEAR);
2131        sp_list = lp;
2132    }
2133    return 0;
2134
2135 Free:
2136    swsusp_free();
2137    return error;
2138}
2139
2140/**
2141 * get_buffer - compute the address that snapshot_write_next() should
2142 * set for its caller to write to.
2143 */
2144
2145static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
2146{
2147    struct pbe *pbe;
2148    struct page *page;
2149    unsigned long pfn = memory_bm_next_pfn(bm);
2150
2151    if (pfn == BM_END_OF_MAP)
2152        return ERR_PTR(-EFAULT);
2153
2154    page = pfn_to_page(pfn);
2155    if (PageHighMem(page))
2156        return get_highmem_page_buffer(page, ca);
2157
2158    if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2159        /* We have allocated the "original" page frame and we can
2160         * use it directly to store the loaded page.
2161         */
2162        return page_address(page);
2163
2164    /* The "original" page frame has not been allocated and we have to
2165     * use a "safe" page frame to store the loaded page.
2166     */
2167    pbe = chain_alloc(ca, sizeof(struct pbe));
2168    if (!pbe) {
2169        swsusp_free();
2170        return ERR_PTR(-ENOMEM);
2171    }
2172    pbe->orig_address = page_address(page);
2173    pbe->address = safe_pages_list;
2174    safe_pages_list = safe_pages_list->next;
2175    pbe->next = restore_pblist;
2176    restore_pblist = pbe;
2177    return pbe->address;
2178}
2179
2180/**
2181 * snapshot_write_next - used for writing the system memory snapshot.
2182 *
2183 * On the first call to it @handle should point to a zeroed
2184 * snapshot_handle structure. The structure gets updated and a pointer
2185 * to it should be passed to this function every next time.
2186 *
2187 * On success the function returns a positive number. Then, the caller
2188 * is allowed to write up to the returned number of bytes to the memory
2189 * location computed by the data_of() macro.
2190 *
2191 * The function returns 0 to indicate the "end of file" condition,
2192 * and a negative number is returned on error. In such cases the
2193 * structure pointed to by @handle is not updated and should not be used
2194 * any more.
2195 */
2196
2197int snapshot_write_next(struct snapshot_handle *handle)
2198{
2199    static struct chain_allocator ca;
2200    int error = 0;
2201
2202    /* Check if we have already loaded the entire image */
2203    if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages)
2204        return 0;
2205
2206    handle->sync_read = 1;
2207
2208    if (!handle->cur) {
2209        if (!buffer)
2210            /* This makes the buffer be freed by swsusp_free() */
2211            buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2212
2213        if (!buffer)
2214            return -ENOMEM;
2215
2216        handle->buffer = buffer;
2217    } else if (handle->cur == 1) {
2218        error = load_header(buffer);
2219        if (error)
2220            return error;
2221
2222        error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2223        if (error)
2224            return error;
2225
2226    } else if (handle->cur <= nr_meta_pages + 1) {
2227        error = unpack_orig_pfns(buffer, &copy_bm);
2228        if (error)
2229            return error;
2230
2231        if (handle->cur == nr_meta_pages + 1) {
2232            error = prepare_image(&orig_bm, &copy_bm);
2233            if (error)
2234                return error;
2235
2236            chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2237            memory_bm_position_reset(&orig_bm);
2238            restore_pblist = NULL;
2239            handle->buffer = get_buffer(&orig_bm, &ca);
2240            handle->sync_read = 0;
2241            if (IS_ERR(handle->buffer))
2242                return PTR_ERR(handle->buffer);
2243        }
2244    } else {
2245        copy_last_highmem_page();
2246        handle->buffer = get_buffer(&orig_bm, &ca);
2247        if (IS_ERR(handle->buffer))
2248            return PTR_ERR(handle->buffer);
2249        if (handle->buffer != buffer)
2250            handle->sync_read = 0;
2251    }
2252    handle->cur++;
2253    return PAGE_SIZE;
2254}
2255
2256/**
2257 * snapshot_write_finalize - must be called after the last call to
2258 * snapshot_write_next() in case the last page in the image happens
2259 * to be a highmem page and its contents should be stored in the
2260 * highmem. Additionally, it releases the memory that will not be
2261 * used any more.
2262 */
2263
2264void snapshot_write_finalize(struct snapshot_handle *handle)
2265{
2266    copy_last_highmem_page();
2267    /* Free only if we have loaded the image entirely */
2268    if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {
2269        memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
2270        free_highmem_data();
2271    }
2272}
2273
2274int snapshot_image_loaded(struct snapshot_handle *handle)
2275{
2276    return !(!nr_copy_pages || !last_highmem_page_copied() ||
2277            handle->cur <= nr_meta_pages + nr_copy_pages);
2278}
2279
2280#ifdef CONFIG_HIGHMEM
2281/* Assumes that @buf is ready and points to a "safe" page */
2282static inline void
2283swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
2284{
2285    void *kaddr1, *kaddr2;
2286
2287    kaddr1 = kmap_atomic(p1, KM_USER0);
2288    kaddr2 = kmap_atomic(p2, KM_USER1);
2289    copy_page(buf, kaddr1);
2290    copy_page(kaddr1, kaddr2);
2291    copy_page(kaddr2, buf);
2292    kunmap_atomic(kaddr2, KM_USER1);
2293    kunmap_atomic(kaddr1, KM_USER0);
2294}
2295
2296/**
2297 * restore_highmem - for each highmem page that was allocated before
2298 * the suspend and included in the suspend image, and also has been
2299 * allocated by the "resume" kernel swap its current (ie. "before
2300 * resume") contents with the previous (ie. "before suspend") one.
2301 *
2302 * If the resume eventually fails, we can call this function once
2303 * again and restore the "before resume" highmem state.
2304 */
2305
2306int restore_highmem(void)
2307{
2308    struct highmem_pbe *pbe = highmem_pblist;
2309    void *buf;
2310
2311    if (!pbe)
2312        return 0;
2313
2314    buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2315    if (!buf)
2316        return -ENOMEM;
2317
2318    while (pbe) {
2319        swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2320        pbe = pbe->next;
2321    }
2322    free_image_page(buf, PG_UNSAFE_CLEAR);
2323    return 0;
2324}
2325#endif /* CONFIG_HIGHMEM */
2326

Archive Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9



interactive