Root/kernel/power/snapshot.c

1/*
2 * linux/kernel/power/snapshot.c
3 *
4 * This file provides system snapshot/restore functionality for swsusp.
5 *
6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8 *
9 * This file is released under the GPLv2.
10 *
11 */
12
13#include <linux/version.h>
14#include <linux/module.h>
15#include <linux/mm.h>
16#include <linux/suspend.h>
17#include <linux/delay.h>
18#include <linux/bitops.h>
19#include <linux/spinlock.h>
20#include <linux/kernel.h>
21#include <linux/pm.h>
22#include <linux/device.h>
23#include <linux/init.h>
24#include <linux/bootmem.h>
25#include <linux/syscalls.h>
26#include <linux/console.h>
27#include <linux/highmem.h>
28#include <linux/list.h>
29#include <linux/slab.h>
30
31#include <asm/uaccess.h>
32#include <asm/mmu_context.h>
33#include <asm/pgtable.h>
34#include <asm/tlbflush.h>
35#include <asm/io.h>
36
37#include "power.h"
38
39static int swsusp_page_is_free(struct page *);
40static void swsusp_set_page_forbidden(struct page *);
41static void swsusp_unset_page_forbidden(struct page *);
42
43/*
44 * Preferred image size in bytes (tunable via /sys/power/image_size).
45 * When it is set to N, swsusp will do its best to ensure the image
46 * size will not exceed N bytes, but if that is impossible, it will
47 * try to create the smallest image possible.
48 */
49unsigned long image_size;
50
51void __init hibernate_image_size_init(void)
52{
53    image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE;
54}
55
56/* List of PBEs needed for restoring the pages that were allocated before
57 * the suspend and included in the suspend image, but have also been
58 * allocated by the "resume" kernel, so their contents cannot be written
59 * directly to their "original" page frames.
60 */
61struct pbe *restore_pblist;
62
63/* Pointer to an auxiliary buffer (1 page) */
64static void *buffer;
65
66/**
67 * @safe_needed - on resume, for storing the PBE list and the image,
68 * we can only use memory pages that do not conflict with the pages
69 * used before suspend. The unsafe pages have PageNosaveFree set
70 * and we count them using unsafe_pages.
71 *
72 * Each allocated image page is marked as PageNosave and PageNosaveFree
73 * so that swsusp_free() can release it.
74 */
75
76#define PG_ANY 0
77#define PG_SAFE 1
78#define PG_UNSAFE_CLEAR 1
79#define PG_UNSAFE_KEEP 0
80
81static unsigned int allocated_unsafe_pages;
82
83static void *get_image_page(gfp_t gfp_mask, int safe_needed)
84{
85    void *res;
86
87    res = (void *)get_zeroed_page(gfp_mask);
88    if (safe_needed)
89        while (res && swsusp_page_is_free(virt_to_page(res))) {
90            /* The page is unsafe, mark it for swsusp_free() */
91            swsusp_set_page_forbidden(virt_to_page(res));
92            allocated_unsafe_pages++;
93            res = (void *)get_zeroed_page(gfp_mask);
94        }
95    if (res) {
96        swsusp_set_page_forbidden(virt_to_page(res));
97        swsusp_set_page_free(virt_to_page(res));
98    }
99    return res;
100}
101
102unsigned long get_safe_page(gfp_t gfp_mask)
103{
104    return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
105}
106
107static struct page *alloc_image_page(gfp_t gfp_mask)
108{
109    struct page *page;
110
111    page = alloc_page(gfp_mask);
112    if (page) {
113        swsusp_set_page_forbidden(page);
114        swsusp_set_page_free(page);
115    }
116    return page;
117}
118
119/**
120 * free_image_page - free page represented by @addr, allocated with
121 * get_image_page (page flags set by it must be cleared)
122 */
123
124static inline void free_image_page(void *addr, int clear_nosave_free)
125{
126    struct page *page;
127
128    BUG_ON(!virt_addr_valid(addr));
129
130    page = virt_to_page(addr);
131
132    swsusp_unset_page_forbidden(page);
133    if (clear_nosave_free)
134        swsusp_unset_page_free(page);
135
136    __free_page(page);
137}
138
139/* struct linked_page is used to build chains of pages */
140
141#define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
142
143struct linked_page {
144    struct linked_page *next;
145    char data[LINKED_PAGE_DATA_SIZE];
146} __attribute__((packed));
147
148static inline void
149free_list_of_pages(struct linked_page *list, int clear_page_nosave)
150{
151    while (list) {
152        struct linked_page *lp = list->next;
153
154        free_image_page(list, clear_page_nosave);
155        list = lp;
156    }
157}
158
159/**
160  * struct chain_allocator is used for allocating small objects out of
161  * a linked list of pages called 'the chain'.
162  *
163  * The chain grows each time when there is no room for a new object in
164  * the current page. The allocated objects cannot be freed individually.
165  * It is only possible to free them all at once, by freeing the entire
166  * chain.
167  *
168  * NOTE: The chain allocator may be inefficient if the allocated objects
169  * are not much smaller than PAGE_SIZE.
170  */
171
172struct chain_allocator {
173    struct linked_page *chain; /* the chain */
174    unsigned int used_space; /* total size of objects allocated out
175                     * of the current page
176                     */
177    gfp_t gfp_mask; /* mask for allocating pages */
178    int safe_needed; /* if set, only "safe" pages are allocated */
179};
180
181static void
182chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
183{
184    ca->chain = NULL;
185    ca->used_space = LINKED_PAGE_DATA_SIZE;
186    ca->gfp_mask = gfp_mask;
187    ca->safe_needed = safe_needed;
188}
189
190static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
191{
192    void *ret;
193
194    if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
195        struct linked_page *lp;
196
197        lp = get_image_page(ca->gfp_mask, ca->safe_needed);
198        if (!lp)
199            return NULL;
200
201        lp->next = ca->chain;
202        ca->chain = lp;
203        ca->used_space = 0;
204    }
205    ret = ca->chain->data + ca->used_space;
206    ca->used_space += size;
207    return ret;
208}
209
210/**
211 * Data types related to memory bitmaps.
212 *
213 * Memory bitmap is a structure consiting of many linked lists of
214 * objects. The main list's elements are of type struct zone_bitmap
215 * and each of them corresonds to one zone. For each zone bitmap
216 * object there is a list of objects of type struct bm_block that
217 * represent each blocks of bitmap in which information is stored.
218 *
219 * struct memory_bitmap contains a pointer to the main list of zone
220 * bitmap objects, a struct bm_position used for browsing the bitmap,
221 * and a pointer to the list of pages used for allocating all of the
222 * zone bitmap objects and bitmap block objects.
223 *
224 * NOTE: It has to be possible to lay out the bitmap in memory
225 * using only allocations of order 0. Additionally, the bitmap is
226 * designed to work with arbitrary number of zones (this is over the
227 * top for now, but let's avoid making unnecessary assumptions ;-).
228 *
229 * struct zone_bitmap contains a pointer to a list of bitmap block
230 * objects and a pointer to the bitmap block object that has been
231 * most recently used for setting bits. Additionally, it contains the
232 * pfns that correspond to the start and end of the represented zone.
233 *
234 * struct bm_block contains a pointer to the memory page in which
235 * information is stored (in the form of a block of bitmap)
236 * It also contains the pfns that correspond to the start and end of
237 * the represented memory area.
238 */
239
240#define BM_END_OF_MAP (~0UL)
241
242#define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE)
243
244struct bm_block {
245    struct list_head hook; /* hook into a list of bitmap blocks */
246    unsigned long start_pfn; /* pfn represented by the first bit */
247    unsigned long end_pfn; /* pfn represented by the last bit plus 1 */
248    unsigned long *data; /* bitmap representing pages */
249};
250
251static inline unsigned long bm_block_bits(struct bm_block *bb)
252{
253    return bb->end_pfn - bb->start_pfn;
254}
255
256/* strcut bm_position is used for browsing memory bitmaps */
257
258struct bm_position {
259    struct bm_block *block;
260    int bit;
261};
262
263struct memory_bitmap {
264    struct list_head blocks; /* list of bitmap blocks */
265    struct linked_page *p_list; /* list of pages used to store zone
266                     * bitmap objects and bitmap block
267                     * objects
268                     */
269    struct bm_position cur; /* most recently used bit position */
270};
271
272/* Functions that operate on memory bitmaps */
273
274static void memory_bm_position_reset(struct memory_bitmap *bm)
275{
276    bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook);
277    bm->cur.bit = 0;
278}
279
280static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
281
282/**
283 * create_bm_block_list - create a list of block bitmap objects
284 * @pages - number of pages to track
285 * @list - list to put the allocated blocks into
286 * @ca - chain allocator to be used for allocating memory
287 */
288static int create_bm_block_list(unsigned long pages,
289                struct list_head *list,
290                struct chain_allocator *ca)
291{
292    unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
293
294    while (nr_blocks-- > 0) {
295        struct bm_block *bb;
296
297        bb = chain_alloc(ca, sizeof(struct bm_block));
298        if (!bb)
299            return -ENOMEM;
300        list_add(&bb->hook, list);
301    }
302
303    return 0;
304}
305
306struct mem_extent {
307    struct list_head hook;
308    unsigned long start;
309    unsigned long end;
310};
311
312/**
313 * free_mem_extents - free a list of memory extents
314 * @list - list of extents to empty
315 */
316static void free_mem_extents(struct list_head *list)
317{
318    struct mem_extent *ext, *aux;
319
320    list_for_each_entry_safe(ext, aux, list, hook) {
321        list_del(&ext->hook);
322        kfree(ext);
323    }
324}
325
326/**
327 * create_mem_extents - create a list of memory extents representing
328 * contiguous ranges of PFNs
329 * @list - list to put the extents into
330 * @gfp_mask - mask to use for memory allocations
331 */
332static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
333{
334    struct zone *zone;
335
336    INIT_LIST_HEAD(list);
337
338    for_each_populated_zone(zone) {
339        unsigned long zone_start, zone_end;
340        struct mem_extent *ext, *cur, *aux;
341
342        zone_start = zone->zone_start_pfn;
343        zone_end = zone->zone_start_pfn + zone->spanned_pages;
344
345        list_for_each_entry(ext, list, hook)
346            if (zone_start <= ext->end)
347                break;
348
349        if (&ext->hook == list || zone_end < ext->start) {
350            /* New extent is necessary */
351            struct mem_extent *new_ext;
352
353            new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask);
354            if (!new_ext) {
355                free_mem_extents(list);
356                return -ENOMEM;
357            }
358            new_ext->start = zone_start;
359            new_ext->end = zone_end;
360            list_add_tail(&new_ext->hook, &ext->hook);
361            continue;
362        }
363
364        /* Merge this zone's range of PFNs with the existing one */
365        if (zone_start < ext->start)
366            ext->start = zone_start;
367        if (zone_end > ext->end)
368            ext->end = zone_end;
369
370        /* More merging may be possible */
371        cur = ext;
372        list_for_each_entry_safe_continue(cur, aux, list, hook) {
373            if (zone_end < cur->start)
374                break;
375            if (zone_end < cur->end)
376                ext->end = cur->end;
377            list_del(&cur->hook);
378            kfree(cur);
379        }
380    }
381
382    return 0;
383}
384
385/**
386  * memory_bm_create - allocate memory for a memory bitmap
387  */
388static int
389memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
390{
391    struct chain_allocator ca;
392    struct list_head mem_extents;
393    struct mem_extent *ext;
394    int error;
395
396    chain_init(&ca, gfp_mask, safe_needed);
397    INIT_LIST_HEAD(&bm->blocks);
398
399    error = create_mem_extents(&mem_extents, gfp_mask);
400    if (error)
401        return error;
402
403    list_for_each_entry(ext, &mem_extents, hook) {
404        struct bm_block *bb;
405        unsigned long pfn = ext->start;
406        unsigned long pages = ext->end - ext->start;
407
408        bb = list_entry(bm->blocks.prev, struct bm_block, hook);
409
410        error = create_bm_block_list(pages, bm->blocks.prev, &ca);
411        if (error)
412            goto Error;
413
414        list_for_each_entry_continue(bb, &bm->blocks, hook) {
415            bb->data = get_image_page(gfp_mask, safe_needed);
416            if (!bb->data) {
417                error = -ENOMEM;
418                goto Error;
419            }
420
421            bb->start_pfn = pfn;
422            if (pages >= BM_BITS_PER_BLOCK) {
423                pfn += BM_BITS_PER_BLOCK;
424                pages -= BM_BITS_PER_BLOCK;
425            } else {
426                /* This is executed only once in the loop */
427                pfn += pages;
428            }
429            bb->end_pfn = pfn;
430        }
431    }
432
433    bm->p_list = ca.chain;
434    memory_bm_position_reset(bm);
435 Exit:
436    free_mem_extents(&mem_extents);
437    return error;
438
439 Error:
440    bm->p_list = ca.chain;
441    memory_bm_free(bm, PG_UNSAFE_CLEAR);
442    goto Exit;
443}
444
445/**
446  * memory_bm_free - free memory occupied by the memory bitmap @bm
447  */
448static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
449{
450    struct bm_block *bb;
451
452    list_for_each_entry(bb, &bm->blocks, hook)
453        if (bb->data)
454            free_image_page(bb->data, clear_nosave_free);
455
456    free_list_of_pages(bm->p_list, clear_nosave_free);
457
458    INIT_LIST_HEAD(&bm->blocks);
459}
460
461/**
462 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
463 * to given pfn. The cur_zone_bm member of @bm and the cur_block member
464 * of @bm->cur_zone_bm are updated.
465 */
466static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
467                void **addr, unsigned int *bit_nr)
468{
469    struct bm_block *bb;
470
471    /*
472     * Check if the pfn corresponds to the current bitmap block and find
473     * the block where it fits if this is not the case.
474     */
475    bb = bm->cur.block;
476    if (pfn < bb->start_pfn)
477        list_for_each_entry_continue_reverse(bb, &bm->blocks, hook)
478            if (pfn >= bb->start_pfn)
479                break;
480
481    if (pfn >= bb->end_pfn)
482        list_for_each_entry_continue(bb, &bm->blocks, hook)
483            if (pfn >= bb->start_pfn && pfn < bb->end_pfn)
484                break;
485
486    if (&bb->hook == &bm->blocks)
487        return -EFAULT;
488
489    /* The block has been found */
490    bm->cur.block = bb;
491    pfn -= bb->start_pfn;
492    bm->cur.bit = pfn + 1;
493    *bit_nr = pfn;
494    *addr = bb->data;
495    return 0;
496}
497
498static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
499{
500    void *addr;
501    unsigned int bit;
502    int error;
503
504    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
505    BUG_ON(error);
506    set_bit(bit, addr);
507}
508
509static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
510{
511    void *addr;
512    unsigned int bit;
513    int error;
514
515    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
516    if (!error)
517        set_bit(bit, addr);
518    return error;
519}
520
521static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
522{
523    void *addr;
524    unsigned int bit;
525    int error;
526
527    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
528    BUG_ON(error);
529    clear_bit(bit, addr);
530}
531
532static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
533{
534    void *addr;
535    unsigned int bit;
536    int error;
537
538    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
539    BUG_ON(error);
540    return test_bit(bit, addr);
541}
542
543static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
544{
545    void *addr;
546    unsigned int bit;
547
548    return !memory_bm_find_bit(bm, pfn, &addr, &bit);
549}
550
551/**
552 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
553 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
554 * returned.
555 *
556 * It is required to run memory_bm_position_reset() before the first call to
557 * this function.
558 */
559
560static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
561{
562    struct bm_block *bb;
563    int bit;
564
565    bb = bm->cur.block;
566    do {
567        bit = bm->cur.bit;
568        bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
569        if (bit < bm_block_bits(bb))
570            goto Return_pfn;
571
572        bb = list_entry(bb->hook.next, struct bm_block, hook);
573        bm->cur.block = bb;
574        bm->cur.bit = 0;
575    } while (&bb->hook != &bm->blocks);
576
577    memory_bm_position_reset(bm);
578    return BM_END_OF_MAP;
579
580 Return_pfn:
581    bm->cur.bit = bit + 1;
582    return bb->start_pfn + bit;
583}
584
585/**
586 * This structure represents a range of page frames the contents of which
587 * should not be saved during the suspend.
588 */
589
590struct nosave_region {
591    struct list_head list;
592    unsigned long start_pfn;
593    unsigned long end_pfn;
594};
595
596static LIST_HEAD(nosave_regions);
597
598/**
599 * register_nosave_region - register a range of page frames the contents
600 * of which should not be saved during the suspend (to be used in the early
601 * initialization code)
602 */
603
604void __init
605__register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
606             int use_kmalloc)
607{
608    struct nosave_region *region;
609
610    if (start_pfn >= end_pfn)
611        return;
612
613    if (!list_empty(&nosave_regions)) {
614        /* Try to extend the previous region (they should be sorted) */
615        region = list_entry(nosave_regions.prev,
616                    struct nosave_region, list);
617        if (region->end_pfn == start_pfn) {
618            region->end_pfn = end_pfn;
619            goto Report;
620        }
621    }
622    if (use_kmalloc) {
623        /* during init, this shouldn't fail */
624        region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
625        BUG_ON(!region);
626    } else
627        /* This allocation cannot fail */
628        region = alloc_bootmem(sizeof(struct nosave_region));
629    region->start_pfn = start_pfn;
630    region->end_pfn = end_pfn;
631    list_add_tail(&region->list, &nosave_regions);
632 Report:
633    printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
634        start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
635}
636
637/*
638 * Set bits in this map correspond to the page frames the contents of which
639 * should not be saved during the suspend.
640 */
641static struct memory_bitmap *forbidden_pages_map;
642
643/* Set bits in this map correspond to free page frames. */
644static struct memory_bitmap *free_pages_map;
645
646/*
647 * Each page frame allocated for creating the image is marked by setting the
648 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
649 */
650
651void swsusp_set_page_free(struct page *page)
652{
653    if (free_pages_map)
654        memory_bm_set_bit(free_pages_map, page_to_pfn(page));
655}
656
657static int swsusp_page_is_free(struct page *page)
658{
659    return free_pages_map ?
660        memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
661}
662
663void swsusp_unset_page_free(struct page *page)
664{
665    if (free_pages_map)
666        memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
667}
668
669static void swsusp_set_page_forbidden(struct page *page)
670{
671    if (forbidden_pages_map)
672        memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
673}
674
675int swsusp_page_is_forbidden(struct page *page)
676{
677    return forbidden_pages_map ?
678        memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
679}
680
681static void swsusp_unset_page_forbidden(struct page *page)
682{
683    if (forbidden_pages_map)
684        memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
685}
686
687/**
688 * mark_nosave_pages - set bits corresponding to the page frames the
689 * contents of which should not be saved in a given bitmap.
690 */
691
692static void mark_nosave_pages(struct memory_bitmap *bm)
693{
694    struct nosave_region *region;
695
696    if (list_empty(&nosave_regions))
697        return;
698
699    list_for_each_entry(region, &nosave_regions, list) {
700        unsigned long pfn;
701
702        pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
703                region->start_pfn << PAGE_SHIFT,
704                region->end_pfn << PAGE_SHIFT);
705
706        for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
707            if (pfn_valid(pfn)) {
708                /*
709                 * It is safe to ignore the result of
710                 * mem_bm_set_bit_check() here, since we won't
711                 * touch the PFNs for which the error is
712                 * returned anyway.
713                 */
714                mem_bm_set_bit_check(bm, pfn);
715            }
716    }
717}
718
719/**
720 * create_basic_memory_bitmaps - create bitmaps needed for marking page
721 * frames that should not be saved and free page frames. The pointers
722 * forbidden_pages_map and free_pages_map are only modified if everything
723 * goes well, because we don't want the bits to be used before both bitmaps
724 * are set up.
725 */
726
727int create_basic_memory_bitmaps(void)
728{
729    struct memory_bitmap *bm1, *bm2;
730    int error = 0;
731
732    BUG_ON(forbidden_pages_map || free_pages_map);
733
734    bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
735    if (!bm1)
736        return -ENOMEM;
737
738    error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
739    if (error)
740        goto Free_first_object;
741
742    bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
743    if (!bm2)
744        goto Free_first_bitmap;
745
746    error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
747    if (error)
748        goto Free_second_object;
749
750    forbidden_pages_map = bm1;
751    free_pages_map = bm2;
752    mark_nosave_pages(forbidden_pages_map);
753
754    pr_debug("PM: Basic memory bitmaps created\n");
755
756    return 0;
757
758 Free_second_object:
759    kfree(bm2);
760 Free_first_bitmap:
761     memory_bm_free(bm1, PG_UNSAFE_CLEAR);
762 Free_first_object:
763    kfree(bm1);
764    return -ENOMEM;
765}
766
767/**
768 * free_basic_memory_bitmaps - free memory bitmaps allocated by
769 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary
770 * so that the bitmaps themselves are not referred to while they are being
771 * freed.
772 */
773
774void free_basic_memory_bitmaps(void)
775{
776    struct memory_bitmap *bm1, *bm2;
777
778    BUG_ON(!(forbidden_pages_map && free_pages_map));
779
780    bm1 = forbidden_pages_map;
781    bm2 = free_pages_map;
782    forbidden_pages_map = NULL;
783    free_pages_map = NULL;
784    memory_bm_free(bm1, PG_UNSAFE_CLEAR);
785    kfree(bm1);
786    memory_bm_free(bm2, PG_UNSAFE_CLEAR);
787    kfree(bm2);
788
789    pr_debug("PM: Basic memory bitmaps freed\n");
790}
791
792/**
793 * snapshot_additional_pages - estimate the number of additional pages
794 * be needed for setting up the suspend image data structures for given
795 * zone (usually the returned value is greater than the exact number)
796 */
797
798unsigned int snapshot_additional_pages(struct zone *zone)
799{
800    unsigned int res;
801
802    res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
803    res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
804    return 2 * res;
805}
806
807#ifdef CONFIG_HIGHMEM
808/**
809 * count_free_highmem_pages - compute the total number of free highmem
810 * pages, system-wide.
811 */
812
813static unsigned int count_free_highmem_pages(void)
814{
815    struct zone *zone;
816    unsigned int cnt = 0;
817
818    for_each_populated_zone(zone)
819        if (is_highmem(zone))
820            cnt += zone_page_state(zone, NR_FREE_PAGES);
821
822    return cnt;
823}
824
825/**
826 * saveable_highmem_page - Determine whether a highmem page should be
827 * included in the suspend image.
828 *
829 * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
830 * and it isn't a part of a free chunk of pages.
831 */
832static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)
833{
834    struct page *page;
835
836    if (!pfn_valid(pfn))
837        return NULL;
838
839    page = pfn_to_page(pfn);
840    if (page_zone(page) != zone)
841        return NULL;
842
843    BUG_ON(!PageHighMem(page));
844
845    if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) ||
846        PageReserved(page))
847        return NULL;
848
849    return page;
850}
851
852/**
853 * count_highmem_pages - compute the total number of saveable highmem
854 * pages.
855 */
856
857static unsigned int count_highmem_pages(void)
858{
859    struct zone *zone;
860    unsigned int n = 0;
861
862    for_each_populated_zone(zone) {
863        unsigned long pfn, max_zone_pfn;
864
865        if (!is_highmem(zone))
866            continue;
867
868        mark_free_pages(zone);
869        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
870        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
871            if (saveable_highmem_page(zone, pfn))
872                n++;
873    }
874    return n;
875}
876#else
877static inline void *saveable_highmem_page(struct zone *z, unsigned long p)
878{
879    return NULL;
880}
881#endif /* CONFIG_HIGHMEM */
882
883/**
884 * saveable_page - Determine whether a non-highmem page should be included
885 * in the suspend image.
886 *
887 * We should save the page if it isn't Nosave, and is not in the range
888 * of pages statically defined as 'unsaveable', and it isn't a part of
889 * a free chunk of pages.
890 */
891static struct page *saveable_page(struct zone *zone, unsigned long pfn)
892{
893    struct page *page;
894
895    if (!pfn_valid(pfn))
896        return NULL;
897
898    page = pfn_to_page(pfn);
899    if (page_zone(page) != zone)
900        return NULL;
901
902    BUG_ON(PageHighMem(page));
903
904    if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
905        return NULL;
906
907    if (PageReserved(page)
908        && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
909        return NULL;
910
911    return page;
912}
913
914/**
915 * count_data_pages - compute the total number of saveable non-highmem
916 * pages.
917 */
918
919static unsigned int count_data_pages(void)
920{
921    struct zone *zone;
922    unsigned long pfn, max_zone_pfn;
923    unsigned int n = 0;
924
925    for_each_populated_zone(zone) {
926        if (is_highmem(zone))
927            continue;
928
929        mark_free_pages(zone);
930        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
931        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
932            if (saveable_page(zone, pfn))
933                n++;
934    }
935    return n;
936}
937
938/* This is needed, because copy_page and memcpy are not usable for copying
939 * task structs.
940 */
941static inline void do_copy_page(long *dst, long *src)
942{
943    int n;
944
945    for (n = PAGE_SIZE / sizeof(long); n; n--)
946        *dst++ = *src++;
947}
948
949
950/**
951 * safe_copy_page - check if the page we are going to copy is marked as
952 * present in the kernel page tables (this always is the case if
953 * CONFIG_DEBUG_PAGEALLOC is not set and in that case
954 * kernel_page_present() always returns 'true').
955 */
956static void safe_copy_page(void *dst, struct page *s_page)
957{
958    if (kernel_page_present(s_page)) {
959        do_copy_page(dst, page_address(s_page));
960    } else {
961        kernel_map_pages(s_page, 1, 1);
962        do_copy_page(dst, page_address(s_page));
963        kernel_map_pages(s_page, 1, 0);
964    }
965}
966
967
968#ifdef CONFIG_HIGHMEM
969static inline struct page *
970page_is_saveable(struct zone *zone, unsigned long pfn)
971{
972    return is_highmem(zone) ?
973        saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
974}
975
976static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
977{
978    struct page *s_page, *d_page;
979    void *src, *dst;
980
981    s_page = pfn_to_page(src_pfn);
982    d_page = pfn_to_page(dst_pfn);
983    if (PageHighMem(s_page)) {
984        src = kmap_atomic(s_page, KM_USER0);
985        dst = kmap_atomic(d_page, KM_USER1);
986        do_copy_page(dst, src);
987        kunmap_atomic(dst, KM_USER1);
988        kunmap_atomic(src, KM_USER0);
989    } else {
990        if (PageHighMem(d_page)) {
991            /* Page pointed to by src may contain some kernel
992             * data modified by kmap_atomic()
993             */
994            safe_copy_page(buffer, s_page);
995            dst = kmap_atomic(d_page, KM_USER0);
996            copy_page(dst, buffer);
997            kunmap_atomic(dst, KM_USER0);
998        } else {
999            safe_copy_page(page_address(d_page), s_page);
1000        }
1001    }
1002}
1003#else
1004#define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
1005
1006static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1007{
1008    safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1009                pfn_to_page(src_pfn));
1010}
1011#endif /* CONFIG_HIGHMEM */
1012
1013static void
1014copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
1015{
1016    struct zone *zone;
1017    unsigned long pfn;
1018
1019    for_each_populated_zone(zone) {
1020        unsigned long max_zone_pfn;
1021
1022        mark_free_pages(zone);
1023        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1024        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1025            if (page_is_saveable(zone, pfn))
1026                memory_bm_set_bit(orig_bm, pfn);
1027    }
1028    memory_bm_position_reset(orig_bm);
1029    memory_bm_position_reset(copy_bm);
1030    for(;;) {
1031        pfn = memory_bm_next_pfn(orig_bm);
1032        if (unlikely(pfn == BM_END_OF_MAP))
1033            break;
1034        copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1035    }
1036}
1037
1038/* Total number of image pages */
1039static unsigned int nr_copy_pages;
1040/* Number of pages needed for saving the original pfns of the image pages */
1041static unsigned int nr_meta_pages;
1042/*
1043 * Numbers of normal and highmem page frames allocated for hibernation image
1044 * before suspending devices.
1045 */
1046unsigned int alloc_normal, alloc_highmem;
1047/*
1048 * Memory bitmap used for marking saveable pages (during hibernation) or
1049 * hibernation image pages (during restore)
1050 */
1051static struct memory_bitmap orig_bm;
1052/*
1053 * Memory bitmap used during hibernation for marking allocated page frames that
1054 * will contain copies of saveable pages. During restore it is initially used
1055 * for marking hibernation image pages, but then the set bits from it are
1056 * duplicated in @orig_bm and it is released. On highmem systems it is next
1057 * used for marking "safe" highmem pages, but it has to be reinitialized for
1058 * this purpose.
1059 */
1060static struct memory_bitmap copy_bm;
1061
1062/**
1063 * swsusp_free - free pages allocated for the suspend.
1064 *
1065 * Suspend pages are alocated before the atomic copy is made, so we
1066 * need to release them after the resume.
1067 */
1068
1069void swsusp_free(void)
1070{
1071    struct zone *zone;
1072    unsigned long pfn, max_zone_pfn;
1073
1074    for_each_populated_zone(zone) {
1075        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1076        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1077            if (pfn_valid(pfn)) {
1078                struct page *page = pfn_to_page(pfn);
1079
1080                if (swsusp_page_is_forbidden(page) &&
1081                    swsusp_page_is_free(page)) {
1082                    swsusp_unset_page_forbidden(page);
1083                    swsusp_unset_page_free(page);
1084                    __free_page(page);
1085                }
1086            }
1087    }
1088    nr_copy_pages = 0;
1089    nr_meta_pages = 0;
1090    restore_pblist = NULL;
1091    buffer = NULL;
1092    alloc_normal = 0;
1093    alloc_highmem = 0;
1094}
1095
1096/* Helper functions used for the shrinking of memory. */
1097
1098#define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN)
1099
1100/**
1101 * preallocate_image_pages - Allocate a number of pages for hibernation image
1102 * @nr_pages: Number of page frames to allocate.
1103 * @mask: GFP flags to use for the allocation.
1104 *
1105 * Return value: Number of page frames actually allocated
1106 */
1107static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1108{
1109    unsigned long nr_alloc = 0;
1110
1111    while (nr_pages > 0) {
1112        struct page *page;
1113
1114        page = alloc_image_page(mask);
1115        if (!page)
1116            break;
1117        memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1118        if (PageHighMem(page))
1119            alloc_highmem++;
1120        else
1121            alloc_normal++;
1122        nr_pages--;
1123        nr_alloc++;
1124    }
1125
1126    return nr_alloc;
1127}
1128
1129static unsigned long preallocate_image_memory(unsigned long nr_pages,
1130                          unsigned long avail_normal)
1131{
1132    unsigned long alloc;
1133
1134    if (avail_normal <= alloc_normal)
1135        return 0;
1136
1137    alloc = avail_normal - alloc_normal;
1138    if (nr_pages < alloc)
1139        alloc = nr_pages;
1140
1141    return preallocate_image_pages(alloc, GFP_IMAGE);
1142}
1143
1144#ifdef CONFIG_HIGHMEM
1145static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1146{
1147    return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
1148}
1149
1150/**
1151 * __fraction - Compute (an approximation of) x * (multiplier / base)
1152 */
1153static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1154{
1155    x *= multiplier;
1156    do_div(x, base);
1157    return (unsigned long)x;
1158}
1159
1160static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1161                        unsigned long highmem,
1162                        unsigned long total)
1163{
1164    unsigned long alloc = __fraction(nr_pages, highmem, total);
1165
1166    return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
1167}
1168#else /* CONFIG_HIGHMEM */
1169static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1170{
1171    return 0;
1172}
1173
1174static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1175                        unsigned long highmem,
1176                        unsigned long total)
1177{
1178    return 0;
1179}
1180#endif /* CONFIG_HIGHMEM */
1181
1182/**
1183 * free_unnecessary_pages - Release preallocated pages not needed for the image
1184 */
1185static void free_unnecessary_pages(void)
1186{
1187    unsigned long save, to_free_normal, to_free_highmem;
1188
1189    save = count_data_pages();
1190    if (alloc_normal >= save) {
1191        to_free_normal = alloc_normal - save;
1192        save = 0;
1193    } else {
1194        to_free_normal = 0;
1195        save -= alloc_normal;
1196    }
1197    save += count_highmem_pages();
1198    if (alloc_highmem >= save) {
1199        to_free_highmem = alloc_highmem - save;
1200    } else {
1201        to_free_highmem = 0;
1202        to_free_normal -= save - alloc_highmem;
1203    }
1204
1205    memory_bm_position_reset(&copy_bm);
1206
1207    while (to_free_normal > 0 || to_free_highmem > 0) {
1208        unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1209        struct page *page = pfn_to_page(pfn);
1210
1211        if (PageHighMem(page)) {
1212            if (!to_free_highmem)
1213                continue;
1214            to_free_highmem--;
1215            alloc_highmem--;
1216        } else {
1217            if (!to_free_normal)
1218                continue;
1219            to_free_normal--;
1220            alloc_normal--;
1221        }
1222        memory_bm_clear_bit(&copy_bm, pfn);
1223        swsusp_unset_page_forbidden(page);
1224        swsusp_unset_page_free(page);
1225        __free_page(page);
1226    }
1227}
1228
1229/**
1230 * minimum_image_size - Estimate the minimum acceptable size of an image
1231 * @saveable: Number of saveable pages in the system.
1232 *
1233 * We want to avoid attempting to free too much memory too hard, so estimate the
1234 * minimum acceptable size of a hibernation image to use as the lower limit for
1235 * preallocating memory.
1236 *
1237 * We assume that the minimum image size should be proportional to
1238 *
1239 * [number of saveable pages] - [number of pages that can be freed in theory]
1240 *
1241 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1242 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
1243 * minus mapped file pages.
1244 */
1245static unsigned long minimum_image_size(unsigned long saveable)
1246{
1247    unsigned long size;
1248
1249    size = global_page_state(NR_SLAB_RECLAIMABLE)
1250        + global_page_state(NR_ACTIVE_ANON)
1251        + global_page_state(NR_INACTIVE_ANON)
1252        + global_page_state(NR_ACTIVE_FILE)
1253        + global_page_state(NR_INACTIVE_FILE)
1254        - global_page_state(NR_FILE_MAPPED);
1255
1256    return saveable <= size ? 0 : saveable - size;
1257}
1258
1259/**
1260 * hibernate_preallocate_memory - Preallocate memory for hibernation image
1261 *
1262 * To create a hibernation image it is necessary to make a copy of every page
1263 * frame in use. We also need a number of page frames to be free during
1264 * hibernation for allocations made while saving the image and for device
1265 * drivers, in case they need to allocate memory from their hibernation
1266 * callbacks (these two numbers are given by PAGES_FOR_IO and SPARE_PAGES,
1267 * respectively, both of which are rough estimates). To make this happen, we
1268 * compute the total number of available page frames and allocate at least
1269 *
1270 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 + 2 * SPARE_PAGES
1271 *
1272 * of them, which corresponds to the maximum size of a hibernation image.
1273 *
1274 * If image_size is set below the number following from the above formula,
1275 * the preallocation of memory is continued until the total number of saveable
1276 * pages in the system is below the requested image size or the minimum
1277 * acceptable image size returned by minimum_image_size(), whichever is greater.
1278 */
1279int hibernate_preallocate_memory(void)
1280{
1281    struct zone *zone;
1282    unsigned long saveable, size, max_size, count, highmem, pages = 0;
1283    unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1284    struct timeval start, stop;
1285    int error;
1286
1287    printk(KERN_INFO "PM: Preallocating image memory... ");
1288    do_gettimeofday(&start);
1289
1290    error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1291    if (error)
1292        goto err_out;
1293
1294    error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1295    if (error)
1296        goto err_out;
1297
1298    alloc_normal = 0;
1299    alloc_highmem = 0;
1300
1301    /* Count the number of saveable data pages. */
1302    save_highmem = count_highmem_pages();
1303    saveable = count_data_pages();
1304
1305    /*
1306     * Compute the total number of page frames we can use (count) and the
1307     * number of pages needed for image metadata (size).
1308     */
1309    count = saveable;
1310    saveable += save_highmem;
1311    highmem = save_highmem;
1312    size = 0;
1313    for_each_populated_zone(zone) {
1314        size += snapshot_additional_pages(zone);
1315        if (is_highmem(zone))
1316            highmem += zone_page_state(zone, NR_FREE_PAGES);
1317        else
1318            count += zone_page_state(zone, NR_FREE_PAGES);
1319    }
1320    avail_normal = count;
1321    count += highmem;
1322    count -= totalreserve_pages;
1323
1324    /* Compute the maximum number of saveable pages to leave in memory. */
1325    max_size = (count - (size + PAGES_FOR_IO)) / 2 - 2 * SPARE_PAGES;
1326    /* Compute the desired number of image pages specified by image_size. */
1327    size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1328    if (size > max_size)
1329        size = max_size;
1330    /*
1331     * If the desired number of image pages is at least as large as the
1332     * current number of saveable pages in memory, allocate page frames for
1333     * the image and we're done.
1334     */
1335    if (size >= saveable) {
1336        pages = preallocate_image_highmem(save_highmem);
1337        pages += preallocate_image_memory(saveable - pages, avail_normal);
1338        goto out;
1339    }
1340
1341    /* Estimate the minimum size of the image. */
1342    pages = minimum_image_size(saveable);
1343    /*
1344     * To avoid excessive pressure on the normal zone, leave room in it to
1345     * accommodate an image of the minimum size (unless it's already too
1346     * small, in which case don't preallocate pages from it at all).
1347     */
1348    if (avail_normal > pages)
1349        avail_normal -= pages;
1350    else
1351        avail_normal = 0;
1352    if (size < pages)
1353        size = min_t(unsigned long, pages, max_size);
1354
1355    /*
1356     * Let the memory management subsystem know that we're going to need a
1357     * large number of page frames to allocate and make it free some memory.
1358     * NOTE: If this is not done, performance will be hurt badly in some
1359     * test cases.
1360     */
1361    shrink_all_memory(saveable - size);
1362
1363    /*
1364     * The number of saveable pages in memory was too high, so apply some
1365     * pressure to decrease it. First, make room for the largest possible
1366     * image and fail if that doesn't work. Next, try to decrease the size
1367     * of the image as much as indicated by 'size' using allocations from
1368     * highmem and non-highmem zones separately.
1369     */
1370    pages_highmem = preallocate_image_highmem(highmem / 2);
1371    alloc = (count - max_size) - pages_highmem;
1372    pages = preallocate_image_memory(alloc, avail_normal);
1373    if (pages < alloc) {
1374        /* We have exhausted non-highmem pages, try highmem. */
1375        alloc -= pages;
1376        pages += pages_highmem;
1377        pages_highmem = preallocate_image_highmem(alloc);
1378        if (pages_highmem < alloc)
1379            goto err_out;
1380        pages += pages_highmem;
1381        /*
1382         * size is the desired number of saveable pages to leave in
1383         * memory, so try to preallocate (all memory - size) pages.
1384         */
1385        alloc = (count - pages) - size;
1386        pages += preallocate_image_highmem(alloc);
1387    } else {
1388        /*
1389         * There are approximately max_size saveable pages at this point
1390         * and we want to reduce this number down to size.
1391         */
1392        alloc = max_size - size;
1393        size = preallocate_highmem_fraction(alloc, highmem, count);
1394        pages_highmem += size;
1395        alloc -= size;
1396        size = preallocate_image_memory(alloc, avail_normal);
1397        pages_highmem += preallocate_image_highmem(alloc - size);
1398        pages += pages_highmem + size;
1399    }
1400
1401    /*
1402     * We only need as many page frames for the image as there are saveable
1403     * pages in memory, but we have allocated more. Release the excessive
1404     * ones now.
1405     */
1406    free_unnecessary_pages();
1407
1408 out:
1409    do_gettimeofday(&stop);
1410    printk(KERN_CONT "done (allocated %lu pages)\n", pages);
1411    swsusp_show_speed(&start, &stop, pages, "Allocated");
1412
1413    return 0;
1414
1415 err_out:
1416    printk(KERN_CONT "\n");
1417    swsusp_free();
1418    return -ENOMEM;
1419}
1420
1421#ifdef CONFIG_HIGHMEM
1422/**
1423  * count_pages_for_highmem - compute the number of non-highmem pages
1424  * that will be necessary for creating copies of highmem pages.
1425  */
1426
1427static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1428{
1429    unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1430
1431    if (free_highmem >= nr_highmem)
1432        nr_highmem = 0;
1433    else
1434        nr_highmem -= free_highmem;
1435
1436    return nr_highmem;
1437}
1438#else
1439static unsigned int
1440count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1441#endif /* CONFIG_HIGHMEM */
1442
1443/**
1444 * enough_free_mem - Make sure we have enough free memory for the
1445 * snapshot image.
1446 */
1447
1448static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1449{
1450    struct zone *zone;
1451    unsigned int free = alloc_normal;
1452
1453    for_each_populated_zone(zone)
1454        if (!is_highmem(zone))
1455            free += zone_page_state(zone, NR_FREE_PAGES);
1456
1457    nr_pages += count_pages_for_highmem(nr_highmem);
1458    pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1459        nr_pages, PAGES_FOR_IO, free);
1460
1461    return free > nr_pages + PAGES_FOR_IO;
1462}
1463
1464#ifdef CONFIG_HIGHMEM
1465/**
1466 * get_highmem_buffer - if there are some highmem pages in the suspend
1467 * image, we may need the buffer to copy them and/or load their data.
1468 */
1469
1470static inline int get_highmem_buffer(int safe_needed)
1471{
1472    buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1473    return buffer ? 0 : -ENOMEM;
1474}
1475
1476/**
1477 * alloc_highmem_image_pages - allocate some highmem pages for the image.
1478 * Try to allocate as many pages as needed, but if the number of free
1479 * highmem pages is lesser than that, allocate them all.
1480 */
1481
1482static inline unsigned int
1483alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1484{
1485    unsigned int to_alloc = count_free_highmem_pages();
1486
1487    if (to_alloc > nr_highmem)
1488        to_alloc = nr_highmem;
1489
1490    nr_highmem -= to_alloc;
1491    while (to_alloc-- > 0) {
1492        struct page *page;
1493
1494        page = alloc_image_page(__GFP_HIGHMEM);
1495        memory_bm_set_bit(bm, page_to_pfn(page));
1496    }
1497    return nr_highmem;
1498}
1499#else
1500static inline int get_highmem_buffer(int safe_needed) { return 0; }
1501
1502static inline unsigned int
1503alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1504#endif /* CONFIG_HIGHMEM */
1505
1506/**
1507 * swsusp_alloc - allocate memory for the suspend image
1508 *
1509 * We first try to allocate as many highmem pages as there are
1510 * saveable highmem pages in the system. If that fails, we allocate
1511 * non-highmem pages for the copies of the remaining highmem ones.
1512 *
1513 * In this approach it is likely that the copies of highmem pages will
1514 * also be located in the high memory, because of the way in which
1515 * copy_data_pages() works.
1516 */
1517
1518static int
1519swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1520        unsigned int nr_pages, unsigned int nr_highmem)
1521{
1522    if (nr_highmem > 0) {
1523        if (get_highmem_buffer(PG_ANY))
1524            goto err_out;
1525        if (nr_highmem > alloc_highmem) {
1526            nr_highmem -= alloc_highmem;
1527            nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
1528        }
1529    }
1530    if (nr_pages > alloc_normal) {
1531        nr_pages -= alloc_normal;
1532        while (nr_pages-- > 0) {
1533            struct page *page;
1534
1535            page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1536            if (!page)
1537                goto err_out;
1538            memory_bm_set_bit(copy_bm, page_to_pfn(page));
1539        }
1540    }
1541
1542    return 0;
1543
1544 err_out:
1545    swsusp_free();
1546    return -ENOMEM;
1547}
1548
1549asmlinkage int swsusp_save(void)
1550{
1551    unsigned int nr_pages, nr_highmem;
1552
1553    printk(KERN_INFO "PM: Creating hibernation image:\n");
1554
1555    drain_local_pages(NULL);
1556    nr_pages = count_data_pages();
1557    nr_highmem = count_highmem_pages();
1558    printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1559
1560    if (!enough_free_mem(nr_pages, nr_highmem)) {
1561        printk(KERN_ERR "PM: Not enough free memory\n");
1562        return -ENOMEM;
1563    }
1564
1565    if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1566        printk(KERN_ERR "PM: Memory allocation failed\n");
1567        return -ENOMEM;
1568    }
1569
1570    /* During allocating of suspend pagedir, new cold pages may appear.
1571     * Kill them.
1572     */
1573    drain_local_pages(NULL);
1574    copy_data_pages(&copy_bm, &orig_bm);
1575
1576    /*
1577     * End of critical section. From now on, we can write to memory,
1578     * but we should not touch disk. This specially means we must _not_
1579     * touch swap space! Except we must write out our image of course.
1580     */
1581
1582    nr_pages += nr_highmem;
1583    nr_copy_pages = nr_pages;
1584    nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1585
1586    printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1587        nr_pages);
1588
1589    return 0;
1590}
1591
1592#ifndef CONFIG_ARCH_HIBERNATION_HEADER
1593static int init_header_complete(struct swsusp_info *info)
1594{
1595    memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1596    info->version_code = LINUX_VERSION_CODE;
1597    return 0;
1598}
1599
1600static char *check_image_kernel(struct swsusp_info *info)
1601{
1602    if (info->version_code != LINUX_VERSION_CODE)
1603        return "kernel version";
1604    if (strcmp(info->uts.sysname,init_utsname()->sysname))
1605        return "system type";
1606    if (strcmp(info->uts.release,init_utsname()->release))
1607        return "kernel release";
1608    if (strcmp(info->uts.version,init_utsname()->version))
1609        return "version";
1610    if (strcmp(info->uts.machine,init_utsname()->machine))
1611        return "machine";
1612    return NULL;
1613}
1614#endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1615
1616unsigned long snapshot_get_image_size(void)
1617{
1618    return nr_copy_pages + nr_meta_pages + 1;
1619}
1620
1621static int init_header(struct swsusp_info *info)
1622{
1623    memset(info, 0, sizeof(struct swsusp_info));
1624    info->num_physpages = num_physpages;
1625    info->image_pages = nr_copy_pages;
1626    info->pages = snapshot_get_image_size();
1627    info->size = info->pages;
1628    info->size <<= PAGE_SHIFT;
1629    return init_header_complete(info);
1630}
1631
1632/**
1633 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1634 * are stored in the array @buf[] (1 page at a time)
1635 */
1636
1637static inline void
1638pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1639{
1640    int j;
1641
1642    for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1643        buf[j] = memory_bm_next_pfn(bm);
1644        if (unlikely(buf[j] == BM_END_OF_MAP))
1645            break;
1646    }
1647}
1648
1649/**
1650 * snapshot_read_next - used for reading the system memory snapshot.
1651 *
1652 * On the first call to it @handle should point to a zeroed
1653 * snapshot_handle structure. The structure gets updated and a pointer
1654 * to it should be passed to this function every next time.
1655 *
1656 * On success the function returns a positive number. Then, the caller
1657 * is allowed to read up to the returned number of bytes from the memory
1658 * location computed by the data_of() macro.
1659 *
1660 * The function returns 0 to indicate the end of data stream condition,
1661 * and a negative number is returned on error. In such cases the
1662 * structure pointed to by @handle is not updated and should not be used
1663 * any more.
1664 */
1665
1666int snapshot_read_next(struct snapshot_handle *handle)
1667{
1668    if (handle->cur > nr_meta_pages + nr_copy_pages)
1669        return 0;
1670
1671    if (!buffer) {
1672        /* This makes the buffer be freed by swsusp_free() */
1673        buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1674        if (!buffer)
1675            return -ENOMEM;
1676    }
1677    if (!handle->cur) {
1678        int error;
1679
1680        error = init_header((struct swsusp_info *)buffer);
1681        if (error)
1682            return error;
1683        handle->buffer = buffer;
1684        memory_bm_position_reset(&orig_bm);
1685        memory_bm_position_reset(&copy_bm);
1686    } else if (handle->cur <= nr_meta_pages) {
1687        clear_page(buffer);
1688        pack_pfns(buffer, &orig_bm);
1689    } else {
1690        struct page *page;
1691
1692        page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1693        if (PageHighMem(page)) {
1694            /* Highmem pages are copied to the buffer,
1695             * because we can't return with a kmapped
1696             * highmem page (we may not be called again).
1697             */
1698            void *kaddr;
1699
1700            kaddr = kmap_atomic(page, KM_USER0);
1701            copy_page(buffer, kaddr);
1702            kunmap_atomic(kaddr, KM_USER0);
1703            handle->buffer = buffer;
1704        } else {
1705            handle->buffer = page_address(page);
1706        }
1707    }
1708    handle->cur++;
1709    return PAGE_SIZE;
1710}
1711
1712/**
1713 * mark_unsafe_pages - mark the pages that cannot be used for storing
1714 * the image during resume, because they conflict with the pages that
1715 * had been used before suspend
1716 */
1717
1718static int mark_unsafe_pages(struct memory_bitmap *bm)
1719{
1720    struct zone *zone;
1721    unsigned long pfn, max_zone_pfn;
1722
1723    /* Clear page flags */
1724    for_each_populated_zone(zone) {
1725        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1726        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1727            if (pfn_valid(pfn))
1728                swsusp_unset_page_free(pfn_to_page(pfn));
1729    }
1730
1731    /* Mark pages that correspond to the "original" pfns as "unsafe" */
1732    memory_bm_position_reset(bm);
1733    do {
1734        pfn = memory_bm_next_pfn(bm);
1735        if (likely(pfn != BM_END_OF_MAP)) {
1736            if (likely(pfn_valid(pfn)))
1737                swsusp_set_page_free(pfn_to_page(pfn));
1738            else
1739                return -EFAULT;
1740        }
1741    } while (pfn != BM_END_OF_MAP);
1742
1743    allocated_unsafe_pages = 0;
1744
1745    return 0;
1746}
1747
1748static void
1749duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1750{
1751    unsigned long pfn;
1752
1753    memory_bm_position_reset(src);
1754    pfn = memory_bm_next_pfn(src);
1755    while (pfn != BM_END_OF_MAP) {
1756        memory_bm_set_bit(dst, pfn);
1757        pfn = memory_bm_next_pfn(src);
1758    }
1759}
1760
1761static int check_header(struct swsusp_info *info)
1762{
1763    char *reason;
1764
1765    reason = check_image_kernel(info);
1766    if (!reason && info->num_physpages != num_physpages)
1767        reason = "memory size";
1768    if (reason) {
1769        printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1770        return -EPERM;
1771    }
1772    return 0;
1773}
1774
1775/**
1776 * load header - check the image header and copy data from it
1777 */
1778
1779static int
1780load_header(struct swsusp_info *info)
1781{
1782    int error;
1783
1784    restore_pblist = NULL;
1785    error = check_header(info);
1786    if (!error) {
1787        nr_copy_pages = info->image_pages;
1788        nr_meta_pages = info->pages - info->image_pages - 1;
1789    }
1790    return error;
1791}
1792
1793/**
1794 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1795 * the corresponding bit in the memory bitmap @bm
1796 */
1797static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1798{
1799    int j;
1800
1801    for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1802        if (unlikely(buf[j] == BM_END_OF_MAP))
1803            break;
1804
1805        if (memory_bm_pfn_present(bm, buf[j]))
1806            memory_bm_set_bit(bm, buf[j]);
1807        else
1808            return -EFAULT;
1809    }
1810
1811    return 0;
1812}
1813
1814/* List of "safe" pages that may be used to store data loaded from the suspend
1815 * image
1816 */
1817static struct linked_page *safe_pages_list;
1818
1819#ifdef CONFIG_HIGHMEM
1820/* struct highmem_pbe is used for creating the list of highmem pages that
1821 * should be restored atomically during the resume from disk, because the page
1822 * frames they have occupied before the suspend are in use.
1823 */
1824struct highmem_pbe {
1825    struct page *copy_page; /* data is here now */
1826    struct page *orig_page; /* data was here before the suspend */
1827    struct highmem_pbe *next;
1828};
1829
1830/* List of highmem PBEs needed for restoring the highmem pages that were
1831 * allocated before the suspend and included in the suspend image, but have
1832 * also been allocated by the "resume" kernel, so their contents cannot be
1833 * written directly to their "original" page frames.
1834 */
1835static struct highmem_pbe *highmem_pblist;
1836
1837/**
1838 * count_highmem_image_pages - compute the number of highmem pages in the
1839 * suspend image. The bits in the memory bitmap @bm that correspond to the
1840 * image pages are assumed to be set.
1841 */
1842
1843static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1844{
1845    unsigned long pfn;
1846    unsigned int cnt = 0;
1847
1848    memory_bm_position_reset(bm);
1849    pfn = memory_bm_next_pfn(bm);
1850    while (pfn != BM_END_OF_MAP) {
1851        if (PageHighMem(pfn_to_page(pfn)))
1852            cnt++;
1853
1854        pfn = memory_bm_next_pfn(bm);
1855    }
1856    return cnt;
1857}
1858
1859/**
1860 * prepare_highmem_image - try to allocate as many highmem pages as
1861 * there are highmem image pages (@nr_highmem_p points to the variable
1862 * containing the number of highmem image pages). The pages that are
1863 * "safe" (ie. will not be overwritten when the suspend image is
1864 * restored) have the corresponding bits set in @bm (it must be
1865 * unitialized).
1866 *
1867 * NOTE: This function should not be called if there are no highmem
1868 * image pages.
1869 */
1870
1871static unsigned int safe_highmem_pages;
1872
1873static struct memory_bitmap *safe_highmem_bm;
1874
1875static int
1876prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1877{
1878    unsigned int to_alloc;
1879
1880    if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1881        return -ENOMEM;
1882
1883    if (get_highmem_buffer(PG_SAFE))
1884        return -ENOMEM;
1885
1886    to_alloc = count_free_highmem_pages();
1887    if (to_alloc > *nr_highmem_p)
1888        to_alloc = *nr_highmem_p;
1889    else
1890        *nr_highmem_p = to_alloc;
1891
1892    safe_highmem_pages = 0;
1893    while (to_alloc-- > 0) {
1894        struct page *page;
1895
1896        page = alloc_page(__GFP_HIGHMEM);
1897        if (!swsusp_page_is_free(page)) {
1898            /* The page is "safe", set its bit the bitmap */
1899            memory_bm_set_bit(bm, page_to_pfn(page));
1900            safe_highmem_pages++;
1901        }
1902        /* Mark the page as allocated */
1903        swsusp_set_page_forbidden(page);
1904        swsusp_set_page_free(page);
1905    }
1906    memory_bm_position_reset(bm);
1907    safe_highmem_bm = bm;
1908    return 0;
1909}
1910
1911/**
1912 * get_highmem_page_buffer - for given highmem image page find the buffer
1913 * that suspend_write_next() should set for its caller to write to.
1914 *
1915 * If the page is to be saved to its "original" page frame or a copy of
1916 * the page is to be made in the highmem, @buffer is returned. Otherwise,
1917 * the copy of the page is to be made in normal memory, so the address of
1918 * the copy is returned.
1919 *
1920 * If @buffer is returned, the caller of suspend_write_next() will write
1921 * the page's contents to @buffer, so they will have to be copied to the
1922 * right location on the next call to suspend_write_next() and it is done
1923 * with the help of copy_last_highmem_page(). For this purpose, if
1924 * @buffer is returned, @last_highmem page is set to the page to which
1925 * the data will have to be copied from @buffer.
1926 */
1927
1928static struct page *last_highmem_page;
1929
1930static void *
1931get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1932{
1933    struct highmem_pbe *pbe;
1934    void *kaddr;
1935
1936    if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1937        /* We have allocated the "original" page frame and we can
1938         * use it directly to store the loaded page.
1939         */
1940        last_highmem_page = page;
1941        return buffer;
1942    }
1943    /* The "original" page frame has not been allocated and we have to
1944     * use a "safe" page frame to store the loaded page.
1945     */
1946    pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1947    if (!pbe) {
1948        swsusp_free();
1949        return ERR_PTR(-ENOMEM);
1950    }
1951    pbe->orig_page = page;
1952    if (safe_highmem_pages > 0) {
1953        struct page *tmp;
1954
1955        /* Copy of the page will be stored in high memory */
1956        kaddr = buffer;
1957        tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1958        safe_highmem_pages--;
1959        last_highmem_page = tmp;
1960        pbe->copy_page = tmp;
1961    } else {
1962        /* Copy of the page will be stored in normal memory */
1963        kaddr = safe_pages_list;
1964        safe_pages_list = safe_pages_list->next;
1965        pbe->copy_page = virt_to_page(kaddr);
1966    }
1967    pbe->next = highmem_pblist;
1968    highmem_pblist = pbe;
1969    return kaddr;
1970}
1971
1972/**
1973 * copy_last_highmem_page - copy the contents of a highmem image from
1974 * @buffer, where the caller of snapshot_write_next() has place them,
1975 * to the right location represented by @last_highmem_page .
1976 */
1977
1978static void copy_last_highmem_page(void)
1979{
1980    if (last_highmem_page) {
1981        void *dst;
1982
1983        dst = kmap_atomic(last_highmem_page, KM_USER0);
1984        copy_page(dst, buffer);
1985        kunmap_atomic(dst, KM_USER0);
1986        last_highmem_page = NULL;
1987    }
1988}
1989
1990static inline int last_highmem_page_copied(void)
1991{
1992    return !last_highmem_page;
1993}
1994
1995static inline void free_highmem_data(void)
1996{
1997    if (safe_highmem_bm)
1998        memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
1999
2000    if (buffer)
2001        free_image_page(buffer, PG_UNSAFE_CLEAR);
2002}
2003#else
2004static inline int get_safe_write_buffer(void) { return 0; }
2005
2006static unsigned int
2007count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2008
2009static inline int
2010prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
2011{
2012    return 0;
2013}
2014
2015static inline void *
2016get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
2017{
2018    return ERR_PTR(-EINVAL);
2019}
2020
2021static inline void copy_last_highmem_page(void) {}
2022static inline int last_highmem_page_copied(void) { return 1; }
2023static inline void free_highmem_data(void) {}
2024#endif /* CONFIG_HIGHMEM */
2025
2026/**
2027 * prepare_image - use the memory bitmap @bm to mark the pages that will
2028 * be overwritten in the process of restoring the system memory state
2029 * from the suspend image ("unsafe" pages) and allocate memory for the
2030 * image.
2031 *
2032 * The idea is to allocate a new memory bitmap first and then allocate
2033 * as many pages as needed for the image data, but not to assign these
2034 * pages to specific tasks initially. Instead, we just mark them as
2035 * allocated and create a lists of "safe" pages that will be used
2036 * later. On systems with high memory a list of "safe" highmem pages is
2037 * also created.
2038 */
2039
2040#define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2041
2042static int
2043prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
2044{
2045    unsigned int nr_pages, nr_highmem;
2046    struct linked_page *sp_list, *lp;
2047    int error;
2048
2049    /* If there is no highmem, the buffer will not be necessary */
2050    free_image_page(buffer, PG_UNSAFE_CLEAR);
2051    buffer = NULL;
2052
2053    nr_highmem = count_highmem_image_pages(bm);
2054    error = mark_unsafe_pages(bm);
2055    if (error)
2056        goto Free;
2057
2058    error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2059    if (error)
2060        goto Free;
2061
2062    duplicate_memory_bitmap(new_bm, bm);
2063    memory_bm_free(bm, PG_UNSAFE_KEEP);
2064    if (nr_highmem > 0) {
2065        error = prepare_highmem_image(bm, &nr_highmem);
2066        if (error)
2067            goto Free;
2068    }
2069    /* Reserve some safe pages for potential later use.
2070     *
2071     * NOTE: This way we make sure there will be enough safe pages for the
2072     * chain_alloc() in get_buffer(). It is a bit wasteful, but
2073     * nr_copy_pages cannot be greater than 50% of the memory anyway.
2074     */
2075    sp_list = NULL;
2076    /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2077    nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2078    nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2079    while (nr_pages > 0) {
2080        lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2081        if (!lp) {
2082            error = -ENOMEM;
2083            goto Free;
2084        }
2085        lp->next = sp_list;
2086        sp_list = lp;
2087        nr_pages--;
2088    }
2089    /* Preallocate memory for the image */
2090    safe_pages_list = NULL;
2091    nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2092    while (nr_pages > 0) {
2093        lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2094        if (!lp) {
2095            error = -ENOMEM;
2096            goto Free;
2097        }
2098        if (!swsusp_page_is_free(virt_to_page(lp))) {
2099            /* The page is "safe", add it to the list */
2100            lp->next = safe_pages_list;
2101            safe_pages_list = lp;
2102        }
2103        /* Mark the page as allocated */
2104        swsusp_set_page_forbidden(virt_to_page(lp));
2105        swsusp_set_page_free(virt_to_page(lp));
2106        nr_pages--;
2107    }
2108    /* Free the reserved safe pages so that chain_alloc() can use them */
2109    while (sp_list) {
2110        lp = sp_list->next;
2111        free_image_page(sp_list, PG_UNSAFE_CLEAR);
2112        sp_list = lp;
2113    }
2114    return 0;
2115
2116 Free:
2117    swsusp_free();
2118    return error;
2119}
2120
2121/**
2122 * get_buffer - compute the address that snapshot_write_next() should
2123 * set for its caller to write to.
2124 */
2125
2126static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
2127{
2128    struct pbe *pbe;
2129    struct page *page;
2130    unsigned long pfn = memory_bm_next_pfn(bm);
2131
2132    if (pfn == BM_END_OF_MAP)
2133        return ERR_PTR(-EFAULT);
2134
2135    page = pfn_to_page(pfn);
2136    if (PageHighMem(page))
2137        return get_highmem_page_buffer(page, ca);
2138
2139    if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2140        /* We have allocated the "original" page frame and we can
2141         * use it directly to store the loaded page.
2142         */
2143        return page_address(page);
2144
2145    /* The "original" page frame has not been allocated and we have to
2146     * use a "safe" page frame to store the loaded page.
2147     */
2148    pbe = chain_alloc(ca, sizeof(struct pbe));
2149    if (!pbe) {
2150        swsusp_free();
2151        return ERR_PTR(-ENOMEM);
2152    }
2153    pbe->orig_address = page_address(page);
2154    pbe->address = safe_pages_list;
2155    safe_pages_list = safe_pages_list->next;
2156    pbe->next = restore_pblist;
2157    restore_pblist = pbe;
2158    return pbe->address;
2159}
2160
2161/**
2162 * snapshot_write_next - used for writing the system memory snapshot.
2163 *
2164 * On the first call to it @handle should point to a zeroed
2165 * snapshot_handle structure. The structure gets updated and a pointer
2166 * to it should be passed to this function every next time.
2167 *
2168 * On success the function returns a positive number. Then, the caller
2169 * is allowed to write up to the returned number of bytes to the memory
2170 * location computed by the data_of() macro.
2171 *
2172 * The function returns 0 to indicate the "end of file" condition,
2173 * and a negative number is returned on error. In such cases the
2174 * structure pointed to by @handle is not updated and should not be used
2175 * any more.
2176 */
2177
2178int snapshot_write_next(struct snapshot_handle *handle)
2179{
2180    static struct chain_allocator ca;
2181    int error = 0;
2182
2183    /* Check if we have already loaded the entire image */
2184    if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages)
2185        return 0;
2186
2187    handle->sync_read = 1;
2188
2189    if (!handle->cur) {
2190        if (!buffer)
2191            /* This makes the buffer be freed by swsusp_free() */
2192            buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2193
2194        if (!buffer)
2195            return -ENOMEM;
2196
2197        handle->buffer = buffer;
2198    } else if (handle->cur == 1) {
2199        error = load_header(buffer);
2200        if (error)
2201            return error;
2202
2203        error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2204        if (error)
2205            return error;
2206
2207    } else if (handle->cur <= nr_meta_pages + 1) {
2208        error = unpack_orig_pfns(buffer, &copy_bm);
2209        if (error)
2210            return error;
2211
2212        if (handle->cur == nr_meta_pages + 1) {
2213            error = prepare_image(&orig_bm, &copy_bm);
2214            if (error)
2215                return error;
2216
2217            chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2218            memory_bm_position_reset(&orig_bm);
2219            restore_pblist = NULL;
2220            handle->buffer = get_buffer(&orig_bm, &ca);
2221            handle->sync_read = 0;
2222            if (IS_ERR(handle->buffer))
2223                return PTR_ERR(handle->buffer);
2224        }
2225    } else {
2226        copy_last_highmem_page();
2227        handle->buffer = get_buffer(&orig_bm, &ca);
2228        if (IS_ERR(handle->buffer))
2229            return PTR_ERR(handle->buffer);
2230        if (handle->buffer != buffer)
2231            handle->sync_read = 0;
2232    }
2233    handle->cur++;
2234    return PAGE_SIZE;
2235}
2236
2237/**
2238 * snapshot_write_finalize - must be called after the last call to
2239 * snapshot_write_next() in case the last page in the image happens
2240 * to be a highmem page and its contents should be stored in the
2241 * highmem. Additionally, it releases the memory that will not be
2242 * used any more.
2243 */
2244
2245void snapshot_write_finalize(struct snapshot_handle *handle)
2246{
2247    copy_last_highmem_page();
2248    /* Free only if we have loaded the image entirely */
2249    if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {
2250        memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
2251        free_highmem_data();
2252    }
2253}
2254
2255int snapshot_image_loaded(struct snapshot_handle *handle)
2256{
2257    return !(!nr_copy_pages || !last_highmem_page_copied() ||
2258            handle->cur <= nr_meta_pages + nr_copy_pages);
2259}
2260
2261#ifdef CONFIG_HIGHMEM
2262/* Assumes that @buf is ready and points to a "safe" page */
2263static inline void
2264swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
2265{
2266    void *kaddr1, *kaddr2;
2267
2268    kaddr1 = kmap_atomic(p1, KM_USER0);
2269    kaddr2 = kmap_atomic(p2, KM_USER1);
2270    copy_page(buf, kaddr1);
2271    copy_page(kaddr1, kaddr2);
2272    copy_page(kaddr2, buf);
2273    kunmap_atomic(kaddr2, KM_USER1);
2274    kunmap_atomic(kaddr1, KM_USER0);
2275}
2276
2277/**
2278 * restore_highmem - for each highmem page that was allocated before
2279 * the suspend and included in the suspend image, and also has been
2280 * allocated by the "resume" kernel swap its current (ie. "before
2281 * resume") contents with the previous (ie. "before suspend") one.
2282 *
2283 * If the resume eventually fails, we can call this function once
2284 * again and restore the "before resume" highmem state.
2285 */
2286
2287int restore_highmem(void)
2288{
2289    struct highmem_pbe *pbe = highmem_pblist;
2290    void *buf;
2291
2292    if (!pbe)
2293        return 0;
2294
2295    buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2296    if (!buf)
2297        return -ENOMEM;
2298
2299    while (pbe) {
2300        swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2301        pbe = pbe->next;
2302    }
2303    free_image_page(buf, PG_UNSAFE_CLEAR);
2304    return 0;
2305}
2306#endif /* CONFIG_HIGHMEM */
2307

Archive Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9



interactive