Root/kernel/power/snapshot.c

1/*
2 * linux/kernel/power/snapshot.c
3 *
4 * This file provides system snapshot/restore functionality for swsusp.
5 *
6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8 *
9 * This file is released under the GPLv2.
10 *
11 */
12
13#include <linux/version.h>
14#include <linux/module.h>
15#include <linux/mm.h>
16#include <linux/suspend.h>
17#include <linux/delay.h>
18#include <linux/bitops.h>
19#include <linux/spinlock.h>
20#include <linux/kernel.h>
21#include <linux/pm.h>
22#include <linux/device.h>
23#include <linux/init.h>
24#include <linux/bootmem.h>
25#include <linux/syscalls.h>
26#include <linux/console.h>
27#include <linux/highmem.h>
28#include <linux/list.h>
29#include <linux/slab.h>
30
31#include <asm/uaccess.h>
32#include <asm/mmu_context.h>
33#include <asm/pgtable.h>
34#include <asm/tlbflush.h>
35#include <asm/io.h>
36
37#include "power.h"
38
39static int swsusp_page_is_free(struct page *);
40static void swsusp_set_page_forbidden(struct page *);
41static void swsusp_unset_page_forbidden(struct page *);
42
43/*
44 * Number of bytes to reserve for memory allocations made by device drivers
45 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't
46 * cause image creation to fail (tunable via /sys/power/reserved_size).
47 */
48unsigned long reserved_size;
49
50void __init hibernate_reserved_size_init(void)
51{
52    reserved_size = SPARE_PAGES * PAGE_SIZE;
53}
54
55/*
56 * Preferred image size in bytes (tunable via /sys/power/image_size).
57 * When it is set to N, swsusp will do its best to ensure the image
58 * size will not exceed N bytes, but if that is impossible, it will
59 * try to create the smallest image possible.
60 */
61unsigned long image_size;
62
63void __init hibernate_image_size_init(void)
64{
65    image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE;
66}
67
68/* List of PBEs needed for restoring the pages that were allocated before
69 * the suspend and included in the suspend image, but have also been
70 * allocated by the "resume" kernel, so their contents cannot be written
71 * directly to their "original" page frames.
72 */
73struct pbe *restore_pblist;
74
75/* Pointer to an auxiliary buffer (1 page) */
76static void *buffer;
77
78/**
79 * @safe_needed - on resume, for storing the PBE list and the image,
80 * we can only use memory pages that do not conflict with the pages
81 * used before suspend. The unsafe pages have PageNosaveFree set
82 * and we count them using unsafe_pages.
83 *
84 * Each allocated image page is marked as PageNosave and PageNosaveFree
85 * so that swsusp_free() can release it.
86 */
87
88#define PG_ANY 0
89#define PG_SAFE 1
90#define PG_UNSAFE_CLEAR 1
91#define PG_UNSAFE_KEEP 0
92
93static unsigned int allocated_unsafe_pages;
94
95static void *get_image_page(gfp_t gfp_mask, int safe_needed)
96{
97    void *res;
98
99    res = (void *)get_zeroed_page(gfp_mask);
100    if (safe_needed)
101        while (res && swsusp_page_is_free(virt_to_page(res))) {
102            /* The page is unsafe, mark it for swsusp_free() */
103            swsusp_set_page_forbidden(virt_to_page(res));
104            allocated_unsafe_pages++;
105            res = (void *)get_zeroed_page(gfp_mask);
106        }
107    if (res) {
108        swsusp_set_page_forbidden(virt_to_page(res));
109        swsusp_set_page_free(virt_to_page(res));
110    }
111    return res;
112}
113
114unsigned long get_safe_page(gfp_t gfp_mask)
115{
116    return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
117}
118
119static struct page *alloc_image_page(gfp_t gfp_mask)
120{
121    struct page *page;
122
123    page = alloc_page(gfp_mask);
124    if (page) {
125        swsusp_set_page_forbidden(page);
126        swsusp_set_page_free(page);
127    }
128    return page;
129}
130
131/**
132 * free_image_page - free page represented by @addr, allocated with
133 * get_image_page (page flags set by it must be cleared)
134 */
135
136static inline void free_image_page(void *addr, int clear_nosave_free)
137{
138    struct page *page;
139
140    BUG_ON(!virt_addr_valid(addr));
141
142    page = virt_to_page(addr);
143
144    swsusp_unset_page_forbidden(page);
145    if (clear_nosave_free)
146        swsusp_unset_page_free(page);
147
148    __free_page(page);
149}
150
151/* struct linked_page is used to build chains of pages */
152
153#define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
154
155struct linked_page {
156    struct linked_page *next;
157    char data[LINKED_PAGE_DATA_SIZE];
158} __attribute__((packed));
159
160static inline void
161free_list_of_pages(struct linked_page *list, int clear_page_nosave)
162{
163    while (list) {
164        struct linked_page *lp = list->next;
165
166        free_image_page(list, clear_page_nosave);
167        list = lp;
168    }
169}
170
171/**
172  * struct chain_allocator is used for allocating small objects out of
173  * a linked list of pages called 'the chain'.
174  *
175  * The chain grows each time when there is no room for a new object in
176  * the current page. The allocated objects cannot be freed individually.
177  * It is only possible to free them all at once, by freeing the entire
178  * chain.
179  *
180  * NOTE: The chain allocator may be inefficient if the allocated objects
181  * are not much smaller than PAGE_SIZE.
182  */
183
184struct chain_allocator {
185    struct linked_page *chain; /* the chain */
186    unsigned int used_space; /* total size of objects allocated out
187                     * of the current page
188                     */
189    gfp_t gfp_mask; /* mask for allocating pages */
190    int safe_needed; /* if set, only "safe" pages are allocated */
191};
192
193static void
194chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
195{
196    ca->chain = NULL;
197    ca->used_space = LINKED_PAGE_DATA_SIZE;
198    ca->gfp_mask = gfp_mask;
199    ca->safe_needed = safe_needed;
200}
201
202static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
203{
204    void *ret;
205
206    if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
207        struct linked_page *lp;
208
209        lp = get_image_page(ca->gfp_mask, ca->safe_needed);
210        if (!lp)
211            return NULL;
212
213        lp->next = ca->chain;
214        ca->chain = lp;
215        ca->used_space = 0;
216    }
217    ret = ca->chain->data + ca->used_space;
218    ca->used_space += size;
219    return ret;
220}
221
222/**
223 * Data types related to memory bitmaps.
224 *
225 * Memory bitmap is a structure consiting of many linked lists of
226 * objects. The main list's elements are of type struct zone_bitmap
227 * and each of them corresonds to one zone. For each zone bitmap
228 * object there is a list of objects of type struct bm_block that
229 * represent each blocks of bitmap in which information is stored.
230 *
231 * struct memory_bitmap contains a pointer to the main list of zone
232 * bitmap objects, a struct bm_position used for browsing the bitmap,
233 * and a pointer to the list of pages used for allocating all of the
234 * zone bitmap objects and bitmap block objects.
235 *
236 * NOTE: It has to be possible to lay out the bitmap in memory
237 * using only allocations of order 0. Additionally, the bitmap is
238 * designed to work with arbitrary number of zones (this is over the
239 * top for now, but let's avoid making unnecessary assumptions ;-).
240 *
241 * struct zone_bitmap contains a pointer to a list of bitmap block
242 * objects and a pointer to the bitmap block object that has been
243 * most recently used for setting bits. Additionally, it contains the
244 * pfns that correspond to the start and end of the represented zone.
245 *
246 * struct bm_block contains a pointer to the memory page in which
247 * information is stored (in the form of a block of bitmap)
248 * It also contains the pfns that correspond to the start and end of
249 * the represented memory area.
250 */
251
252#define BM_END_OF_MAP (~0UL)
253
254#define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE)
255
256struct bm_block {
257    struct list_head hook; /* hook into a list of bitmap blocks */
258    unsigned long start_pfn; /* pfn represented by the first bit */
259    unsigned long end_pfn; /* pfn represented by the last bit plus 1 */
260    unsigned long *data; /* bitmap representing pages */
261};
262
263static inline unsigned long bm_block_bits(struct bm_block *bb)
264{
265    return bb->end_pfn - bb->start_pfn;
266}
267
268/* strcut bm_position is used for browsing memory bitmaps */
269
270struct bm_position {
271    struct bm_block *block;
272    int bit;
273};
274
275struct memory_bitmap {
276    struct list_head blocks; /* list of bitmap blocks */
277    struct linked_page *p_list; /* list of pages used to store zone
278                     * bitmap objects and bitmap block
279                     * objects
280                     */
281    struct bm_position cur; /* most recently used bit position */
282};
283
284/* Functions that operate on memory bitmaps */
285
286static void memory_bm_position_reset(struct memory_bitmap *bm)
287{
288    bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook);
289    bm->cur.bit = 0;
290}
291
292static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
293
294/**
295 * create_bm_block_list - create a list of block bitmap objects
296 * @pages - number of pages to track
297 * @list - list to put the allocated blocks into
298 * @ca - chain allocator to be used for allocating memory
299 */
300static int create_bm_block_list(unsigned long pages,
301                struct list_head *list,
302                struct chain_allocator *ca)
303{
304    unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
305
306    while (nr_blocks-- > 0) {
307        struct bm_block *bb;
308
309        bb = chain_alloc(ca, sizeof(struct bm_block));
310        if (!bb)
311            return -ENOMEM;
312        list_add(&bb->hook, list);
313    }
314
315    return 0;
316}
317
318struct mem_extent {
319    struct list_head hook;
320    unsigned long start;
321    unsigned long end;
322};
323
324/**
325 * free_mem_extents - free a list of memory extents
326 * @list - list of extents to empty
327 */
328static void free_mem_extents(struct list_head *list)
329{
330    struct mem_extent *ext, *aux;
331
332    list_for_each_entry_safe(ext, aux, list, hook) {
333        list_del(&ext->hook);
334        kfree(ext);
335    }
336}
337
338/**
339 * create_mem_extents - create a list of memory extents representing
340 * contiguous ranges of PFNs
341 * @list - list to put the extents into
342 * @gfp_mask - mask to use for memory allocations
343 */
344static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
345{
346    struct zone *zone;
347
348    INIT_LIST_HEAD(list);
349
350    for_each_populated_zone(zone) {
351        unsigned long zone_start, zone_end;
352        struct mem_extent *ext, *cur, *aux;
353
354        zone_start = zone->zone_start_pfn;
355        zone_end = zone->zone_start_pfn + zone->spanned_pages;
356
357        list_for_each_entry(ext, list, hook)
358            if (zone_start <= ext->end)
359                break;
360
361        if (&ext->hook == list || zone_end < ext->start) {
362            /* New extent is necessary */
363            struct mem_extent *new_ext;
364
365            new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask);
366            if (!new_ext) {
367                free_mem_extents(list);
368                return -ENOMEM;
369            }
370            new_ext->start = zone_start;
371            new_ext->end = zone_end;
372            list_add_tail(&new_ext->hook, &ext->hook);
373            continue;
374        }
375
376        /* Merge this zone's range of PFNs with the existing one */
377        if (zone_start < ext->start)
378            ext->start = zone_start;
379        if (zone_end > ext->end)
380            ext->end = zone_end;
381
382        /* More merging may be possible */
383        cur = ext;
384        list_for_each_entry_safe_continue(cur, aux, list, hook) {
385            if (zone_end < cur->start)
386                break;
387            if (zone_end < cur->end)
388                ext->end = cur->end;
389            list_del(&cur->hook);
390            kfree(cur);
391        }
392    }
393
394    return 0;
395}
396
397/**
398  * memory_bm_create - allocate memory for a memory bitmap
399  */
400static int
401memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
402{
403    struct chain_allocator ca;
404    struct list_head mem_extents;
405    struct mem_extent *ext;
406    int error;
407
408    chain_init(&ca, gfp_mask, safe_needed);
409    INIT_LIST_HEAD(&bm->blocks);
410
411    error = create_mem_extents(&mem_extents, gfp_mask);
412    if (error)
413        return error;
414
415    list_for_each_entry(ext, &mem_extents, hook) {
416        struct bm_block *bb;
417        unsigned long pfn = ext->start;
418        unsigned long pages = ext->end - ext->start;
419
420        bb = list_entry(bm->blocks.prev, struct bm_block, hook);
421
422        error = create_bm_block_list(pages, bm->blocks.prev, &ca);
423        if (error)
424            goto Error;
425
426        list_for_each_entry_continue(bb, &bm->blocks, hook) {
427            bb->data = get_image_page(gfp_mask, safe_needed);
428            if (!bb->data) {
429                error = -ENOMEM;
430                goto Error;
431            }
432
433            bb->start_pfn = pfn;
434            if (pages >= BM_BITS_PER_BLOCK) {
435                pfn += BM_BITS_PER_BLOCK;
436                pages -= BM_BITS_PER_BLOCK;
437            } else {
438                /* This is executed only once in the loop */
439                pfn += pages;
440            }
441            bb->end_pfn = pfn;
442        }
443    }
444
445    bm->p_list = ca.chain;
446    memory_bm_position_reset(bm);
447 Exit:
448    free_mem_extents(&mem_extents);
449    return error;
450
451 Error:
452    bm->p_list = ca.chain;
453    memory_bm_free(bm, PG_UNSAFE_CLEAR);
454    goto Exit;
455}
456
457/**
458  * memory_bm_free - free memory occupied by the memory bitmap @bm
459  */
460static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
461{
462    struct bm_block *bb;
463
464    list_for_each_entry(bb, &bm->blocks, hook)
465        if (bb->data)
466            free_image_page(bb->data, clear_nosave_free);
467
468    free_list_of_pages(bm->p_list, clear_nosave_free);
469
470    INIT_LIST_HEAD(&bm->blocks);
471}
472
473/**
474 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
475 * to given pfn. The cur_zone_bm member of @bm and the cur_block member
476 * of @bm->cur_zone_bm are updated.
477 */
478static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
479                void **addr, unsigned int *bit_nr)
480{
481    struct bm_block *bb;
482
483    /*
484     * Check if the pfn corresponds to the current bitmap block and find
485     * the block where it fits if this is not the case.
486     */
487    bb = bm->cur.block;
488    if (pfn < bb->start_pfn)
489        list_for_each_entry_continue_reverse(bb, &bm->blocks, hook)
490            if (pfn >= bb->start_pfn)
491                break;
492
493    if (pfn >= bb->end_pfn)
494        list_for_each_entry_continue(bb, &bm->blocks, hook)
495            if (pfn >= bb->start_pfn && pfn < bb->end_pfn)
496                break;
497
498    if (&bb->hook == &bm->blocks)
499        return -EFAULT;
500
501    /* The block has been found */
502    bm->cur.block = bb;
503    pfn -= bb->start_pfn;
504    bm->cur.bit = pfn + 1;
505    *bit_nr = pfn;
506    *addr = bb->data;
507    return 0;
508}
509
510static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
511{
512    void *addr;
513    unsigned int bit;
514    int error;
515
516    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
517    BUG_ON(error);
518    set_bit(bit, addr);
519}
520
521static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
522{
523    void *addr;
524    unsigned int bit;
525    int error;
526
527    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
528    if (!error)
529        set_bit(bit, addr);
530    return error;
531}
532
533static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
534{
535    void *addr;
536    unsigned int bit;
537    int error;
538
539    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
540    BUG_ON(error);
541    clear_bit(bit, addr);
542}
543
544static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
545{
546    void *addr;
547    unsigned int bit;
548    int error;
549
550    error = memory_bm_find_bit(bm, pfn, &addr, &bit);
551    BUG_ON(error);
552    return test_bit(bit, addr);
553}
554
555static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
556{
557    void *addr;
558    unsigned int bit;
559
560    return !memory_bm_find_bit(bm, pfn, &addr, &bit);
561}
562
563/**
564 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
565 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
566 * returned.
567 *
568 * It is required to run memory_bm_position_reset() before the first call to
569 * this function.
570 */
571
572static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
573{
574    struct bm_block *bb;
575    int bit;
576
577    bb = bm->cur.block;
578    do {
579        bit = bm->cur.bit;
580        bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
581        if (bit < bm_block_bits(bb))
582            goto Return_pfn;
583
584        bb = list_entry(bb->hook.next, struct bm_block, hook);
585        bm->cur.block = bb;
586        bm->cur.bit = 0;
587    } while (&bb->hook != &bm->blocks);
588
589    memory_bm_position_reset(bm);
590    return BM_END_OF_MAP;
591
592 Return_pfn:
593    bm->cur.bit = bit + 1;
594    return bb->start_pfn + bit;
595}
596
597/**
598 * This structure represents a range of page frames the contents of which
599 * should not be saved during the suspend.
600 */
601
602struct nosave_region {
603    struct list_head list;
604    unsigned long start_pfn;
605    unsigned long end_pfn;
606};
607
608static LIST_HEAD(nosave_regions);
609
610/**
611 * register_nosave_region - register a range of page frames the contents
612 * of which should not be saved during the suspend (to be used in the early
613 * initialization code)
614 */
615
616void __init
617__register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
618             int use_kmalloc)
619{
620    struct nosave_region *region;
621
622    if (start_pfn >= end_pfn)
623        return;
624
625    if (!list_empty(&nosave_regions)) {
626        /* Try to extend the previous region (they should be sorted) */
627        region = list_entry(nosave_regions.prev,
628                    struct nosave_region, list);
629        if (region->end_pfn == start_pfn) {
630            region->end_pfn = end_pfn;
631            goto Report;
632        }
633    }
634    if (use_kmalloc) {
635        /* during init, this shouldn't fail */
636        region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
637        BUG_ON(!region);
638    } else
639        /* This allocation cannot fail */
640        region = alloc_bootmem(sizeof(struct nosave_region));
641    region->start_pfn = start_pfn;
642    region->end_pfn = end_pfn;
643    list_add_tail(&region->list, &nosave_regions);
644 Report:
645    printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
646        start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
647}
648
649/*
650 * Set bits in this map correspond to the page frames the contents of which
651 * should not be saved during the suspend.
652 */
653static struct memory_bitmap *forbidden_pages_map;
654
655/* Set bits in this map correspond to free page frames. */
656static struct memory_bitmap *free_pages_map;
657
658/*
659 * Each page frame allocated for creating the image is marked by setting the
660 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
661 */
662
663void swsusp_set_page_free(struct page *page)
664{
665    if (free_pages_map)
666        memory_bm_set_bit(free_pages_map, page_to_pfn(page));
667}
668
669static int swsusp_page_is_free(struct page *page)
670{
671    return free_pages_map ?
672        memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
673}
674
675void swsusp_unset_page_free(struct page *page)
676{
677    if (free_pages_map)
678        memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
679}
680
681static void swsusp_set_page_forbidden(struct page *page)
682{
683    if (forbidden_pages_map)
684        memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
685}
686
687int swsusp_page_is_forbidden(struct page *page)
688{
689    return forbidden_pages_map ?
690        memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
691}
692
693static void swsusp_unset_page_forbidden(struct page *page)
694{
695    if (forbidden_pages_map)
696        memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
697}
698
699/**
700 * mark_nosave_pages - set bits corresponding to the page frames the
701 * contents of which should not be saved in a given bitmap.
702 */
703
704static void mark_nosave_pages(struct memory_bitmap *bm)
705{
706    struct nosave_region *region;
707
708    if (list_empty(&nosave_regions))
709        return;
710
711    list_for_each_entry(region, &nosave_regions, list) {
712        unsigned long pfn;
713
714        pr_debug("PM: Marking nosave pages: [mem %#010llx-%#010llx]\n",
715             (unsigned long long) region->start_pfn << PAGE_SHIFT,
716             ((unsigned long long) region->end_pfn << PAGE_SHIFT)
717                - 1);
718
719        for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
720            if (pfn_valid(pfn)) {
721                /*
722                 * It is safe to ignore the result of
723                 * mem_bm_set_bit_check() here, since we won't
724                 * touch the PFNs for which the error is
725                 * returned anyway.
726                 */
727                mem_bm_set_bit_check(bm, pfn);
728            }
729    }
730}
731
732/**
733 * create_basic_memory_bitmaps - create bitmaps needed for marking page
734 * frames that should not be saved and free page frames. The pointers
735 * forbidden_pages_map and free_pages_map are only modified if everything
736 * goes well, because we don't want the bits to be used before both bitmaps
737 * are set up.
738 */
739
740int create_basic_memory_bitmaps(void)
741{
742    struct memory_bitmap *bm1, *bm2;
743    int error = 0;
744
745    BUG_ON(forbidden_pages_map || free_pages_map);
746
747    bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
748    if (!bm1)
749        return -ENOMEM;
750
751    error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
752    if (error)
753        goto Free_first_object;
754
755    bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
756    if (!bm2)
757        goto Free_first_bitmap;
758
759    error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
760    if (error)
761        goto Free_second_object;
762
763    forbidden_pages_map = bm1;
764    free_pages_map = bm2;
765    mark_nosave_pages(forbidden_pages_map);
766
767    pr_debug("PM: Basic memory bitmaps created\n");
768
769    return 0;
770
771 Free_second_object:
772    kfree(bm2);
773 Free_first_bitmap:
774     memory_bm_free(bm1, PG_UNSAFE_CLEAR);
775 Free_first_object:
776    kfree(bm1);
777    return -ENOMEM;
778}
779
780/**
781 * free_basic_memory_bitmaps - free memory bitmaps allocated by
782 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary
783 * so that the bitmaps themselves are not referred to while they are being
784 * freed.
785 */
786
787void free_basic_memory_bitmaps(void)
788{
789    struct memory_bitmap *bm1, *bm2;
790
791    BUG_ON(!(forbidden_pages_map && free_pages_map));
792
793    bm1 = forbidden_pages_map;
794    bm2 = free_pages_map;
795    forbidden_pages_map = NULL;
796    free_pages_map = NULL;
797    memory_bm_free(bm1, PG_UNSAFE_CLEAR);
798    kfree(bm1);
799    memory_bm_free(bm2, PG_UNSAFE_CLEAR);
800    kfree(bm2);
801
802    pr_debug("PM: Basic memory bitmaps freed\n");
803}
804
805/**
806 * snapshot_additional_pages - estimate the number of additional pages
807 * be needed for setting up the suspend image data structures for given
808 * zone (usually the returned value is greater than the exact number)
809 */
810
811unsigned int snapshot_additional_pages(struct zone *zone)
812{
813    unsigned int res;
814
815    res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
816    res += DIV_ROUND_UP(res * sizeof(struct bm_block),
817                LINKED_PAGE_DATA_SIZE);
818    return 2 * res;
819}
820
821#ifdef CONFIG_HIGHMEM
822/**
823 * count_free_highmem_pages - compute the total number of free highmem
824 * pages, system-wide.
825 */
826
827static unsigned int count_free_highmem_pages(void)
828{
829    struct zone *zone;
830    unsigned int cnt = 0;
831
832    for_each_populated_zone(zone)
833        if (is_highmem(zone))
834            cnt += zone_page_state(zone, NR_FREE_PAGES);
835
836    return cnt;
837}
838
839/**
840 * saveable_highmem_page - Determine whether a highmem page should be
841 * included in the suspend image.
842 *
843 * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
844 * and it isn't a part of a free chunk of pages.
845 */
846static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)
847{
848    struct page *page;
849
850    if (!pfn_valid(pfn))
851        return NULL;
852
853    page = pfn_to_page(pfn);
854    if (page_zone(page) != zone)
855        return NULL;
856
857    BUG_ON(!PageHighMem(page));
858
859    if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) ||
860        PageReserved(page))
861        return NULL;
862
863    if (page_is_guard(page))
864        return NULL;
865
866    return page;
867}
868
869/**
870 * count_highmem_pages - compute the total number of saveable highmem
871 * pages.
872 */
873
874static unsigned int count_highmem_pages(void)
875{
876    struct zone *zone;
877    unsigned int n = 0;
878
879    for_each_populated_zone(zone) {
880        unsigned long pfn, max_zone_pfn;
881
882        if (!is_highmem(zone))
883            continue;
884
885        mark_free_pages(zone);
886        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
887        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
888            if (saveable_highmem_page(zone, pfn))
889                n++;
890    }
891    return n;
892}
893#else
894static inline void *saveable_highmem_page(struct zone *z, unsigned long p)
895{
896    return NULL;
897}
898#endif /* CONFIG_HIGHMEM */
899
900/**
901 * saveable_page - Determine whether a non-highmem page should be included
902 * in the suspend image.
903 *
904 * We should save the page if it isn't Nosave, and is not in the range
905 * of pages statically defined as 'unsaveable', and it isn't a part of
906 * a free chunk of pages.
907 */
908static struct page *saveable_page(struct zone *zone, unsigned long pfn)
909{
910    struct page *page;
911
912    if (!pfn_valid(pfn))
913        return NULL;
914
915    page = pfn_to_page(pfn);
916    if (page_zone(page) != zone)
917        return NULL;
918
919    BUG_ON(PageHighMem(page));
920
921    if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
922        return NULL;
923
924    if (PageReserved(page)
925        && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
926        return NULL;
927
928    if (page_is_guard(page))
929        return NULL;
930
931    return page;
932}
933
934/**
935 * count_data_pages - compute the total number of saveable non-highmem
936 * pages.
937 */
938
939static unsigned int count_data_pages(void)
940{
941    struct zone *zone;
942    unsigned long pfn, max_zone_pfn;
943    unsigned int n = 0;
944
945    for_each_populated_zone(zone) {
946        if (is_highmem(zone))
947            continue;
948
949        mark_free_pages(zone);
950        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
951        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
952            if (saveable_page(zone, pfn))
953                n++;
954    }
955    return n;
956}
957
958/* This is needed, because copy_page and memcpy are not usable for copying
959 * task structs.
960 */
961static inline void do_copy_page(long *dst, long *src)
962{
963    int n;
964
965    for (n = PAGE_SIZE / sizeof(long); n; n--)
966        *dst++ = *src++;
967}
968
969
970/**
971 * safe_copy_page - check if the page we are going to copy is marked as
972 * present in the kernel page tables (this always is the case if
973 * CONFIG_DEBUG_PAGEALLOC is not set and in that case
974 * kernel_page_present() always returns 'true').
975 */
976static void safe_copy_page(void *dst, struct page *s_page)
977{
978    if (kernel_page_present(s_page)) {
979        do_copy_page(dst, page_address(s_page));
980    } else {
981        kernel_map_pages(s_page, 1, 1);
982        do_copy_page(dst, page_address(s_page));
983        kernel_map_pages(s_page, 1, 0);
984    }
985}
986
987
988#ifdef CONFIG_HIGHMEM
989static inline struct page *
990page_is_saveable(struct zone *zone, unsigned long pfn)
991{
992    return is_highmem(zone) ?
993        saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
994}
995
996static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
997{
998    struct page *s_page, *d_page;
999    void *src, *dst;
1000
1001    s_page = pfn_to_page(src_pfn);
1002    d_page = pfn_to_page(dst_pfn);
1003    if (PageHighMem(s_page)) {
1004        src = kmap_atomic(s_page);
1005        dst = kmap_atomic(d_page);
1006        do_copy_page(dst, src);
1007        kunmap_atomic(dst);
1008        kunmap_atomic(src);
1009    } else {
1010        if (PageHighMem(d_page)) {
1011            /* Page pointed to by src may contain some kernel
1012             * data modified by kmap_atomic()
1013             */
1014            safe_copy_page(buffer, s_page);
1015            dst = kmap_atomic(d_page);
1016            copy_page(dst, buffer);
1017            kunmap_atomic(dst);
1018        } else {
1019            safe_copy_page(page_address(d_page), s_page);
1020        }
1021    }
1022}
1023#else
1024#define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
1025
1026static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1027{
1028    safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1029                pfn_to_page(src_pfn));
1030}
1031#endif /* CONFIG_HIGHMEM */
1032
1033static void
1034copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
1035{
1036    struct zone *zone;
1037    unsigned long pfn;
1038
1039    for_each_populated_zone(zone) {
1040        unsigned long max_zone_pfn;
1041
1042        mark_free_pages(zone);
1043        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1044        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1045            if (page_is_saveable(zone, pfn))
1046                memory_bm_set_bit(orig_bm, pfn);
1047    }
1048    memory_bm_position_reset(orig_bm);
1049    memory_bm_position_reset(copy_bm);
1050    for(;;) {
1051        pfn = memory_bm_next_pfn(orig_bm);
1052        if (unlikely(pfn == BM_END_OF_MAP))
1053            break;
1054        copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1055    }
1056}
1057
1058/* Total number of image pages */
1059static unsigned int nr_copy_pages;
1060/* Number of pages needed for saving the original pfns of the image pages */
1061static unsigned int nr_meta_pages;
1062/*
1063 * Numbers of normal and highmem page frames allocated for hibernation image
1064 * before suspending devices.
1065 */
1066unsigned int alloc_normal, alloc_highmem;
1067/*
1068 * Memory bitmap used for marking saveable pages (during hibernation) or
1069 * hibernation image pages (during restore)
1070 */
1071static struct memory_bitmap orig_bm;
1072/*
1073 * Memory bitmap used during hibernation for marking allocated page frames that
1074 * will contain copies of saveable pages. During restore it is initially used
1075 * for marking hibernation image pages, but then the set bits from it are
1076 * duplicated in @orig_bm and it is released. On highmem systems it is next
1077 * used for marking "safe" highmem pages, but it has to be reinitialized for
1078 * this purpose.
1079 */
1080static struct memory_bitmap copy_bm;
1081
1082/**
1083 * swsusp_free - free pages allocated for the suspend.
1084 *
1085 * Suspend pages are alocated before the atomic copy is made, so we
1086 * need to release them after the resume.
1087 */
1088
1089void swsusp_free(void)
1090{
1091    struct zone *zone;
1092    unsigned long pfn, max_zone_pfn;
1093
1094    for_each_populated_zone(zone) {
1095        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1096        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1097            if (pfn_valid(pfn)) {
1098                struct page *page = pfn_to_page(pfn);
1099
1100                if (swsusp_page_is_forbidden(page) &&
1101                    swsusp_page_is_free(page)) {
1102                    swsusp_unset_page_forbidden(page);
1103                    swsusp_unset_page_free(page);
1104                    __free_page(page);
1105                }
1106            }
1107    }
1108    nr_copy_pages = 0;
1109    nr_meta_pages = 0;
1110    restore_pblist = NULL;
1111    buffer = NULL;
1112    alloc_normal = 0;
1113    alloc_highmem = 0;
1114}
1115
1116/* Helper functions used for the shrinking of memory. */
1117
1118#define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN)
1119
1120/**
1121 * preallocate_image_pages - Allocate a number of pages for hibernation image
1122 * @nr_pages: Number of page frames to allocate.
1123 * @mask: GFP flags to use for the allocation.
1124 *
1125 * Return value: Number of page frames actually allocated
1126 */
1127static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1128{
1129    unsigned long nr_alloc = 0;
1130
1131    while (nr_pages > 0) {
1132        struct page *page;
1133
1134        page = alloc_image_page(mask);
1135        if (!page)
1136            break;
1137        memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1138        if (PageHighMem(page))
1139            alloc_highmem++;
1140        else
1141            alloc_normal++;
1142        nr_pages--;
1143        nr_alloc++;
1144    }
1145
1146    return nr_alloc;
1147}
1148
1149static unsigned long preallocate_image_memory(unsigned long nr_pages,
1150                          unsigned long avail_normal)
1151{
1152    unsigned long alloc;
1153
1154    if (avail_normal <= alloc_normal)
1155        return 0;
1156
1157    alloc = avail_normal - alloc_normal;
1158    if (nr_pages < alloc)
1159        alloc = nr_pages;
1160
1161    return preallocate_image_pages(alloc, GFP_IMAGE);
1162}
1163
1164#ifdef CONFIG_HIGHMEM
1165static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1166{
1167    return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
1168}
1169
1170/**
1171 * __fraction - Compute (an approximation of) x * (multiplier / base)
1172 */
1173static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1174{
1175    x *= multiplier;
1176    do_div(x, base);
1177    return (unsigned long)x;
1178}
1179
1180static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1181                        unsigned long highmem,
1182                        unsigned long total)
1183{
1184    unsigned long alloc = __fraction(nr_pages, highmem, total);
1185
1186    return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
1187}
1188#else /* CONFIG_HIGHMEM */
1189static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1190{
1191    return 0;
1192}
1193
1194static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1195                        unsigned long highmem,
1196                        unsigned long total)
1197{
1198    return 0;
1199}
1200#endif /* CONFIG_HIGHMEM */
1201
1202/**
1203 * free_unnecessary_pages - Release preallocated pages not needed for the image
1204 */
1205static void free_unnecessary_pages(void)
1206{
1207    unsigned long save, to_free_normal, to_free_highmem;
1208
1209    save = count_data_pages();
1210    if (alloc_normal >= save) {
1211        to_free_normal = alloc_normal - save;
1212        save = 0;
1213    } else {
1214        to_free_normal = 0;
1215        save -= alloc_normal;
1216    }
1217    save += count_highmem_pages();
1218    if (alloc_highmem >= save) {
1219        to_free_highmem = alloc_highmem - save;
1220    } else {
1221        to_free_highmem = 0;
1222        save -= alloc_highmem;
1223        if (to_free_normal > save)
1224            to_free_normal -= save;
1225        else
1226            to_free_normal = 0;
1227    }
1228
1229    memory_bm_position_reset(&copy_bm);
1230
1231    while (to_free_normal > 0 || to_free_highmem > 0) {
1232        unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1233        struct page *page = pfn_to_page(pfn);
1234
1235        if (PageHighMem(page)) {
1236            if (!to_free_highmem)
1237                continue;
1238            to_free_highmem--;
1239            alloc_highmem--;
1240        } else {
1241            if (!to_free_normal)
1242                continue;
1243            to_free_normal--;
1244            alloc_normal--;
1245        }
1246        memory_bm_clear_bit(&copy_bm, pfn);
1247        swsusp_unset_page_forbidden(page);
1248        swsusp_unset_page_free(page);
1249        __free_page(page);
1250    }
1251}
1252
1253/**
1254 * minimum_image_size - Estimate the minimum acceptable size of an image
1255 * @saveable: Number of saveable pages in the system.
1256 *
1257 * We want to avoid attempting to free too much memory too hard, so estimate the
1258 * minimum acceptable size of a hibernation image to use as the lower limit for
1259 * preallocating memory.
1260 *
1261 * We assume that the minimum image size should be proportional to
1262 *
1263 * [number of saveable pages] - [number of pages that can be freed in theory]
1264 *
1265 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1266 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
1267 * minus mapped file pages.
1268 */
1269static unsigned long minimum_image_size(unsigned long saveable)
1270{
1271    unsigned long size;
1272
1273    size = global_page_state(NR_SLAB_RECLAIMABLE)
1274        + global_page_state(NR_ACTIVE_ANON)
1275        + global_page_state(NR_INACTIVE_ANON)
1276        + global_page_state(NR_ACTIVE_FILE)
1277        + global_page_state(NR_INACTIVE_FILE)
1278        - global_page_state(NR_FILE_MAPPED);
1279
1280    return saveable <= size ? 0 : saveable - size;
1281}
1282
1283/**
1284 * hibernate_preallocate_memory - Preallocate memory for hibernation image
1285 *
1286 * To create a hibernation image it is necessary to make a copy of every page
1287 * frame in use. We also need a number of page frames to be free during
1288 * hibernation for allocations made while saving the image and for device
1289 * drivers, in case they need to allocate memory from their hibernation
1290 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1291 * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through
1292 * /sys/power/reserved_size, respectively). To make this happen, we compute the
1293 * total number of available page frames and allocate at least
1294 *
1295 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2
1296 * + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1297 *
1298 * of them, which corresponds to the maximum size of a hibernation image.
1299 *
1300 * If image_size is set below the number following from the above formula,
1301 * the preallocation of memory is continued until the total number of saveable
1302 * pages in the system is below the requested image size or the minimum
1303 * acceptable image size returned by minimum_image_size(), whichever is greater.
1304 */
1305int hibernate_preallocate_memory(void)
1306{
1307    struct zone *zone;
1308    unsigned long saveable, size, max_size, count, highmem, pages = 0;
1309    unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1310    struct timeval start, stop;
1311    int error;
1312
1313    printk(KERN_INFO "PM: Preallocating image memory... ");
1314    do_gettimeofday(&start);
1315
1316    error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1317    if (error)
1318        goto err_out;
1319
1320    error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1321    if (error)
1322        goto err_out;
1323
1324    alloc_normal = 0;
1325    alloc_highmem = 0;
1326
1327    /* Count the number of saveable data pages. */
1328    save_highmem = count_highmem_pages();
1329    saveable = count_data_pages();
1330
1331    /*
1332     * Compute the total number of page frames we can use (count) and the
1333     * number of pages needed for image metadata (size).
1334     */
1335    count = saveable;
1336    saveable += save_highmem;
1337    highmem = save_highmem;
1338    size = 0;
1339    for_each_populated_zone(zone) {
1340        size += snapshot_additional_pages(zone);
1341        if (is_highmem(zone))
1342            highmem += zone_page_state(zone, NR_FREE_PAGES);
1343        else
1344            count += zone_page_state(zone, NR_FREE_PAGES);
1345    }
1346    avail_normal = count;
1347    count += highmem;
1348    count -= totalreserve_pages;
1349
1350    /* Add number of pages required for page keys (s390 only). */
1351    size += page_key_additional_pages(saveable);
1352
1353    /* Compute the maximum number of saveable pages to leave in memory. */
1354    max_size = (count - (size + PAGES_FOR_IO)) / 2
1355            - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
1356    /* Compute the desired number of image pages specified by image_size. */
1357    size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1358    if (size > max_size)
1359        size = max_size;
1360    /*
1361     * If the desired number of image pages is at least as large as the
1362     * current number of saveable pages in memory, allocate page frames for
1363     * the image and we're done.
1364     */
1365    if (size >= saveable) {
1366        pages = preallocate_image_highmem(save_highmem);
1367        pages += preallocate_image_memory(saveable - pages, avail_normal);
1368        goto out;
1369    }
1370
1371    /* Estimate the minimum size of the image. */
1372    pages = minimum_image_size(saveable);
1373    /*
1374     * To avoid excessive pressure on the normal zone, leave room in it to
1375     * accommodate an image of the minimum size (unless it's already too
1376     * small, in which case don't preallocate pages from it at all).
1377     */
1378    if (avail_normal > pages)
1379        avail_normal -= pages;
1380    else
1381        avail_normal = 0;
1382    if (size < pages)
1383        size = min_t(unsigned long, pages, max_size);
1384
1385    /*
1386     * Let the memory management subsystem know that we're going to need a
1387     * large number of page frames to allocate and make it free some memory.
1388     * NOTE: If this is not done, performance will be hurt badly in some
1389     * test cases.
1390     */
1391    shrink_all_memory(saveable - size);
1392
1393    /*
1394     * The number of saveable pages in memory was too high, so apply some
1395     * pressure to decrease it. First, make room for the largest possible
1396     * image and fail if that doesn't work. Next, try to decrease the size
1397     * of the image as much as indicated by 'size' using allocations from
1398     * highmem and non-highmem zones separately.
1399     */
1400    pages_highmem = preallocate_image_highmem(highmem / 2);
1401    alloc = (count - max_size) - pages_highmem;
1402    pages = preallocate_image_memory(alloc, avail_normal);
1403    if (pages < alloc) {
1404        /* We have exhausted non-highmem pages, try highmem. */
1405        alloc -= pages;
1406        pages += pages_highmem;
1407        pages_highmem = preallocate_image_highmem(alloc);
1408        if (pages_highmem < alloc)
1409            goto err_out;
1410        pages += pages_highmem;
1411        /*
1412         * size is the desired number of saveable pages to leave in
1413         * memory, so try to preallocate (all memory - size) pages.
1414         */
1415        alloc = (count - pages) - size;
1416        pages += preallocate_image_highmem(alloc);
1417    } else {
1418        /*
1419         * There are approximately max_size saveable pages at this point
1420         * and we want to reduce this number down to size.
1421         */
1422        alloc = max_size - size;
1423        size = preallocate_highmem_fraction(alloc, highmem, count);
1424        pages_highmem += size;
1425        alloc -= size;
1426        size = preallocate_image_memory(alloc, avail_normal);
1427        pages_highmem += preallocate_image_highmem(alloc - size);
1428        pages += pages_highmem + size;
1429    }
1430
1431    /*
1432     * We only need as many page frames for the image as there are saveable
1433     * pages in memory, but we have allocated more. Release the excessive
1434     * ones now.
1435     */
1436    free_unnecessary_pages();
1437
1438 out:
1439    do_gettimeofday(&stop);
1440    printk(KERN_CONT "done (allocated %lu pages)\n", pages);
1441    swsusp_show_speed(&start, &stop, pages, "Allocated");
1442
1443    return 0;
1444
1445 err_out:
1446    printk(KERN_CONT "\n");
1447    swsusp_free();
1448    return -ENOMEM;
1449}
1450
1451#ifdef CONFIG_HIGHMEM
1452/**
1453  * count_pages_for_highmem - compute the number of non-highmem pages
1454  * that will be necessary for creating copies of highmem pages.
1455  */
1456
1457static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1458{
1459    unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1460
1461    if (free_highmem >= nr_highmem)
1462        nr_highmem = 0;
1463    else
1464        nr_highmem -= free_highmem;
1465
1466    return nr_highmem;
1467}
1468#else
1469static unsigned int
1470count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1471#endif /* CONFIG_HIGHMEM */
1472
1473/**
1474 * enough_free_mem - Make sure we have enough free memory for the
1475 * snapshot image.
1476 */
1477
1478static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1479{
1480    struct zone *zone;
1481    unsigned int free = alloc_normal;
1482
1483    for_each_populated_zone(zone)
1484        if (!is_highmem(zone))
1485            free += zone_page_state(zone, NR_FREE_PAGES);
1486
1487    nr_pages += count_pages_for_highmem(nr_highmem);
1488    pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1489        nr_pages, PAGES_FOR_IO, free);
1490
1491    return free > nr_pages + PAGES_FOR_IO;
1492}
1493
1494#ifdef CONFIG_HIGHMEM
1495/**
1496 * get_highmem_buffer - if there are some highmem pages in the suspend
1497 * image, we may need the buffer to copy them and/or load their data.
1498 */
1499
1500static inline int get_highmem_buffer(int safe_needed)
1501{
1502    buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1503    return buffer ? 0 : -ENOMEM;
1504}
1505
1506/**
1507 * alloc_highmem_image_pages - allocate some highmem pages for the image.
1508 * Try to allocate as many pages as needed, but if the number of free
1509 * highmem pages is lesser than that, allocate them all.
1510 */
1511
1512static inline unsigned int
1513alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1514{
1515    unsigned int to_alloc = count_free_highmem_pages();
1516
1517    if (to_alloc > nr_highmem)
1518        to_alloc = nr_highmem;
1519
1520    nr_highmem -= to_alloc;
1521    while (to_alloc-- > 0) {
1522        struct page *page;
1523
1524        page = alloc_image_page(__GFP_HIGHMEM);
1525        memory_bm_set_bit(bm, page_to_pfn(page));
1526    }
1527    return nr_highmem;
1528}
1529#else
1530static inline int get_highmem_buffer(int safe_needed) { return 0; }
1531
1532static inline unsigned int
1533alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1534#endif /* CONFIG_HIGHMEM */
1535
1536/**
1537 * swsusp_alloc - allocate memory for the suspend image
1538 *
1539 * We first try to allocate as many highmem pages as there are
1540 * saveable highmem pages in the system. If that fails, we allocate
1541 * non-highmem pages for the copies of the remaining highmem ones.
1542 *
1543 * In this approach it is likely that the copies of highmem pages will
1544 * also be located in the high memory, because of the way in which
1545 * copy_data_pages() works.
1546 */
1547
1548static int
1549swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1550        unsigned int nr_pages, unsigned int nr_highmem)
1551{
1552    if (nr_highmem > 0) {
1553        if (get_highmem_buffer(PG_ANY))
1554            goto err_out;
1555        if (nr_highmem > alloc_highmem) {
1556            nr_highmem -= alloc_highmem;
1557            nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
1558        }
1559    }
1560    if (nr_pages > alloc_normal) {
1561        nr_pages -= alloc_normal;
1562        while (nr_pages-- > 0) {
1563            struct page *page;
1564
1565            page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1566            if (!page)
1567                goto err_out;
1568            memory_bm_set_bit(copy_bm, page_to_pfn(page));
1569        }
1570    }
1571
1572    return 0;
1573
1574 err_out:
1575    swsusp_free();
1576    return -ENOMEM;
1577}
1578
1579asmlinkage int swsusp_save(void)
1580{
1581    unsigned int nr_pages, nr_highmem;
1582
1583    printk(KERN_INFO "PM: Creating hibernation image:\n");
1584
1585    drain_local_pages(NULL);
1586    nr_pages = count_data_pages();
1587    nr_highmem = count_highmem_pages();
1588    printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1589
1590    if (!enough_free_mem(nr_pages, nr_highmem)) {
1591        printk(KERN_ERR "PM: Not enough free memory\n");
1592        return -ENOMEM;
1593    }
1594
1595    if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1596        printk(KERN_ERR "PM: Memory allocation failed\n");
1597        return -ENOMEM;
1598    }
1599
1600    /* During allocating of suspend pagedir, new cold pages may appear.
1601     * Kill them.
1602     */
1603    drain_local_pages(NULL);
1604    copy_data_pages(&copy_bm, &orig_bm);
1605
1606    /*
1607     * End of critical section. From now on, we can write to memory,
1608     * but we should not touch disk. This specially means we must _not_
1609     * touch swap space! Except we must write out our image of course.
1610     */
1611
1612    nr_pages += nr_highmem;
1613    nr_copy_pages = nr_pages;
1614    nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1615
1616    printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1617        nr_pages);
1618
1619    return 0;
1620}
1621
1622#ifndef CONFIG_ARCH_HIBERNATION_HEADER
1623static int init_header_complete(struct swsusp_info *info)
1624{
1625    memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1626    info->version_code = LINUX_VERSION_CODE;
1627    return 0;
1628}
1629
1630static char *check_image_kernel(struct swsusp_info *info)
1631{
1632    if (info->version_code != LINUX_VERSION_CODE)
1633        return "kernel version";
1634    if (strcmp(info->uts.sysname,init_utsname()->sysname))
1635        return "system type";
1636    if (strcmp(info->uts.release,init_utsname()->release))
1637        return "kernel release";
1638    if (strcmp(info->uts.version,init_utsname()->version))
1639        return "version";
1640    if (strcmp(info->uts.machine,init_utsname()->machine))
1641        return "machine";
1642    return NULL;
1643}
1644#endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1645
1646unsigned long snapshot_get_image_size(void)
1647{
1648    return nr_copy_pages + nr_meta_pages + 1;
1649}
1650
1651static int init_header(struct swsusp_info *info)
1652{
1653    memset(info, 0, sizeof(struct swsusp_info));
1654    info->num_physpages = num_physpages;
1655    info->image_pages = nr_copy_pages;
1656    info->pages = snapshot_get_image_size();
1657    info->size = info->pages;
1658    info->size <<= PAGE_SHIFT;
1659    return init_header_complete(info);
1660}
1661
1662/**
1663 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1664 * are stored in the array @buf[] (1 page at a time)
1665 */
1666
1667static inline void
1668pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1669{
1670    int j;
1671
1672    for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1673        buf[j] = memory_bm_next_pfn(bm);
1674        if (unlikely(buf[j] == BM_END_OF_MAP))
1675            break;
1676        /* Save page key for data page (s390 only). */
1677        page_key_read(buf + j);
1678    }
1679}
1680
1681/**
1682 * snapshot_read_next - used for reading the system memory snapshot.
1683 *
1684 * On the first call to it @handle should point to a zeroed
1685 * snapshot_handle structure. The structure gets updated and a pointer
1686 * to it should be passed to this function every next time.
1687 *
1688 * On success the function returns a positive number. Then, the caller
1689 * is allowed to read up to the returned number of bytes from the memory
1690 * location computed by the data_of() macro.
1691 *
1692 * The function returns 0 to indicate the end of data stream condition,
1693 * and a negative number is returned on error. In such cases the
1694 * structure pointed to by @handle is not updated and should not be used
1695 * any more.
1696 */
1697
1698int snapshot_read_next(struct snapshot_handle *handle)
1699{
1700    if (handle->cur > nr_meta_pages + nr_copy_pages)
1701        return 0;
1702
1703    if (!buffer) {
1704        /* This makes the buffer be freed by swsusp_free() */
1705        buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1706        if (!buffer)
1707            return -ENOMEM;
1708    }
1709    if (!handle->cur) {
1710        int error;
1711
1712        error = init_header((struct swsusp_info *)buffer);
1713        if (error)
1714            return error;
1715        handle->buffer = buffer;
1716        memory_bm_position_reset(&orig_bm);
1717        memory_bm_position_reset(&copy_bm);
1718    } else if (handle->cur <= nr_meta_pages) {
1719        clear_page(buffer);
1720        pack_pfns(buffer, &orig_bm);
1721    } else {
1722        struct page *page;
1723
1724        page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1725        if (PageHighMem(page)) {
1726            /* Highmem pages are copied to the buffer,
1727             * because we can't return with a kmapped
1728             * highmem page (we may not be called again).
1729             */
1730            void *kaddr;
1731
1732            kaddr = kmap_atomic(page);
1733            copy_page(buffer, kaddr);
1734            kunmap_atomic(kaddr);
1735            handle->buffer = buffer;
1736        } else {
1737            handle->buffer = page_address(page);
1738        }
1739    }
1740    handle->cur++;
1741    return PAGE_SIZE;
1742}
1743
1744/**
1745 * mark_unsafe_pages - mark the pages that cannot be used for storing
1746 * the image during resume, because they conflict with the pages that
1747 * had been used before suspend
1748 */
1749
1750static int mark_unsafe_pages(struct memory_bitmap *bm)
1751{
1752    struct zone *zone;
1753    unsigned long pfn, max_zone_pfn;
1754
1755    /* Clear page flags */
1756    for_each_populated_zone(zone) {
1757        max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1758        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1759            if (pfn_valid(pfn))
1760                swsusp_unset_page_free(pfn_to_page(pfn));
1761    }
1762
1763    /* Mark pages that correspond to the "original" pfns as "unsafe" */
1764    memory_bm_position_reset(bm);
1765    do {
1766        pfn = memory_bm_next_pfn(bm);
1767        if (likely(pfn != BM_END_OF_MAP)) {
1768            if (likely(pfn_valid(pfn)))
1769                swsusp_set_page_free(pfn_to_page(pfn));
1770            else
1771                return -EFAULT;
1772        }
1773    } while (pfn != BM_END_OF_MAP);
1774
1775    allocated_unsafe_pages = 0;
1776
1777    return 0;
1778}
1779
1780static void
1781duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1782{
1783    unsigned long pfn;
1784
1785    memory_bm_position_reset(src);
1786    pfn = memory_bm_next_pfn(src);
1787    while (pfn != BM_END_OF_MAP) {
1788        memory_bm_set_bit(dst, pfn);
1789        pfn = memory_bm_next_pfn(src);
1790    }
1791}
1792
1793static int check_header(struct swsusp_info *info)
1794{
1795    char *reason;
1796
1797    reason = check_image_kernel(info);
1798    if (!reason && info->num_physpages != num_physpages)
1799        reason = "memory size";
1800    if (reason) {
1801        printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1802        return -EPERM;
1803    }
1804    return 0;
1805}
1806
1807/**
1808 * load header - check the image header and copy data from it
1809 */
1810
1811static int
1812load_header(struct swsusp_info *info)
1813{
1814    int error;
1815
1816    restore_pblist = NULL;
1817    error = check_header(info);
1818    if (!error) {
1819        nr_copy_pages = info->image_pages;
1820        nr_meta_pages = info->pages - info->image_pages - 1;
1821    }
1822    return error;
1823}
1824
1825/**
1826 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1827 * the corresponding bit in the memory bitmap @bm
1828 */
1829static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1830{
1831    int j;
1832
1833    for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1834        if (unlikely(buf[j] == BM_END_OF_MAP))
1835            break;
1836
1837        /* Extract and buffer page key for data page (s390 only). */
1838        page_key_memorize(buf + j);
1839
1840        if (memory_bm_pfn_present(bm, buf[j]))
1841            memory_bm_set_bit(bm, buf[j]);
1842        else
1843            return -EFAULT;
1844    }
1845
1846    return 0;
1847}
1848
1849/* List of "safe" pages that may be used to store data loaded from the suspend
1850 * image
1851 */
1852static struct linked_page *safe_pages_list;
1853
1854#ifdef CONFIG_HIGHMEM
1855/* struct highmem_pbe is used for creating the list of highmem pages that
1856 * should be restored atomically during the resume from disk, because the page
1857 * frames they have occupied before the suspend are in use.
1858 */
1859struct highmem_pbe {
1860    struct page *copy_page; /* data is here now */
1861    struct page *orig_page; /* data was here before the suspend */
1862    struct highmem_pbe *next;
1863};
1864
1865/* List of highmem PBEs needed for restoring the highmem pages that were
1866 * allocated before the suspend and included in the suspend image, but have
1867 * also been allocated by the "resume" kernel, so their contents cannot be
1868 * written directly to their "original" page frames.
1869 */
1870static struct highmem_pbe *highmem_pblist;
1871
1872/**
1873 * count_highmem_image_pages - compute the number of highmem pages in the
1874 * suspend image. The bits in the memory bitmap @bm that correspond to the
1875 * image pages are assumed to be set.
1876 */
1877
1878static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1879{
1880    unsigned long pfn;
1881    unsigned int cnt = 0;
1882
1883    memory_bm_position_reset(bm);
1884    pfn = memory_bm_next_pfn(bm);
1885    while (pfn != BM_END_OF_MAP) {
1886        if (PageHighMem(pfn_to_page(pfn)))
1887            cnt++;
1888
1889        pfn = memory_bm_next_pfn(bm);
1890    }
1891    return cnt;
1892}
1893
1894/**
1895 * prepare_highmem_image - try to allocate as many highmem pages as
1896 * there are highmem image pages (@nr_highmem_p points to the variable
1897 * containing the number of highmem image pages). The pages that are
1898 * "safe" (ie. will not be overwritten when the suspend image is
1899 * restored) have the corresponding bits set in @bm (it must be
1900 * unitialized).
1901 *
1902 * NOTE: This function should not be called if there are no highmem
1903 * image pages.
1904 */
1905
1906static unsigned int safe_highmem_pages;
1907
1908static struct memory_bitmap *safe_highmem_bm;
1909
1910static int
1911prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1912{
1913    unsigned int to_alloc;
1914
1915    if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1916        return -ENOMEM;
1917
1918    if (get_highmem_buffer(PG_SAFE))
1919        return -ENOMEM;
1920
1921    to_alloc = count_free_highmem_pages();
1922    if (to_alloc > *nr_highmem_p)
1923        to_alloc = *nr_highmem_p;
1924    else
1925        *nr_highmem_p = to_alloc;
1926
1927    safe_highmem_pages = 0;
1928    while (to_alloc-- > 0) {
1929        struct page *page;
1930
1931        page = alloc_page(__GFP_HIGHMEM);
1932        if (!swsusp_page_is_free(page)) {
1933            /* The page is "safe", set its bit the bitmap */
1934            memory_bm_set_bit(bm, page_to_pfn(page));
1935            safe_highmem_pages++;
1936        }
1937        /* Mark the page as allocated */
1938        swsusp_set_page_forbidden(page);
1939        swsusp_set_page_free(page);
1940    }
1941    memory_bm_position_reset(bm);
1942    safe_highmem_bm = bm;
1943    return 0;
1944}
1945
1946/**
1947 * get_highmem_page_buffer - for given highmem image page find the buffer
1948 * that suspend_write_next() should set for its caller to write to.
1949 *
1950 * If the page is to be saved to its "original" page frame or a copy of
1951 * the page is to be made in the highmem, @buffer is returned. Otherwise,
1952 * the copy of the page is to be made in normal memory, so the address of
1953 * the copy is returned.
1954 *
1955 * If @buffer is returned, the caller of suspend_write_next() will write
1956 * the page's contents to @buffer, so they will have to be copied to the
1957 * right location on the next call to suspend_write_next() and it is done
1958 * with the help of copy_last_highmem_page(). For this purpose, if
1959 * @buffer is returned, @last_highmem page is set to the page to which
1960 * the data will have to be copied from @buffer.
1961 */
1962
1963static struct page *last_highmem_page;
1964
1965static void *
1966get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1967{
1968    struct highmem_pbe *pbe;
1969    void *kaddr;
1970
1971    if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1972        /* We have allocated the "original" page frame and we can
1973         * use it directly to store the loaded page.
1974         */
1975        last_highmem_page = page;
1976        return buffer;
1977    }
1978    /* The "original" page frame has not been allocated and we have to
1979     * use a "safe" page frame to store the loaded page.
1980     */
1981    pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1982    if (!pbe) {
1983        swsusp_free();
1984        return ERR_PTR(-ENOMEM);
1985    }
1986    pbe->orig_page = page;
1987    if (safe_highmem_pages > 0) {
1988        struct page *tmp;
1989
1990        /* Copy of the page will be stored in high memory */
1991        kaddr = buffer;
1992        tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1993        safe_highmem_pages--;
1994        last_highmem_page = tmp;
1995        pbe->copy_page = tmp;
1996    } else {
1997        /* Copy of the page will be stored in normal memory */
1998        kaddr = safe_pages_list;
1999        safe_pages_list = safe_pages_list->next;
2000        pbe->copy_page = virt_to_page(kaddr);
2001    }
2002    pbe->next = highmem_pblist;
2003    highmem_pblist = pbe;
2004    return kaddr;
2005}
2006
2007/**
2008 * copy_last_highmem_page - copy the contents of a highmem image from
2009 * @buffer, where the caller of snapshot_write_next() has place them,
2010 * to the right location represented by @last_highmem_page .
2011 */
2012
2013static void copy_last_highmem_page(void)
2014{
2015    if (last_highmem_page) {
2016        void *dst;
2017
2018        dst = kmap_atomic(last_highmem_page);
2019        copy_page(dst, buffer);
2020        kunmap_atomic(dst);
2021        last_highmem_page = NULL;
2022    }
2023}
2024
2025static inline int last_highmem_page_copied(void)
2026{
2027    return !last_highmem_page;
2028}
2029
2030static inline void free_highmem_data(void)
2031{
2032    if (safe_highmem_bm)
2033        memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
2034
2035    if (buffer)
2036        free_image_page(buffer, PG_UNSAFE_CLEAR);
2037}
2038#else
2039static inline int get_safe_write_buffer(void) { return 0; }
2040
2041static unsigned int
2042count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2043
2044static inline int
2045prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
2046{
2047    return 0;
2048}
2049
2050static inline void *
2051get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
2052{
2053    return ERR_PTR(-EINVAL);
2054}
2055
2056static inline void copy_last_highmem_page(void) {}
2057static inline int last_highmem_page_copied(void) { return 1; }
2058static inline void free_highmem_data(void) {}
2059#endif /* CONFIG_HIGHMEM */
2060
2061/**
2062 * prepare_image - use the memory bitmap @bm to mark the pages that will
2063 * be overwritten in the process of restoring the system memory state
2064 * from the suspend image ("unsafe" pages) and allocate memory for the
2065 * image.
2066 *
2067 * The idea is to allocate a new memory bitmap first and then allocate
2068 * as many pages as needed for the image data, but not to assign these
2069 * pages to specific tasks initially. Instead, we just mark them as
2070 * allocated and create a lists of "safe" pages that will be used
2071 * later. On systems with high memory a list of "safe" highmem pages is
2072 * also created.
2073 */
2074
2075#define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2076
2077static int
2078prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
2079{
2080    unsigned int nr_pages, nr_highmem;
2081    struct linked_page *sp_list, *lp;
2082    int error;
2083
2084    /* If there is no highmem, the buffer will not be necessary */
2085    free_image_page(buffer, PG_UNSAFE_CLEAR);
2086    buffer = NULL;
2087
2088    nr_highmem = count_highmem_image_pages(bm);
2089    error = mark_unsafe_pages(bm);
2090    if (error)
2091        goto Free;
2092
2093    error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2094    if (error)
2095        goto Free;
2096
2097    duplicate_memory_bitmap(new_bm, bm);
2098    memory_bm_free(bm, PG_UNSAFE_KEEP);
2099    if (nr_highmem > 0) {
2100        error = prepare_highmem_image(bm, &nr_highmem);
2101        if (error)
2102            goto Free;
2103    }
2104    /* Reserve some safe pages for potential later use.
2105     *
2106     * NOTE: This way we make sure there will be enough safe pages for the
2107     * chain_alloc() in get_buffer(). It is a bit wasteful, but
2108     * nr_copy_pages cannot be greater than 50% of the memory anyway.
2109     */
2110    sp_list = NULL;
2111    /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2112    nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2113    nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2114    while (nr_pages > 0) {
2115        lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2116        if (!lp) {
2117            error = -ENOMEM;
2118            goto Free;
2119        }
2120        lp->next = sp_list;
2121        sp_list = lp;
2122        nr_pages--;
2123    }
2124    /* Preallocate memory for the image */
2125    safe_pages_list = NULL;
2126    nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2127    while (nr_pages > 0) {
2128        lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2129        if (!lp) {
2130            error = -ENOMEM;
2131            goto Free;
2132        }
2133        if (!swsusp_page_is_free(virt_to_page(lp))) {
2134            /* The page is "safe", add it to the list */
2135            lp->next = safe_pages_list;
2136            safe_pages_list = lp;
2137        }
2138        /* Mark the page as allocated */
2139        swsusp_set_page_forbidden(virt_to_page(lp));
2140        swsusp_set_page_free(virt_to_page(lp));
2141        nr_pages--;
2142    }
2143    /* Free the reserved safe pages so that chain_alloc() can use them */
2144    while (sp_list) {
2145        lp = sp_list->next;
2146        free_image_page(sp_list, PG_UNSAFE_CLEAR);
2147        sp_list = lp;
2148    }
2149    return 0;
2150
2151 Free:
2152    swsusp_free();
2153    return error;
2154}
2155
2156/**
2157 * get_buffer - compute the address that snapshot_write_next() should
2158 * set for its caller to write to.
2159 */
2160
2161static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
2162{
2163    struct pbe *pbe;
2164    struct page *page;
2165    unsigned long pfn = memory_bm_next_pfn(bm);
2166
2167    if (pfn == BM_END_OF_MAP)
2168        return ERR_PTR(-EFAULT);
2169
2170    page = pfn_to_page(pfn);
2171    if (PageHighMem(page))
2172        return get_highmem_page_buffer(page, ca);
2173
2174    if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2175        /* We have allocated the "original" page frame and we can
2176         * use it directly to store the loaded page.
2177         */
2178        return page_address(page);
2179
2180    /* The "original" page frame has not been allocated and we have to
2181     * use a "safe" page frame to store the loaded page.
2182     */
2183    pbe = chain_alloc(ca, sizeof(struct pbe));
2184    if (!pbe) {
2185        swsusp_free();
2186        return ERR_PTR(-ENOMEM);
2187    }
2188    pbe->orig_address = page_address(page);
2189    pbe->address = safe_pages_list;
2190    safe_pages_list = safe_pages_list->next;
2191    pbe->next = restore_pblist;
2192    restore_pblist = pbe;
2193    return pbe->address;
2194}
2195
2196/**
2197 * snapshot_write_next - used for writing the system memory snapshot.
2198 *
2199 * On the first call to it @handle should point to a zeroed
2200 * snapshot_handle structure. The structure gets updated and a pointer
2201 * to it should be passed to this function every next time.
2202 *
2203 * On success the function returns a positive number. Then, the caller
2204 * is allowed to write up to the returned number of bytes to the memory
2205 * location computed by the data_of() macro.
2206 *
2207 * The function returns 0 to indicate the "end of file" condition,
2208 * and a negative number is returned on error. In such cases the
2209 * structure pointed to by @handle is not updated and should not be used
2210 * any more.
2211 */
2212
2213int snapshot_write_next(struct snapshot_handle *handle)
2214{
2215    static struct chain_allocator ca;
2216    int error = 0;
2217
2218    /* Check if we have already loaded the entire image */
2219    if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages)
2220        return 0;
2221
2222    handle->sync_read = 1;
2223
2224    if (!handle->cur) {
2225        if (!buffer)
2226            /* This makes the buffer be freed by swsusp_free() */
2227            buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2228
2229        if (!buffer)
2230            return -ENOMEM;
2231
2232        handle->buffer = buffer;
2233    } else if (handle->cur == 1) {
2234        error = load_header(buffer);
2235        if (error)
2236            return error;
2237
2238        error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2239        if (error)
2240            return error;
2241
2242        /* Allocate buffer for page keys. */
2243        error = page_key_alloc(nr_copy_pages);
2244        if (error)
2245            return error;
2246
2247    } else if (handle->cur <= nr_meta_pages + 1) {
2248        error = unpack_orig_pfns(buffer, &copy_bm);
2249        if (error)
2250            return error;
2251
2252        if (handle->cur == nr_meta_pages + 1) {
2253            error = prepare_image(&orig_bm, &copy_bm);
2254            if (error)
2255                return error;
2256
2257            chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2258            memory_bm_position_reset(&orig_bm);
2259            restore_pblist = NULL;
2260            handle->buffer = get_buffer(&orig_bm, &ca);
2261            handle->sync_read = 0;
2262            if (IS_ERR(handle->buffer))
2263                return PTR_ERR(handle->buffer);
2264        }
2265    } else {
2266        copy_last_highmem_page();
2267        /* Restore page key for data page (s390 only). */
2268        page_key_write(handle->buffer);
2269        handle->buffer = get_buffer(&orig_bm, &ca);
2270        if (IS_ERR(handle->buffer))
2271            return PTR_ERR(handle->buffer);
2272        if (handle->buffer != buffer)
2273            handle->sync_read = 0;
2274    }
2275    handle->cur++;
2276    return PAGE_SIZE;
2277}
2278
2279/**
2280 * snapshot_write_finalize - must be called after the last call to
2281 * snapshot_write_next() in case the last page in the image happens
2282 * to be a highmem page and its contents should be stored in the
2283 * highmem. Additionally, it releases the memory that will not be
2284 * used any more.
2285 */
2286
2287void snapshot_write_finalize(struct snapshot_handle *handle)
2288{
2289    copy_last_highmem_page();
2290    /* Restore page key for data page (s390 only). */
2291    page_key_write(handle->buffer);
2292    page_key_free();
2293    /* Free only if we have loaded the image entirely */
2294    if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {
2295        memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
2296        free_highmem_data();
2297    }
2298}
2299
2300int snapshot_image_loaded(struct snapshot_handle *handle)
2301{
2302    return !(!nr_copy_pages || !last_highmem_page_copied() ||
2303            handle->cur <= nr_meta_pages + nr_copy_pages);
2304}
2305
2306#ifdef CONFIG_HIGHMEM
2307/* Assumes that @buf is ready and points to a "safe" page */
2308static inline void
2309swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
2310{
2311    void *kaddr1, *kaddr2;
2312
2313    kaddr1 = kmap_atomic(p1);
2314    kaddr2 = kmap_atomic(p2);
2315    copy_page(buf, kaddr1);
2316    copy_page(kaddr1, kaddr2);
2317    copy_page(kaddr2, buf);
2318    kunmap_atomic(kaddr2);
2319    kunmap_atomic(kaddr1);
2320}
2321
2322/**
2323 * restore_highmem - for each highmem page that was allocated before
2324 * the suspend and included in the suspend image, and also has been
2325 * allocated by the "resume" kernel swap its current (ie. "before
2326 * resume") contents with the previous (ie. "before suspend") one.
2327 *
2328 * If the resume eventually fails, we can call this function once
2329 * again and restore the "before resume" highmem state.
2330 */
2331
2332int restore_highmem(void)
2333{
2334    struct highmem_pbe *pbe = highmem_pblist;
2335    void *buf;
2336
2337    if (!pbe)
2338        return 0;
2339
2340    buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2341    if (!buf)
2342        return -ENOMEM;
2343
2344    while (pbe) {
2345        swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2346        pbe = pbe->next;
2347    }
2348    free_image_page(buf, PG_UNSAFE_CLEAR);
2349    return 0;
2350}
2351#endif /* CONFIG_HIGHMEM */
2352

Archive Download this file



interactive