Root/mm/nobootmem.c

1/*
2 * bootmem - A boot-time physical memory allocator and configurator
3 *
4 * Copyright (C) 1999 Ingo Molnar
5 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
7 *
8 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
10 */
11#include <linux/init.h>
12#include <linux/pfn.h>
13#include <linux/slab.h>
14#include <linux/bootmem.h>
15#include <linux/export.h>
16#include <linux/kmemleak.h>
17#include <linux/range.h>
18#include <linux/memblock.h>
19
20#include <asm/bug.h>
21#include <asm/io.h>
22#include <asm/processor.h>
23
24#include "internal.h"
25
26#ifndef CONFIG_NEED_MULTIPLE_NODES
27struct pglist_data __refdata contig_page_data;
28EXPORT_SYMBOL(contig_page_data);
29#endif
30
31unsigned long max_low_pfn;
32unsigned long min_low_pfn;
33unsigned long max_pfn;
34
35static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
36                    u64 goal, u64 limit)
37{
38    void *ptr;
39    u64 addr;
40
41    if (limit > memblock.current_limit)
42        limit = memblock.current_limit;
43
44    addr = memblock_find_in_range_node(goal, limit, size, align, nid);
45    if (!addr)
46        return NULL;
47
48    memblock_reserve(addr, size);
49    ptr = phys_to_virt(addr);
50    memset(ptr, 0, size);
51    /*
52     * The min_count is set to 0 so that bootmem allocated blocks
53     * are never reported as leaks.
54     */
55    kmemleak_alloc(ptr, size, 0, 0);
56    return ptr;
57}
58
59/*
60 * free_bootmem_late - free bootmem pages directly to page allocator
61 * @addr: starting address of the range
62 * @size: size of the range in bytes
63 *
64 * This is only useful when the bootmem allocator has already been torn
65 * down, but we are still initializing the system. Pages are given directly
66 * to the page allocator, no bootmem metadata is updated because it is gone.
67 */
68void __init free_bootmem_late(unsigned long addr, unsigned long size)
69{
70    unsigned long cursor, end;
71
72    kmemleak_free_part(__va(addr), size);
73
74    cursor = PFN_UP(addr);
75    end = PFN_DOWN(addr + size);
76
77    for (; cursor < end; cursor++) {
78        __free_pages_bootmem(pfn_to_page(cursor), 0);
79        totalram_pages++;
80    }
81}
82
83static void __init __free_pages_memory(unsigned long start, unsigned long end)
84{
85    unsigned long i, start_aligned, end_aligned;
86    int order = ilog2(BITS_PER_LONG);
87
88    start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
89    end_aligned = end & ~(BITS_PER_LONG - 1);
90
91    if (end_aligned <= start_aligned) {
92        for (i = start; i < end; i++)
93            __free_pages_bootmem(pfn_to_page(i), 0);
94
95        return;
96    }
97
98    for (i = start; i < start_aligned; i++)
99        __free_pages_bootmem(pfn_to_page(i), 0);
100
101    for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
102        __free_pages_bootmem(pfn_to_page(i), order);
103
104    for (i = end_aligned; i < end; i++)
105        __free_pages_bootmem(pfn_to_page(i), 0);
106}
107
108static unsigned long __init __free_memory_core(phys_addr_t start,
109                 phys_addr_t end)
110{
111    unsigned long start_pfn = PFN_UP(start);
112    unsigned long end_pfn = min_t(unsigned long,
113                      PFN_DOWN(end), max_low_pfn);
114
115    if (start_pfn > end_pfn)
116        return 0;
117
118    __free_pages_memory(start_pfn, end_pfn);
119
120    return end_pfn - start_pfn;
121}
122
123static unsigned long __init free_low_memory_core_early(void)
124{
125    unsigned long count = 0;
126    phys_addr_t start, end, size;
127    u64 i;
128
129    for_each_free_mem_range(i, MAX_NUMNODES, &start, &end, NULL)
130        count += __free_memory_core(start, end);
131
132    /* free range that is used for reserved array if we allocate it */
133    size = get_allocated_memblock_reserved_regions_info(&start);
134    if (size)
135        count += __free_memory_core(start, start + size);
136
137    return count;
138}
139
140static int reset_managed_pages_done __initdata;
141
142static inline void __init reset_node_managed_pages(pg_data_t *pgdat)
143{
144    struct zone *z;
145
146    if (reset_managed_pages_done)
147        return;
148    for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
149        z->managed_pages = 0;
150}
151
152void __init reset_all_zones_managed_pages(void)
153{
154    struct pglist_data *pgdat;
155
156    for_each_online_pgdat(pgdat)
157        reset_node_managed_pages(pgdat);
158    reset_managed_pages_done = 1;
159}
160
161/**
162 * free_all_bootmem - release free pages to the buddy allocator
163 *
164 * Returns the number of pages actually released.
165 */
166unsigned long __init free_all_bootmem(void)
167{
168    unsigned long pages;
169
170    reset_all_zones_managed_pages();
171
172    /*
173     * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
174     * because in some case like Node0 doesn't have RAM installed
175     * low ram will be on Node1
176     */
177    pages = free_low_memory_core_early();
178    totalram_pages += pages;
179
180    return pages;
181}
182
183/**
184 * free_bootmem_node - mark a page range as usable
185 * @pgdat: node the range resides on
186 * @physaddr: starting address of the range
187 * @size: size of the range in bytes
188 *
189 * Partial pages will be considered reserved and left as they are.
190 *
191 * The range must reside completely on the specified node.
192 */
193void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
194                  unsigned long size)
195{
196    kmemleak_free_part(__va(physaddr), size);
197    memblock_free(physaddr, size);
198}
199
200/**
201 * free_bootmem - mark a page range as usable
202 * @addr: starting address of the range
203 * @size: size of the range in bytes
204 *
205 * Partial pages will be considered reserved and left as they are.
206 *
207 * The range must be contiguous but may span node boundaries.
208 */
209void __init free_bootmem(unsigned long addr, unsigned long size)
210{
211    kmemleak_free_part(__va(addr), size);
212    memblock_free(addr, size);
213}
214
215static void * __init ___alloc_bootmem_nopanic(unsigned long size,
216                    unsigned long align,
217                    unsigned long goal,
218                    unsigned long limit)
219{
220    void *ptr;
221
222    if (WARN_ON_ONCE(slab_is_available()))
223        return kzalloc(size, GFP_NOWAIT);
224
225restart:
226
227    ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
228
229    if (ptr)
230        return ptr;
231
232    if (goal != 0) {
233        goal = 0;
234        goto restart;
235    }
236
237    return NULL;
238}
239
240/**
241 * __alloc_bootmem_nopanic - allocate boot memory without panicking
242 * @size: size of the request in bytes
243 * @align: alignment of the region
244 * @goal: preferred starting address of the region
245 *
246 * The goal is dropped if it can not be satisfied and the allocation will
247 * fall back to memory below @goal.
248 *
249 * Allocation may happen on any node in the system.
250 *
251 * Returns NULL on failure.
252 */
253void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
254                    unsigned long goal)
255{
256    unsigned long limit = -1UL;
257
258    return ___alloc_bootmem_nopanic(size, align, goal, limit);
259}
260
261static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
262                    unsigned long goal, unsigned long limit)
263{
264    void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
265
266    if (mem)
267        return mem;
268    /*
269     * Whoops, we cannot satisfy the allocation request.
270     */
271    printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
272    panic("Out of memory");
273    return NULL;
274}
275
276/**
277 * __alloc_bootmem - allocate boot memory
278 * @size: size of the request in bytes
279 * @align: alignment of the region
280 * @goal: preferred starting address of the region
281 *
282 * The goal is dropped if it can not be satisfied and the allocation will
283 * fall back to memory below @goal.
284 *
285 * Allocation may happen on any node in the system.
286 *
287 * The function panics if the request can not be satisfied.
288 */
289void * __init __alloc_bootmem(unsigned long size, unsigned long align,
290                  unsigned long goal)
291{
292    unsigned long limit = -1UL;
293
294    return ___alloc_bootmem(size, align, goal, limit);
295}
296
297void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
298                           unsigned long size,
299                           unsigned long align,
300                           unsigned long goal,
301                           unsigned long limit)
302{
303    void *ptr;
304
305again:
306    ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
307                    goal, limit);
308    if (ptr)
309        return ptr;
310
311    ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
312                    goal, limit);
313    if (ptr)
314        return ptr;
315
316    if (goal) {
317        goal = 0;
318        goto again;
319    }
320
321    return NULL;
322}
323
324void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
325                   unsigned long align, unsigned long goal)
326{
327    if (WARN_ON_ONCE(slab_is_available()))
328        return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
329
330    return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
331}
332
333void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
334                    unsigned long align, unsigned long goal,
335                    unsigned long limit)
336{
337    void *ptr;
338
339    ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit);
340    if (ptr)
341        return ptr;
342
343    printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
344    panic("Out of memory");
345    return NULL;
346}
347
348/**
349 * __alloc_bootmem_node - allocate boot memory from a specific node
350 * @pgdat: node to allocate from
351 * @size: size of the request in bytes
352 * @align: alignment of the region
353 * @goal: preferred starting address of the region
354 *
355 * The goal is dropped if it can not be satisfied and the allocation will
356 * fall back to memory below @goal.
357 *
358 * Allocation may fall back to any node in the system if the specified node
359 * can not hold the requested memory.
360 *
361 * The function panics if the request can not be satisfied.
362 */
363void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
364                   unsigned long align, unsigned long goal)
365{
366    if (WARN_ON_ONCE(slab_is_available()))
367        return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
368
369    return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
370}
371
372void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
373                   unsigned long align, unsigned long goal)
374{
375    return __alloc_bootmem_node(pgdat, size, align, goal);
376}
377
378#ifndef ARCH_LOW_ADDRESS_LIMIT
379#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
380#endif
381
382/**
383 * __alloc_bootmem_low - allocate low boot memory
384 * @size: size of the request in bytes
385 * @align: alignment of the region
386 * @goal: preferred starting address of the region
387 *
388 * The goal is dropped if it can not be satisfied and the allocation will
389 * fall back to memory below @goal.
390 *
391 * Allocation may happen on any node in the system.
392 *
393 * The function panics if the request can not be satisfied.
394 */
395void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
396                  unsigned long goal)
397{
398    return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
399}
400
401void * __init __alloc_bootmem_low_nopanic(unsigned long size,
402                      unsigned long align,
403                      unsigned long goal)
404{
405    return ___alloc_bootmem_nopanic(size, align, goal,
406                    ARCH_LOW_ADDRESS_LIMIT);
407}
408
409/**
410 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
411 * @pgdat: node to allocate from
412 * @size: size of the request in bytes
413 * @align: alignment of the region
414 * @goal: preferred starting address of the region
415 *
416 * The goal is dropped if it can not be satisfied and the allocation will
417 * fall back to memory below @goal.
418 *
419 * Allocation may fall back to any node in the system if the specified node
420 * can not hold the requested memory.
421 *
422 * The function panics if the request can not be satisfied.
423 */
424void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
425                       unsigned long align, unsigned long goal)
426{
427    if (WARN_ON_ONCE(slab_is_available()))
428        return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
429
430    return ___alloc_bootmem_node(pgdat, size, align, goal,
431                     ARCH_LOW_ADDRESS_LIMIT);
432}
433

Archive Download this file



interactive