Root/mm/nobootmem.c

1/*
2 * bootmem - A boot-time physical memory allocator and configurator
3 *
4 * Copyright (C) 1999 Ingo Molnar
5 * 1999 Kanoj Sarcar, SGI
6 * 2008 Johannes Weiner
7 *
8 * Access to this subsystem has to be serialized externally (which is true
9 * for the boot process anyway).
10 */
11#include <linux/init.h>
12#include <linux/pfn.h>
13#include <linux/slab.h>
14#include <linux/bootmem.h>
15#include <linux/export.h>
16#include <linux/kmemleak.h>
17#include <linux/range.h>
18#include <linux/memblock.h>
19
20#include <asm/bug.h>
21#include <asm/io.h>
22#include <asm/processor.h>
23
24#include "internal.h"
25
26#ifndef CONFIG_NEED_MULTIPLE_NODES
27struct pglist_data __refdata contig_page_data;
28EXPORT_SYMBOL(contig_page_data);
29#endif
30
31unsigned long max_low_pfn;
32unsigned long min_low_pfn;
33unsigned long max_pfn;
34
35static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
36                    u64 goal, u64 limit)
37{
38    void *ptr;
39    u64 addr;
40
41    if (limit > memblock.current_limit)
42        limit = memblock.current_limit;
43
44    addr = memblock_find_in_range_node(goal, limit, size, align, nid);
45    if (!addr)
46        return NULL;
47
48    ptr = phys_to_virt(addr);
49    memset(ptr, 0, size);
50    memblock_reserve(addr, size);
51    /*
52     * The min_count is set to 0 so that bootmem allocated blocks
53     * are never reported as leaks.
54     */
55    kmemleak_alloc(ptr, size, 0, 0);
56    return ptr;
57}
58
59/*
60 * free_bootmem_late - free bootmem pages directly to page allocator
61 * @addr: starting address of the range
62 * @size: size of the range in bytes
63 *
64 * This is only useful when the bootmem allocator has already been torn
65 * down, but we are still initializing the system. Pages are given directly
66 * to the page allocator, no bootmem metadata is updated because it is gone.
67 */
68void __init free_bootmem_late(unsigned long addr, unsigned long size)
69{
70    unsigned long cursor, end;
71
72    kmemleak_free_part(__va(addr), size);
73
74    cursor = PFN_UP(addr);
75    end = PFN_DOWN(addr + size);
76
77    for (; cursor < end; cursor++) {
78        __free_pages_bootmem(pfn_to_page(cursor), 0);
79        totalram_pages++;
80    }
81}
82
83static void __init __free_pages_memory(unsigned long start, unsigned long end)
84{
85    unsigned long i, start_aligned, end_aligned;
86    int order = ilog2(BITS_PER_LONG);
87
88    start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
89    end_aligned = end & ~(BITS_PER_LONG - 1);
90
91    if (end_aligned <= start_aligned) {
92        for (i = start; i < end; i++)
93            __free_pages_bootmem(pfn_to_page(i), 0);
94
95        return;
96    }
97
98    for (i = start; i < start_aligned; i++)
99        __free_pages_bootmem(pfn_to_page(i), 0);
100
101    for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
102        __free_pages_bootmem(pfn_to_page(i), order);
103
104    for (i = end_aligned; i < end; i++)
105        __free_pages_bootmem(pfn_to_page(i), 0);
106}
107
108unsigned long __init free_low_memory_core_early(int nodeid)
109{
110    unsigned long count = 0;
111    phys_addr_t start, end;
112    u64 i;
113
114    /* free reserved array temporarily so that it's treated as free area */
115    memblock_free_reserved_regions();
116
117    for_each_free_mem_range(i, MAX_NUMNODES, &start, &end, NULL) {
118        unsigned long start_pfn = PFN_UP(start);
119        unsigned long end_pfn = min_t(unsigned long,
120                          PFN_DOWN(end), max_low_pfn);
121        if (start_pfn < end_pfn) {
122            __free_pages_memory(start_pfn, end_pfn);
123            count += end_pfn - start_pfn;
124        }
125    }
126
127    /* put region array back? */
128    memblock_reserve_reserved_regions();
129    return count;
130}
131
132/**
133 * free_all_bootmem_node - release a node's free pages to the buddy allocator
134 * @pgdat: node to be released
135 *
136 * Returns the number of pages actually released.
137 */
138unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
139{
140    register_page_bootmem_info_node(pgdat);
141
142    /* free_low_memory_core_early(MAX_NUMNODES) will be called later */
143    return 0;
144}
145
146/**
147 * free_all_bootmem - release free pages to the buddy allocator
148 *
149 * Returns the number of pages actually released.
150 */
151unsigned long __init free_all_bootmem(void)
152{
153    /*
154     * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
155     * because in some case like Node0 doesn't have RAM installed
156     * low ram will be on Node1
157     * Use MAX_NUMNODES will make sure all ranges in early_node_map[]
158     * will be used instead of only Node0 related
159     */
160    return free_low_memory_core_early(MAX_NUMNODES);
161}
162
163/**
164 * free_bootmem_node - mark a page range as usable
165 * @pgdat: node the range resides on
166 * @physaddr: starting address of the range
167 * @size: size of the range in bytes
168 *
169 * Partial pages will be considered reserved and left as they are.
170 *
171 * The range must reside completely on the specified node.
172 */
173void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
174                  unsigned long size)
175{
176    kmemleak_free_part(__va(physaddr), size);
177    memblock_free(physaddr, size);
178}
179
180/**
181 * free_bootmem - mark a page range as usable
182 * @addr: starting address of the range
183 * @size: size of the range in bytes
184 *
185 * Partial pages will be considered reserved and left as they are.
186 *
187 * The range must be contiguous but may span node boundaries.
188 */
189void __init free_bootmem(unsigned long addr, unsigned long size)
190{
191    kmemleak_free_part(__va(addr), size);
192    memblock_free(addr, size);
193}
194
195static void * __init ___alloc_bootmem_nopanic(unsigned long size,
196                    unsigned long align,
197                    unsigned long goal,
198                    unsigned long limit)
199{
200    void *ptr;
201
202    if (WARN_ON_ONCE(slab_is_available()))
203        return kzalloc(size, GFP_NOWAIT);
204
205restart:
206
207    ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
208
209    if (ptr)
210        return ptr;
211
212    if (goal != 0) {
213        goal = 0;
214        goto restart;
215    }
216
217    return NULL;
218}
219
220/**
221 * __alloc_bootmem_nopanic - allocate boot memory without panicking
222 * @size: size of the request in bytes
223 * @align: alignment of the region
224 * @goal: preferred starting address of the region
225 *
226 * The goal is dropped if it can not be satisfied and the allocation will
227 * fall back to memory below @goal.
228 *
229 * Allocation may happen on any node in the system.
230 *
231 * Returns NULL on failure.
232 */
233void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
234                    unsigned long goal)
235{
236    unsigned long limit = -1UL;
237
238    return ___alloc_bootmem_nopanic(size, align, goal, limit);
239}
240
241static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
242                    unsigned long goal, unsigned long limit)
243{
244    void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
245
246    if (mem)
247        return mem;
248    /*
249     * Whoops, we cannot satisfy the allocation request.
250     */
251    printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
252    panic("Out of memory");
253    return NULL;
254}
255
256/**
257 * __alloc_bootmem - allocate boot memory
258 * @size: size of the request in bytes
259 * @align: alignment of the region
260 * @goal: preferred starting address of the region
261 *
262 * The goal is dropped if it can not be satisfied and the allocation will
263 * fall back to memory below @goal.
264 *
265 * Allocation may happen on any node in the system.
266 *
267 * The function panics if the request can not be satisfied.
268 */
269void * __init __alloc_bootmem(unsigned long size, unsigned long align,
270                  unsigned long goal)
271{
272    unsigned long limit = -1UL;
273
274    return ___alloc_bootmem(size, align, goal, limit);
275}
276
277/**
278 * __alloc_bootmem_node - allocate boot memory from a specific node
279 * @pgdat: node to allocate from
280 * @size: size of the request in bytes
281 * @align: alignment of the region
282 * @goal: preferred starting address of the region
283 *
284 * The goal is dropped if it can not be satisfied and the allocation will
285 * fall back to memory below @goal.
286 *
287 * Allocation may fall back to any node in the system if the specified node
288 * can not hold the requested memory.
289 *
290 * The function panics if the request can not be satisfied.
291 */
292void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
293                   unsigned long align, unsigned long goal)
294{
295    void *ptr;
296
297    if (WARN_ON_ONCE(slab_is_available()))
298        return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
299
300again:
301    ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
302                     goal, -1ULL);
303    if (ptr)
304        return ptr;
305
306    ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
307                    goal, -1ULL);
308    if (!ptr && goal) {
309        goal = 0;
310        goto again;
311    }
312    return ptr;
313}
314
315void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
316                   unsigned long align, unsigned long goal)
317{
318    return __alloc_bootmem_node(pgdat, size, align, goal);
319}
320
321#ifdef CONFIG_SPARSEMEM
322/**
323 * alloc_bootmem_section - allocate boot memory from a specific section
324 * @size: size of the request in bytes
325 * @section_nr: sparse map section to allocate from
326 *
327 * Return NULL on failure.
328 */
329void * __init alloc_bootmem_section(unsigned long size,
330                    unsigned long section_nr)
331{
332    unsigned long pfn, goal, limit;
333
334    pfn = section_nr_to_pfn(section_nr);
335    goal = pfn << PAGE_SHIFT;
336    limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
337
338    return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
339                     SMP_CACHE_BYTES, goal, limit);
340}
341#endif
342
343void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
344                   unsigned long align, unsigned long goal)
345{
346    void *ptr;
347
348    if (WARN_ON_ONCE(slab_is_available()))
349        return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
350
351    ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
352                         goal, -1ULL);
353    if (ptr)
354        return ptr;
355
356    return __alloc_bootmem_nopanic(size, align, goal);
357}
358
359#ifndef ARCH_LOW_ADDRESS_LIMIT
360#define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL
361#endif
362
363/**
364 * __alloc_bootmem_low - allocate low boot memory
365 * @size: size of the request in bytes
366 * @align: alignment of the region
367 * @goal: preferred starting address of the region
368 *
369 * The goal is dropped if it can not be satisfied and the allocation will
370 * fall back to memory below @goal.
371 *
372 * Allocation may happen on any node in the system.
373 *
374 * The function panics if the request can not be satisfied.
375 */
376void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
377                  unsigned long goal)
378{
379    return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
380}
381
382/**
383 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
384 * @pgdat: node to allocate from
385 * @size: size of the request in bytes
386 * @align: alignment of the region
387 * @goal: preferred starting address of the region
388 *
389 * The goal is dropped if it can not be satisfied and the allocation will
390 * fall back to memory below @goal.
391 *
392 * Allocation may fall back to any node in the system if the specified node
393 * can not hold the requested memory.
394 *
395 * The function panics if the request can not be satisfied.
396 */
397void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
398                       unsigned long align, unsigned long goal)
399{
400    void *ptr;
401
402    if (WARN_ON_ONCE(slab_is_available()))
403        return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
404
405    ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
406                goal, ARCH_LOW_ADDRESS_LIMIT);
407    if (ptr)
408        return ptr;
409
410    return __alloc_memory_core_early(MAX_NUMNODES, size, align,
411                goal, ARCH_LOW_ADDRESS_LIMIT);
412}
413

Archive Download this file



interactive