Root/mm/slub.c

1/*
2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists.
4 *
5 * The allocator synchronizes using per slab locks or atomic operatios
6 * and only uses a centralized lock to manage a pool of partial slabs.
7 *
8 * (C) 2007 SGI, Christoph Lameter
9 * (C) 2011 Linux Foundation, Christoph Lameter
10 */
11
12#include <linux/mm.h>
13#include <linux/swap.h> /* struct reclaim_state */
14#include <linux/module.h>
15#include <linux/bit_spinlock.h>
16#include <linux/interrupt.h>
17#include <linux/bitops.h>
18#include <linux/slab.h>
19#include "slab.h"
20#include <linux/proc_fs.h>
21#include <linux/seq_file.h>
22#include <linux/kmemcheck.h>
23#include <linux/cpu.h>
24#include <linux/cpuset.h>
25#include <linux/mempolicy.h>
26#include <linux/ctype.h>
27#include <linux/debugobjects.h>
28#include <linux/kallsyms.h>
29#include <linux/memory.h>
30#include <linux/math64.h>
31#include <linux/fault-inject.h>
32#include <linux/stacktrace.h>
33#include <linux/prefetch.h>
34
35#include <trace/events/kmem.h>
36
37#include "internal.h"
38
39/*
40 * Lock order:
41 * 1. slab_mutex (Global Mutex)
42 * 2. node->list_lock
43 * 3. slab_lock(page) (Only on some arches and for debugging)
44 *
45 * slab_mutex
46 *
47 * The role of the slab_mutex is to protect the list of all the slabs
48 * and to synchronize major metadata changes to slab cache structures.
49 *
50 * The slab_lock is only used for debugging and on arches that do not
51 * have the ability to do a cmpxchg_double. It only protects the second
52 * double word in the page struct. Meaning
53 * A. page->freelist -> List of object free in a page
54 * B. page->counters -> Counters of objects
55 * C. page->frozen -> frozen state
56 *
57 * If a slab is frozen then it is exempt from list management. It is not
58 * on any list. The processor that froze the slab is the one who can
59 * perform list operations on the page. Other processors may put objects
60 * onto the freelist but the processor that froze the slab is the only
61 * one that can retrieve the objects from the page's freelist.
62 *
63 * The list_lock protects the partial and full list on each node and
64 * the partial slab counter. If taken then no new slabs may be added or
65 * removed from the lists nor make the number of partial slabs be modified.
66 * (Note that the total number of slabs is an atomic value that may be
67 * modified without taking the list lock).
68 *
69 * The list_lock is a centralized lock and thus we avoid taking it as
70 * much as possible. As long as SLUB does not have to handle partial
71 * slabs, operations can continue without any centralized lock. F.e.
72 * allocating a long series of objects that fill up slabs does not require
73 * the list lock.
74 * Interrupts are disabled during allocation and deallocation in order to
75 * make the slab allocator safe to use in the context of an irq. In addition
76 * interrupts are disabled to ensure that the processor does not change
77 * while handling per_cpu slabs, due to kernel preemption.
78 *
79 * SLUB assigns one slab for allocation to each processor.
80 * Allocations only occur from these slabs called cpu slabs.
81 *
82 * Slabs with free elements are kept on a partial list and during regular
83 * operations no list for full slabs is used. If an object in a full slab is
84 * freed then the slab will show up again on the partial lists.
85 * We track full slabs for debugging purposes though because otherwise we
86 * cannot scan all objects.
87 *
88 * Slabs are freed when they become empty. Teardown and setup is
89 * minimal so we rely on the page allocators per cpu caches for
90 * fast frees and allocs.
91 *
92 * Overloading of page flags that are otherwise used for LRU management.
93 *
94 * PageActive The slab is frozen and exempt from list processing.
95 * This means that the slab is dedicated to a purpose
96 * such as satisfying allocations for a specific
97 * processor. Objects may be freed in the slab while
98 * it is frozen but slab_free will then skip the usual
99 * list operations. It is up to the processor holding
100 * the slab to integrate the slab into the slab lists
101 * when the slab is no longer needed.
102 *
103 * One use of this flag is to mark slabs that are
104 * used for allocations. Then such a slab becomes a cpu
105 * slab. The cpu slab may be equipped with an additional
106 * freelist that allows lockless access to
107 * free objects in addition to the regular freelist
108 * that requires the slab lock.
109 *
110 * PageError Slab requires special handling due to debug
111 * options set. This moves slab handling out of
112 * the fast path and disables lockless freelists.
113 */
114
115#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
116        SLAB_TRACE | SLAB_DEBUG_FREE)
117
118static inline int kmem_cache_debug(struct kmem_cache *s)
119{
120#ifdef CONFIG_SLUB_DEBUG
121    return unlikely(s->flags & SLAB_DEBUG_FLAGS);
122#else
123    return 0;
124#endif
125}
126
127/*
128 * Issues still to be resolved:
129 *
130 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
131 *
132 * - Variable sizing of the per node arrays
133 */
134
135/* Enable to test recovery from slab corruption on boot */
136#undef SLUB_RESILIENCY_TEST
137
138/* Enable to log cmpxchg failures */
139#undef SLUB_DEBUG_CMPXCHG
140
141/*
142 * Mininum number of partial slabs. These will be left on the partial
143 * lists even if they are empty. kmem_cache_shrink may reclaim them.
144 */
145#define MIN_PARTIAL 5
146
147/*
148 * Maximum number of desirable partial slabs.
149 * The existence of more partial slabs makes kmem_cache_shrink
150 * sort the partial list by the number of objects in the.
151 */
152#define MAX_PARTIAL 10
153
154#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
155                SLAB_POISON | SLAB_STORE_USER)
156
157/*
158 * Debugging flags that require metadata to be stored in the slab. These get
159 * disabled when slub_debug=O is used and a cache's min order increases with
160 * metadata.
161 */
162#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
163
164/*
165 * Set of flags that will prevent slab merging
166 */
167#define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
168        SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
169        SLAB_FAILSLAB)
170
171#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
172        SLAB_CACHE_DMA | SLAB_NOTRACK)
173
174#define OO_SHIFT 16
175#define OO_MASK ((1 << OO_SHIFT) - 1)
176#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
177
178/* Internal SLUB flags */
179#define __OBJECT_POISON 0x80000000UL /* Poison object */
180#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
181
182static int kmem_size = sizeof(struct kmem_cache);
183
184#ifdef CONFIG_SMP
185static struct notifier_block slab_notifier;
186#endif
187
188/*
189 * Tracking user of a slab.
190 */
191#define TRACK_ADDRS_COUNT 16
192struct track {
193    unsigned long addr; /* Called from address */
194#ifdef CONFIG_STACKTRACE
195    unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
196#endif
197    int cpu; /* Was running on cpu */
198    int pid; /* Pid context */
199    unsigned long when; /* When did the operation occur */
200};
201
202enum track_item { TRACK_ALLOC, TRACK_FREE };
203
204#ifdef CONFIG_SYSFS
205static int sysfs_slab_add(struct kmem_cache *);
206static int sysfs_slab_alias(struct kmem_cache *, const char *);
207static void sysfs_slab_remove(struct kmem_cache *);
208
209#else
210static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
211static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
212                            { return 0; }
213static inline void sysfs_slab_remove(struct kmem_cache *s)
214{
215    kfree(s->name);
216    kfree(s);
217}
218
219#endif
220
221static inline void stat(const struct kmem_cache *s, enum stat_item si)
222{
223#ifdef CONFIG_SLUB_STATS
224    __this_cpu_inc(s->cpu_slab->stat[si]);
225#endif
226}
227
228/********************************************************************
229 * Core slab cache functions
230 *******************************************************************/
231
232static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
233{
234    return s->node[node];
235}
236
237/* Verify that a pointer has an address that is valid within a slab page */
238static inline int check_valid_pointer(struct kmem_cache *s,
239                struct page *page, const void *object)
240{
241    void *base;
242
243    if (!object)
244        return 1;
245
246    base = page_address(page);
247    if (object < base || object >= base + page->objects * s->size ||
248        (object - base) % s->size) {
249        return 0;
250    }
251
252    return 1;
253}
254
255static inline void *get_freepointer(struct kmem_cache *s, void *object)
256{
257    return *(void **)(object + s->offset);
258}
259
260static void prefetch_freepointer(const struct kmem_cache *s, void *object)
261{
262    prefetch(object + s->offset);
263}
264
265static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
266{
267    void *p;
268
269#ifdef CONFIG_DEBUG_PAGEALLOC
270    probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
271#else
272    p = get_freepointer(s, object);
273#endif
274    return p;
275}
276
277static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
278{
279    *(void **)(object + s->offset) = fp;
280}
281
282/* Loop over all objects in a slab */
283#define for_each_object(__p, __s, __addr, __objects) \
284    for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
285            __p += (__s)->size)
286
287/* Determine object index from a given position */
288static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
289{
290    return (p - addr) / s->size;
291}
292
293static inline size_t slab_ksize(const struct kmem_cache *s)
294{
295#ifdef CONFIG_SLUB_DEBUG
296    /*
297     * Debugging requires use of the padding between object
298     * and whatever may come after it.
299     */
300    if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
301        return s->object_size;
302
303#endif
304    /*
305     * If we have the need to store the freelist pointer
306     * back there or track user information then we can
307     * only use the space before that information.
308     */
309    if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
310        return s->inuse;
311    /*
312     * Else we can use all the padding etc for the allocation
313     */
314    return s->size;
315}
316
317static inline int order_objects(int order, unsigned long size, int reserved)
318{
319    return ((PAGE_SIZE << order) - reserved) / size;
320}
321
322static inline struct kmem_cache_order_objects oo_make(int order,
323        unsigned long size, int reserved)
324{
325    struct kmem_cache_order_objects x = {
326        (order << OO_SHIFT) + order_objects(order, size, reserved)
327    };
328
329    return x;
330}
331
332static inline int oo_order(struct kmem_cache_order_objects x)
333{
334    return x.x >> OO_SHIFT;
335}
336
337static inline int oo_objects(struct kmem_cache_order_objects x)
338{
339    return x.x & OO_MASK;
340}
341
342/*
343 * Per slab locking using the pagelock
344 */
345static __always_inline void slab_lock(struct page *page)
346{
347    bit_spin_lock(PG_locked, &page->flags);
348}
349
350static __always_inline void slab_unlock(struct page *page)
351{
352    __bit_spin_unlock(PG_locked, &page->flags);
353}
354
355/* Interrupts must be disabled (for the fallback code to work right) */
356static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
357        void *freelist_old, unsigned long counters_old,
358        void *freelist_new, unsigned long counters_new,
359        const char *n)
360{
361    VM_BUG_ON(!irqs_disabled());
362#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
363    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
364    if (s->flags & __CMPXCHG_DOUBLE) {
365        if (cmpxchg_double(&page->freelist, &page->counters,
366            freelist_old, counters_old,
367            freelist_new, counters_new))
368        return 1;
369    } else
370#endif
371    {
372        slab_lock(page);
373        if (page->freelist == freelist_old && page->counters == counters_old) {
374            page->freelist = freelist_new;
375            page->counters = counters_new;
376            slab_unlock(page);
377            return 1;
378        }
379        slab_unlock(page);
380    }
381
382    cpu_relax();
383    stat(s, CMPXCHG_DOUBLE_FAIL);
384
385#ifdef SLUB_DEBUG_CMPXCHG
386    printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
387#endif
388
389    return 0;
390}
391
392static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
393        void *freelist_old, unsigned long counters_old,
394        void *freelist_new, unsigned long counters_new,
395        const char *n)
396{
397#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
398    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
399    if (s->flags & __CMPXCHG_DOUBLE) {
400        if (cmpxchg_double(&page->freelist, &page->counters,
401            freelist_old, counters_old,
402            freelist_new, counters_new))
403        return 1;
404    } else
405#endif
406    {
407        unsigned long flags;
408
409        local_irq_save(flags);
410        slab_lock(page);
411        if (page->freelist == freelist_old && page->counters == counters_old) {
412            page->freelist = freelist_new;
413            page->counters = counters_new;
414            slab_unlock(page);
415            local_irq_restore(flags);
416            return 1;
417        }
418        slab_unlock(page);
419        local_irq_restore(flags);
420    }
421
422    cpu_relax();
423    stat(s, CMPXCHG_DOUBLE_FAIL);
424
425#ifdef SLUB_DEBUG_CMPXCHG
426    printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
427#endif
428
429    return 0;
430}
431
432#ifdef CONFIG_SLUB_DEBUG
433/*
434 * Determine a map of object in use on a page.
435 *
436 * Node listlock must be held to guarantee that the page does
437 * not vanish from under us.
438 */
439static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
440{
441    void *p;
442    void *addr = page_address(page);
443
444    for (p = page->freelist; p; p = get_freepointer(s, p))
445        set_bit(slab_index(p, s, addr), map);
446}
447
448/*
449 * Debug settings:
450 */
451#ifdef CONFIG_SLUB_DEBUG_ON
452static int slub_debug = DEBUG_DEFAULT_FLAGS;
453#else
454static int slub_debug;
455#endif
456
457static char *slub_debug_slabs;
458static int disable_higher_order_debug;
459
460/*
461 * Object debugging
462 */
463static void print_section(char *text, u8 *addr, unsigned int length)
464{
465    print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
466            length, 1);
467}
468
469static struct track *get_track(struct kmem_cache *s, void *object,
470    enum track_item alloc)
471{
472    struct track *p;
473
474    if (s->offset)
475        p = object + s->offset + sizeof(void *);
476    else
477        p = object + s->inuse;
478
479    return p + alloc;
480}
481
482static void set_track(struct kmem_cache *s, void *object,
483            enum track_item alloc, unsigned long addr)
484{
485    struct track *p = get_track(s, object, alloc);
486
487    if (addr) {
488#ifdef CONFIG_STACKTRACE
489        struct stack_trace trace;
490        int i;
491
492        trace.nr_entries = 0;
493        trace.max_entries = TRACK_ADDRS_COUNT;
494        trace.entries = p->addrs;
495        trace.skip = 3;
496        save_stack_trace(&trace);
497
498        /* See rant in lockdep.c */
499        if (trace.nr_entries != 0 &&
500            trace.entries[trace.nr_entries - 1] == ULONG_MAX)
501            trace.nr_entries--;
502
503        for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
504            p->addrs[i] = 0;
505#endif
506        p->addr = addr;
507        p->cpu = smp_processor_id();
508        p->pid = current->pid;
509        p->when = jiffies;
510    } else
511        memset(p, 0, sizeof(struct track));
512}
513
514static void init_tracking(struct kmem_cache *s, void *object)
515{
516    if (!(s->flags & SLAB_STORE_USER))
517        return;
518
519    set_track(s, object, TRACK_FREE, 0UL);
520    set_track(s, object, TRACK_ALLOC, 0UL);
521}
522
523static void print_track(const char *s, struct track *t)
524{
525    if (!t->addr)
526        return;
527
528    printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
529        s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
530#ifdef CONFIG_STACKTRACE
531    {
532        int i;
533        for (i = 0; i < TRACK_ADDRS_COUNT; i++)
534            if (t->addrs[i])
535                printk(KERN_ERR "\t%pS\n", (void *)t->addrs[i]);
536            else
537                break;
538    }
539#endif
540}
541
542static void print_tracking(struct kmem_cache *s, void *object)
543{
544    if (!(s->flags & SLAB_STORE_USER))
545        return;
546
547    print_track("Allocated", get_track(s, object, TRACK_ALLOC));
548    print_track("Freed", get_track(s, object, TRACK_FREE));
549}
550
551static void print_page_info(struct page *page)
552{
553    printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
554        page, page->objects, page->inuse, page->freelist, page->flags);
555
556}
557
558static void slab_bug(struct kmem_cache *s, char *fmt, ...)
559{
560    va_list args;
561    char buf[100];
562
563    va_start(args, fmt);
564    vsnprintf(buf, sizeof(buf), fmt, args);
565    va_end(args);
566    printk(KERN_ERR "========================================"
567            "=====================================\n");
568    printk(KERN_ERR "BUG %s (%s): %s\n", s->name, print_tainted(), buf);
569    printk(KERN_ERR "----------------------------------------"
570            "-------------------------------------\n\n");
571}
572
573static void slab_fix(struct kmem_cache *s, char *fmt, ...)
574{
575    va_list args;
576    char buf[100];
577
578    va_start(args, fmt);
579    vsnprintf(buf, sizeof(buf), fmt, args);
580    va_end(args);
581    printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
582}
583
584static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
585{
586    unsigned int off; /* Offset of last byte */
587    u8 *addr = page_address(page);
588
589    print_tracking(s, p);
590
591    print_page_info(page);
592
593    printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
594            p, p - addr, get_freepointer(s, p));
595
596    if (p > addr + 16)
597        print_section("Bytes b4 ", p - 16, 16);
598
599    print_section("Object ", p, min_t(unsigned long, s->object_size,
600                PAGE_SIZE));
601    if (s->flags & SLAB_RED_ZONE)
602        print_section("Redzone ", p + s->object_size,
603            s->inuse - s->object_size);
604
605    if (s->offset)
606        off = s->offset + sizeof(void *);
607    else
608        off = s->inuse;
609
610    if (s->flags & SLAB_STORE_USER)
611        off += 2 * sizeof(struct track);
612
613    if (off != s->size)
614        /* Beginning of the filler is the free pointer */
615        print_section("Padding ", p + off, s->size - off);
616
617    dump_stack();
618}
619
620static void object_err(struct kmem_cache *s, struct page *page,
621            u8 *object, char *reason)
622{
623    slab_bug(s, "%s", reason);
624    print_trailer(s, page, object);
625}
626
627static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
628{
629    va_list args;
630    char buf[100];
631
632    va_start(args, fmt);
633    vsnprintf(buf, sizeof(buf), fmt, args);
634    va_end(args);
635    slab_bug(s, "%s", buf);
636    print_page_info(page);
637    dump_stack();
638}
639
640static void init_object(struct kmem_cache *s, void *object, u8 val)
641{
642    u8 *p = object;
643
644    if (s->flags & __OBJECT_POISON) {
645        memset(p, POISON_FREE, s->object_size - 1);
646        p[s->object_size - 1] = POISON_END;
647    }
648
649    if (s->flags & SLAB_RED_ZONE)
650        memset(p + s->object_size, val, s->inuse - s->object_size);
651}
652
653static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
654                        void *from, void *to)
655{
656    slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
657    memset(from, data, to - from);
658}
659
660static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
661            u8 *object, char *what,
662            u8 *start, unsigned int value, unsigned int bytes)
663{
664    u8 *fault;
665    u8 *end;
666
667    fault = memchr_inv(start, value, bytes);
668    if (!fault)
669        return 1;
670
671    end = start + bytes;
672    while (end > fault && end[-1] == value)
673        end--;
674
675    slab_bug(s, "%s overwritten", what);
676    printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
677                    fault, end - 1, fault[0], value);
678    print_trailer(s, page, object);
679
680    restore_bytes(s, what, value, fault, end);
681    return 0;
682}
683
684/*
685 * Object layout:
686 *
687 * object address
688 * Bytes of the object to be managed.
689 * If the freepointer may overlay the object then the free
690 * pointer is the first word of the object.
691 *
692 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
693 * 0xa5 (POISON_END)
694 *
695 * object + s->object_size
696 * Padding to reach word boundary. This is also used for Redzoning.
697 * Padding is extended by another word if Redzoning is enabled and
698 * object_size == inuse.
699 *
700 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
701 * 0xcc (RED_ACTIVE) for objects in use.
702 *
703 * object + s->inuse
704 * Meta data starts here.
705 *
706 * A. Free pointer (if we cannot overwrite object on free)
707 * B. Tracking data for SLAB_STORE_USER
708 * C. Padding to reach required alignment boundary or at mininum
709 * one word if debugging is on to be able to detect writes
710 * before the word boundary.
711 *
712 * Padding is done using 0x5a (POISON_INUSE)
713 *
714 * object + s->size
715 * Nothing is used beyond s->size.
716 *
717 * If slabcaches are merged then the object_size and inuse boundaries are mostly
718 * ignored. And therefore no slab options that rely on these boundaries
719 * may be used with merged slabcaches.
720 */
721
722static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
723{
724    unsigned long off = s->inuse; /* The end of info */
725
726    if (s->offset)
727        /* Freepointer is placed after the object. */
728        off += sizeof(void *);
729
730    if (s->flags & SLAB_STORE_USER)
731        /* We also have user information there */
732        off += 2 * sizeof(struct track);
733
734    if (s->size == off)
735        return 1;
736
737    return check_bytes_and_report(s, page, p, "Object padding",
738                p + off, POISON_INUSE, s->size - off);
739}
740
741/* Check the pad bytes at the end of a slab page */
742static int slab_pad_check(struct kmem_cache *s, struct page *page)
743{
744    u8 *start;
745    u8 *fault;
746    u8 *end;
747    int length;
748    int remainder;
749
750    if (!(s->flags & SLAB_POISON))
751        return 1;
752
753    start = page_address(page);
754    length = (PAGE_SIZE << compound_order(page)) - s->reserved;
755    end = start + length;
756    remainder = length % s->size;
757    if (!remainder)
758        return 1;
759
760    fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
761    if (!fault)
762        return 1;
763    while (end > fault && end[-1] == POISON_INUSE)
764        end--;
765
766    slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
767    print_section("Padding ", end - remainder, remainder);
768
769    restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
770    return 0;
771}
772
773static int check_object(struct kmem_cache *s, struct page *page,
774                    void *object, u8 val)
775{
776    u8 *p = object;
777    u8 *endobject = object + s->object_size;
778
779    if (s->flags & SLAB_RED_ZONE) {
780        if (!check_bytes_and_report(s, page, object, "Redzone",
781            endobject, val, s->inuse - s->object_size))
782            return 0;
783    } else {
784        if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
785            check_bytes_and_report(s, page, p, "Alignment padding",
786                endobject, POISON_INUSE, s->inuse - s->object_size);
787        }
788    }
789
790    if (s->flags & SLAB_POISON) {
791        if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
792            (!check_bytes_and_report(s, page, p, "Poison", p,
793                    POISON_FREE, s->object_size - 1) ||
794             !check_bytes_and_report(s, page, p, "Poison",
795                p + s->object_size - 1, POISON_END, 1)))
796            return 0;
797        /*
798         * check_pad_bytes cleans up on its own.
799         */
800        check_pad_bytes(s, page, p);
801    }
802
803    if (!s->offset && val == SLUB_RED_ACTIVE)
804        /*
805         * Object and freepointer overlap. Cannot check
806         * freepointer while object is allocated.
807         */
808        return 1;
809
810    /* Check free pointer validity */
811    if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
812        object_err(s, page, p, "Freepointer corrupt");
813        /*
814         * No choice but to zap it and thus lose the remainder
815         * of the free objects in this slab. May cause
816         * another error because the object count is now wrong.
817         */
818        set_freepointer(s, p, NULL);
819        return 0;
820    }
821    return 1;
822}
823
824static int check_slab(struct kmem_cache *s, struct page *page)
825{
826    int maxobj;
827
828    VM_BUG_ON(!irqs_disabled());
829
830    if (!PageSlab(page)) {
831        slab_err(s, page, "Not a valid slab page");
832        return 0;
833    }
834
835    maxobj = order_objects(compound_order(page), s->size, s->reserved);
836    if (page->objects > maxobj) {
837        slab_err(s, page, "objects %u > max %u",
838            s->name, page->objects, maxobj);
839        return 0;
840    }
841    if (page->inuse > page->objects) {
842        slab_err(s, page, "inuse %u > max %u",
843            s->name, page->inuse, page->objects);
844        return 0;
845    }
846    /* Slab_pad_check fixes things up after itself */
847    slab_pad_check(s, page);
848    return 1;
849}
850
851/*
852 * Determine if a certain object on a page is on the freelist. Must hold the
853 * slab lock to guarantee that the chains are in a consistent state.
854 */
855static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
856{
857    int nr = 0;
858    void *fp;
859    void *object = NULL;
860    unsigned long max_objects;
861
862    fp = page->freelist;
863    while (fp && nr <= page->objects) {
864        if (fp == search)
865            return 1;
866        if (!check_valid_pointer(s, page, fp)) {
867            if (object) {
868                object_err(s, page, object,
869                    "Freechain corrupt");
870                set_freepointer(s, object, NULL);
871                break;
872            } else {
873                slab_err(s, page, "Freepointer corrupt");
874                page->freelist = NULL;
875                page->inuse = page->objects;
876                slab_fix(s, "Freelist cleared");
877                return 0;
878            }
879            break;
880        }
881        object = fp;
882        fp = get_freepointer(s, object);
883        nr++;
884    }
885
886    max_objects = order_objects(compound_order(page), s->size, s->reserved);
887    if (max_objects > MAX_OBJS_PER_PAGE)
888        max_objects = MAX_OBJS_PER_PAGE;
889
890    if (page->objects != max_objects) {
891        slab_err(s, page, "Wrong number of objects. Found %d but "
892            "should be %d", page->objects, max_objects);
893        page->objects = max_objects;
894        slab_fix(s, "Number of objects adjusted.");
895    }
896    if (page->inuse != page->objects - nr) {
897        slab_err(s, page, "Wrong object count. Counter is %d but "
898            "counted were %d", page->inuse, page->objects - nr);
899        page->inuse = page->objects - nr;
900        slab_fix(s, "Object count adjusted.");
901    }
902    return search == NULL;
903}
904
905static void trace(struct kmem_cache *s, struct page *page, void *object,
906                                int alloc)
907{
908    if (s->flags & SLAB_TRACE) {
909        printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
910            s->name,
911            alloc ? "alloc" : "free",
912            object, page->inuse,
913            page->freelist);
914
915        if (!alloc)
916            print_section("Object ", (void *)object, s->object_size);
917
918        dump_stack();
919    }
920}
921
922/*
923 * Hooks for other subsystems that check memory allocations. In a typical
924 * production configuration these hooks all should produce no code at all.
925 */
926static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
927{
928    flags &= gfp_allowed_mask;
929    lockdep_trace_alloc(flags);
930    might_sleep_if(flags & __GFP_WAIT);
931
932    return should_failslab(s->object_size, flags, s->flags);
933}
934
935static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, void *object)
936{
937    flags &= gfp_allowed_mask;
938    kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
939    kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
940}
941
942static inline void slab_free_hook(struct kmem_cache *s, void *x)
943{
944    kmemleak_free_recursive(x, s->flags);
945
946    /*
947     * Trouble is that we may no longer disable interupts in the fast path
948     * So in order to make the debug calls that expect irqs to be
949     * disabled we need to disable interrupts temporarily.
950     */
951#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP)
952    {
953        unsigned long flags;
954
955        local_irq_save(flags);
956        kmemcheck_slab_free(s, x, s->object_size);
957        debug_check_no_locks_freed(x, s->object_size);
958        local_irq_restore(flags);
959    }
960#endif
961    if (!(s->flags & SLAB_DEBUG_OBJECTS))
962        debug_check_no_obj_freed(x, s->object_size);
963}
964
965/*
966 * Tracking of fully allocated slabs for debugging purposes.
967 *
968 * list_lock must be held.
969 */
970static void add_full(struct kmem_cache *s,
971    struct kmem_cache_node *n, struct page *page)
972{
973    if (!(s->flags & SLAB_STORE_USER))
974        return;
975
976    list_add(&page->lru, &n->full);
977}
978
979/*
980 * list_lock must be held.
981 */
982static void remove_full(struct kmem_cache *s, struct page *page)
983{
984    if (!(s->flags & SLAB_STORE_USER))
985        return;
986
987    list_del(&page->lru);
988}
989
990/* Tracking of the number of slabs for debugging purposes */
991static inline unsigned long slabs_node(struct kmem_cache *s, int node)
992{
993    struct kmem_cache_node *n = get_node(s, node);
994
995    return atomic_long_read(&n->nr_slabs);
996}
997
998static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
999{
1000    return atomic_long_read(&n->nr_slabs);
1001}
1002
1003static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1004{
1005    struct kmem_cache_node *n = get_node(s, node);
1006
1007    /*
1008     * May be called early in order to allocate a slab for the
1009     * kmem_cache_node structure. Solve the chicken-egg
1010     * dilemma by deferring the increment of the count during
1011     * bootstrap (see early_kmem_cache_node_alloc).
1012     */
1013    if (n) {
1014        atomic_long_inc(&n->nr_slabs);
1015        atomic_long_add(objects, &n->total_objects);
1016    }
1017}
1018static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1019{
1020    struct kmem_cache_node *n = get_node(s, node);
1021
1022    atomic_long_dec(&n->nr_slabs);
1023    atomic_long_sub(objects, &n->total_objects);
1024}
1025
1026/* Object debug checks for alloc/free paths */
1027static void setup_object_debug(struct kmem_cache *s, struct page *page,
1028                                void *object)
1029{
1030    if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
1031        return;
1032
1033    init_object(s, object, SLUB_RED_INACTIVE);
1034    init_tracking(s, object);
1035}
1036
1037static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *page,
1038                    void *object, unsigned long addr)
1039{
1040    if (!check_slab(s, page))
1041        goto bad;
1042
1043    if (!check_valid_pointer(s, page, object)) {
1044        object_err(s, page, object, "Freelist Pointer check fails");
1045        goto bad;
1046    }
1047
1048    if (!check_object(s, page, object, SLUB_RED_INACTIVE))
1049        goto bad;
1050
1051    /* Success perform special debug activities for allocs */
1052    if (s->flags & SLAB_STORE_USER)
1053        set_track(s, object, TRACK_ALLOC, addr);
1054    trace(s, page, object, 1);
1055    init_object(s, object, SLUB_RED_ACTIVE);
1056    return 1;
1057
1058bad:
1059    if (PageSlab(page)) {
1060        /*
1061         * If this is a slab page then lets do the best we can
1062         * to avoid issues in the future. Marking all objects
1063         * as used avoids touching the remaining objects.
1064         */
1065        slab_fix(s, "Marking all objects used");
1066        page->inuse = page->objects;
1067        page->freelist = NULL;
1068    }
1069    return 0;
1070}
1071
1072static noinline int free_debug_processing(struct kmem_cache *s,
1073         struct page *page, void *object, unsigned long addr)
1074{
1075    unsigned long flags;
1076    int rc = 0;
1077
1078    local_irq_save(flags);
1079    slab_lock(page);
1080
1081    if (!check_slab(s, page))
1082        goto fail;
1083
1084    if (!check_valid_pointer(s, page, object)) {
1085        slab_err(s, page, "Invalid object pointer 0x%p", object);
1086        goto fail;
1087    }
1088
1089    if (on_freelist(s, page, object)) {
1090        object_err(s, page, object, "Object already free");
1091        goto fail;
1092    }
1093
1094    if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1095        goto out;
1096
1097    if (unlikely(s != page->slab)) {
1098        if (!PageSlab(page)) {
1099            slab_err(s, page, "Attempt to free object(0x%p) "
1100                "outside of slab", object);
1101        } else if (!page->slab) {
1102            printk(KERN_ERR
1103                "SLUB <none>: no slab for object 0x%p.\n",
1104                        object);
1105            dump_stack();
1106        } else
1107            object_err(s, page, object,
1108                    "page slab pointer corrupt.");
1109        goto fail;
1110    }
1111
1112    if (s->flags & SLAB_STORE_USER)
1113        set_track(s, object, TRACK_FREE, addr);
1114    trace(s, page, object, 0);
1115    init_object(s, object, SLUB_RED_INACTIVE);
1116    rc = 1;
1117out:
1118    slab_unlock(page);
1119    local_irq_restore(flags);
1120    return rc;
1121
1122fail:
1123    slab_fix(s, "Object at 0x%p not freed", object);
1124    goto out;
1125}
1126
1127static int __init setup_slub_debug(char *str)
1128{
1129    slub_debug = DEBUG_DEFAULT_FLAGS;
1130    if (*str++ != '=' || !*str)
1131        /*
1132         * No options specified. Switch on full debugging.
1133         */
1134        goto out;
1135
1136    if (*str == ',')
1137        /*
1138         * No options but restriction on slabs. This means full
1139         * debugging for slabs matching a pattern.
1140         */
1141        goto check_slabs;
1142
1143    if (tolower(*str) == 'o') {
1144        /*
1145         * Avoid enabling debugging on caches if its minimum order
1146         * would increase as a result.
1147         */
1148        disable_higher_order_debug = 1;
1149        goto out;
1150    }
1151
1152    slub_debug = 0;
1153    if (*str == '-')
1154        /*
1155         * Switch off all debugging measures.
1156         */
1157        goto out;
1158
1159    /*
1160     * Determine which debug features should be switched on
1161     */
1162    for (; *str && *str != ','; str++) {
1163        switch (tolower(*str)) {
1164        case 'f':
1165            slub_debug |= SLAB_DEBUG_FREE;
1166            break;
1167        case 'z':
1168            slub_debug |= SLAB_RED_ZONE;
1169            break;
1170        case 'p':
1171            slub_debug |= SLAB_POISON;
1172            break;
1173        case 'u':
1174            slub_debug |= SLAB_STORE_USER;
1175            break;
1176        case 't':
1177            slub_debug |= SLAB_TRACE;
1178            break;
1179        case 'a':
1180            slub_debug |= SLAB_FAILSLAB;
1181            break;
1182        default:
1183            printk(KERN_ERR "slub_debug option '%c' "
1184                "unknown. skipped\n", *str);
1185        }
1186    }
1187
1188check_slabs:
1189    if (*str == ',')
1190        slub_debug_slabs = str + 1;
1191out:
1192    return 1;
1193}
1194
1195__setup("slub_debug", setup_slub_debug);
1196
1197static unsigned long kmem_cache_flags(unsigned long object_size,
1198    unsigned long flags, const char *name,
1199    void (*ctor)(void *))
1200{
1201    /*
1202     * Enable debugging if selected on the kernel commandline.
1203     */
1204    if (slub_debug && (!slub_debug_slabs ||
1205        !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))
1206        flags |= slub_debug;
1207
1208    return flags;
1209}
1210#else
1211static inline void setup_object_debug(struct kmem_cache *s,
1212            struct page *page, void *object) {}
1213
1214static inline int alloc_debug_processing(struct kmem_cache *s,
1215    struct page *page, void *object, unsigned long addr) { return 0; }
1216
1217static inline int free_debug_processing(struct kmem_cache *s,
1218    struct page *page, void *object, unsigned long addr) { return 0; }
1219
1220static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1221            { return 1; }
1222static inline int check_object(struct kmem_cache *s, struct page *page,
1223            void *object, u8 val) { return 1; }
1224static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
1225                    struct page *page) {}
1226static inline void remove_full(struct kmem_cache *s, struct page *page) {}
1227static inline unsigned long kmem_cache_flags(unsigned long object_size,
1228    unsigned long flags, const char *name,
1229    void (*ctor)(void *))
1230{
1231    return flags;
1232}
1233#define slub_debug 0
1234
1235#define disable_higher_order_debug 0
1236
1237static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1238                            { return 0; }
1239static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1240                            { return 0; }
1241static inline void inc_slabs_node(struct kmem_cache *s, int node,
1242                            int objects) {}
1243static inline void dec_slabs_node(struct kmem_cache *s, int node,
1244                            int objects) {}
1245
1246static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
1247                            { return 0; }
1248
1249static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
1250        void *object) {}
1251
1252static inline void slab_free_hook(struct kmem_cache *s, void *x) {}
1253
1254#endif /* CONFIG_SLUB_DEBUG */
1255
1256/*
1257 * Slab allocation and freeing
1258 */
1259static inline struct page *alloc_slab_page(gfp_t flags, int node,
1260                    struct kmem_cache_order_objects oo)
1261{
1262    int order = oo_order(oo);
1263
1264    flags |= __GFP_NOTRACK;
1265
1266    if (node == NUMA_NO_NODE)
1267        return alloc_pages(flags, order);
1268    else
1269        return alloc_pages_exact_node(node, flags, order);
1270}
1271
1272static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1273{
1274    struct page *page;
1275    struct kmem_cache_order_objects oo = s->oo;
1276    gfp_t alloc_gfp;
1277
1278    flags &= gfp_allowed_mask;
1279
1280    if (flags & __GFP_WAIT)
1281        local_irq_enable();
1282
1283    flags |= s->allocflags;
1284
1285    /*
1286     * Let the initial higher-order allocation fail under memory pressure
1287     * so we fall-back to the minimum order allocation.
1288     */
1289    alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
1290
1291    page = alloc_slab_page(alloc_gfp, node, oo);
1292    if (unlikely(!page)) {
1293        oo = s->min;
1294        /*
1295         * Allocation may have failed due to fragmentation.
1296         * Try a lower order alloc if possible
1297         */
1298        page = alloc_slab_page(flags, node, oo);
1299
1300        if (page)
1301            stat(s, ORDER_FALLBACK);
1302    }
1303
1304    if (kmemcheck_enabled && page
1305        && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
1306        int pages = 1 << oo_order(oo);
1307
1308        kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
1309
1310        /*
1311         * Objects from caches that have a constructor don't get
1312         * cleared when they're allocated, so we need to do it here.
1313         */
1314        if (s->ctor)
1315            kmemcheck_mark_uninitialized_pages(page, pages);
1316        else
1317            kmemcheck_mark_unallocated_pages(page, pages);
1318    }
1319
1320    if (flags & __GFP_WAIT)
1321        local_irq_disable();
1322    if (!page)
1323        return NULL;
1324
1325    page->objects = oo_objects(oo);
1326    mod_zone_page_state(page_zone(page),
1327        (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1328        NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1329        1 << oo_order(oo));
1330
1331    return page;
1332}
1333
1334static void setup_object(struct kmem_cache *s, struct page *page,
1335                void *object)
1336{
1337    setup_object_debug(s, page, object);
1338    if (unlikely(s->ctor))
1339        s->ctor(object);
1340}
1341
1342static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1343{
1344    struct page *page;
1345    void *start;
1346    void *last;
1347    void *p;
1348
1349    BUG_ON(flags & GFP_SLAB_BUG_MASK);
1350
1351    page = allocate_slab(s,
1352        flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
1353    if (!page)
1354        goto out;
1355
1356    inc_slabs_node(s, page_to_nid(page), page->objects);
1357    page->slab = s;
1358    __SetPageSlab(page);
1359    if (page->pfmemalloc)
1360        SetPageSlabPfmemalloc(page);
1361
1362    start = page_address(page);
1363
1364    if (unlikely(s->flags & SLAB_POISON))
1365        memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
1366
1367    last = start;
1368    for_each_object(p, s, start, page->objects) {
1369        setup_object(s, page, last);
1370        set_freepointer(s, last, p);
1371        last = p;
1372    }
1373    setup_object(s, page, last);
1374    set_freepointer(s, last, NULL);
1375
1376    page->freelist = start;
1377    page->inuse = page->objects;
1378    page->frozen = 1;
1379out:
1380    return page;
1381}
1382
1383static void __free_slab(struct kmem_cache *s, struct page *page)
1384{
1385    int order = compound_order(page);
1386    int pages = 1 << order;
1387
1388    if (kmem_cache_debug(s)) {
1389        void *p;
1390
1391        slab_pad_check(s, page);
1392        for_each_object(p, s, page_address(page),
1393                        page->objects)
1394            check_object(s, page, p, SLUB_RED_INACTIVE);
1395    }
1396
1397    kmemcheck_free_shadow(page, compound_order(page));
1398
1399    mod_zone_page_state(page_zone(page),
1400        (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1401        NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1402        -pages);
1403
1404    __ClearPageSlabPfmemalloc(page);
1405    __ClearPageSlab(page);
1406    reset_page_mapcount(page);
1407    if (current->reclaim_state)
1408        current->reclaim_state->reclaimed_slab += pages;
1409    __free_pages(page, order);
1410}
1411
1412#define need_reserve_slab_rcu \
1413    (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
1414
1415static void rcu_free_slab(struct rcu_head *h)
1416{
1417    struct page *page;
1418
1419    if (need_reserve_slab_rcu)
1420        page = virt_to_head_page(h);
1421    else
1422        page = container_of((struct list_head *)h, struct page, lru);
1423
1424    __free_slab(page->slab, page);
1425}
1426
1427static void free_slab(struct kmem_cache *s, struct page *page)
1428{
1429    if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1430        struct rcu_head *head;
1431
1432        if (need_reserve_slab_rcu) {
1433            int order = compound_order(page);
1434            int offset = (PAGE_SIZE << order) - s->reserved;
1435
1436            VM_BUG_ON(s->reserved != sizeof(*head));
1437            head = page_address(page) + offset;
1438        } else {
1439            /*
1440             * RCU free overloads the RCU head over the LRU
1441             */
1442            head = (void *)&page->lru;
1443        }
1444
1445        call_rcu(head, rcu_free_slab);
1446    } else
1447        __free_slab(s, page);
1448}
1449
1450static void discard_slab(struct kmem_cache *s, struct page *page)
1451{
1452    dec_slabs_node(s, page_to_nid(page), page->objects);
1453    free_slab(s, page);
1454}
1455
1456/*
1457 * Management of partially allocated slabs.
1458 *
1459 * list_lock must be held.
1460 */
1461static inline void add_partial(struct kmem_cache_node *n,
1462                struct page *page, int tail)
1463{
1464    n->nr_partial++;
1465    if (tail == DEACTIVATE_TO_TAIL)
1466        list_add_tail(&page->lru, &n->partial);
1467    else
1468        list_add(&page->lru, &n->partial);
1469}
1470
1471/*
1472 * list_lock must be held.
1473 */
1474static inline void remove_partial(struct kmem_cache_node *n,
1475                    struct page *page)
1476{
1477    list_del(&page->lru);
1478    n->nr_partial--;
1479}
1480
1481/*
1482 * Remove slab from the partial list, freeze it and
1483 * return the pointer to the freelist.
1484 *
1485 * Returns a list of objects or NULL if it fails.
1486 *
1487 * Must hold list_lock since we modify the partial list.
1488 */
1489static inline void *acquire_slab(struct kmem_cache *s,
1490        struct kmem_cache_node *n, struct page *page,
1491        int mode)
1492{
1493    void *freelist;
1494    unsigned long counters;
1495    struct page new;
1496
1497    /*
1498     * Zap the freelist and set the frozen bit.
1499     * The old freelist is the list of objects for the
1500     * per cpu allocation list.
1501     */
1502    freelist = page->freelist;
1503    counters = page->counters;
1504    new.counters = counters;
1505    if (mode) {
1506        new.inuse = page->objects;
1507        new.freelist = NULL;
1508    } else {
1509        new.freelist = freelist;
1510    }
1511
1512    VM_BUG_ON(new.frozen);
1513    new.frozen = 1;
1514
1515    if (!__cmpxchg_double_slab(s, page,
1516            freelist, counters,
1517            new.freelist, new.counters,
1518            "acquire_slab"))
1519        return NULL;
1520
1521    remove_partial(n, page);
1522    WARN_ON(!freelist);
1523    return freelist;
1524}
1525
1526static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
1527
1528/*
1529 * Try to allocate a partial slab from a specific node.
1530 */
1531static void *get_partial_node(struct kmem_cache *s,
1532        struct kmem_cache_node *n, struct kmem_cache_cpu *c)
1533{
1534    struct page *page, *page2;
1535    void *object = NULL;
1536
1537    /*
1538     * Racy check. If we mistakenly see no partial slabs then we
1539     * just allocate an empty slab. If we mistakenly try to get a
1540     * partial slab and there is none available then get_partials()
1541     * will return NULL.
1542     */
1543    if (!n || !n->nr_partial)
1544        return NULL;
1545
1546    spin_lock(&n->list_lock);
1547    list_for_each_entry_safe(page, page2, &n->partial, lru) {
1548        void *t = acquire_slab(s, n, page, object == NULL);
1549        int available;
1550
1551        if (!t)
1552            break;
1553
1554        if (!object) {
1555            c->page = page;
1556            stat(s, ALLOC_FROM_PARTIAL);
1557            object = t;
1558            available = page->objects - page->inuse;
1559        } else {
1560            available = put_cpu_partial(s, page, 0);
1561            stat(s, CPU_PARTIAL_NODE);
1562        }
1563        if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
1564            break;
1565
1566    }
1567    spin_unlock(&n->list_lock);
1568    return object;
1569}
1570
1571/*
1572 * Get a page from somewhere. Search in increasing NUMA distances.
1573 */
1574static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
1575        struct kmem_cache_cpu *c)
1576{
1577#ifdef CONFIG_NUMA
1578    struct zonelist *zonelist;
1579    struct zoneref *z;
1580    struct zone *zone;
1581    enum zone_type high_zoneidx = gfp_zone(flags);
1582    void *object;
1583    unsigned int cpuset_mems_cookie;
1584
1585    /*
1586     * The defrag ratio allows a configuration of the tradeoffs between
1587     * inter node defragmentation and node local allocations. A lower
1588     * defrag_ratio increases the tendency to do local allocations
1589     * instead of attempting to obtain partial slabs from other nodes.
1590     *
1591     * If the defrag_ratio is set to 0 then kmalloc() always
1592     * returns node local objects. If the ratio is higher then kmalloc()
1593     * may return off node objects because partial slabs are obtained
1594     * from other nodes and filled up.
1595     *
1596     * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
1597     * defrag_ratio = 1000) then every (well almost) allocation will
1598     * first attempt to defrag slab caches on other nodes. This means
1599     * scanning over all nodes to look for partial slabs which may be
1600     * expensive if we do it every time we are trying to find a slab
1601     * with available objects.
1602     */
1603    if (!s->remote_node_defrag_ratio ||
1604            get_cycles() % 1024 > s->remote_node_defrag_ratio)
1605        return NULL;
1606
1607    do {
1608        cpuset_mems_cookie = get_mems_allowed();
1609        zonelist = node_zonelist(slab_node(), flags);
1610        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1611            struct kmem_cache_node *n;
1612
1613            n = get_node(s, zone_to_nid(zone));
1614
1615            if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
1616                    n->nr_partial > s->min_partial) {
1617                object = get_partial_node(s, n, c);
1618                if (object) {
1619                    /*
1620                     * Return the object even if
1621                     * put_mems_allowed indicated that
1622                     * the cpuset mems_allowed was
1623                     * updated in parallel. It's a
1624                     * harmless race between the alloc
1625                     * and the cpuset update.
1626                     */
1627                    put_mems_allowed(cpuset_mems_cookie);
1628                    return object;
1629                }
1630            }
1631        }
1632    } while (!put_mems_allowed(cpuset_mems_cookie));
1633#endif
1634    return NULL;
1635}
1636
1637/*
1638 * Get a partial page, lock it and return it.
1639 */
1640static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
1641        struct kmem_cache_cpu *c)
1642{
1643    void *object;
1644    int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
1645
1646    object = get_partial_node(s, get_node(s, searchnode), c);
1647    if (object || node != NUMA_NO_NODE)
1648        return object;
1649
1650    return get_any_partial(s, flags, c);
1651}
1652
1653#ifdef CONFIG_PREEMPT
1654/*
1655 * Calculate the next globally unique transaction for disambiguiation
1656 * during cmpxchg. The transactions start with the cpu number and are then
1657 * incremented by CONFIG_NR_CPUS.
1658 */
1659#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
1660#else
1661/*
1662 * No preemption supported therefore also no need to check for
1663 * different cpus.
1664 */
1665#define TID_STEP 1
1666#endif
1667
1668static inline unsigned long next_tid(unsigned long tid)
1669{
1670    return tid + TID_STEP;
1671}
1672
1673static inline unsigned int tid_to_cpu(unsigned long tid)
1674{
1675    return tid % TID_STEP;
1676}
1677
1678static inline unsigned long tid_to_event(unsigned long tid)
1679{
1680    return tid / TID_STEP;
1681}
1682
1683static inline unsigned int init_tid(int cpu)
1684{
1685    return cpu;
1686}
1687
1688static inline void note_cmpxchg_failure(const char *n,
1689        const struct kmem_cache *s, unsigned long tid)
1690{
1691#ifdef SLUB_DEBUG_CMPXCHG
1692    unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
1693
1694    printk(KERN_INFO "%s %s: cmpxchg redo ", n, s->name);
1695
1696#ifdef CONFIG_PREEMPT
1697    if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
1698        printk("due to cpu change %d -> %d\n",
1699            tid_to_cpu(tid), tid_to_cpu(actual_tid));
1700    else
1701#endif
1702    if (tid_to_event(tid) != tid_to_event(actual_tid))
1703        printk("due to cpu running other code. Event %ld->%ld\n",
1704            tid_to_event(tid), tid_to_event(actual_tid));
1705    else
1706        printk("for unknown reason: actual=%lx was=%lx target=%lx\n",
1707            actual_tid, tid, next_tid(tid));
1708#endif
1709    stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
1710}
1711
1712void init_kmem_cache_cpus(struct kmem_cache *s)
1713{
1714    int cpu;
1715
1716    for_each_possible_cpu(cpu)
1717        per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
1718}
1719
1720/*
1721 * Remove the cpu slab
1722 */
1723static void deactivate_slab(struct kmem_cache *s, struct page *page, void *freelist)
1724{
1725    enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
1726    struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1727    int lock = 0;
1728    enum slab_modes l = M_NONE, m = M_NONE;
1729    void *nextfree;
1730    int tail = DEACTIVATE_TO_HEAD;
1731    struct page new;
1732    struct page old;
1733
1734    if (page->freelist) {
1735        stat(s, DEACTIVATE_REMOTE_FREES);
1736        tail = DEACTIVATE_TO_TAIL;
1737    }
1738
1739    /*
1740     * Stage one: Free all available per cpu objects back
1741     * to the page freelist while it is still frozen. Leave the
1742     * last one.
1743     *
1744     * There is no need to take the list->lock because the page
1745     * is still frozen.
1746     */
1747    while (freelist && (nextfree = get_freepointer(s, freelist))) {
1748        void *prior;
1749        unsigned long counters;
1750
1751        do {
1752            prior = page->freelist;
1753            counters = page->counters;
1754            set_freepointer(s, freelist, prior);
1755            new.counters = counters;
1756            new.inuse--;
1757            VM_BUG_ON(!new.frozen);
1758
1759        } while (!__cmpxchg_double_slab(s, page,
1760            prior, counters,
1761            freelist, new.counters,
1762            "drain percpu freelist"));
1763
1764        freelist = nextfree;
1765    }
1766
1767    /*
1768     * Stage two: Ensure that the page is unfrozen while the
1769     * list presence reflects the actual number of objects
1770     * during unfreeze.
1771     *
1772     * We setup the list membership and then perform a cmpxchg
1773     * with the count. If there is a mismatch then the page
1774     * is not unfrozen but the page is on the wrong list.
1775     *
1776     * Then we restart the process which may have to remove
1777     * the page from the list that we just put it on again
1778     * because the number of objects in the slab may have
1779     * changed.
1780     */
1781redo:
1782
1783    old.freelist = page->freelist;
1784    old.counters = page->counters;
1785    VM_BUG_ON(!old.frozen);
1786
1787    /* Determine target state of the slab */
1788    new.counters = old.counters;
1789    if (freelist) {
1790        new.inuse--;
1791        set_freepointer(s, freelist, old.freelist);
1792        new.freelist = freelist;
1793    } else
1794        new.freelist = old.freelist;
1795
1796    new.frozen = 0;
1797
1798    if (!new.inuse && n->nr_partial > s->min_partial)
1799        m = M_FREE;
1800    else if (new.freelist) {
1801        m = M_PARTIAL;
1802        if (!lock) {
1803            lock = 1;
1804            /*
1805             * Taking the spinlock removes the possiblity
1806             * that acquire_slab() will see a slab page that
1807             * is frozen
1808             */
1809            spin_lock(&n->list_lock);
1810        }
1811    } else {
1812        m = M_FULL;
1813        if (kmem_cache_debug(s) && !lock) {
1814            lock = 1;
1815            /*
1816             * This also ensures that the scanning of full
1817             * slabs from diagnostic functions will not see
1818             * any frozen slabs.
1819             */
1820            spin_lock(&n->list_lock);
1821        }
1822    }
1823
1824    if (l != m) {
1825
1826        if (l == M_PARTIAL)
1827
1828            remove_partial(n, page);
1829
1830        else if (l == M_FULL)
1831
1832            remove_full(s, page);
1833
1834        if (m == M_PARTIAL) {
1835
1836            add_partial(n, page, tail);
1837            stat(s, tail);
1838
1839        } else if (m == M_FULL) {
1840
1841            stat(s, DEACTIVATE_FULL);
1842            add_full(s, n, page);
1843
1844        }
1845    }
1846
1847    l = m;
1848    if (!__cmpxchg_double_slab(s, page,
1849                old.freelist, old.counters,
1850                new.freelist, new.counters,
1851                "unfreezing slab"))
1852        goto redo;
1853
1854    if (lock)
1855        spin_unlock(&n->list_lock);
1856
1857    if (m == M_FREE) {
1858        stat(s, DEACTIVATE_EMPTY);
1859        discard_slab(s, page);
1860        stat(s, FREE_SLAB);
1861    }
1862}
1863
1864/*
1865 * Unfreeze all the cpu partial slabs.
1866 *
1867 * This function must be called with interrupt disabled.
1868 */
1869static void unfreeze_partials(struct kmem_cache *s)
1870{
1871    struct kmem_cache_node *n = NULL, *n2 = NULL;
1872    struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
1873    struct page *page, *discard_page = NULL;
1874
1875    while ((page = c->partial)) {
1876        struct page new;
1877        struct page old;
1878
1879        c->partial = page->next;
1880
1881        n2 = get_node(s, page_to_nid(page));
1882        if (n != n2) {
1883            if (n)
1884                spin_unlock(&n->list_lock);
1885
1886            n = n2;
1887            spin_lock(&n->list_lock);
1888        }
1889
1890        do {
1891
1892            old.freelist = page->freelist;
1893            old.counters = page->counters;
1894            VM_BUG_ON(!old.frozen);
1895
1896            new.counters = old.counters;
1897            new.freelist = old.freelist;
1898
1899            new.frozen = 0;
1900
1901        } while (!__cmpxchg_double_slab(s, page,
1902                old.freelist, old.counters,
1903                new.freelist, new.counters,
1904                "unfreezing slab"));
1905
1906        if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) {
1907            page->next = discard_page;
1908            discard_page = page;
1909        } else {
1910            add_partial(n, page, DEACTIVATE_TO_TAIL);
1911            stat(s, FREE_ADD_PARTIAL);
1912        }
1913    }
1914
1915    if (n)
1916        spin_unlock(&n->list_lock);
1917
1918    while (discard_page) {
1919        page = discard_page;
1920        discard_page = discard_page->next;
1921
1922        stat(s, DEACTIVATE_EMPTY);
1923        discard_slab(s, page);
1924        stat(s, FREE_SLAB);
1925    }
1926}
1927
1928/*
1929 * Put a page that was just frozen (in __slab_free) into a partial page
1930 * slot if available. This is done without interrupts disabled and without
1931 * preemption disabled. The cmpxchg is racy and may put the partial page
1932 * onto a random cpus partial slot.
1933 *
1934 * If we did not find a slot then simply move all the partials to the
1935 * per node partial list.
1936 */
1937int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
1938{
1939    struct page *oldpage;
1940    int pages;
1941    int pobjects;
1942
1943    do {
1944        pages = 0;
1945        pobjects = 0;
1946        oldpage = this_cpu_read(s->cpu_slab->partial);
1947
1948        if (oldpage) {
1949            pobjects = oldpage->pobjects;
1950            pages = oldpage->pages;
1951            if (drain && pobjects > s->cpu_partial) {
1952                unsigned long flags;
1953                /*
1954                 * partial array is full. Move the existing
1955                 * set to the per node partial list.
1956                 */
1957                local_irq_save(flags);
1958                unfreeze_partials(s);
1959                local_irq_restore(flags);
1960                pobjects = 0;
1961                pages = 0;
1962                stat(s, CPU_PARTIAL_DRAIN);
1963            }
1964        }
1965
1966        pages++;
1967        pobjects += page->objects - page->inuse;
1968
1969        page->pages = pages;
1970        page->pobjects = pobjects;
1971        page->next = oldpage;
1972
1973    } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
1974    return pobjects;
1975}
1976
1977static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
1978{
1979    stat(s, CPUSLAB_FLUSH);
1980    deactivate_slab(s, c->page, c->freelist);
1981
1982    c->tid = next_tid(c->tid);
1983    c->page = NULL;
1984    c->freelist = NULL;
1985}
1986
1987/*
1988 * Flush cpu slab.
1989 *
1990 * Called from IPI handler with interrupts disabled.
1991 */
1992static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
1993{
1994    struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
1995
1996    if (likely(c)) {
1997        if (c->page)
1998            flush_slab(s, c);
1999
2000        unfreeze_partials(s);
2001    }
2002}
2003
2004static void flush_cpu_slab(void *d)
2005{
2006    struct kmem_cache *s = d;
2007
2008    __flush_cpu_slab(s, smp_processor_id());
2009}
2010
2011static bool has_cpu_slab(int cpu, void *info)
2012{
2013    struct kmem_cache *s = info;
2014    struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2015
2016    return c->page || c->partial;
2017}
2018
2019static void flush_all(struct kmem_cache *s)
2020{
2021    on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
2022}
2023
2024/*
2025 * Check if the objects in a per cpu structure fit numa
2026 * locality expectations.
2027 */
2028static inline int node_match(struct page *page, int node)
2029{
2030#ifdef CONFIG_NUMA
2031    if (node != NUMA_NO_NODE && page_to_nid(page) != node)
2032        return 0;
2033#endif
2034    return 1;
2035}
2036
2037static int count_free(struct page *page)
2038{
2039    return page->objects - page->inuse;
2040}
2041
2042static unsigned long count_partial(struct kmem_cache_node *n,
2043                    int (*get_count)(struct page *))
2044{
2045    unsigned long flags;
2046    unsigned long x = 0;
2047    struct page *page;
2048
2049    spin_lock_irqsave(&n->list_lock, flags);
2050    list_for_each_entry(page, &n->partial, lru)
2051        x += get_count(page);
2052    spin_unlock_irqrestore(&n->list_lock, flags);
2053    return x;
2054}
2055
2056static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2057{
2058#ifdef CONFIG_SLUB_DEBUG
2059    return atomic_long_read(&n->total_objects);
2060#else
2061    return 0;
2062#endif
2063}
2064
2065static noinline void
2066slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2067{
2068    int node;
2069
2070    printk(KERN_WARNING
2071        "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
2072        nid, gfpflags);
2073    printk(KERN_WARNING " cache: %s, object size: %d, buffer size: %d, "
2074        "default order: %d, min order: %d\n", s->name, s->object_size,
2075        s->size, oo_order(s->oo), oo_order(s->min));
2076
2077    if (oo_order(s->min) > get_order(s->object_size))
2078        printk(KERN_WARNING " %s debugging increased min order, use "
2079               "slub_debug=O to disable.\n", s->name);
2080
2081    for_each_online_node(node) {
2082        struct kmem_cache_node *n = get_node(s, node);
2083        unsigned long nr_slabs;
2084        unsigned long nr_objs;
2085        unsigned long nr_free;
2086
2087        if (!n)
2088            continue;
2089
2090        nr_free = count_partial(n, count_free);
2091        nr_slabs = node_nr_slabs(n);
2092        nr_objs = node_nr_objs(n);
2093
2094        printk(KERN_WARNING
2095            " node %d: slabs: %ld, objs: %ld, free: %ld\n",
2096            node, nr_slabs, nr_objs, nr_free);
2097    }
2098}
2099
2100static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
2101            int node, struct kmem_cache_cpu **pc)
2102{
2103    void *freelist;
2104    struct kmem_cache_cpu *c = *pc;
2105    struct page *page;
2106
2107    freelist = get_partial(s, flags, node, c);
2108
2109    if (freelist)
2110        return freelist;
2111
2112    page = new_slab(s, flags, node);
2113    if (page) {
2114        c = __this_cpu_ptr(s->cpu_slab);
2115        if (c->page)
2116            flush_slab(s, c);
2117
2118        /*
2119         * No other reference to the page yet so we can
2120         * muck around with it freely without cmpxchg
2121         */
2122        freelist = page->freelist;
2123        page->freelist = NULL;
2124
2125        stat(s, ALLOC_SLAB);
2126        c->page = page;
2127        *pc = c;
2128    } else
2129        freelist = NULL;
2130
2131    return freelist;
2132}
2133
2134static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
2135{
2136    if (unlikely(PageSlabPfmemalloc(page)))
2137        return gfp_pfmemalloc_allowed(gfpflags);
2138
2139    return true;
2140}
2141
2142/*
2143 * Check the page->freelist of a page and either transfer the freelist to the per cpu freelist
2144 * or deactivate the page.
2145 *
2146 * The page is still frozen if the return value is not NULL.
2147 *
2148 * If this function returns NULL then the page has been unfrozen.
2149 *
2150 * This function must be called with interrupt disabled.
2151 */
2152static inline void *get_freelist(struct kmem_cache *s, struct page *page)
2153{
2154    struct page new;
2155    unsigned long counters;
2156    void *freelist;
2157
2158    do {
2159        freelist = page->freelist;
2160        counters = page->counters;
2161
2162        new.counters = counters;
2163        VM_BUG_ON(!new.frozen);
2164
2165        new.inuse = page->objects;
2166        new.frozen = freelist != NULL;
2167
2168    } while (!__cmpxchg_double_slab(s, page,
2169        freelist, counters,
2170        NULL, new.counters,
2171        "get_freelist"));
2172
2173    return freelist;
2174}
2175
2176/*
2177 * Slow path. The lockless freelist is empty or we need to perform
2178 * debugging duties.
2179 *
2180 * Processing is still very fast if new objects have been freed to the
2181 * regular freelist. In that case we simply take over the regular freelist
2182 * as the lockless freelist and zap the regular freelist.
2183 *
2184 * If that is not working then we fall back to the partial lists. We take the
2185 * first element of the freelist as the object to allocate now and move the
2186 * rest of the freelist to the lockless freelist.
2187 *
2188 * And if we were unable to get a new slab from the partial slab lists then
2189 * we need to allocate a new slab. This is the slowest path since it involves
2190 * a call to the page allocator and the setup of a new slab.
2191 */
2192static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
2193              unsigned long addr, struct kmem_cache_cpu *c)
2194{
2195    void *freelist;
2196    struct page *page;
2197    unsigned long flags;
2198
2199    local_irq_save(flags);
2200#ifdef CONFIG_PREEMPT
2201    /*
2202     * We may have been preempted and rescheduled on a different
2203     * cpu before disabling interrupts. Need to reload cpu area
2204     * pointer.
2205     */
2206    c = this_cpu_ptr(s->cpu_slab);
2207#endif
2208
2209    page = c->page;
2210    if (!page)
2211        goto new_slab;
2212redo:
2213
2214    if (unlikely(!node_match(page, node))) {
2215        stat(s, ALLOC_NODE_MISMATCH);
2216        deactivate_slab(s, page, c->freelist);
2217        c->page = NULL;
2218        c->freelist = NULL;
2219        goto new_slab;
2220    }
2221
2222    /*
2223     * By rights, we should be searching for a slab page that was
2224     * PFMEMALLOC but right now, we are losing the pfmemalloc
2225     * information when the page leaves the per-cpu allocator
2226     */
2227    if (unlikely(!pfmemalloc_match(page, gfpflags))) {
2228        deactivate_slab(s, page, c->freelist);
2229        c->page = NULL;
2230        c->freelist = NULL;
2231        goto new_slab;
2232    }
2233
2234    /* must check again c->freelist in case of cpu migration or IRQ */
2235    freelist = c->freelist;
2236    if (freelist)
2237        goto load_freelist;
2238
2239    stat(s, ALLOC_SLOWPATH);
2240
2241    freelist = get_freelist(s, page);
2242
2243    if (!freelist) {
2244        c->page = NULL;
2245        stat(s, DEACTIVATE_BYPASS);
2246        goto new_slab;
2247    }
2248
2249    stat(s, ALLOC_REFILL);
2250
2251load_freelist:
2252    /*
2253     * freelist is pointing to the list of objects to be used.
2254     * page is pointing to the page from which the objects are obtained.
2255     * That page must be frozen for per cpu allocations to work.
2256     */
2257    VM_BUG_ON(!c->page->frozen);
2258    c->freelist = get_freepointer(s, freelist);
2259    c->tid = next_tid(c->tid);
2260    local_irq_restore(flags);
2261    return freelist;
2262
2263new_slab:
2264
2265    if (c->partial) {
2266        page = c->page = c->partial;
2267        c->partial = page->next;
2268        stat(s, CPU_PARTIAL_ALLOC);
2269        c->freelist = NULL;
2270        goto redo;
2271    }
2272
2273    freelist = new_slab_objects(s, gfpflags, node, &c);
2274
2275    if (unlikely(!freelist)) {
2276        if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
2277            slab_out_of_memory(s, gfpflags, node);
2278
2279        local_irq_restore(flags);
2280        return NULL;
2281    }
2282
2283    page = c->page;
2284    if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
2285        goto load_freelist;
2286
2287    /* Only entered in the debug case */
2288    if (kmem_cache_debug(s) && !alloc_debug_processing(s, page, freelist, addr))
2289        goto new_slab; /* Slab failed checks. Next slab needed */
2290
2291    deactivate_slab(s, page, get_freepointer(s, freelist));
2292    c->page = NULL;
2293    c->freelist = NULL;
2294    local_irq_restore(flags);
2295    return freelist;
2296}
2297
2298/*
2299 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2300 * have the fastpath folded into their functions. So no function call
2301 * overhead for requests that can be satisfied on the fastpath.
2302 *
2303 * The fastpath works by first checking if the lockless freelist can be used.
2304 * If not then __slab_alloc is called for slow processing.
2305 *
2306 * Otherwise we can simply pick the next object from the lockless free list.
2307 */
2308static __always_inline void *slab_alloc(struct kmem_cache *s,
2309        gfp_t gfpflags, int node, unsigned long addr)
2310{
2311    void **object;
2312    struct kmem_cache_cpu *c;
2313    struct page *page;
2314    unsigned long tid;
2315
2316    if (slab_pre_alloc_hook(s, gfpflags))
2317        return NULL;
2318
2319redo:
2320
2321    /*
2322     * Must read kmem_cache cpu data via this cpu ptr. Preemption is
2323     * enabled. We may switch back and forth between cpus while
2324     * reading from one cpu area. That does not matter as long
2325     * as we end up on the original cpu again when doing the cmpxchg.
2326     */
2327    c = __this_cpu_ptr(s->cpu_slab);
2328
2329    /*
2330     * The transaction ids are globally unique per cpu and per operation on
2331     * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2332     * occurs on the right processor and that there was no operation on the
2333     * linked list in between.
2334     */
2335    tid = c->tid;
2336    barrier();
2337
2338    object = c->freelist;
2339    page = c->page;
2340    if (unlikely(!object || !node_match(page, node)))
2341        object = __slab_alloc(s, gfpflags, node, addr, c);
2342
2343    else {
2344        void *next_object = get_freepointer_safe(s, object);
2345
2346        /*
2347         * The cmpxchg will only match if there was no additional
2348         * operation and if we are on the right processor.
2349         *
2350         * The cmpxchg does the following atomically (without lock semantics!)
2351         * 1. Relocate first pointer to the current per cpu area.
2352         * 2. Verify that tid and freelist have not been changed
2353         * 3. If they were not changed replace tid and freelist
2354         *
2355         * Since this is without lock semantics the protection is only against
2356         * code executing on this cpu *not* from access by other cpus.
2357         */
2358        if (unlikely(!this_cpu_cmpxchg_double(
2359                s->cpu_slab->freelist, s->cpu_slab->tid,
2360                object, tid,
2361                next_object, next_tid(tid)))) {
2362
2363            note_cmpxchg_failure("slab_alloc", s, tid);
2364            goto redo;
2365        }
2366        prefetch_freepointer(s, next_object);
2367        stat(s, ALLOC_FASTPATH);
2368    }
2369
2370    if (unlikely(gfpflags & __GFP_ZERO) && object)
2371        memset(object, 0, s->object_size);
2372
2373    slab_post_alloc_hook(s, gfpflags, object);
2374
2375    return object;
2376}
2377
2378void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
2379{
2380    void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
2381
2382    trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, gfpflags);
2383
2384    return ret;
2385}
2386EXPORT_SYMBOL(kmem_cache_alloc);
2387
2388#ifdef CONFIG_TRACING
2389void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
2390{
2391    void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
2392    trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2393    return ret;
2394}
2395EXPORT_SYMBOL(kmem_cache_alloc_trace);
2396
2397void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
2398{
2399    void *ret = kmalloc_order(size, flags, order);
2400    trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
2401    return ret;
2402}
2403EXPORT_SYMBOL(kmalloc_order_trace);
2404#endif
2405
2406#ifdef CONFIG_NUMA
2407void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
2408{
2409    void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
2410
2411    trace_kmem_cache_alloc_node(_RET_IP_, ret,
2412                    s->object_size, s->size, gfpflags, node);
2413
2414    return ret;
2415}
2416EXPORT_SYMBOL(kmem_cache_alloc_node);
2417
2418#ifdef CONFIG_TRACING
2419void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
2420                    gfp_t gfpflags,
2421                    int node, size_t size)
2422{
2423    void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
2424
2425    trace_kmalloc_node(_RET_IP_, ret,
2426               size, s->size, gfpflags, node);
2427    return ret;
2428}
2429EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
2430#endif
2431#endif
2432
2433/*
2434 * Slow patch handling. This may still be called frequently since objects
2435 * have a longer lifetime than the cpu slabs in most processing loads.
2436 *
2437 * So we still attempt to reduce cache line usage. Just take the slab
2438 * lock and free the item. If there is no additional partial page
2439 * handling required then we can return immediately.
2440 */
2441static void __slab_free(struct kmem_cache *s, struct page *page,
2442            void *x, unsigned long addr)
2443{
2444    void *prior;
2445    void **object = (void *)x;
2446    int was_frozen;
2447    int inuse;
2448    struct page new;
2449    unsigned long counters;
2450    struct kmem_cache_node *n = NULL;
2451    unsigned long uninitialized_var(flags);
2452
2453    stat(s, FREE_SLOWPATH);
2454
2455    if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
2456        return;
2457
2458    do {
2459        prior = page->freelist;
2460        counters = page->counters;
2461        set_freepointer(s, object, prior);
2462        new.counters = counters;
2463        was_frozen = new.frozen;
2464        new.inuse--;
2465        if ((!new.inuse || !prior) && !was_frozen && !n) {
2466
2467            if (!kmem_cache_debug(s) && !prior)
2468
2469                /*
2470                 * Slab was on no list before and will be partially empty
2471                 * We can defer the list move and instead freeze it.
2472                 */
2473                new.frozen = 1;
2474
2475            else { /* Needs to be taken off a list */
2476
2477                            n = get_node(s, page_to_nid(page));
2478                /*
2479                 * Speculatively acquire the list_lock.
2480                 * If the cmpxchg does not succeed then we may
2481                 * drop the list_lock without any processing.
2482                 *
2483                 * Otherwise the list_lock will synchronize with
2484                 * other processors updating the list of slabs.
2485                 */
2486                spin_lock_irqsave(&n->list_lock, flags);
2487
2488            }
2489        }
2490        inuse = new.inuse;
2491
2492    } while (!cmpxchg_double_slab(s, page,
2493        prior, counters,
2494        object, new.counters,
2495        "__slab_free"));
2496
2497    if (likely(!n)) {
2498
2499        /*
2500         * If we just froze the page then put it onto the
2501         * per cpu partial list.
2502         */
2503        if (new.frozen && !was_frozen) {
2504            put_cpu_partial(s, page, 1);
2505            stat(s, CPU_PARTIAL_FREE);
2506        }
2507        /*
2508         * The list lock was not taken therefore no list
2509         * activity can be necessary.
2510         */
2511                if (was_frozen)
2512                        stat(s, FREE_FROZEN);
2513                return;
2514        }
2515
2516    /*
2517     * was_frozen may have been set after we acquired the list_lock in
2518     * an earlier loop. So we need to check it here again.
2519     */
2520    if (was_frozen)
2521        stat(s, FREE_FROZEN);
2522    else {
2523        if (unlikely(!inuse && n->nr_partial > s->min_partial))
2524                        goto slab_empty;
2525
2526        /*
2527         * Objects left in the slab. If it was not on the partial list before
2528         * then add it.
2529         */
2530        if (unlikely(!prior)) {
2531            remove_full(s, page);
2532            add_partial(n, page, DEACTIVATE_TO_TAIL);
2533            stat(s, FREE_ADD_PARTIAL);
2534        }
2535    }
2536    spin_unlock_irqrestore(&n->list_lock, flags);
2537    return;
2538
2539slab_empty:
2540    if (prior) {
2541        /*
2542         * Slab on the partial list.
2543         */
2544        remove_partial(n, page);
2545        stat(s, FREE_REMOVE_PARTIAL);
2546    } else
2547        /* Slab must be on the full list */
2548        remove_full(s, page);
2549
2550    spin_unlock_irqrestore(&n->list_lock, flags);
2551    stat(s, FREE_SLAB);
2552    discard_slab(s, page);
2553}
2554
2555/*
2556 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
2557 * can perform fastpath freeing without additional function calls.
2558 *
2559 * The fastpath is only possible if we are freeing to the current cpu slab
2560 * of this processor. This typically the case if we have just allocated
2561 * the item before.
2562 *
2563 * If fastpath is not possible then fall back to __slab_free where we deal
2564 * with all sorts of special processing.
2565 */
2566static __always_inline void slab_free(struct kmem_cache *s,
2567            struct page *page, void *x, unsigned long addr)
2568{
2569    void **object = (void *)x;
2570    struct kmem_cache_cpu *c;
2571    unsigned long tid;
2572
2573    slab_free_hook(s, x);
2574
2575redo:
2576    /*
2577     * Determine the currently cpus per cpu slab.
2578     * The cpu may change afterward. However that does not matter since
2579     * data is retrieved via this pointer. If we are on the same cpu
2580     * during the cmpxchg then the free will succedd.
2581     */
2582    c = __this_cpu_ptr(s->cpu_slab);
2583
2584    tid = c->tid;
2585    barrier();
2586
2587    if (likely(page == c->page)) {
2588        set_freepointer(s, object, c->freelist);
2589
2590        if (unlikely(!this_cpu_cmpxchg_double(
2591                s->cpu_slab->freelist, s->cpu_slab->tid,
2592                c->freelist, tid,
2593                object, next_tid(tid)))) {
2594
2595            note_cmpxchg_failure("slab_free", s, tid);
2596            goto redo;
2597        }
2598        stat(s, FREE_FASTPATH);
2599    } else
2600        __slab_free(s, page, x, addr);
2601
2602}
2603
2604void kmem_cache_free(struct kmem_cache *s, void *x)
2605{
2606    struct page *page;
2607
2608    page = virt_to_head_page(x);
2609
2610    slab_free(s, page, x, _RET_IP_);
2611
2612    trace_kmem_cache_free(_RET_IP_, x);
2613}
2614EXPORT_SYMBOL(kmem_cache_free);
2615
2616/*
2617 * Object placement in a slab is made very easy because we always start at
2618 * offset 0. If we tune the size of the object to the alignment then we can
2619 * get the required alignment by putting one properly sized object after
2620 * another.
2621 *
2622 * Notice that the allocation order determines the sizes of the per cpu
2623 * caches. Each processor has always one slab available for allocations.
2624 * Increasing the allocation order reduces the number of times that slabs
2625 * must be moved on and off the partial lists and is therefore a factor in
2626 * locking overhead.
2627 */
2628
2629/*
2630 * Mininum / Maximum order of slab pages. This influences locking overhead
2631 * and slab fragmentation. A higher order reduces the number of partial slabs
2632 * and increases the number of allocations possible without having to
2633 * take the list_lock.
2634 */
2635static int slub_min_order;
2636static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
2637static int slub_min_objects;
2638
2639/*
2640 * Merge control. If this is set then no merging of slab caches will occur.
2641 * (Could be removed. This was introduced to pacify the merge skeptics.)
2642 */
2643static int slub_nomerge;
2644
2645/*
2646 * Calculate the order of allocation given an slab object size.
2647 *
2648 * The order of allocation has significant impact on performance and other
2649 * system components. Generally order 0 allocations should be preferred since
2650 * order 0 does not cause fragmentation in the page allocator. Larger objects
2651 * be problematic to put into order 0 slabs because there may be too much
2652 * unused space left. We go to a higher order if more than 1/16th of the slab
2653 * would be wasted.
2654 *
2655 * In order to reach satisfactory performance we must ensure that a minimum
2656 * number of objects is in one slab. Otherwise we may generate too much
2657 * activity on the partial lists which requires taking the list_lock. This is
2658 * less a concern for large slabs though which are rarely used.
2659 *
2660 * slub_max_order specifies the order where we begin to stop considering the
2661 * number of objects in a slab as critical. If we reach slub_max_order then
2662 * we try to keep the page order as low as possible. So we accept more waste
2663 * of space in favor of a small page order.
2664 *
2665 * Higher order allocations also allow the placement of more objects in a
2666 * slab and thereby reduce object handling overhead. If the user has
2667 * requested a higher mininum order then we start with that one instead of
2668 * the smallest order which will fit the object.
2669 */
2670static inline int slab_order(int size, int min_objects,
2671                int max_order, int fract_leftover, int reserved)
2672{
2673    int order;
2674    int rem;
2675    int min_order = slub_min_order;
2676
2677    if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
2678        return get_order(size * MAX_OBJS_PER_PAGE) - 1;
2679
2680    for (order = max(min_order,
2681                fls(min_objects * size - 1) - PAGE_SHIFT);
2682            order <= max_order; order++) {
2683
2684        unsigned long slab_size = PAGE_SIZE << order;
2685
2686        if (slab_size < min_objects * size + reserved)
2687            continue;
2688
2689        rem = (slab_size - reserved) % size;
2690
2691        if (rem <= slab_size / fract_leftover)
2692            break;
2693
2694    }
2695
2696    return order;
2697}
2698
2699static inline int calculate_order(int size, int reserved)
2700{
2701    int order;
2702    int min_objects;
2703    int fraction;
2704    int max_objects;
2705
2706    /*
2707     * Attempt to find best configuration for a slab. This
2708     * works by first attempting to generate a layout with
2709     * the best configuration and backing off gradually.
2710     *
2711     * First we reduce the acceptable waste in a slab. Then
2712     * we reduce the minimum objects required in a slab.
2713     */
2714    min_objects = slub_min_objects;
2715    if (!min_objects)
2716        min_objects = 4 * (fls(nr_cpu_ids) + 1);
2717    max_objects = order_objects(slub_max_order, size, reserved);
2718    min_objects = min(min_objects, max_objects);
2719
2720    while (min_objects > 1) {
2721        fraction = 16;
2722        while (fraction >= 4) {
2723            order = slab_order(size, min_objects,
2724                    slub_max_order, fraction, reserved);
2725            if (order <= slub_max_order)
2726                return order;
2727            fraction /= 2;
2728        }
2729        min_objects--;
2730    }
2731
2732    /*
2733     * We were unable to place multiple objects in a slab. Now
2734     * lets see if we can place a single object there.
2735     */
2736    order = slab_order(size, 1, slub_max_order, 1, reserved);
2737    if (order <= slub_max_order)
2738        return order;
2739
2740    /*
2741     * Doh this slab cannot be placed using slub_max_order.
2742     */
2743    order = slab_order(size, 1, MAX_ORDER, 1, reserved);
2744    if (order < MAX_ORDER)
2745        return order;
2746    return -ENOSYS;
2747}
2748
2749/*
2750 * Figure out what the alignment of the objects will be.
2751 */
2752static unsigned long calculate_alignment(unsigned long flags,
2753        unsigned long align, unsigned long size)
2754{
2755    /*
2756     * If the user wants hardware cache aligned objects then follow that
2757     * suggestion if the object is sufficiently large.
2758     *
2759     * The hardware cache alignment cannot override the specified
2760     * alignment though. If that is greater then use it.
2761     */
2762    if (flags & SLAB_HWCACHE_ALIGN) {
2763        unsigned long ralign = cache_line_size();
2764        while (size <= ralign / 2)
2765            ralign /= 2;
2766        align = max(align, ralign);
2767    }
2768
2769    if (align < ARCH_SLAB_MINALIGN)
2770        align = ARCH_SLAB_MINALIGN;
2771
2772    return ALIGN(align, sizeof(void *));
2773}
2774
2775static void
2776init_kmem_cache_node(struct kmem_cache_node *n)
2777{
2778    n->nr_partial = 0;
2779    spin_lock_init(&n->list_lock);
2780    INIT_LIST_HEAD(&n->partial);
2781#ifdef CONFIG_SLUB_DEBUG
2782    atomic_long_set(&n->nr_slabs, 0);
2783    atomic_long_set(&n->total_objects, 0);
2784    INIT_LIST_HEAD(&n->full);
2785#endif
2786}
2787
2788static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
2789{
2790    BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
2791            SLUB_PAGE_SHIFT * sizeof(struct kmem_cache_cpu));
2792
2793    /*
2794     * Must align to double word boundary for the double cmpxchg
2795     * instructions to work; see __pcpu_double_call_return_bool().
2796     */
2797    s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
2798                     2 * sizeof(void *));
2799
2800    if (!s->cpu_slab)
2801        return 0;
2802
2803    init_kmem_cache_cpus(s);
2804
2805    return 1;
2806}
2807
2808static struct kmem_cache *kmem_cache_node;
2809
2810/*
2811 * No kmalloc_node yet so do it by hand. We know that this is the first
2812 * slab on the node for this slabcache. There are no concurrent accesses
2813 * possible.
2814 *
2815 * Note that this function only works on the kmalloc_node_cache
2816 * when allocating for the kmalloc_node_cache. This is used for bootstrapping
2817 * memory on a fresh node that has no slab structures yet.
2818 */
2819static void early_kmem_cache_node_alloc(int node)
2820{
2821    struct page *page;
2822    struct kmem_cache_node *n;
2823
2824    BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
2825
2826    page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
2827
2828    BUG_ON(!page);
2829    if (page_to_nid(page) != node) {
2830        printk(KERN_ERR "SLUB: Unable to allocate memory from "
2831                "node %d\n", node);
2832        printk(KERN_ERR "SLUB: Allocating a useless per node structure "
2833                "in order to be able to continue\n");
2834    }
2835
2836    n = page->freelist;
2837    BUG_ON(!n);
2838    page->freelist = get_freepointer(kmem_cache_node, n);
2839    page->inuse = 1;
2840    page->frozen = 0;
2841    kmem_cache_node->node[node] = n;
2842#ifdef CONFIG_SLUB_DEBUG
2843    init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
2844    init_tracking(kmem_cache_node, n);
2845#endif
2846    init_kmem_cache_node(n);
2847    inc_slabs_node(kmem_cache_node, node, page->objects);
2848
2849    add_partial(n, page, DEACTIVATE_TO_HEAD);
2850}
2851
2852static void free_kmem_cache_nodes(struct kmem_cache *s)
2853{
2854    int node;
2855
2856    for_each_node_state(node, N_NORMAL_MEMORY) {
2857        struct kmem_cache_node *n = s->node[node];
2858
2859        if (n)
2860            kmem_cache_free(kmem_cache_node, n);
2861
2862        s->node[node] = NULL;
2863    }
2864}
2865
2866static int init_kmem_cache_nodes(struct kmem_cache *s)
2867{
2868    int node;
2869
2870    for_each_node_state(node, N_NORMAL_MEMORY) {
2871        struct kmem_cache_node *n;
2872
2873        if (slab_state == DOWN) {
2874            early_kmem_cache_node_alloc(node);
2875            continue;
2876        }
2877        n = kmem_cache_alloc_node(kmem_cache_node,
2878                        GFP_KERNEL, node);
2879
2880        if (!n) {
2881            free_kmem_cache_nodes(s);
2882            return 0;
2883        }
2884
2885        s->node[node] = n;
2886        init_kmem_cache_node(n);
2887    }
2888    return 1;
2889}
2890
2891static void set_min_partial(struct kmem_cache *s, unsigned long min)
2892{
2893    if (min < MIN_PARTIAL)
2894        min = MIN_PARTIAL;
2895    else if (min > MAX_PARTIAL)
2896        min = MAX_PARTIAL;
2897    s->min_partial = min;
2898}
2899
2900/*
2901 * calculate_sizes() determines the order and the distribution of data within
2902 * a slab object.
2903 */
2904static int calculate_sizes(struct kmem_cache *s, int forced_order)
2905{
2906    unsigned long flags = s->flags;
2907    unsigned long size = s->object_size;
2908    unsigned long align = s->align;
2909    int order;
2910
2911    /*
2912     * Round up object size to the next word boundary. We can only
2913     * place the free pointer at word boundaries and this determines
2914     * the possible location of the free pointer.
2915     */
2916    size = ALIGN(size, sizeof(void *));
2917
2918#ifdef CONFIG_SLUB_DEBUG
2919    /*
2920     * Determine if we can poison the object itself. If the user of
2921     * the slab may touch the object after free or before allocation
2922     * then we should never poison the object itself.
2923     */
2924    if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
2925            !s->ctor)
2926        s->flags |= __OBJECT_POISON;
2927    else
2928        s->flags &= ~__OBJECT_POISON;
2929
2930
2931    /*
2932     * If we are Redzoning then check if there is some space between the
2933     * end of the object and the free pointer. If not then add an
2934     * additional word to have some bytes to store Redzone information.
2935     */
2936    if ((flags & SLAB_RED_ZONE) && size == s->object_size)
2937        size += sizeof(void *);
2938#endif
2939
2940    /*
2941     * With that we have determined the number of bytes in actual use
2942     * by the object. This is the potential offset to the free pointer.
2943     */
2944    s->inuse = size;
2945
2946    if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
2947        s->ctor)) {
2948        /*
2949         * Relocate free pointer after the object if it is not
2950         * permitted to overwrite the first word of the object on
2951         * kmem_cache_free.
2952         *
2953         * This is the case if we do RCU, have a constructor or
2954         * destructor or are poisoning the objects.
2955         */
2956        s->offset = size;
2957        size += sizeof(void *);
2958    }
2959
2960#ifdef CONFIG_SLUB_DEBUG
2961    if (flags & SLAB_STORE_USER)
2962        /*
2963         * Need to store information about allocs and frees after
2964         * the object.
2965         */
2966        size += 2 * sizeof(struct track);
2967
2968    if (flags & SLAB_RED_ZONE)
2969        /*
2970         * Add some empty padding so that we can catch
2971         * overwrites from earlier objects rather than let
2972         * tracking information or the free pointer be
2973         * corrupted if a user writes before the start
2974         * of the object.
2975         */
2976        size += sizeof(void *);
2977#endif
2978
2979    /*
2980     * Determine the alignment based on various parameters that the
2981     * user specified and the dynamic determination of cache line size
2982     * on bootup.
2983     */
2984    align = calculate_alignment(flags, align, s->object_size);
2985    s->align = align;
2986
2987    /*
2988     * SLUB stores one object immediately after another beginning from
2989     * offset 0. In order to align the objects we have to simply size
2990     * each object to conform to the alignment.
2991     */
2992    size = ALIGN(size, align);
2993    s->size = size;
2994    if (forced_order >= 0)
2995        order = forced_order;
2996    else
2997        order = calculate_order(size, s->reserved);
2998
2999    if (order < 0)
3000        return 0;
3001
3002    s->allocflags = 0;
3003    if (order)
3004        s->allocflags |= __GFP_COMP;
3005
3006    if (s->flags & SLAB_CACHE_DMA)
3007        s->allocflags |= SLUB_DMA;
3008
3009    if (s->flags & SLAB_RECLAIM_ACCOUNT)
3010        s->allocflags |= __GFP_RECLAIMABLE;
3011
3012    /*
3013     * Determine the number of objects per slab
3014     */
3015    s->oo = oo_make(order, size, s->reserved);
3016    s->min = oo_make(get_order(size), size, s->reserved);
3017    if (oo_objects(s->oo) > oo_objects(s->max))
3018        s->max = s->oo;
3019
3020    return !!oo_objects(s->oo);
3021
3022}
3023
3024static int kmem_cache_open(struct kmem_cache *s,
3025        const char *name, size_t size,
3026        size_t align, unsigned long flags,
3027        void (*ctor)(void *))
3028{
3029    memset(s, 0, kmem_size);
3030    s->name = name;
3031    s->ctor = ctor;
3032    s->object_size = size;
3033    s->align = align;
3034    s->flags = kmem_cache_flags(size, flags, name, ctor);
3035    s->reserved = 0;
3036
3037    if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
3038        s->reserved = sizeof(struct rcu_head);
3039
3040    if (!calculate_sizes(s, -1))
3041        goto error;
3042    if (disable_higher_order_debug) {
3043        /*
3044         * Disable debugging flags that store metadata if the min slab
3045         * order increased.
3046         */
3047        if (get_order(s->size) > get_order(s->object_size)) {
3048            s->flags &= ~DEBUG_METADATA_FLAGS;
3049            s->offset = 0;
3050            if (!calculate_sizes(s, -1))
3051                goto error;
3052        }
3053    }
3054
3055#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3056    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
3057    if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
3058        /* Enable fast mode */
3059        s->flags |= __CMPXCHG_DOUBLE;
3060#endif
3061
3062    /*
3063     * The larger the object size is, the more pages we want on the partial
3064     * list to avoid pounding the page allocator excessively.
3065     */
3066    set_min_partial(s, ilog2(s->size) / 2);
3067
3068    /*
3069     * cpu_partial determined the maximum number of objects kept in the
3070     * per cpu partial lists of a processor.
3071     *
3072     * Per cpu partial lists mainly contain slabs that just have one
3073     * object freed. If they are used for allocation then they can be
3074     * filled up again with minimal effort. The slab will never hit the
3075     * per node partial lists and therefore no locking will be required.
3076     *
3077     * This setting also determines
3078     *
3079     * A) The number of objects from per cpu partial slabs dumped to the
3080     * per node list when we reach the limit.
3081     * B) The number of objects in cpu partial slabs to extract from the
3082     * per node list when we run out of per cpu objects. We only fetch 50%
3083     * to keep some capacity around for frees.
3084     */
3085    if (kmem_cache_debug(s))
3086        s->cpu_partial = 0;
3087    else if (s->size >= PAGE_SIZE)
3088        s->cpu_partial = 2;
3089    else if (s->size >= 1024)
3090        s->cpu_partial = 6;
3091    else if (s->size >= 256)
3092        s->cpu_partial = 13;
3093    else
3094        s->cpu_partial = 30;
3095
3096    s->refcount = 1;
3097#ifdef CONFIG_NUMA
3098    s->remote_node_defrag_ratio = 1000;
3099#endif
3100    if (!init_kmem_cache_nodes(s))
3101        goto error;
3102
3103    if (alloc_kmem_cache_cpus(s))
3104        return 1;
3105
3106    free_kmem_cache_nodes(s);
3107error:
3108    if (flags & SLAB_PANIC)
3109        panic("Cannot create slab %s size=%lu realsize=%u "
3110            "order=%u offset=%u flags=%lx\n",
3111            s->name, (unsigned long)size, s->size, oo_order(s->oo),
3112            s->offset, flags);
3113    return 0;
3114}
3115
3116/*
3117 * Determine the size of a slab object
3118 */
3119unsigned int kmem_cache_size(struct kmem_cache *s)
3120{
3121    return s->object_size;
3122}
3123EXPORT_SYMBOL(kmem_cache_size);
3124
3125static void list_slab_objects(struct kmem_cache *s, struct page *page,
3126                            const char *text)
3127{
3128#ifdef CONFIG_SLUB_DEBUG
3129    void *addr = page_address(page);
3130    void *p;
3131    unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) *
3132                     sizeof(long), GFP_ATOMIC);
3133    if (!map)
3134        return;
3135    slab_err(s, page, "%s", text);
3136    slab_lock(page);
3137
3138    get_map(s, page, map);
3139    for_each_object(p, s, addr, page->objects) {
3140
3141        if (!test_bit(slab_index(p, s, addr), map)) {
3142            printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n",
3143                            p, p - addr);
3144            print_tracking(s, p);
3145        }
3146    }
3147    slab_unlock(page);
3148    kfree(map);
3149#endif
3150}
3151
3152/*
3153 * Attempt to free all partial slabs on a node.
3154 * This is called from kmem_cache_close(). We must be the last thread
3155 * using the cache and therefore we do not need to lock anymore.
3156 */
3157static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
3158{
3159    struct page *page, *h;
3160
3161    list_for_each_entry_safe(page, h, &n->partial, lru) {
3162        if (!page->inuse) {
3163            remove_partial(n, page);
3164            discard_slab(s, page);
3165        } else {
3166            list_slab_objects(s, page,
3167                "Objects remaining on kmem_cache_close()");
3168        }
3169    }
3170}
3171
3172/*
3173 * Release all resources used by a slab cache.
3174 */
3175static inline int kmem_cache_close(struct kmem_cache *s)
3176{
3177    int node;
3178
3179    flush_all(s);
3180    free_percpu(s->cpu_slab);
3181    /* Attempt to free all objects */
3182    for_each_node_state(node, N_NORMAL_MEMORY) {
3183        struct kmem_cache_node *n = get_node(s, node);
3184
3185        free_partial(s, n);
3186        if (n->nr_partial || slabs_node(s, node))
3187            return 1;
3188    }
3189    free_kmem_cache_nodes(s);
3190    return 0;
3191}
3192
3193/*
3194 * Close a cache and release the kmem_cache structure
3195 * (must be used for caches created using kmem_cache_create)
3196 */
3197void kmem_cache_destroy(struct kmem_cache *s)
3198{
3199    mutex_lock(&slab_mutex);
3200    s->refcount--;
3201    if (!s->refcount) {
3202        list_del(&s->list);
3203        mutex_unlock(&slab_mutex);
3204        if (kmem_cache_close(s)) {
3205            printk(KERN_ERR "SLUB %s: %s called for cache that "
3206                "still has objects.\n", s->name, __func__);
3207            dump_stack();
3208        }
3209        if (s->flags & SLAB_DESTROY_BY_RCU)
3210            rcu_barrier();
3211        sysfs_slab_remove(s);
3212    } else
3213        mutex_unlock(&slab_mutex);
3214}
3215EXPORT_SYMBOL(kmem_cache_destroy);
3216
3217/********************************************************************
3218 * Kmalloc subsystem
3219 *******************************************************************/
3220
3221struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
3222EXPORT_SYMBOL(kmalloc_caches);
3223
3224static struct kmem_cache *kmem_cache;
3225
3226#ifdef CONFIG_ZONE_DMA
3227static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
3228#endif
3229
3230static int __init setup_slub_min_order(char *str)
3231{
3232    get_option(&str, &slub_min_order);
3233
3234    return 1;
3235}
3236
3237__setup("slub_min_order=", setup_slub_min_order);
3238
3239static int __init setup_slub_max_order(char *str)
3240{
3241    get_option(&str, &slub_max_order);
3242    slub_max_order = min(slub_max_order, MAX_ORDER - 1);
3243
3244    return 1;
3245}
3246
3247__setup("slub_max_order=", setup_slub_max_order);
3248
3249static int __init setup_slub_min_objects(char *str)
3250{
3251    get_option(&str, &slub_min_objects);
3252
3253    return 1;
3254}
3255
3256__setup("slub_min_objects=", setup_slub_min_objects);
3257
3258static int __init setup_slub_nomerge(char *str)
3259{
3260    slub_nomerge = 1;
3261    return 1;
3262}
3263
3264__setup("slub_nomerge", setup_slub_nomerge);
3265
3266static struct kmem_cache *__init create_kmalloc_cache(const char *name,
3267                        int size, unsigned int flags)
3268{
3269    struct kmem_cache *s;
3270
3271    s = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3272
3273    /*
3274     * This function is called with IRQs disabled during early-boot on
3275     * single CPU so there's no need to take slab_mutex here.
3276     */
3277    if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
3278                                flags, NULL))
3279        goto panic;
3280
3281    list_add(&s->list, &slab_caches);
3282    return s;
3283
3284panic:
3285    panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
3286    return NULL;
3287}
3288
3289/*
3290 * Conversion table for small slabs sizes / 8 to the index in the
3291 * kmalloc array. This is necessary for slabs < 192 since we have non power
3292 * of two cache sizes there. The size of larger slabs can be determined using
3293 * fls.
3294 */
3295static s8 size_index[24] = {
3296    3, /* 8 */
3297    4, /* 16 */
3298    5, /* 24 */
3299    5, /* 32 */
3300    6, /* 40 */
3301    6, /* 48 */
3302    6, /* 56 */
3303    6, /* 64 */
3304    1, /* 72 */
3305    1, /* 80 */
3306    1, /* 88 */
3307    1, /* 96 */
3308    7, /* 104 */
3309    7, /* 112 */
3310    7, /* 120 */
3311    7, /* 128 */
3312    2, /* 136 */
3313    2, /* 144 */
3314    2, /* 152 */
3315    2, /* 160 */
3316    2, /* 168 */
3317    2, /* 176 */
3318    2, /* 184 */
3319    2 /* 192 */
3320};
3321
3322static inline int size_index_elem(size_t bytes)
3323{
3324    return (bytes - 1) / 8;
3325}
3326
3327static struct kmem_cache *get_slab(size_t size, gfp_t flags)
3328{
3329    int index;
3330
3331    if (size <= 192) {
3332        if (!size)
3333            return ZERO_SIZE_PTR;
3334
3335        index = size_index[size_index_elem(size)];
3336    } else
3337        index = fls(size - 1);
3338
3339#ifdef CONFIG_ZONE_DMA
3340    if (unlikely((flags & SLUB_DMA)))
3341        return kmalloc_dma_caches[index];
3342
3343#endif
3344    return kmalloc_caches[index];
3345}
3346
3347void *__kmalloc(size_t size, gfp_t flags)
3348{
3349    struct kmem_cache *s;
3350    void *ret;
3351
3352    if (unlikely(size > SLUB_MAX_SIZE))
3353        return kmalloc_large(size, flags);
3354
3355    s = get_slab(size, flags);
3356
3357    if (unlikely(ZERO_OR_NULL_PTR(s)))
3358        return s;
3359
3360    ret = slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_);
3361
3362    trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
3363
3364    return ret;
3365}
3366EXPORT_SYMBOL(__kmalloc);
3367
3368#ifdef CONFIG_NUMA
3369static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3370{
3371    struct page *page;
3372    void *ptr = NULL;
3373
3374    flags |= __GFP_COMP | __GFP_NOTRACK;
3375    page = alloc_pages_node(node, flags, get_order(size));
3376    if (page)
3377        ptr = page_address(page);
3378
3379    kmemleak_alloc(ptr, size, 1, flags);
3380    return ptr;
3381}
3382
3383void *__kmalloc_node(size_t size, gfp_t flags, int node)
3384{
3385    struct kmem_cache *s;
3386    void *ret;
3387
3388    if (unlikely(size > SLUB_MAX_SIZE)) {
3389        ret = kmalloc_large_node(size, flags, node);
3390
3391        trace_kmalloc_node(_RET_IP_, ret,
3392                   size, PAGE_SIZE << get_order(size),
3393                   flags, node);
3394
3395        return ret;
3396    }
3397
3398    s = get_slab(size, flags);
3399
3400    if (unlikely(ZERO_OR_NULL_PTR(s)))
3401        return s;
3402
3403    ret = slab_alloc(s, flags, node, _RET_IP_);
3404
3405    trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
3406
3407    return ret;
3408}
3409EXPORT_SYMBOL(__kmalloc_node);
3410#endif
3411
3412size_t ksize(const void *object)
3413{
3414    struct page *page;
3415
3416    if (unlikely(object == ZERO_SIZE_PTR))
3417        return 0;
3418
3419    page = virt_to_head_page(object);
3420
3421    if (unlikely(!PageSlab(page))) {
3422        WARN_ON(!PageCompound(page));
3423        return PAGE_SIZE << compound_order(page);
3424    }
3425
3426    return slab_ksize(page->slab);
3427}
3428EXPORT_SYMBOL(ksize);
3429
3430#ifdef CONFIG_SLUB_DEBUG
3431bool verify_mem_not_deleted(const void *x)
3432{
3433    struct page *page;
3434    void *object = (void *)x;
3435    unsigned long flags;
3436    bool rv;
3437
3438    if (unlikely(ZERO_OR_NULL_PTR(x)))
3439        return false;
3440
3441    local_irq_save(flags);
3442
3443    page = virt_to_head_page(x);
3444    if (unlikely(!PageSlab(page))) {
3445        /* maybe it was from stack? */
3446        rv = true;
3447        goto out_unlock;
3448    }
3449
3450    slab_lock(page);
3451    if (on_freelist(page->slab, page, object)) {
3452        object_err(page->slab, page, object, "Object is on free-list");
3453        rv = false;
3454    } else {
3455        rv = true;
3456    }
3457    slab_unlock(page);
3458
3459out_unlock:
3460    local_irq_restore(flags);
3461    return rv;
3462}
3463EXPORT_SYMBOL(verify_mem_not_deleted);
3464#endif
3465
3466void kfree(const void *x)
3467{
3468    struct page *page;
3469    void *object = (void *)x;
3470
3471    trace_kfree(_RET_IP_, x);
3472
3473    if (unlikely(ZERO_OR_NULL_PTR(x)))
3474        return;
3475
3476    page = virt_to_head_page(x);
3477    if (unlikely(!PageSlab(page))) {
3478        BUG_ON(!PageCompound(page));
3479        kmemleak_free(x);
3480        put_page(page);
3481        return;
3482    }
3483    slab_free(page->slab, page, object, _RET_IP_);
3484}
3485EXPORT_SYMBOL(kfree);
3486
3487/*
3488 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
3489 * the remaining slabs by the number of items in use. The slabs with the
3490 * most items in use come first. New allocations will then fill those up
3491 * and thus they can be removed from the partial lists.
3492 *
3493 * The slabs with the least items are placed last. This results in them
3494 * being allocated from last increasing the chance that the last objects
3495 * are freed in them.
3496 */
3497int kmem_cache_shrink(struct kmem_cache *s)
3498{
3499    int node;
3500    int i;
3501    struct kmem_cache_node *n;
3502    struct page *page;
3503    struct page *t;
3504    int objects = oo_objects(s->max);
3505    struct list_head *slabs_by_inuse =
3506        kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
3507    unsigned long flags;
3508
3509    if (!slabs_by_inuse)
3510        return -ENOMEM;
3511
3512    flush_all(s);
3513    for_each_node_state(node, N_NORMAL_MEMORY) {
3514        n = get_node(s, node);
3515
3516        if (!n->nr_partial)
3517            continue;
3518
3519        for (i = 0; i < objects; i++)
3520            INIT_LIST_HEAD(slabs_by_inuse + i);
3521
3522        spin_lock_irqsave(&n->list_lock, flags);
3523
3524        /*
3525         * Build lists indexed by the items in use in each slab.
3526         *
3527         * Note that concurrent frees may occur while we hold the
3528         * list_lock. page->inuse here is the upper limit.
3529         */
3530        list_for_each_entry_safe(page, t, &n->partial, lru) {
3531            list_move(&page->lru, slabs_by_inuse + page->inuse);
3532            if (!page->inuse)
3533                n->nr_partial--;
3534        }
3535
3536        /*
3537         * Rebuild the partial list with the slabs filled up most
3538         * first and the least used slabs at the end.
3539         */
3540        for (i = objects - 1; i > 0; i--)
3541            list_splice(slabs_by_inuse + i, n->partial.prev);
3542
3543        spin_unlock_irqrestore(&n->list_lock, flags);
3544
3545        /* Release empty slabs */
3546        list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
3547            discard_slab(s, page);
3548    }
3549
3550    kfree(slabs_by_inuse);
3551    return 0;
3552}
3553EXPORT_SYMBOL(kmem_cache_shrink);
3554
3555#if defined(CONFIG_MEMORY_HOTPLUG)
3556static int slab_mem_going_offline_callback(void *arg)
3557{
3558    struct kmem_cache *s;
3559
3560    mutex_lock(&slab_mutex);
3561    list_for_each_entry(s, &slab_caches, list)
3562        kmem_cache_shrink(s);
3563    mutex_unlock(&slab_mutex);
3564
3565    return 0;
3566}
3567
3568static void slab_mem_offline_callback(void *arg)
3569{
3570    struct kmem_cache_node *n;
3571    struct kmem_cache *s;
3572    struct memory_notify *marg = arg;
3573    int offline_node;
3574
3575    offline_node = marg->status_change_nid;
3576
3577    /*
3578     * If the node still has available memory. we need kmem_cache_node
3579     * for it yet.
3580     */
3581    if (offline_node < 0)
3582        return;
3583
3584    mutex_lock(&slab_mutex);
3585    list_for_each_entry(s, &slab_caches, list) {
3586        n = get_node(s, offline_node);
3587        if (n) {
3588            /*
3589             * if n->nr_slabs > 0, slabs still exist on the node
3590             * that is going down. We were unable to free them,
3591             * and offline_pages() function shouldn't call this
3592             * callback. So, we must fail.
3593             */
3594            BUG_ON(slabs_node(s, offline_node));
3595
3596            s->node[offline_node] = NULL;
3597            kmem_cache_free(kmem_cache_node, n);
3598        }
3599    }
3600    mutex_unlock(&slab_mutex);
3601}
3602
3603static int slab_mem_going_online_callback(void *arg)
3604{
3605    struct kmem_cache_node *n;
3606    struct kmem_cache *s;
3607    struct memory_notify *marg = arg;
3608    int nid = marg->status_change_nid;
3609    int ret = 0;
3610
3611    /*
3612     * If the node's memory is already available, then kmem_cache_node is
3613     * already created. Nothing to do.
3614     */
3615    if (nid < 0)
3616        return 0;
3617
3618    /*
3619     * We are bringing a node online. No memory is available yet. We must
3620     * allocate a kmem_cache_node structure in order to bring the node
3621     * online.
3622     */
3623    mutex_lock(&slab_mutex);
3624    list_for_each_entry(s, &slab_caches, list) {
3625        /*
3626         * XXX: kmem_cache_alloc_node will fallback to other nodes
3627         * since memory is not yet available from the node that
3628         * is brought up.
3629         */
3630        n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
3631        if (!n) {
3632            ret = -ENOMEM;
3633            goto out;
3634        }
3635        init_kmem_cache_node(n);
3636        s->node[nid] = n;
3637    }
3638out:
3639    mutex_unlock(&slab_mutex);
3640    return ret;
3641}
3642
3643static int slab_memory_callback(struct notifier_block *self,
3644                unsigned long action, void *arg)
3645{
3646    int ret = 0;
3647
3648    switch (action) {
3649    case MEM_GOING_ONLINE:
3650        ret = slab_mem_going_online_callback(arg);
3651        break;
3652    case MEM_GOING_OFFLINE:
3653        ret = slab_mem_going_offline_callback(arg);
3654        break;
3655    case MEM_OFFLINE:
3656    case MEM_CANCEL_ONLINE:
3657        slab_mem_offline_callback(arg);
3658        break;
3659    case MEM_ONLINE:
3660    case MEM_CANCEL_OFFLINE:
3661        break;
3662    }
3663    if (ret)
3664        ret = notifier_from_errno(ret);
3665    else
3666        ret = NOTIFY_OK;
3667    return ret;
3668}
3669
3670#endif /* CONFIG_MEMORY_HOTPLUG */
3671
3672/********************************************************************
3673 * Basic setup of slabs
3674 *******************************************************************/
3675
3676/*
3677 * Used for early kmem_cache structures that were allocated using
3678 * the page allocator
3679 */
3680
3681static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
3682{
3683    int node;
3684
3685    list_add(&s->list, &slab_caches);
3686    s->refcount = -1;
3687
3688    for_each_node_state(node, N_NORMAL_MEMORY) {
3689        struct kmem_cache_node *n = get_node(s, node);
3690        struct page *p;
3691
3692        if (n) {
3693            list_for_each_entry(p, &n->partial, lru)
3694                p->slab = s;
3695
3696#ifdef CONFIG_SLUB_DEBUG
3697            list_for_each_entry(p, &n->full, lru)
3698                p->slab = s;
3699#endif
3700        }
3701    }
3702}
3703
3704void __init kmem_cache_init(void)
3705{
3706    int i;
3707    int caches = 0;
3708    struct kmem_cache *temp_kmem_cache;
3709    int order;
3710    struct kmem_cache *temp_kmem_cache_node;
3711    unsigned long kmalloc_size;
3712
3713    if (debug_guardpage_minorder())
3714        slub_max_order = 0;
3715
3716    kmem_size = offsetof(struct kmem_cache, node) +
3717                nr_node_ids * sizeof(struct kmem_cache_node *);
3718
3719    /* Allocate two kmem_caches from the page allocator */
3720    kmalloc_size = ALIGN(kmem_size, cache_line_size());
3721    order = get_order(2 * kmalloc_size);
3722    kmem_cache = (void *)__get_free_pages(GFP_NOWAIT, order);
3723
3724    /*
3725     * Must first have the slab cache available for the allocations of the
3726     * struct kmem_cache_node's. There is special bootstrap code in
3727     * kmem_cache_open for slab_state == DOWN.
3728     */
3729    kmem_cache_node = (void *)kmem_cache + kmalloc_size;
3730
3731    kmem_cache_open(kmem_cache_node, "kmem_cache_node",
3732        sizeof(struct kmem_cache_node),
3733        0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
3734
3735    hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
3736
3737    /* Able to allocate the per node structures */
3738    slab_state = PARTIAL;
3739
3740    temp_kmem_cache = kmem_cache;
3741    kmem_cache_open(kmem_cache, "kmem_cache", kmem_size,
3742        0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
3743    kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3744    memcpy(kmem_cache, temp_kmem_cache, kmem_size);
3745
3746    /*
3747     * Allocate kmem_cache_node properly from the kmem_cache slab.
3748     * kmem_cache_node is separately allocated so no need to
3749     * update any list pointers.
3750     */
3751    temp_kmem_cache_node = kmem_cache_node;
3752
3753    kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3754    memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
3755
3756    kmem_cache_bootstrap_fixup(kmem_cache_node);
3757
3758    caches++;
3759    kmem_cache_bootstrap_fixup(kmem_cache);
3760    caches++;
3761    /* Free temporary boot structure */
3762    free_pages((unsigned long)temp_kmem_cache, order);
3763
3764    /* Now we can use the kmem_cache to allocate kmalloc slabs */
3765
3766    /*
3767     * Patch up the size_index table if we have strange large alignment
3768     * requirements for the kmalloc array. This is only the case for
3769     * MIPS it seems. The standard arches will not generate any code here.
3770     *
3771     * Largest permitted alignment is 256 bytes due to the way we
3772     * handle the index determination for the smaller caches.
3773     *
3774     * Make sure that nothing crazy happens if someone starts tinkering
3775     * around with ARCH_KMALLOC_MINALIGN
3776     */
3777    BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
3778        (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
3779
3780    for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
3781        int elem = size_index_elem(i);
3782        if (elem >= ARRAY_SIZE(size_index))
3783            break;
3784        size_index[elem] = KMALLOC_SHIFT_LOW;
3785    }
3786
3787    if (KMALLOC_MIN_SIZE == 64) {
3788        /*
3789         * The 96 byte size cache is not used if the alignment
3790         * is 64 byte.
3791         */
3792        for (i = 64 + 8; i <= 96; i += 8)
3793            size_index[size_index_elem(i)] = 7;
3794    } else if (KMALLOC_MIN_SIZE == 128) {
3795        /*
3796         * The 192 byte sized cache is not used if the alignment
3797         * is 128 byte. Redirect kmalloc to use the 256 byte cache
3798         * instead.
3799         */
3800        for (i = 128 + 8; i <= 192; i += 8)
3801            size_index[size_index_elem(i)] = 8;
3802    }
3803
3804    /* Caches that are not of the two-to-the-power-of size */
3805    if (KMALLOC_MIN_SIZE <= 32) {
3806        kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
3807        caches++;
3808    }
3809
3810    if (KMALLOC_MIN_SIZE <= 64) {
3811        kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
3812        caches++;
3813    }
3814
3815    for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
3816        kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
3817        caches++;
3818    }
3819
3820    slab_state = UP;
3821
3822    /* Provide the correct kmalloc names now that the caches are up */
3823    if (KMALLOC_MIN_SIZE <= 32) {
3824        kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
3825        BUG_ON(!kmalloc_caches[1]->name);
3826    }
3827
3828    if (KMALLOC_MIN_SIZE <= 64) {
3829        kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
3830        BUG_ON(!kmalloc_caches[2]->name);
3831    }
3832
3833    for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
3834        char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
3835
3836        BUG_ON(!s);
3837        kmalloc_caches[i]->name = s;
3838    }
3839
3840#ifdef CONFIG_SMP
3841    register_cpu_notifier(&slab_notifier);
3842#endif
3843
3844#ifdef CONFIG_ZONE_DMA
3845    for (i = 0; i < SLUB_PAGE_SHIFT; i++) {
3846        struct kmem_cache *s = kmalloc_caches[i];
3847
3848        if (s && s->size) {
3849            char *name = kasprintf(GFP_NOWAIT,
3850                 "dma-kmalloc-%d", s->object_size);
3851
3852            BUG_ON(!name);
3853            kmalloc_dma_caches[i] = create_kmalloc_cache(name,
3854                s->object_size, SLAB_CACHE_DMA);
3855        }
3856    }
3857#endif
3858    printk(KERN_INFO
3859        "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
3860        " CPUs=%d, Nodes=%d\n",
3861        caches, cache_line_size(),
3862        slub_min_order, slub_max_order, slub_min_objects,
3863        nr_cpu_ids, nr_node_ids);
3864}
3865
3866void __init kmem_cache_init_late(void)
3867{
3868}
3869
3870/*
3871 * Find a mergeable slab cache
3872 */
3873static int slab_unmergeable(struct kmem_cache *s)
3874{
3875    if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
3876        return 1;
3877
3878    if (s->ctor)
3879        return 1;
3880
3881    /*
3882     * We may have set a slab to be unmergeable during bootstrap.
3883     */
3884    if (s->refcount < 0)
3885        return 1;
3886
3887    return 0;
3888}
3889
3890static struct kmem_cache *find_mergeable(size_t size,
3891        size_t align, unsigned long flags, const char *name,
3892        void (*ctor)(void *))
3893{
3894    struct kmem_cache *s;
3895
3896    if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
3897        return NULL;
3898
3899    if (ctor)
3900        return NULL;
3901
3902    size = ALIGN(size, sizeof(void *));
3903    align = calculate_alignment(flags, align, size);
3904    size = ALIGN(size, align);
3905    flags = kmem_cache_flags(size, flags, name, NULL);
3906
3907    list_for_each_entry(s, &slab_caches, list) {
3908        if (slab_unmergeable(s))
3909            continue;
3910
3911        if (size > s->size)
3912            continue;
3913
3914        if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
3915                continue;
3916        /*
3917         * Check if alignment is compatible.
3918         * Courtesy of Adrian Drzewiecki
3919         */
3920        if ((s->size & ~(align - 1)) != s->size)
3921            continue;
3922
3923        if (s->size - size >= sizeof(void *))
3924            continue;
3925
3926        return s;
3927    }
3928    return NULL;
3929}
3930
3931struct kmem_cache *__kmem_cache_create(const char *name, size_t size,
3932        size_t align, unsigned long flags, void (*ctor)(void *))
3933{
3934    struct kmem_cache *s;
3935    char *n;
3936
3937    s = find_mergeable(size, align, flags, name, ctor);
3938    if (s) {
3939        s->refcount++;
3940        /*
3941         * Adjust the object sizes so that we clear
3942         * the complete object on kzalloc.
3943         */
3944        s->object_size = max(s->object_size, (int)size);
3945        s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
3946
3947        if (sysfs_slab_alias(s, name)) {
3948            s->refcount--;
3949            return NULL;
3950        }
3951        return s;
3952    }
3953
3954    n = kstrdup(name, GFP_KERNEL);
3955    if (!n)
3956        return NULL;
3957
3958    s = kmalloc(kmem_size, GFP_KERNEL);
3959    if (s) {
3960        if (kmem_cache_open(s, n,
3961                size, align, flags, ctor)) {
3962            int r;
3963
3964            list_add(&s->list, &slab_caches);
3965            mutex_unlock(&slab_mutex);
3966            r = sysfs_slab_add(s);
3967            mutex_lock(&slab_mutex);
3968
3969            if (!r)
3970                return s;
3971
3972            list_del(&s->list);
3973            kmem_cache_close(s);
3974        }
3975        kfree(s);
3976    }
3977    kfree(n);
3978    return NULL;
3979}
3980
3981#ifdef CONFIG_SMP
3982/*
3983 * Use the cpu notifier to insure that the cpu slabs are flushed when
3984 * necessary.
3985 */
3986static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
3987        unsigned long action, void *hcpu)
3988{
3989    long cpu = (long)hcpu;
3990    struct kmem_cache *s;
3991    unsigned long flags;
3992
3993    switch (action) {
3994    case CPU_UP_CANCELED:
3995    case CPU_UP_CANCELED_FROZEN:
3996    case CPU_DEAD:
3997    case CPU_DEAD_FROZEN:
3998        mutex_lock(&slab_mutex);
3999        list_for_each_entry(s, &slab_caches, list) {
4000            local_irq_save(flags);
4001            __flush_cpu_slab(s, cpu);
4002            local_irq_restore(flags);
4003        }
4004        mutex_unlock(&slab_mutex);
4005        break;
4006    default:
4007        break;
4008    }
4009    return NOTIFY_OK;
4010}
4011
4012static struct notifier_block __cpuinitdata slab_notifier = {
4013    .notifier_call = slab_cpuup_callback
4014};
4015
4016#endif
4017
4018void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
4019{
4020    struct kmem_cache *s;
4021    void *ret;
4022
4023    if (unlikely(size > SLUB_MAX_SIZE))
4024        return kmalloc_large(size, gfpflags);
4025
4026    s = get_slab(size, gfpflags);
4027
4028    if (unlikely(ZERO_OR_NULL_PTR(s)))
4029        return s;
4030
4031    ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, caller);
4032
4033    /* Honor the call site pointer we received. */
4034    trace_kmalloc(caller, ret, size, s->size, gfpflags);
4035
4036    return ret;
4037}
4038
4039#ifdef CONFIG_NUMA
4040void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4041                    int node, unsigned long caller)
4042{
4043    struct kmem_cache *s;
4044    void *ret;
4045
4046    if (unlikely(size > SLUB_MAX_SIZE)) {
4047        ret = kmalloc_large_node(size, gfpflags, node);
4048
4049        trace_kmalloc_node(caller, ret,
4050                   size, PAGE_SIZE << get_order(size),
4051                   gfpflags, node);
4052
4053        return ret;
4054    }
4055
4056    s = get_slab(size, gfpflags);
4057
4058    if (unlikely(ZERO_OR_NULL_PTR(s)))
4059        return s;
4060
4061    ret = slab_alloc(s, gfpflags, node, caller);
4062
4063    /* Honor the call site pointer we received. */
4064    trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
4065
4066    return ret;
4067}
4068#endif
4069
4070#ifdef CONFIG_SYSFS
4071static int count_inuse(struct page *page)
4072{
4073    return page->inuse;
4074}
4075
4076static int count_total(struct page *page)
4077{
4078    return page->objects;
4079}
4080#endif
4081
4082#ifdef CONFIG_SLUB_DEBUG
4083static int validate_slab(struct kmem_cache *s, struct page *page,
4084                        unsigned long *map)
4085{
4086    void *p;
4087    void *addr = page_address(page);
4088
4089    if (!check_slab(s, page) ||
4090            !on_freelist(s, page, NULL))
4091        return 0;
4092
4093    /* Now we know that a valid freelist exists */
4094    bitmap_zero(map, page->objects);
4095
4096    get_map(s, page, map);
4097    for_each_object(p, s, addr, page->objects) {
4098        if (test_bit(slab_index(p, s, addr), map))
4099            if (!check_object(s, page, p, SLUB_RED_INACTIVE))
4100                return 0;
4101    }
4102
4103    for_each_object(p, s, addr, page->objects)
4104        if (!test_bit(slab_index(p, s, addr), map))
4105            if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4106                return 0;
4107    return 1;
4108}
4109
4110static void validate_slab_slab(struct kmem_cache *s, struct page *page,
4111                        unsigned long *map)
4112{
4113    slab_lock(page);
4114    validate_slab(s, page, map);
4115    slab_unlock(page);
4116}
4117
4118static int validate_slab_node(struct kmem_cache *s,
4119        struct kmem_cache_node *n, unsigned long *map)
4120{
4121    unsigned long count = 0;
4122    struct page *page;
4123    unsigned long flags;
4124
4125    spin_lock_irqsave(&n->list_lock, flags);
4126
4127    list_for_each_entry(page, &n->partial, lru) {
4128        validate_slab_slab(s, page, map);
4129        count++;
4130    }
4131    if (count != n->nr_partial)
4132        printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
4133            "counter=%ld\n", s->name, count, n->nr_partial);
4134
4135    if (!(s->flags & SLAB_STORE_USER))
4136        goto out;
4137
4138    list_for_each_entry(page, &n->full, lru) {
4139        validate_slab_slab(s, page, map);
4140        count++;
4141    }
4142    if (count != atomic_long_read(&n->nr_slabs))
4143        printk(KERN_ERR "SLUB: %s %ld slabs counted but "
4144            "counter=%ld\n", s->name, count,
4145            atomic_long_read(&n->nr_slabs));
4146
4147out:
4148    spin_unlock_irqrestore(&n->list_lock, flags);
4149    return count;
4150}
4151
4152static long validate_slab_cache(struct kmem_cache *s)
4153{
4154    int node;
4155    unsigned long count = 0;
4156    unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4157                sizeof(unsigned long), GFP_KERNEL);
4158
4159    if (!map)
4160        return -ENOMEM;
4161
4162    flush_all(s);
4163    for_each_node_state(node, N_NORMAL_MEMORY) {
4164        struct kmem_cache_node *n = get_node(s, node);
4165
4166        count += validate_slab_node(s, n, map);
4167    }
4168    kfree(map);
4169    return count;
4170}
4171/*
4172 * Generate lists of code addresses where slabcache objects are allocated
4173 * and freed.
4174 */
4175
4176struct location {
4177    unsigned long count;
4178    unsigned long addr;
4179    long long sum_time;
4180    long min_time;
4181    long max_time;
4182    long min_pid;
4183    long max_pid;
4184    DECLARE_BITMAP(cpus, NR_CPUS);
4185    nodemask_t nodes;
4186};
4187
4188struct loc_track {
4189    unsigned long max;
4190    unsigned long count;
4191    struct location *loc;
4192};
4193
4194static void free_loc_track(struct loc_track *t)
4195{
4196    if (t->max)
4197        free_pages((unsigned long)t->loc,
4198            get_order(sizeof(struct location) * t->max));
4199}
4200
4201static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4202{
4203    struct location *l;
4204    int order;
4205
4206    order = get_order(sizeof(struct location) * max);
4207
4208    l = (void *)__get_free_pages(flags, order);
4209    if (!l)
4210        return 0;
4211
4212    if (t->count) {
4213        memcpy(l, t->loc, sizeof(struct location) * t->count);
4214        free_loc_track(t);
4215    }
4216    t->max = max;
4217    t->loc = l;
4218    return 1;
4219}
4220
4221static int add_location(struct loc_track *t, struct kmem_cache *s,
4222                const struct track *track)
4223{
4224    long start, end, pos;
4225    struct location *l;
4226    unsigned long caddr;
4227    unsigned long age = jiffies - track->when;
4228
4229    start = -1;
4230    end = t->count;
4231
4232    for ( ; ; ) {
4233        pos = start + (end - start + 1) / 2;
4234
4235        /*
4236         * There is nothing at "end". If we end up there
4237         * we need to add something to before end.
4238         */
4239        if (pos == end)
4240            break;
4241
4242        caddr = t->loc[pos].addr;
4243        if (track->addr == caddr) {
4244
4245            l = &t->loc[pos];
4246            l->count++;
4247            if (track->when) {
4248                l->sum_time += age;
4249                if (age < l->min_time)
4250                    l->min_time = age;
4251                if (age > l->max_time)
4252                    l->max_time = age;
4253
4254                if (track->pid < l->min_pid)
4255                    l->min_pid = track->pid;
4256                if (track->pid > l->max_pid)
4257                    l->max_pid = track->pid;
4258
4259                cpumask_set_cpu(track->cpu,
4260                        to_cpumask(l->cpus));
4261            }
4262            node_set(page_to_nid(virt_to_page(track)), l->nodes);
4263            return 1;
4264        }
4265
4266        if (track->addr < caddr)
4267            end = pos;
4268        else
4269            start = pos;
4270    }
4271
4272    /*
4273     * Not found. Insert new tracking element.
4274     */
4275    if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
4276        return 0;
4277
4278    l = t->loc + pos;
4279    if (pos < t->count)
4280        memmove(l + 1, l,
4281            (t->count - pos) * sizeof(struct location));
4282    t->count++;
4283    l->count = 1;
4284    l->addr = track->addr;
4285    l->sum_time = age;
4286    l->min_time = age;
4287    l->max_time = age;
4288    l->min_pid = track->pid;
4289    l->max_pid = track->pid;
4290    cpumask_clear(to_cpumask(l->cpus));
4291    cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
4292    nodes_clear(l->nodes);
4293    node_set(page_to_nid(virt_to_page(track)), l->nodes);
4294    return 1;
4295}
4296
4297static void process_slab(struct loc_track *t, struct kmem_cache *s,
4298        struct page *page, enum track_item alloc,
4299        unsigned long *map)
4300{
4301    void *addr = page_address(page);
4302    void *p;
4303
4304    bitmap_zero(map, page->objects);
4305    get_map(s, page, map);
4306
4307    for_each_object(p, s, addr, page->objects)
4308        if (!test_bit(slab_index(p, s, addr), map))
4309            add_location(t, s, get_track(s, p, alloc));
4310}
4311
4312static int list_locations(struct kmem_cache *s, char *buf,
4313                    enum track_item alloc)
4314{
4315    int len = 0;
4316    unsigned long i;
4317    struct loc_track t = { 0, 0, NULL };
4318    int node;
4319    unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
4320                     sizeof(unsigned long), GFP_KERNEL);
4321
4322    if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4323                     GFP_TEMPORARY)) {
4324        kfree(map);
4325        return sprintf(buf, "Out of memory\n");
4326    }
4327    /* Push back cpu slabs */
4328    flush_all(s);
4329
4330    for_each_node_state(node, N_NORMAL_MEMORY) {
4331        struct kmem_cache_node *n = get_node(s, node);
4332        unsigned long flags;
4333        struct page *page;
4334
4335        if (!atomic_long_read(&n->nr_slabs))
4336            continue;
4337
4338        spin_lock_irqsave(&n->list_lock, flags);
4339        list_for_each_entry(page, &n->partial, lru)
4340            process_slab(&t, s, page, alloc, map);
4341        list_for_each_entry(page, &n->full, lru)
4342            process_slab(&t, s, page, alloc, map);
4343        spin_unlock_irqrestore(&n->list_lock, flags);
4344    }
4345
4346    for (i = 0; i < t.count; i++) {
4347        struct location *l = &t.loc[i];
4348
4349        if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
4350            break;
4351        len += sprintf(buf + len, "%7ld ", l->count);
4352
4353        if (l->addr)
4354            len += sprintf(buf + len, "%pS", (void *)l->addr);
4355        else
4356            len += sprintf(buf + len, "<not-available>");
4357
4358        if (l->sum_time != l->min_time) {
4359            len += sprintf(buf + len, " age=%ld/%ld/%ld",
4360                l->min_time,
4361                (long)div_u64(l->sum_time, l->count),
4362                l->max_time);
4363        } else
4364            len += sprintf(buf + len, " age=%ld",
4365                l->min_time);
4366
4367        if (l->min_pid != l->max_pid)
4368            len += sprintf(buf + len, " pid=%ld-%ld",
4369                l->min_pid, l->max_pid);
4370        else
4371            len += sprintf(buf + len, " pid=%ld",
4372                l->min_pid);
4373
4374        if (num_online_cpus() > 1 &&
4375                !cpumask_empty(to_cpumask(l->cpus)) &&
4376                len < PAGE_SIZE - 60) {
4377            len += sprintf(buf + len, " cpus=");
4378            len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50,
4379                         to_cpumask(l->cpus));
4380        }
4381
4382        if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
4383                len < PAGE_SIZE - 60) {
4384            len += sprintf(buf + len, " nodes=");
4385            len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
4386                    l->nodes);
4387        }
4388
4389        len += sprintf(buf + len, "\n");
4390    }
4391
4392    free_loc_track(&t);
4393    kfree(map);
4394    if (!t.count)
4395        len += sprintf(buf, "No data\n");
4396    return len;
4397}
4398#endif
4399
4400#ifdef SLUB_RESILIENCY_TEST
4401static void resiliency_test(void)
4402{
4403    u8 *p;
4404
4405    BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || SLUB_PAGE_SHIFT < 10);
4406
4407    printk(KERN_ERR "SLUB resiliency testing\n");
4408    printk(KERN_ERR "-----------------------\n");
4409    printk(KERN_ERR "A. Corruption after allocation\n");
4410
4411    p = kzalloc(16, GFP_KERNEL);
4412    p[16] = 0x12;
4413    printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
4414            " 0x12->0x%p\n\n", p + 16);
4415
4416    validate_slab_cache(kmalloc_caches[4]);
4417
4418    /* Hmmm... The next two are dangerous */
4419    p = kzalloc(32, GFP_KERNEL);
4420    p[32 + sizeof(void *)] = 0x34;
4421    printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
4422            " 0x34 -> -0x%p\n", p);
4423    printk(KERN_ERR
4424        "If allocated object is overwritten then not detectable\n\n");
4425
4426    validate_slab_cache(kmalloc_caches[5]);
4427    p = kzalloc(64, GFP_KERNEL);
4428    p += 64 + (get_cycles() & 0xff) * sizeof(void *);
4429    *p = 0x56;
4430    printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
4431                                    p);
4432    printk(KERN_ERR
4433        "If allocated object is overwritten then not detectable\n\n");
4434    validate_slab_cache(kmalloc_caches[6]);
4435
4436    printk(KERN_ERR "\nB. Corruption after free\n");
4437    p = kzalloc(128, GFP_KERNEL);
4438    kfree(p);
4439    *p = 0x78;
4440    printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
4441    validate_slab_cache(kmalloc_caches[7]);
4442
4443    p = kzalloc(256, GFP_KERNEL);
4444    kfree(p);
4445    p[50] = 0x9a;
4446    printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n",
4447            p);
4448    validate_slab_cache(kmalloc_caches[8]);
4449
4450    p = kzalloc(512, GFP_KERNEL);
4451    kfree(p);
4452    p[512] = 0xab;
4453    printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
4454    validate_slab_cache(kmalloc_caches[9]);
4455}
4456#else
4457#ifdef CONFIG_SYSFS
4458static void resiliency_test(void) {};
4459#endif
4460#endif
4461
4462#ifdef CONFIG_SYSFS
4463enum slab_stat_type {
4464    SL_ALL, /* All slabs */
4465    SL_PARTIAL, /* Only partially allocated slabs */
4466    SL_CPU, /* Only slabs used for cpu caches */
4467    SL_OBJECTS, /* Determine allocated objects not slabs */
4468    SL_TOTAL /* Determine object capacity not slabs */
4469};
4470
4471#define SO_ALL (1 << SL_ALL)
4472#define SO_PARTIAL (1 << SL_PARTIAL)
4473#define SO_CPU (1 << SL_CPU)
4474#define SO_OBJECTS (1 << SL_OBJECTS)
4475#define SO_TOTAL (1 << SL_TOTAL)
4476
4477static ssize_t show_slab_objects(struct kmem_cache *s,
4478                char *buf, unsigned long flags)
4479{
4480    unsigned long total = 0;
4481    int node;
4482    int x;
4483    unsigned long *nodes;
4484    unsigned long *per_cpu;
4485
4486    nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
4487    if (!nodes)
4488        return -ENOMEM;
4489    per_cpu = nodes + nr_node_ids;
4490
4491    if (flags & SO_CPU) {
4492        int cpu;
4493
4494        for_each_possible_cpu(cpu) {
4495            struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
4496            int node;
4497            struct page *page;
4498
4499            page = ACCESS_ONCE(c->page);
4500            if (!page)
4501                continue;
4502
4503            node = page_to_nid(page);
4504            if (flags & SO_TOTAL)
4505                x = page->objects;
4506            else if (flags & SO_OBJECTS)
4507                x = page->inuse;
4508            else
4509                x = 1;
4510
4511            total += x;
4512            nodes[node] += x;
4513
4514            page = ACCESS_ONCE(c->partial);
4515            if (page) {
4516                x = page->pobjects;
4517                total += x;
4518                nodes[node] += x;
4519            }
4520
4521            per_cpu[node]++;
4522        }
4523    }
4524
4525    lock_memory_hotplug();
4526#ifdef CONFIG_SLUB_DEBUG
4527    if (flags & SO_ALL) {
4528        for_each_node_state(node, N_NORMAL_MEMORY) {
4529            struct kmem_cache_node *n = get_node(s, node);
4530
4531        if (flags & SO_TOTAL)
4532            x = atomic_long_read(&n->total_objects);
4533        else if (flags & SO_OBJECTS)
4534            x = atomic_long_read(&n->total_objects) -
4535                count_partial(n, count_free);
4536
4537            else
4538                x = atomic_long_read(&n->nr_slabs);
4539            total += x;
4540            nodes[node] += x;
4541        }
4542
4543    } else
4544#endif
4545    if (flags & SO_PARTIAL) {
4546        for_each_node_state(node, N_NORMAL_MEMORY) {
4547            struct kmem_cache_node *n = get_node(s, node);
4548
4549            if (flags & SO_TOTAL)
4550                x = count_partial(n, count_total);
4551            else if (flags & SO_OBJECTS)
4552                x = count_partial(n, count_inuse);
4553            else
4554                x = n->nr_partial;
4555            total += x;
4556            nodes[node] += x;
4557        }
4558    }
4559    x = sprintf(buf, "%lu", total);
4560#ifdef CONFIG_NUMA
4561    for_each_node_state(node, N_NORMAL_MEMORY)
4562        if (nodes[node])
4563            x += sprintf(buf + x, " N%d=%lu",
4564                    node, nodes[node]);
4565#endif
4566    unlock_memory_hotplug();
4567    kfree(nodes);
4568    return x + sprintf(buf + x, "\n");
4569}
4570
4571#ifdef CONFIG_SLUB_DEBUG
4572static int any_slab_objects(struct kmem_cache *s)
4573{
4574    int node;
4575
4576    for_each_online_node(node) {
4577        struct kmem_cache_node *n = get_node(s, node);
4578
4579        if (!n)
4580            continue;
4581
4582        if (atomic_long_read(&n->total_objects))
4583            return 1;
4584    }
4585    return 0;
4586}
4587#endif
4588
4589#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
4590#define to_slab(n) container_of(n, struct kmem_cache, kobj)
4591
4592struct slab_attribute {
4593    struct attribute attr;
4594    ssize_t (*show)(struct kmem_cache *s, char *buf);
4595    ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
4596};
4597
4598#define SLAB_ATTR_RO(_name) \
4599    static struct slab_attribute _name##_attr = \
4600    __ATTR(_name, 0400, _name##_show, NULL)
4601
4602#define SLAB_ATTR(_name) \
4603    static struct slab_attribute _name##_attr = \
4604    __ATTR(_name, 0600, _name##_show, _name##_store)
4605
4606static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
4607{
4608    return sprintf(buf, "%d\n", s->size);
4609}
4610SLAB_ATTR_RO(slab_size);
4611
4612static ssize_t align_show(struct kmem_cache *s, char *buf)
4613{
4614    return sprintf(buf, "%d\n", s->align);
4615}
4616SLAB_ATTR_RO(align);
4617
4618static ssize_t object_size_show(struct kmem_cache *s, char *buf)
4619{
4620    return sprintf(buf, "%d\n", s->object_size);
4621}
4622SLAB_ATTR_RO(object_size);
4623
4624static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
4625{
4626    return sprintf(buf, "%d\n", oo_objects(s->oo));
4627}
4628SLAB_ATTR_RO(objs_per_slab);
4629
4630static ssize_t order_store(struct kmem_cache *s,
4631                const char *buf, size_t length)
4632{
4633    unsigned long order;
4634    int err;
4635
4636    err = strict_strtoul(buf, 10, &order);
4637    if (err)
4638        return err;
4639
4640    if (order > slub_max_order || order < slub_min_order)
4641        return -EINVAL;
4642
4643    calculate_sizes(s, order);
4644    return length;
4645}
4646
4647static ssize_t order_show(struct kmem_cache *s, char *buf)
4648{
4649    return sprintf(buf, "%d\n", oo_order(s->oo));
4650}
4651SLAB_ATTR(order);
4652
4653static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
4654{
4655    return sprintf(buf, "%lu\n", s->min_partial);
4656}
4657
4658static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
4659                 size_t length)
4660{
4661    unsigned long min;
4662    int err;
4663
4664    err = strict_strtoul(buf, 10, &min);
4665    if (err)
4666        return err;
4667
4668    set_min_partial(s, min);
4669    return length;
4670}
4671SLAB_ATTR(min_partial);
4672
4673static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
4674{
4675    return sprintf(buf, "%u\n", s->cpu_partial);
4676}
4677
4678static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
4679                 size_t length)
4680{
4681    unsigned long objects;
4682    int err;
4683
4684    err = strict_strtoul(buf, 10, &objects);
4685    if (err)
4686        return err;
4687    if (objects && kmem_cache_debug(s))
4688        return -EINVAL;
4689
4690    s->cpu_partial = objects;
4691    flush_all(s);
4692    return length;
4693}
4694SLAB_ATTR(cpu_partial);
4695
4696static ssize_t ctor_show(struct kmem_cache *s, char *buf)
4697{
4698    if (!s->ctor)
4699        return 0;
4700    return sprintf(buf, "%pS\n", s->ctor);
4701}
4702SLAB_ATTR_RO(ctor);
4703
4704static ssize_t aliases_show(struct kmem_cache *s, char *buf)
4705{
4706    return sprintf(buf, "%d\n", s->refcount - 1);
4707}
4708SLAB_ATTR_RO(aliases);
4709
4710static ssize_t partial_show(struct kmem_cache *s, char *buf)
4711{
4712    return show_slab_objects(s, buf, SO_PARTIAL);
4713}
4714SLAB_ATTR_RO(partial);
4715
4716static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
4717{
4718    return show_slab_objects(s, buf, SO_CPU);
4719}
4720SLAB_ATTR_RO(cpu_slabs);
4721
4722static ssize_t objects_show(struct kmem_cache *s, char *buf)
4723{
4724    return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
4725}
4726SLAB_ATTR_RO(objects);
4727
4728static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
4729{
4730    return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
4731}
4732SLAB_ATTR_RO(objects_partial);
4733
4734static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
4735{
4736    int objects = 0;
4737    int pages = 0;
4738    int cpu;
4739    int len;
4740
4741    for_each_online_cpu(cpu) {
4742        struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
4743
4744        if (page) {
4745            pages += page->pages;
4746            objects += page->pobjects;
4747        }
4748    }
4749
4750    len = sprintf(buf, "%d(%d)", objects, pages);
4751
4752#ifdef CONFIG_SMP
4753    for_each_online_cpu(cpu) {
4754        struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
4755
4756        if (page && len < PAGE_SIZE - 20)
4757            len += sprintf(buf + len, " C%d=%d(%d)", cpu,
4758                page->pobjects, page->pages);
4759    }
4760#endif
4761    return len + sprintf(buf + len, "\n");
4762}
4763SLAB_ATTR_RO(slabs_cpu_partial);
4764
4765static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4766{
4767    return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4768}
4769
4770static ssize_t reclaim_account_store(struct kmem_cache *s,
4771                const char *buf, size_t length)
4772{
4773    s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4774    if (buf[0] == '1')
4775        s->flags |= SLAB_RECLAIM_ACCOUNT;
4776    return length;
4777}
4778SLAB_ATTR(reclaim_account);
4779
4780static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4781{
4782    return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
4783}
4784SLAB_ATTR_RO(hwcache_align);
4785
4786#ifdef CONFIG_ZONE_DMA
4787static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4788{
4789    return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4790}
4791SLAB_ATTR_RO(cache_dma);
4792#endif
4793
4794static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4795{
4796    return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4797}
4798SLAB_ATTR_RO(destroy_by_rcu);
4799
4800static ssize_t reserved_show(struct kmem_cache *s, char *buf)
4801{
4802    return sprintf(buf, "%d\n", s->reserved);
4803}
4804SLAB_ATTR_RO(reserved);
4805
4806#ifdef CONFIG_SLUB_DEBUG
4807static ssize_t slabs_show(struct kmem_cache *s, char *buf)
4808{
4809    return show_slab_objects(s, buf, SO_ALL);
4810}
4811SLAB_ATTR_RO(slabs);
4812
4813static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
4814{
4815    return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
4816}
4817SLAB_ATTR_RO(total_objects);
4818
4819static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
4820{
4821    return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
4822}
4823
4824static ssize_t sanity_checks_store(struct kmem_cache *s,
4825                const char *buf, size_t length)
4826{
4827    s->flags &= ~SLAB_DEBUG_FREE;
4828    if (buf[0] == '1') {
4829        s->flags &= ~__CMPXCHG_DOUBLE;
4830        s->flags |= SLAB_DEBUG_FREE;
4831    }
4832    return length;
4833}
4834SLAB_ATTR(sanity_checks);
4835
4836static ssize_t trace_show(struct kmem_cache *s, char *buf)
4837{
4838    return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
4839}
4840
4841static ssize_t trace_store(struct kmem_cache *s, const char *buf,
4842                            size_t length)
4843{
4844    s->flags &= ~SLAB_TRACE;
4845    if (buf[0] == '1') {
4846        s->flags &= ~__CMPXCHG_DOUBLE;
4847        s->flags |= SLAB_TRACE;
4848    }
4849    return length;
4850}
4851SLAB_ATTR(trace);
4852
4853static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
4854{
4855    return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
4856}
4857
4858static ssize_t red_zone_store(struct kmem_cache *s,
4859                const char *buf, size_t length)
4860{
4861    if (any_slab_objects(s))
4862        return -EBUSY;
4863
4864    s->flags &= ~SLAB_RED_ZONE;
4865    if (buf[0] == '1') {
4866        s->flags &= ~__CMPXCHG_DOUBLE;
4867        s->flags |= SLAB_RED_ZONE;
4868    }
4869    calculate_sizes(s, -1);
4870    return length;
4871}
4872SLAB_ATTR(red_zone);
4873
4874static ssize_t poison_show(struct kmem_cache *s, char *buf)
4875{
4876    return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
4877}
4878
4879static ssize_t poison_store(struct kmem_cache *s,
4880                const char *buf, size_t length)
4881{
4882    if (any_slab_objects(s))
4883        return -EBUSY;
4884
4885    s->flags &= ~SLAB_POISON;
4886    if (buf[0] == '1') {
4887        s->flags &= ~__CMPXCHG_DOUBLE;
4888        s->flags |= SLAB_POISON;
4889    }
4890    calculate_sizes(s, -1);
4891    return length;
4892}
4893SLAB_ATTR(poison);
4894
4895static ssize_t store_user_show(struct kmem_cache *s, char *buf)
4896{
4897    return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
4898}
4899
4900static ssize_t store_user_store(struct kmem_cache *s,
4901                const char *buf, size_t length)
4902{
4903    if (any_slab_objects(s))
4904        return -EBUSY;
4905
4906    s->flags &= ~SLAB_STORE_USER;
4907    if (buf[0] == '1') {
4908        s->flags &= ~__CMPXCHG_DOUBLE;
4909        s->flags |= SLAB_STORE_USER;
4910    }
4911    calculate_sizes(s, -1);
4912    return length;
4913}
4914SLAB_ATTR(store_user);
4915
4916static ssize_t validate_show(struct kmem_cache *s, char *buf)
4917{
4918    return 0;
4919}
4920
4921static ssize_t validate_store(struct kmem_cache *s,
4922            const char *buf, size_t length)
4923{
4924    int ret = -EINVAL;
4925
4926    if (buf[0] == '1') {
4927        ret = validate_slab_cache(s);
4928        if (ret >= 0)
4929            ret = length;
4930    }
4931    return ret;
4932}
4933SLAB_ATTR(validate);
4934
4935static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
4936{
4937    if (!(s->flags & SLAB_STORE_USER))
4938        return -ENOSYS;
4939    return list_locations(s, buf, TRACK_ALLOC);
4940}
4941SLAB_ATTR_RO(alloc_calls);
4942
4943static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
4944{
4945    if (!(s->flags & SLAB_STORE_USER))
4946        return -ENOSYS;
4947    return list_locations(s, buf, TRACK_FREE);
4948}
4949SLAB_ATTR_RO(free_calls);
4950#endif /* CONFIG_SLUB_DEBUG */
4951
4952#ifdef CONFIG_FAILSLAB
4953static ssize_t failslab_show(struct kmem_cache *s, char *buf)
4954{
4955    return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
4956}
4957
4958static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
4959                            size_t length)
4960{
4961    s->flags &= ~SLAB_FAILSLAB;
4962    if (buf[0] == '1')
4963        s->flags |= SLAB_FAILSLAB;
4964    return length;
4965}
4966SLAB_ATTR(failslab);
4967#endif
4968
4969static ssize_t shrink_show(struct kmem_cache *s, char *buf)
4970{
4971    return 0;
4972}
4973
4974static ssize_t shrink_store(struct kmem_cache *s,
4975            const char *buf, size_t length)
4976{
4977    if (buf[0] == '1') {
4978        int rc = kmem_cache_shrink(s);
4979
4980        if (rc)
4981            return rc;
4982    } else
4983        return -EINVAL;
4984    return length;
4985}
4986SLAB_ATTR(shrink);
4987
4988#ifdef CONFIG_NUMA
4989static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
4990{
4991    return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
4992}
4993
4994static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
4995                const char *buf, size_t length)
4996{
4997    unsigned long ratio;
4998    int err;
4999
5000    err = strict_strtoul(buf, 10, &ratio);
5001    if (err)
5002        return err;
5003
5004    if (ratio <= 100)
5005        s->remote_node_defrag_ratio = ratio * 10;
5006
5007    return length;
5008}
5009SLAB_ATTR(remote_node_defrag_ratio);
5010#endif
5011
5012#ifdef CONFIG_SLUB_STATS
5013static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
5014{
5015    unsigned long sum = 0;
5016    int cpu;
5017    int len;
5018    int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
5019
5020    if (!data)
5021        return -ENOMEM;
5022
5023    for_each_online_cpu(cpu) {
5024        unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5025
5026        data[cpu] = x;
5027        sum += x;
5028    }
5029
5030    len = sprintf(buf, "%lu", sum);
5031
5032#ifdef CONFIG_SMP
5033    for_each_online_cpu(cpu) {
5034        if (data[cpu] && len < PAGE_SIZE - 20)
5035            len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
5036    }
5037#endif
5038    kfree(data);
5039    return len + sprintf(buf + len, "\n");
5040}
5041
5042static void clear_stat(struct kmem_cache *s, enum stat_item si)
5043{
5044    int cpu;
5045
5046    for_each_online_cpu(cpu)
5047        per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
5048}
5049
5050#define STAT_ATTR(si, text) \
5051static ssize_t text##_show(struct kmem_cache *s, char *buf) \
5052{ \
5053    return show_stat(s, buf, si); \
5054} \
5055static ssize_t text##_store(struct kmem_cache *s, \
5056                const char *buf, size_t length) \
5057{ \
5058    if (buf[0] != '0') \
5059        return -EINVAL; \
5060    clear_stat(s, si); \
5061    return length; \
5062} \
5063SLAB_ATTR(text); \
5064
5065STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5066STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5067STAT_ATTR(FREE_FASTPATH, free_fastpath);
5068STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5069STAT_ATTR(FREE_FROZEN, free_frozen);
5070STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5071STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5072STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5073STAT_ATTR(ALLOC_SLAB, alloc_slab);
5074STAT_ATTR(ALLOC_REFILL, alloc_refill);
5075STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5076STAT_ATTR(FREE_SLAB, free_slab);
5077STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5078STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5079STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5080STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5081STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5082STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5083STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5084STAT_ATTR(ORDER_FALLBACK, order_fallback);
5085STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5086STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5087STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5088STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5089STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5090STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5091#endif
5092
5093static struct attribute *slab_attrs[] = {
5094    &slab_size_attr.attr,
5095    &object_size_attr.attr,
5096    &objs_per_slab_attr.attr,
5097    &order_attr.attr,
5098    &min_partial_attr.attr,
5099    &cpu_partial_attr.attr,
5100    &objects_attr.attr,
5101    &objects_partial_attr.attr,
5102    &partial_attr.attr,
5103    &cpu_slabs_attr.attr,
5104    &ctor_attr.attr,
5105    &aliases_attr.attr,
5106    &align_attr.attr,
5107    &hwcache_align_attr.attr,
5108    &reclaim_account_attr.attr,
5109    &destroy_by_rcu_attr.attr,
5110    &shrink_attr.attr,
5111    &reserved_attr.attr,
5112    &slabs_cpu_partial_attr.attr,
5113#ifdef CONFIG_SLUB_DEBUG
5114    &total_objects_attr.attr,
5115    &slabs_attr.attr,
5116    &sanity_checks_attr.attr,
5117    &trace_attr.attr,
5118    &red_zone_attr.attr,
5119    &poison_attr.attr,
5120    &store_user_attr.attr,
5121    &validate_attr.attr,
5122    &alloc_calls_attr.attr,
5123    &free_calls_attr.attr,
5124#endif
5125#ifdef CONFIG_ZONE_DMA
5126    &cache_dma_attr.attr,
5127#endif
5128#ifdef CONFIG_NUMA
5129    &remote_node_defrag_ratio_attr.attr,
5130#endif
5131#ifdef CONFIG_SLUB_STATS
5132    &alloc_fastpath_attr.attr,
5133    &alloc_slowpath_attr.attr,
5134    &free_fastpath_attr.attr,
5135    &free_slowpath_attr.attr,
5136    &free_frozen_attr.attr,
5137    &free_add_partial_attr.attr,
5138    &free_remove_partial_attr.attr,
5139    &alloc_from_partial_attr.attr,
5140    &alloc_slab_attr.attr,
5141    &alloc_refill_attr.attr,
5142    &alloc_node_mismatch_attr.attr,
5143    &free_slab_attr.attr,
5144    &cpuslab_flush_attr.attr,
5145    &deactivate_full_attr.attr,
5146    &deactivate_empty_attr.attr,
5147    &deactivate_to_head_attr.attr,
5148    &deactivate_to_tail_attr.attr,
5149    &deactivate_remote_frees_attr.attr,
5150    &deactivate_bypass_attr.attr,
5151    &order_fallback_attr.attr,
5152    &cmpxchg_double_fail_attr.attr,
5153    &cmpxchg_double_cpu_fail_attr.attr,
5154    &cpu_partial_alloc_attr.attr,
5155    &cpu_partial_free_attr.attr,
5156    &cpu_partial_node_attr.attr,
5157    &cpu_partial_drain_attr.attr,
5158#endif
5159#ifdef CONFIG_FAILSLAB
5160    &failslab_attr.attr,
5161#endif
5162
5163    NULL
5164};
5165
5166static struct attribute_group slab_attr_group = {
5167    .attrs = slab_attrs,
5168};
5169
5170static ssize_t slab_attr_show(struct kobject *kobj,
5171                struct attribute *attr,
5172                char *buf)
5173{
5174    struct slab_attribute *attribute;
5175    struct kmem_cache *s;
5176    int err;
5177
5178    attribute = to_slab_attr(attr);
5179    s = to_slab(kobj);
5180
5181    if (!attribute->show)
5182        return -EIO;
5183
5184    err = attribute->show(s, buf);
5185
5186    return err;
5187}
5188
5189static ssize_t slab_attr_store(struct kobject *kobj,
5190                struct attribute *attr,
5191                const char *buf, size_t len)
5192{
5193    struct slab_attribute *attribute;
5194    struct kmem_cache *s;
5195    int err;
5196
5197    attribute = to_slab_attr(attr);
5198    s = to_slab(kobj);
5199
5200    if (!attribute->store)
5201        return -EIO;
5202
5203    err = attribute->store(s, buf, len);
5204
5205    return err;
5206}
5207
5208static void kmem_cache_release(struct kobject *kobj)
5209{
5210    struct kmem_cache *s = to_slab(kobj);
5211
5212    kfree(s->name);
5213    kfree(s);
5214}
5215
5216static const struct sysfs_ops slab_sysfs_ops = {
5217    .show = slab_attr_show,
5218    .store = slab_attr_store,
5219};
5220
5221static struct kobj_type slab_ktype = {
5222    .sysfs_ops = &slab_sysfs_ops,
5223    .release = kmem_cache_release
5224};
5225
5226static int uevent_filter(struct kset *kset, struct kobject *kobj)
5227{
5228    struct kobj_type *ktype = get_ktype(kobj);
5229
5230    if (ktype == &slab_ktype)
5231        return 1;
5232    return 0;
5233}
5234
5235static const struct kset_uevent_ops slab_uevent_ops = {
5236    .filter = uevent_filter,
5237};
5238
5239static struct kset *slab_kset;
5240
5241#define ID_STR_LENGTH 64
5242
5243/* Create a unique string id for a slab cache:
5244 *
5245 * Format :[flags-]size
5246 */
5247static char *create_unique_id(struct kmem_cache *s)
5248{
5249    char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5250    char *p = name;
5251
5252    BUG_ON(!name);
5253
5254    *p++ = ':';
5255    /*
5256     * First flags affecting slabcache operations. We will only
5257     * get here for aliasable slabs so we do not need to support
5258     * too many flags. The flags here must cover all flags that
5259     * are matched during merging to guarantee that the id is
5260     * unique.
5261     */
5262    if (s->flags & SLAB_CACHE_DMA)
5263        *p++ = 'd';
5264    if (s->flags & SLAB_RECLAIM_ACCOUNT)
5265        *p++ = 'a';
5266    if (s->flags & SLAB_DEBUG_FREE)
5267        *p++ = 'F';
5268    if (!(s->flags & SLAB_NOTRACK))
5269        *p++ = 't';
5270    if (p != name + 1)
5271        *p++ = '-';
5272    p += sprintf(p, "%07d", s->size);
5273    BUG_ON(p > name + ID_STR_LENGTH - 1);
5274    return name;
5275}
5276
5277static int sysfs_slab_add(struct kmem_cache *s)
5278{
5279    int err;
5280    const char *name;
5281    int unmergeable;
5282
5283    if (slab_state < FULL)
5284        /* Defer until later */
5285        return 0;
5286
5287    unmergeable = slab_unmergeable(s);
5288    if (unmergeable) {
5289        /*
5290         * Slabcache can never be merged so we can use the name proper.
5291         * This is typically the case for debug situations. In that
5292         * case we can catch duplicate names easily.
5293         */
5294        sysfs_remove_link(&slab_kset->kobj, s->name);
5295        name = s->name;
5296    } else {
5297        /*
5298         * Create a unique name for the slab as a target
5299         * for the symlinks.
5300         */
5301        name = create_unique_id(s);
5302    }
5303
5304    s->kobj.kset = slab_kset;
5305    err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
5306    if (err) {
5307        kobject_put(&s->kobj);
5308        return err;
5309    }
5310
5311    err = sysfs_create_group(&s->kobj, &slab_attr_group);
5312    if (err) {
5313        kobject_del(&s->kobj);
5314        kobject_put(&s->kobj);
5315        return err;
5316    }
5317    kobject_uevent(&s->kobj, KOBJ_ADD);
5318    if (!unmergeable) {
5319        /* Setup first alias */
5320        sysfs_slab_alias(s, s->name);
5321        kfree(name);
5322    }
5323    return 0;
5324}
5325
5326static void sysfs_slab_remove(struct kmem_cache *s)
5327{
5328    if (slab_state < FULL)
5329        /*
5330         * Sysfs has not been setup yet so no need to remove the
5331         * cache from sysfs.
5332         */
5333        return;
5334
5335    kobject_uevent(&s->kobj, KOBJ_REMOVE);
5336    kobject_del(&s->kobj);
5337    kobject_put(&s->kobj);
5338}
5339
5340/*
5341 * Need to buffer aliases during bootup until sysfs becomes
5342 * available lest we lose that information.
5343 */
5344struct saved_alias {
5345    struct kmem_cache *s;
5346    const char *name;
5347    struct saved_alias *next;
5348};
5349
5350static struct saved_alias *alias_list;
5351
5352static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
5353{
5354    struct saved_alias *al;
5355
5356    if (slab_state == FULL) {
5357        /*
5358         * If we have a leftover link then remove it.
5359         */
5360        sysfs_remove_link(&slab_kset->kobj, name);
5361        return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
5362    }
5363
5364    al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5365    if (!al)
5366        return -ENOMEM;
5367
5368    al->s = s;
5369    al->name = name;
5370    al->next = alias_list;
5371    alias_list = al;
5372    return 0;
5373}
5374
5375static int __init slab_sysfs_init(void)
5376{
5377    struct kmem_cache *s;
5378    int err;
5379
5380    mutex_lock(&slab_mutex);
5381
5382    slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
5383    if (!slab_kset) {
5384        mutex_unlock(&slab_mutex);
5385        printk(KERN_ERR "Cannot register slab subsystem.\n");
5386        return -ENOSYS;
5387    }
5388
5389    slab_state = FULL;
5390
5391    list_for_each_entry(s, &slab_caches, list) {
5392        err = sysfs_slab_add(s);
5393        if (err)
5394            printk(KERN_ERR "SLUB: Unable to add boot slab %s"
5395                        " to sysfs\n", s->name);
5396    }
5397
5398    while (alias_list) {
5399        struct saved_alias *al = alias_list;
5400
5401        alias_list = alias_list->next;
5402        err = sysfs_slab_alias(al->s, al->name);
5403        if (err)
5404            printk(KERN_ERR "SLUB: Unable to add boot slab alias"
5405                    " %s to sysfs\n", al->name);
5406        kfree(al);
5407    }
5408
5409    mutex_unlock(&slab_mutex);
5410    resiliency_test();
5411    return 0;
5412}
5413
5414__initcall(slab_sysfs_init);
5415#endif /* CONFIG_SYSFS */
5416
5417/*
5418 * The /proc/slabinfo ABI
5419 */
5420#ifdef CONFIG_SLABINFO
5421static void print_slabinfo_header(struct seq_file *m)
5422{
5423    seq_puts(m, "slabinfo - version: 2.1\n");
5424    seq_puts(m, "# name <active_objs> <num_objs> <object_size> "
5425         "<objperslab> <pagesperslab>");
5426    seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
5427    seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
5428    seq_putc(m, '\n');
5429}
5430
5431static void *s_start(struct seq_file *m, loff_t *pos)
5432{
5433    loff_t n = *pos;
5434
5435    mutex_lock(&slab_mutex);
5436    if (!n)
5437        print_slabinfo_header(m);
5438
5439    return seq_list_start(&slab_caches, *pos);
5440}
5441
5442static void *s_next(struct seq_file *m, void *p, loff_t *pos)
5443{
5444    return seq_list_next(p, &slab_caches, pos);
5445}
5446
5447static void s_stop(struct seq_file *m, void *p)
5448{
5449    mutex_unlock(&slab_mutex);
5450}
5451
5452static int s_show(struct seq_file *m, void *p)
5453{
5454    unsigned long nr_partials = 0;
5455    unsigned long nr_slabs = 0;
5456    unsigned long nr_inuse = 0;
5457    unsigned long nr_objs = 0;
5458    unsigned long nr_free = 0;
5459    struct kmem_cache *s;
5460    int node;
5461
5462    s = list_entry(p, struct kmem_cache, list);
5463
5464    for_each_online_node(node) {
5465        struct kmem_cache_node *n = get_node(s, node);
5466
5467        if (!n)
5468            continue;
5469
5470        nr_partials += n->nr_partial;
5471        nr_slabs += atomic_long_read(&n->nr_slabs);
5472        nr_objs += atomic_long_read(&n->total_objects);
5473        nr_free += count_partial(n, count_free);
5474    }
5475
5476    nr_inuse = nr_objs - nr_free;
5477
5478    seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
5479           nr_objs, s->size, oo_objects(s->oo),
5480           (1 << oo_order(s->oo)));
5481    seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0);
5482    seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
5483           0UL);
5484    seq_putc(m, '\n');
5485    return 0;
5486}
5487
5488static const struct seq_operations slabinfo_op = {
5489    .start = s_start,
5490    .next = s_next,
5491    .stop = s_stop,
5492    .show = s_show,
5493};
5494
5495static int slabinfo_open(struct inode *inode, struct file *file)
5496{
5497    return seq_open(file, &slabinfo_op);
5498}
5499
5500static const struct file_operations proc_slabinfo_operations = {
5501    .open = slabinfo_open,
5502    .read = seq_read,
5503    .llseek = seq_lseek,
5504    .release = seq_release,
5505};
5506
5507static int __init slab_proc_init(void)
5508{
5509    proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations);
5510    return 0;
5511}
5512module_init(slab_proc_init);
5513#endif /* CONFIG_SLABINFO */
5514

Archive Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9



interactive