Kernel

Kernel Git Source Tree

Root/mm/slub.c

1	/*
2	* SLUB: A slab allocator that limits cache line use instead of queuing
3	* objects in per cpu and per node lists.
4	*
5	* The allocator synchronizes using per slab locks or atomic operatios
6	* and only uses a centralized lock to manage a pool of partial slabs.
7	*
8	* (C) 2007 SGI, Christoph Lameter
9	* (C) 2011 Linux Foundation, Christoph Lameter
10	*/
11
12	#include <linux/mm.h>
13	#include <linux/swap.h> /* struct reclaim_state */
14	#include <linux/module.h>
15	#include <linux/bit_spinlock.h>
16	#include <linux/interrupt.h>
17	#include <linux/bitops.h>
18	#include <linux/slab.h>
19	#include <linux/proc_fs.h>
20	#include <linux/seq_file.h>
21	#include <linux/kmemcheck.h>
22	#include <linux/cpu.h>
23	#include <linux/cpuset.h>
24	#include <linux/mempolicy.h>
25	#include <linux/ctype.h>
26	#include <linux/debugobjects.h>
27	#include <linux/kallsyms.h>
28	#include <linux/memory.h>
29	#include <linux/math64.h>
30	#include <linux/fault-inject.h>
31	#include <linux/stacktrace.h>
32	#include <linux/prefetch.h>
33
34	#include <trace/events/kmem.h>
35
36	/*
37	* Lock order:
38	* 1. slub_lock (Global Semaphore)
39	* 2. node->list_lock
40	* 3. slab_lock(page) (Only on some arches and for debugging)
41	*
42	* slub_lock
43	*
44	* The role of the slub_lock is to protect the list of all the slabs
45	* and to synchronize major metadata changes to slab cache structures.
46	*
47	* The slab_lock is only used for debugging and on arches that do not
48	* have the ability to do a cmpxchg_double. It only protects the second
49	* double word in the page struct. Meaning
50	* A. page->freelist -> List of object free in a page
51	* B. page->counters -> Counters of objects
52	* C. page->frozen -> frozen state
53	*
54	* If a slab is frozen then it is exempt from list management. It is not
55	* on any list. The processor that froze the slab is the one who can
56	* perform list operations on the page. Other processors may put objects
57	* onto the freelist but the processor that froze the slab is the only
58	* one that can retrieve the objects from the page's freelist.
59	*
60	* The list_lock protects the partial and full list on each node and
61	* the partial slab counter. If taken then no new slabs may be added or
62	* removed from the lists nor make the number of partial slabs be modified.
63	* (Note that the total number of slabs is an atomic value that may be
64	* modified without taking the list lock).
65	*
66	* The list_lock is a centralized lock and thus we avoid taking it as
67	* much as possible. As long as SLUB does not have to handle partial
68	* slabs, operations can continue without any centralized lock. F.e.
69	* allocating a long series of objects that fill up slabs does not require
70	* the list lock.
71	* Interrupts are disabled during allocation and deallocation in order to
72	* make the slab allocator safe to use in the context of an irq. In addition
73	* interrupts are disabled to ensure that the processor does not change
74	* while handling per_cpu slabs, due to kernel preemption.
75	*
76	* SLUB assigns one slab for allocation to each processor.
77	* Allocations only occur from these slabs called cpu slabs.
78	*
79	* Slabs with free elements are kept on a partial list and during regular
80	* operations no list for full slabs is used. If an object in a full slab is
81	* freed then the slab will show up again on the partial lists.
82	* We track full slabs for debugging purposes though because otherwise we
83	* cannot scan all objects.
84	*
85	* Slabs are freed when they become empty. Teardown and setup is
86	* minimal so we rely on the page allocators per cpu caches for
87	* fast frees and allocs.
88	*
89	* Overloading of page flags that are otherwise used for LRU management.
90	*
91	* PageActive The slab is frozen and exempt from list processing.
92	* This means that the slab is dedicated to a purpose
93	* such as satisfying allocations for a specific
94	* processor. Objects may be freed in the slab while
95	* it is frozen but slab_free will then skip the usual
96	* list operations. It is up to the processor holding
97	* the slab to integrate the slab into the slab lists
98	* when the slab is no longer needed.
99	*
100	* One use of this flag is to mark slabs that are
101	* used for allocations. Then such a slab becomes a cpu
102	* slab. The cpu slab may be equipped with an additional
103	* freelist that allows lockless access to
104	* free objects in addition to the regular freelist
105	* that requires the slab lock.
106	*
107	* PageError Slab requires special handling due to debug
108	* options set. This moves slab handling out of
109	* the fast path and disables lockless freelists.
110	*/
111
112	#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER \| \
113	SLAB_TRACE \| SLAB_DEBUG_FREE)
114
115	static inline int kmem_cache_debug(struct kmem_cache *s)
116	{
117	#ifdef CONFIG_SLUB_DEBUG
118	return unlikely(s->flags & SLAB_DEBUG_FLAGS);
119	#else
120	return 0;
121	#endif
122	}
123
124	/*
125	* Issues still to be resolved:
126	*
127	* - Support PAGE_ALLOC_DEBUG. Should be easy to do.
128	*
129	* - Variable sizing of the per node arrays
130	*/
131
132	/* Enable to test recovery from slab corruption on boot */
133	#undef SLUB_RESILIENCY_TEST
134
135	/* Enable to log cmpxchg failures */
136	#undef SLUB_DEBUG_CMPXCHG
137
138	/*
139	* Mininum number of partial slabs. These will be left on the partial
140	* lists even if they are empty. kmem_cache_shrink may reclaim them.
141	*/
142	#define MIN_PARTIAL 5
143
144	/*
145	* Maximum number of desirable partial slabs.
146	* The existence of more partial slabs makes kmem_cache_shrink
147	* sort the partial list by the number of objects in the.
148	*/
149	#define MAX_PARTIAL 10
150
151	#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE \| SLAB_RED_ZONE \| \
152	SLAB_POISON \| SLAB_STORE_USER)
153
154	/*
155	* Debugging flags that require metadata to be stored in the slab. These get
156	* disabled when slub_debug=O is used and a cache's min order increases with
157	* metadata.
158	*/
159	#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER)
160
161	/*
162	* Set of flags that will prevent slab merging
163	*/
164	#define SLUB_NEVER_MERGE (SLAB_RED_ZONE \| SLAB_POISON \| SLAB_STORE_USER \| \
165	SLAB_TRACE \| SLAB_DESTROY_BY_RCU \| SLAB_NOLEAKTRACE \| \
166	SLAB_FAILSLAB)
167
168	#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE \| SLAB_RECLAIM_ACCOUNT \| \
169	SLAB_CACHE_DMA \| SLAB_NOTRACK)
170
171	#define OO_SHIFT 16
172	#define OO_MASK ((1 << OO_SHIFT) - 1)
173	#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
174
175	/* Internal SLUB flags */
176	#define __OBJECT_POISON 0x80000000UL /* Poison object */
177	#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
178
179	static int kmem_size = sizeof(struct kmem_cache);
180
181	#ifdef CONFIG_SMP
182	static struct notifier_block slab_notifier;
183	#endif
184
185	static enum {
186	DOWN, /* No slab functionality available */
187	PARTIAL, /* Kmem_cache_node works */
188	UP, /* Everything works but does not show up in sysfs */
189	SYSFS /* Sysfs up */
190	} slab_state = DOWN;
191
192	/* A list of all slab caches on the system */
193	static DECLARE_RWSEM(slub_lock);
194	static LIST_HEAD(slab_caches);
195
196	/*
197	* Tracking user of a slab.
198	*/
199	#define TRACK_ADDRS_COUNT 16
200	struct track {
201	unsigned long addr; /* Called from address */
202	#ifdef CONFIG_STACKTRACE
203	unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */
204	#endif
205	int cpu; /* Was running on cpu */
206	int pid; /* Pid context */
207	unsigned long when; /* When did the operation occur */
208	};
209
210	enum track_item { TRACK_ALLOC, TRACK_FREE };
211
212	#ifdef CONFIG_SYSFS
213	static int sysfs_slab_add(struct kmem_cache *);
214	static int sysfs_slab_alias(struct kmem_cache , const char );
215	static void sysfs_slab_remove(struct kmem_cache *);
216
217	#else
218	static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
219	static inline int sysfs_slab_alias(struct kmem_cache s, const char p)
220	{ return 0; }
221	static inline void sysfs_slab_remove(struct kmem_cache *s)
222	{
223	kfree(s->name);
224	kfree(s);
225	}
226
227	#endif
228
229	static inline void stat(const struct kmem_cache *s, enum stat_item si)
230	{
231	#ifdef CONFIG_SLUB_STATS
232	__this_cpu_inc(s->cpu_slab->stat[si]);
233	#endif
234	}
235
236	/********************************************************************
237	* Core slab cache functions
238	*******************************************************************/
239
240	int slab_is_available(void)
241	{
242	return slab_state >= UP;
243	}
244
245	static inline struct kmem_cache_node get_node(struct kmem_cache s, int node)
246	{
247	return s->node[node];
248	}
249
250	/* Verify that a pointer has an address that is valid within a slab page */
251	static inline int check_valid_pointer(struct kmem_cache *s,
252	struct page page, const void object)
253	{
254	void *base;
255
256	if (!object)
257	return 1;
258
259	base = page_address(page);
260	if (object < base \|\| object >= base + page->objects * s->size \|\|
261	(object - base) % s->size) {
262	return 0;
263	}
264
265	return 1;
266	}
267
268	static inline void get_freepointer(struct kmem_cache s, void *object)
269	{
270	return (void *)(object + s->offset);
271	}
272
273	static void prefetch_freepointer(const struct kmem_cache s, void object)
274	{
275	prefetch(object + s->offset);
276	}
277
278	static inline void get_freepointer_safe(struct kmem_cache s, void *object)
279	{
280	void *p;
281
282	#ifdef CONFIG_DEBUG_PAGEALLOC
283	probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p));
284	#else
285	p = get_freepointer(s, object);
286	#endif
287	return p;
288	}
289
290	static inline void set_freepointer(struct kmem_cache s, void object, void *fp)
291	{
292	(void *)(object + s->offset) = fp;
293	}
294
295	/* Loop over all objects in a slab */
296	#define for_each_object(__p, __s, __addr, __objects) \
297	for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
298	__p += (__s)->size)
299
300	/* Determine object index from a given position */
301	static inline int slab_index(void p, struct kmem_cache s, void *addr)
302	{
303	return (p - addr) / s->size;
304	}
305
306	static inline size_t slab_ksize(const struct kmem_cache *s)
307	{
308	#ifdef CONFIG_SLUB_DEBUG
309	/*
310	* Debugging requires use of the padding between object
311	* and whatever may come after it.
312	*/
313	if (s->flags & (SLAB_RED_ZONE \| SLAB_POISON))
314	return s->objsize;
315
316	#endif
317	/*
318	* If we have the need to store the freelist pointer
319	* back there or track user information then we can
320	* only use the space before that information.
321	*/
322	if (s->flags & (SLAB_DESTROY_BY_RCU \| SLAB_STORE_USER))
323	return s->inuse;
324	/*
325	* Else we can use all the padding etc for the allocation
326	*/
327	return s->size;
328	}
329
330	static inline int order_objects(int order, unsigned long size, int reserved)
331	{
332	return ((PAGE_SIZE << order) - reserved) / size;
333	}
334
335	static inline struct kmem_cache_order_objects oo_make(int order,
336	unsigned long size, int reserved)
337	{
338	struct kmem_cache_order_objects x = {
339	(order << OO_SHIFT) + order_objects(order, size, reserved)
340	};
341
342	return x;
343	}
344
345	static inline int oo_order(struct kmem_cache_order_objects x)
346	{
347	return x.x >> OO_SHIFT;
348	}
349
350	static inline int oo_objects(struct kmem_cache_order_objects x)
351	{
352	return x.x & OO_MASK;
353	}
354
355	/*
356	* Per slab locking using the pagelock
357	*/
358	static __always_inline void slab_lock(struct page *page)
359	{
360	bit_spin_lock(PG_locked, &page->flags);
361	}
362
363	static __always_inline void slab_unlock(struct page *page)
364	{
365	__bit_spin_unlock(PG_locked, &page->flags);
366	}
367
368	/* Interrupts must be disabled (for the fallback code to work right) */
369	static inline bool __cmpxchg_double_slab(struct kmem_cache s, struct page page,
370	void *freelist_old, unsigned long counters_old,
371	void *freelist_new, unsigned long counters_new,
372	const char *n)
373	{
374	VM_BUG_ON(!irqs_disabled());
375	#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
376	defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
377	if (s->flags & __CMPXCHG_DOUBLE) {
378	if (cmpxchg_double(&page->freelist, &page->counters,
379	freelist_old, counters_old,
380	freelist_new, counters_new))
381	return 1;
382	} else
383	#endif
384	{
385	slab_lock(page);
386	if (page->freelist == freelist_old && page->counters == counters_old) {
387	page->freelist = freelist_new;
388	page->counters = counters_new;
389	slab_unlock(page);
390	return 1;
391	}
392	slab_unlock(page);
393	}
394
395	cpu_relax();
396	stat(s, CMPXCHG_DOUBLE_FAIL);
397
398	#ifdef SLUB_DEBUG_CMPXCHG
399	printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
400	#endif
401
402	return 0;
403	}
404
405	static inline bool cmpxchg_double_slab(struct kmem_cache s, struct page page,
406	void *freelist_old, unsigned long counters_old,
407	void *freelist_new, unsigned long counters_new,
408	const char *n)
409	{
410	#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
411	defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
412	if (s->flags & __CMPXCHG_DOUBLE) {
413	if (cmpxchg_double(&page->freelist, &page->counters,
414	freelist_old, counters_old,
415	freelist_new, counters_new))
416	return 1;
417	} else
418	#endif
419	{
420	unsigned long flags;
421
422	local_irq_save(flags);
423	slab_lock(page);
424	if (page->freelist == freelist_old && page->counters == counters_old) {
425	page->freelist = freelist_new;
426	page->counters = counters_new;
427	slab_unlock(page);
428	local_irq_restore(flags);
429	return 1;
430	}
431	slab_unlock(page);
432	local_irq_restore(flags);
433	}
434
435	cpu_relax();
436	stat(s, CMPXCHG_DOUBLE_FAIL);
437
438	#ifdef SLUB_DEBUG_CMPXCHG
439	printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name);
440	#endif
441
442	return 0;
443	}
444
445	#ifdef CONFIG_SLUB_DEBUG
446	/*
447	* Determine a map of object in use on a page.
448	*
449	* Node listlock must be held to guarantee that the page does
450	* not vanish from under us.
451	*/
452	static void get_map(struct kmem_cache s, struct page page, unsigned long *map)
453	{
454	void *p;
455	void *addr = page_address(page);
456
457	for (p = page->freelist; p; p = get_freepointer(s, p))
458	set_bit(slab_index(p, s, addr), map);
459	}
460
461	/*
462	* Debug settings:
463	*/
464	#ifdef CONFIG_SLUB_DEBUG_ON
465	static int slub_debug = DEBUG_DEFAULT_FLAGS;
466	#else
467	static int slub_debug;
468	#endif
469
470	static char *slub_debug_slabs;
471	static int disable_higher_order_debug;
472
473	/*
474	* Object debugging
475	*/
476	static void print_section(char text, u8 addr, unsigned int length)
477	{
478	print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
479	length, 1);
480	}
481
482	static struct track get_track(struct kmem_cache s, void *object,
483	enum track_item alloc)
484	{
485	struct track *p;
486
487	if (s->offset)
488	p = object + s->offset + sizeof(void *);
489	else
490	p = object + s->inuse;
491
492	return p + alloc;
493	}
494
495	static void set_track(struct kmem_cache s, void object,
496	enum track_item alloc, unsigned long addr)
497	{
498	struct track *p = get_track(s, object, alloc);
499
500	if (addr) {
501	#ifdef CONFIG_STACKTRACE
502	struct stack_trace trace;
503	int i;
504
505	trace.nr_entries = 0;
506	trace.max_entries = TRACK_ADDRS_COUNT;
507	trace.entries = p->addrs;
508	trace.skip = 3;
509	save_stack_trace(&trace);
510
511	/* See rant in lockdep.c */
512	if (trace.nr_entries != 0 &&
513	trace.entries[trace.nr_entries - 1] == ULONG_MAX)
514	trace.nr_entries--;
515
516	for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
517	p->addrs[i] = 0;
518	#endif
519	p->addr = addr;
520	p->cpu = smp_processor_id();
521	p->pid = current->pid;
522	p->when = jiffies;
523	} else
524	memset(p, 0, sizeof(struct track));
525	}
526
527	static void init_tracking(struct kmem_cache s, void object)
528	{
529	if (!(s->flags & SLAB_STORE_USER))
530	return;
531
532	set_track(s, object, TRACK_FREE, 0UL);
533	set_track(s, object, TRACK_ALLOC, 0UL);
534	}
535
536	static void print_track(const char s, struct track t)
537	{
538	if (!t->addr)
539	return;
540
541	printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
542	s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
543	#ifdef CONFIG_STACKTRACE
544	{
545	int i;
546	for (i = 0; i < TRACK_ADDRS_COUNT; i++)
547	if (t->addrs[i])
548	printk(KERN_ERR "\t%pS\n", (void *)t->addrs[i]);
549	else
550	break;
551	}
552	#endif
553	}
554
555	static void print_tracking(struct kmem_cache s, void object)
556	{
557	if (!(s->flags & SLAB_STORE_USER))
558	return;
559
560	print_track("Allocated", get_track(s, object, TRACK_ALLOC));
561	print_track("Freed", get_track(s, object, TRACK_FREE));
562	}
563
564	static void print_page_info(struct page *page)
565	{
566	printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
567	page, page->objects, page->inuse, page->freelist, page->flags);
568
569	}
570
571	static void slab_bug(struct kmem_cache s, char fmt, ...)
572	{
573	va_list args;
574	char buf[100];
575
576	va_start(args, fmt);
577	vsnprintf(buf, sizeof(buf), fmt, args);
578	va_end(args);
579	printk(KERN_ERR "========================================"
580	"=====================================\n");
581	printk(KERN_ERR "BUG %s (%s): %s\n", s->name, print_tainted(), buf);
582	printk(KERN_ERR "----------------------------------------"
583	"-------------------------------------\n\n");
584	}
585
586	static void slab_fix(struct kmem_cache s, char fmt, ...)
587	{
588	va_list args;
589	char buf[100];
590
591	va_start(args, fmt);
592	vsnprintf(buf, sizeof(buf), fmt, args);
593	va_end(args);
594	printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
595	}
596
597	static void print_trailer(struct kmem_cache s, struct page page, u8 *p)
598	{
599	unsigned int off; /* Offset of last byte */
600	u8 *addr = page_address(page);
601
602	print_tracking(s, p);
603
604	print_page_info(page);
605
606	printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
607	p, p - addr, get_freepointer(s, p));
608
609	if (p > addr + 16)
610	print_section("Bytes b4 ", p - 16, 16);
611
612	print_section("Object ", p, min_t(unsigned long, s->objsize,
613	PAGE_SIZE));
614	if (s->flags & SLAB_RED_ZONE)
615	print_section("Redzone ", p + s->objsize,
616	s->inuse - s->objsize);
617
618	if (s->offset)
619	off = s->offset + sizeof(void *);
620	else
621	off = s->inuse;
622
623	if (s->flags & SLAB_STORE_USER)
624	off += 2 * sizeof(struct track);
625
626	if (off != s->size)
627	/* Beginning of the filler is the free pointer */
628	print_section("Padding ", p + off, s->size - off);
629
630	dump_stack();
631	}
632
633	static void object_err(struct kmem_cache s, struct page page,
634	u8 object, char reason)
635	{
636	slab_bug(s, "%s", reason);
637	print_trailer(s, page, object);
638	}
639
640	static void slab_err(struct kmem_cache s, struct page page, char *fmt, ...)
641	{
642	va_list args;
643	char buf[100];
644
645	va_start(args, fmt);
646	vsnprintf(buf, sizeof(buf), fmt, args);
647	va_end(args);
648	slab_bug(s, "%s", buf);
649	print_page_info(page);
650	dump_stack();
651	}
652
653	static void init_object(struct kmem_cache s, void object, u8 val)
654	{
655	u8 *p = object;
656
657	if (s->flags & __OBJECT_POISON) {
658	memset(p, POISON_FREE, s->objsize - 1);
659	p[s->objsize - 1] = POISON_END;
660	}
661
662	if (s->flags & SLAB_RED_ZONE)
663	memset(p + s->objsize, val, s->inuse - s->objsize);
664	}
665
666	static void restore_bytes(struct kmem_cache s, char message, u8 data,
667	void from, void to)
668	{
669	slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
670	memset(from, data, to - from);
671	}
672
673	static int check_bytes_and_report(struct kmem_cache s, struct page page,
674	u8 object, char what,
675	u8 *start, unsigned int value, unsigned int bytes)
676	{
677	u8 *fault;
678	u8 *end;
679
680	fault = memchr_inv(start, value, bytes);
681	if (!fault)
682	return 1;
683
684	end = start + bytes;
685	while (end > fault && end[-1] == value)
686	end--;
687
688	slab_bug(s, "%s overwritten", what);
689	printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
690	fault, end - 1, fault[0], value);
691	print_trailer(s, page, object);
692
693	restore_bytes(s, what, value, fault, end);
694	return 0;
695	}
696
697	/*
698	* Object layout:
699	*
700	* object address
701	* Bytes of the object to be managed.
702	* If the freepointer may overlay the object then the free
703	* pointer is the first word of the object.
704	*
705	* Poisoning uses 0x6b (POISON_FREE) and the last byte is
706	* 0xa5 (POISON_END)
707	*
708	* object + s->objsize
709	* Padding to reach word boundary. This is also used for Redzoning.
710	* Padding is extended by another word if Redzoning is enabled and
711	* objsize == inuse.
712	*
713	* We fill with 0xbb (RED_INACTIVE) for inactive objects and with
714	* 0xcc (RED_ACTIVE) for objects in use.
715	*
716	* object + s->inuse
717	* Meta data starts here.
718	*
719	* A. Free pointer (if we cannot overwrite object on free)
720	* B. Tracking data for SLAB_STORE_USER
721	* C. Padding to reach required alignment boundary or at mininum
722	* one word if debugging is on to be able to detect writes
723	* before the word boundary.
724	*
725	* Padding is done using 0x5a (POISON_INUSE)
726	*
727	* object + s->size
728	* Nothing is used beyond s->size.
729	*
730	* If slabcaches are merged then the objsize and inuse boundaries are mostly
731	* ignored. And therefore no slab options that rely on these boundaries
732	* may be used with merged slabcaches.
733	*/
734
735	static int check_pad_bytes(struct kmem_cache s, struct page page, u8 *p)
736	{
737	unsigned long off = s->inuse; /* The end of info */
738
739	if (s->offset)
740	/* Freepointer is placed after the object. */
741	off += sizeof(void *);
742
743	if (s->flags & SLAB_STORE_USER)
744	/* We also have user information there */
745	off += 2 * sizeof(struct track);
746
747	if (s->size == off)
748	return 1;
749
750	return check_bytes_and_report(s, page, p, "Object padding",
751	p + off, POISON_INUSE, s->size - off);
752	}
753
754	/* Check the pad bytes at the end of a slab page */
755	static int slab_pad_check(struct kmem_cache s, struct page page)
756	{
757	u8 *start;
758	u8 *fault;
759	u8 *end;
760	int length;
761	int remainder;
762
763	if (!(s->flags & SLAB_POISON))
764	return 1;
765
766	start = page_address(page);
767	length = (PAGE_SIZE << compound_order(page)) - s->reserved;
768	end = start + length;
769	remainder = length % s->size;
770	if (!remainder)
771	return 1;
772
773	fault = memchr_inv(end - remainder, POISON_INUSE, remainder);
774	if (!fault)
775	return 1;
776	while (end > fault && end[-1] == POISON_INUSE)
777	end--;
778
779	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
780	print_section("Padding ", end - remainder, remainder);
781
782	restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end);
783	return 0;
784	}
785
786	static int check_object(struct kmem_cache s, struct page page,
787	void *object, u8 val)
788	{
789	u8 *p = object;
790	u8 *endobject = object + s->objsize;
791
792	if (s->flags & SLAB_RED_ZONE) {
793	if (!check_bytes_and_report(s, page, object, "Redzone",
794	endobject, val, s->inuse - s->objsize))
795	return 0;
796	} else {
797	if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
798	check_bytes_and_report(s, page, p, "Alignment padding",
799	endobject, POISON_INUSE, s->inuse - s->objsize);
800	}
801	}
802
803	if (s->flags & SLAB_POISON) {
804	if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
805	(!check_bytes_and_report(s, page, p, "Poison", p,
806	POISON_FREE, s->objsize - 1) \|\|
807	!check_bytes_and_report(s, page, p, "Poison",
808	p + s->objsize - 1, POISON_END, 1)))
809	return 0;
810	/*
811	* check_pad_bytes cleans up on its own.
812	*/
813	check_pad_bytes(s, page, p);
814	}
815
816	if (!s->offset && val == SLUB_RED_ACTIVE)
817	/*
818	* Object and freepointer overlap. Cannot check
819	* freepointer while object is allocated.
820	*/
821	return 1;
822
823	/* Check free pointer validity */
824	if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
825	object_err(s, page, p, "Freepointer corrupt");
826	/*
827	* No choice but to zap it and thus lose the remainder
828	* of the free objects in this slab. May cause
829	* another error because the object count is now wrong.
830	*/
831	set_freepointer(s, p, NULL);
832	return 0;
833	}
834	return 1;
835	}
836
837	static int check_slab(struct kmem_cache s, struct page page)
838	{
839	int maxobj;
840
841	VM_BUG_ON(!irqs_disabled());
842
843	if (!PageSlab(page)) {
844	slab_err(s, page, "Not a valid slab page");
845	return 0;
846	}
847
848	maxobj = order_objects(compound_order(page), s->size, s->reserved);
849	if (page->objects > maxobj) {
850	slab_err(s, page, "objects %u > max %u",
851	s->name, page->objects, maxobj);
852	return 0;
853	}
854	if (page->inuse > page->objects) {
855	slab_err(s, page, "inuse %u > max %u",
856	s->name, page->inuse, page->objects);
857	return 0;
858	}
859	/* Slab_pad_check fixes things up after itself */
860	slab_pad_check(s, page);
861	return 1;
862	}
863
864	/*
865	* Determine if a certain object on a page is on the freelist. Must hold the
866	* slab lock to guarantee that the chains are in a consistent state.
867	*/
868	static int on_freelist(struct kmem_cache s, struct page page, void *search)
869	{
870	int nr = 0;
871	void *fp;
872	void *object = NULL;
873	unsigned long max_objects;
874
875	fp = page->freelist;
876	while (fp && nr <= page->objects) {
877	if (fp == search)
878	return 1;
879	if (!check_valid_pointer(s, page, fp)) {
880	if (object) {
881	object_err(s, page, object,
882	"Freechain corrupt");
883	set_freepointer(s, object, NULL);
884	break;
885	} else {
886	slab_err(s, page, "Freepointer corrupt");
887	page->freelist = NULL;
888	page->inuse = page->objects;
889	slab_fix(s, "Freelist cleared");
890	return 0;
891	}
892	break;
893	}
894	object = fp;
895	fp = get_freepointer(s, object);
896	nr++;
897	}
898
899	max_objects = order_objects(compound_order(page), s->size, s->reserved);
900	if (max_objects > MAX_OBJS_PER_PAGE)
901	max_objects = MAX_OBJS_PER_PAGE;
902
903	if (page->objects != max_objects) {
904	slab_err(s, page, "Wrong number of objects. Found %d but "
905	"should be %d", page->objects, max_objects);
906	page->objects = max_objects;
907	slab_fix(s, "Number of objects adjusted.");
908	}
909	if (page->inuse != page->objects - nr) {
910	slab_err(s, page, "Wrong object count. Counter is %d but "
911	"counted were %d", page->inuse, page->objects - nr);
912	page->inuse = page->objects - nr;
913	slab_fix(s, "Object count adjusted.");
914	}
915	return search == NULL;
916	}
917
918	static void trace(struct kmem_cache s, struct page page, void *object,
919	int alloc)
920	{
921	if (s->flags & SLAB_TRACE) {
922	printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
923	s->name,
924	alloc ? "alloc" : "free",
925	object, page->inuse,
926	page->freelist);
927
928	if (!alloc)
929	print_section("Object ", (void *)object, s->objsize);
930
931	dump_stack();
932	}
933	}
934
935	/*
936	* Hooks for other subsystems that check memory allocations. In a typical
937	* production configuration these hooks all should produce no code at all.
938	*/
939	static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
940	{
941	flags &= gfp_allowed_mask;
942	lockdep_trace_alloc(flags);
943	might_sleep_if(flags & __GFP_WAIT);
944
945	return should_failslab(s->objsize, flags, s->flags);
946	}
947
948	static inline void slab_post_alloc_hook(struct kmem_cache s, gfp_t flags, void object)
949	{
950	flags &= gfp_allowed_mask;
951	kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
952	kmemleak_alloc_recursive(object, s->objsize, 1, s->flags, flags);
953	}
954
955	static inline void slab_free_hook(struct kmem_cache s, void x)
956	{
957	kmemleak_free_recursive(x, s->flags);
958
959	/*
960	* Trouble is that we may no longer disable interupts in the fast path
961	* So in order to make the debug calls that expect irqs to be
962	* disabled we need to disable interrupts temporarily.
963	*/
964	#if defined(CONFIG_KMEMCHECK) \|\| defined(CONFIG_LOCKDEP)
965	{
966	unsigned long flags;
967
968	local_irq_save(flags);
969	kmemcheck_slab_free(s, x, s->objsize);
970	debug_check_no_locks_freed(x, s->objsize);
971	local_irq_restore(flags);
972	}
973	#endif
974	if (!(s->flags & SLAB_DEBUG_OBJECTS))
975	debug_check_no_obj_freed(x, s->objsize);
976	}
977
978	/*
979	* Tracking of fully allocated slabs for debugging purposes.
980	*
981	* list_lock must be held.
982	*/
983	static void add_full(struct kmem_cache *s,
984	struct kmem_cache_node n, struct page page)
985	{
986	if (!(s->flags & SLAB_STORE_USER))
987	return;
988
989	list_add(&page->lru, &n->full);
990	}
991
992	/*
993	* list_lock must be held.
994	*/
995	static void remove_full(struct kmem_cache s, struct page page)
996	{
997	if (!(s->flags & SLAB_STORE_USER))
998	return;
999
1000	list_del(&page->lru);
1001	}
1002
1003	/* Tracking of the number of slabs for debugging purposes */
1004	static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1005	{
1006	struct kmem_cache_node *n = get_node(s, node);
1007
1008	return atomic_long_read(&n->nr_slabs);
1009	}
1010
1011	static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1012	{
1013	return atomic_long_read(&n->nr_slabs);
1014	}
1015
1016	static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1017	{
1018	struct kmem_cache_node *n = get_node(s, node);
1019
1020	/*
1021	* May be called early in order to allocate a slab for the
1022	* kmem_cache_node structure. Solve the chicken-egg
1023	* dilemma by deferring the increment of the count during
1024	* bootstrap (see early_kmem_cache_node_alloc).
1025	*/
1026	if (n) {
1027	atomic_long_inc(&n->nr_slabs);
1028	atomic_long_add(objects, &n->total_objects);
1029	}
1030	}
1031	static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1032	{
1033	struct kmem_cache_node *n = get_node(s, node);
1034
1035	atomic_long_dec(&n->nr_slabs);
1036	atomic_long_sub(objects, &n->total_objects);
1037	}
1038
1039	/* Object debug checks for alloc/free paths */
1040	static void setup_object_debug(struct kmem_cache s, struct page page,
1041	void *object)
1042	{
1043	if (!(s->flags & (SLAB_STORE_USER\|SLAB_RED_ZONE\|__OBJECT_POISON)))
1044	return;
1045
1046	init_object(s, object, SLUB_RED_INACTIVE);
1047	init_tracking(s, object);
1048	}
1049
1050	static noinline int alloc_debug_processing(struct kmem_cache s, struct page page,
1051	void *object, unsigned long addr)
1052	{
1053	if (!check_slab(s, page))
1054	goto bad;
1055
1056	if (!check_valid_pointer(s, page, object)) {
1057	object_err(s, page, object, "Freelist Pointer check fails");
1058	goto bad;
1059	}
1060
1061	if (!check_object(s, page, object, SLUB_RED_INACTIVE))
1062	goto bad;
1063
1064	/* Success perform special debug activities for allocs */
1065	if (s->flags & SLAB_STORE_USER)
1066	set_track(s, object, TRACK_ALLOC, addr);
1067	trace(s, page, object, 1);
1068	init_object(s, object, SLUB_RED_ACTIVE);
1069	return 1;
1070
1071	bad:
1072	if (PageSlab(page)) {
1073	/*
1074	* If this is a slab page then lets do the best we can
1075	* to avoid issues in the future. Marking all objects
1076	* as used avoids touching the remaining objects.
1077	*/
1078	slab_fix(s, "Marking all objects used");
1079	page->inuse = page->objects;
1080	page->freelist = NULL;
1081	}
1082	return 0;
1083	}
1084
1085	static noinline int free_debug_processing(struct kmem_cache *s,
1086	struct page page, void object, unsigned long addr)
1087	{
1088	unsigned long flags;
1089	int rc = 0;
1090
1091	local_irq_save(flags);
1092	slab_lock(page);
1093
1094	if (!check_slab(s, page))
1095	goto fail;
1096
1097	if (!check_valid_pointer(s, page, object)) {
1098	slab_err(s, page, "Invalid object pointer 0x%p", object);
1099	goto fail;
1100	}
1101
1102	if (on_freelist(s, page, object)) {
1103	object_err(s, page, object, "Object already free");
1104	goto fail;
1105	}
1106
1107	if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1108	goto out;
1109
1110	if (unlikely(s != page->slab)) {
1111	if (!PageSlab(page)) {
1112	slab_err(s, page, "Attempt to free object(0x%p) "
1113	"outside of slab", object);
1114	} else if (!page->slab) {
1115	printk(KERN_ERR
1116	"SLUB <none>: no slab for object 0x%p.\n",
1117	object);
1118	dump_stack();
1119	} else
1120	object_err(s, page, object,
1121	"page slab pointer corrupt.");
1122	goto fail;
1123	}
1124
1125	if (s->flags & SLAB_STORE_USER)
1126	set_track(s, object, TRACK_FREE, addr);
1127	trace(s, page, object, 0);
1128	init_object(s, object, SLUB_RED_INACTIVE);
1129	rc = 1;
1130	out:
1131	slab_unlock(page);
1132	local_irq_restore(flags);
1133	return rc;
1134
1135	fail:
1136	slab_fix(s, "Object at 0x%p not freed", object);
1137	goto out;
1138	}
1139
1140	static int __init setup_slub_debug(char *str)
1141	{
1142	slub_debug = DEBUG_DEFAULT_FLAGS;
1143	if (str++ != '=' \|\| !str)
1144	/*
1145	* No options specified. Switch on full debugging.
1146	*/
1147	goto out;
1148
1149	if (*str == ',')
1150	/*
1151	* No options but restriction on slabs. This means full
1152	* debugging for slabs matching a pattern.
1153	*/
1154	goto check_slabs;
1155
1156	if (tolower(*str) == 'o') {
1157	/*
1158	* Avoid enabling debugging on caches if its minimum order
1159	* would increase as a result.
1160	*/
1161	disable_higher_order_debug = 1;
1162	goto out;
1163	}
1164
1165	slub_debug = 0;
1166	if (*str == '-')
1167	/*
1168	* Switch off all debugging measures.
1169	*/
1170	goto out;
1171
1172	/*
1173	* Determine which debug features should be switched on
1174	*/
1175	for (; str && str != ','; str++) {
1176	switch (tolower(*str)) {
1177	case 'f':
1178	slub_debug \|= SLAB_DEBUG_FREE;
1179	break;
1180	case 'z':
1181	slub_debug \|= SLAB_RED_ZONE;
1182	break;
1183	case 'p':
1184	slub_debug \|= SLAB_POISON;
1185	break;
1186	case 'u':
1187	slub_debug \|= SLAB_STORE_USER;
1188	break;
1189	case 't':
1190	slub_debug \|= SLAB_TRACE;
1191	break;
1192	case 'a':
1193	slub_debug \|= SLAB_FAILSLAB;
1194	break;
1195	default:
1196	printk(KERN_ERR "slub_debug option '%c' "
1197	"unknown. skipped\n", *str);
1198	}
1199	}
1200
1201	check_slabs:
1202	if (*str == ',')
1203	slub_debug_slabs = str + 1;
1204	out:
1205	return 1;
1206	}
1207
1208	__setup("slub_debug", setup_slub_debug);
1209
1210	static unsigned long kmem_cache_flags(unsigned long objsize,
1211	unsigned long flags, const char *name,
1212	void (ctor)(void ))
1213	{
1214	/*
1215	* Enable debugging if selected on the kernel commandline.
1216	*/
1217	if (slub_debug && (!slub_debug_slabs \|\|
1218	!strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))
1219	flags \|= slub_debug;
1220
1221	return flags;
1222	}
1223	#else
1224	static inline void setup_object_debug(struct kmem_cache *s,
1225	struct page page, void object) {}
1226
1227	static inline int alloc_debug_processing(struct kmem_cache *s,
1228	struct page page, void object, unsigned long addr) { return 0; }
1229
1230	static inline int free_debug_processing(struct kmem_cache *s,
1231	struct page page, void object, unsigned long addr) { return 0; }
1232
1233	static inline int slab_pad_check(struct kmem_cache s, struct page page)
1234	{ return 1; }
1235	static inline int check_object(struct kmem_cache s, struct page page,
1236	void *object, u8 val) { return 1; }
1237	static inline void add_full(struct kmem_cache s, struct kmem_cache_node n,
1238	struct page *page) {}
1239	static inline void remove_full(struct kmem_cache s, struct page page) {}
1240	static inline unsigned long kmem_cache_flags(unsigned long objsize,
1241	unsigned long flags, const char *name,
1242	void (ctor)(void ))
1243	{
1244	return flags;
1245	}
1246	#define slub_debug 0
1247
1248	#define disable_higher_order_debug 0
1249
1250	static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1251	{ return 0; }
1252	static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
1253	{ return 0; }
1254	static inline void inc_slabs_node(struct kmem_cache *s, int node,
1255	int objects) {}
1256	static inline void dec_slabs_node(struct kmem_cache *s, int node,
1257	int objects) {}
1258
1259	static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
1260	{ return 0; }
1261
1262	static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
1263	void *object) {}
1264
1265	static inline void slab_free_hook(struct kmem_cache s, void x) {}
1266
1267	#endif /* CONFIG_SLUB_DEBUG */
1268
1269	/*
1270	* Slab allocation and freeing
1271	*/
1272	static inline struct page *alloc_slab_page(gfp_t flags, int node,
1273	struct kmem_cache_order_objects oo)
1274	{
1275	int order = oo_order(oo);
1276
1277	flags \|= __GFP_NOTRACK;
1278
1279	if (node == NUMA_NO_NODE)
1280	return alloc_pages(flags, order);
1281	else
1282	return alloc_pages_exact_node(node, flags, order);
1283	}
1284
1285	static struct page allocate_slab(struct kmem_cache s, gfp_t flags, int node)
1286	{
1287	struct page *page;
1288	struct kmem_cache_order_objects oo = s->oo;
1289	gfp_t alloc_gfp;
1290
1291	flags &= gfp_allowed_mask;
1292
1293	if (flags & __GFP_WAIT)
1294	local_irq_enable();
1295
1296	flags \|= s->allocflags;
1297
1298	/*
1299	* Let the initial higher-order allocation fail under memory pressure
1300	* so we fall-back to the minimum order allocation.
1301	*/
1302	alloc_gfp = (flags \| __GFP_NOWARN \| __GFP_NORETRY) & ~__GFP_NOFAIL;
1303
1304	page = alloc_slab_page(alloc_gfp, node, oo);
1305	if (unlikely(!page)) {
1306	oo = s->min;
1307	/*
1308	* Allocation may have failed due to fragmentation.
1309	* Try a lower order alloc if possible
1310	*/
1311	page = alloc_slab_page(flags, node, oo);
1312
1313	if (page)
1314	stat(s, ORDER_FALLBACK);
1315	}
1316
1317	if (flags & __GFP_WAIT)
1318	local_irq_disable();
1319
1320	if (!page)
1321	return NULL;
1322
1323	if (kmemcheck_enabled
1324	&& !(s->flags & (SLAB_NOTRACK \| DEBUG_DEFAULT_FLAGS))) {
1325	int pages = 1 << oo_order(oo);
1326
1327	kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
1328
1329	/*
1330	* Objects from caches that have a constructor don't get
1331	* cleared when they're allocated, so we need to do it here.
1332	*/
1333	if (s->ctor)
1334	kmemcheck_mark_uninitialized_pages(page, pages);
1335	else
1336	kmemcheck_mark_unallocated_pages(page, pages);
1337	}
1338
1339	page->objects = oo_objects(oo);
1340	mod_zone_page_state(page_zone(page),
1341	(s->flags & SLAB_RECLAIM_ACCOUNT) ?
1342	NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1343	1 << oo_order(oo));
1344
1345	return page;
1346	}
1347
1348	static void setup_object(struct kmem_cache s, struct page page,
1349	void *object)
1350	{
1351	setup_object_debug(s, page, object);
1352	if (unlikely(s->ctor))
1353	s->ctor(object);
1354	}
1355
1356	static struct page new_slab(struct kmem_cache s, gfp_t flags, int node)
1357	{
1358	struct page *page;
1359	void *start;
1360	void *last;
1361	void *p;
1362
1363	BUG_ON(flags & GFP_SLAB_BUG_MASK);
1364
1365	page = allocate_slab(s,
1366	flags & (GFP_RECLAIM_MASK \| GFP_CONSTRAINT_MASK), node);
1367	if (!page)
1368	goto out;
1369
1370	inc_slabs_node(s, page_to_nid(page), page->objects);
1371	page->slab = s;
1372	page->flags \|= 1 << PG_slab;
1373
1374	start = page_address(page);
1375
1376	if (unlikely(s->flags & SLAB_POISON))
1377	memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
1378
1379	last = start;
1380	for_each_object(p, s, start, page->objects) {
1381	setup_object(s, page, last);
1382	set_freepointer(s, last, p);
1383	last = p;
1384	}
1385	setup_object(s, page, last);
1386	set_freepointer(s, last, NULL);
1387
1388	page->freelist = start;
1389	page->inuse = page->objects;
1390	page->frozen = 1;
1391	out:
1392	return page;
1393	}
1394
1395	static void __free_slab(struct kmem_cache s, struct page page)
1396	{
1397	int order = compound_order(page);
1398	int pages = 1 << order;
1399
1400	if (kmem_cache_debug(s)) {
1401	void *p;
1402
1403	slab_pad_check(s, page);
1404	for_each_object(p, s, page_address(page),
1405	page->objects)
1406	check_object(s, page, p, SLUB_RED_INACTIVE);
1407	}
1408
1409	kmemcheck_free_shadow(page, compound_order(page));
1410
1411	mod_zone_page_state(page_zone(page),
1412	(s->flags & SLAB_RECLAIM_ACCOUNT) ?
1413	NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1414	-pages);
1415
1416	__ClearPageSlab(page);
1417	reset_page_mapcount(page);
1418	if (current->reclaim_state)
1419	current->reclaim_state->reclaimed_slab += pages;
1420	__free_pages(page, order);
1421	}
1422
1423	#define need_reserve_slab_rcu \
1424	(sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
1425
1426	static void rcu_free_slab(struct rcu_head *h)
1427	{
1428	struct page *page;
1429
1430	if (need_reserve_slab_rcu)
1431	page = virt_to_head_page(h);
1432	else
1433	page = container_of((struct list_head *)h, struct page, lru);
1434
1435	__free_slab(page->slab, page);
1436	}
1437
1438	static void free_slab(struct kmem_cache s, struct page page)
1439	{
1440	if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1441	struct rcu_head *head;
1442
1443	if (need_reserve_slab_rcu) {
1444	int order = compound_order(page);
1445	int offset = (PAGE_SIZE << order) - s->reserved;
1446
1447	VM_BUG_ON(s->reserved != sizeof(*head));
1448	head = page_address(page) + offset;
1449	} else {
1450	/*
1451	* RCU free overloads the RCU head over the LRU
1452	*/
1453	head = (void *)&page->lru;
1454	}
1455
1456	call_rcu(head, rcu_free_slab);
1457	} else
1458	__free_slab(s, page);
1459	}
1460
1461	static void discard_slab(struct kmem_cache s, struct page page)
1462	{
1463	dec_slabs_node(s, page_to_nid(page), page->objects);
1464	free_slab(s, page);
1465	}
1466
1467	/*
1468	* Management of partially allocated slabs.
1469	*
1470	* list_lock must be held.
1471	*/
1472	static inline void add_partial(struct kmem_cache_node *n,
1473	struct page *page, int tail)
1474	{
1475	n->nr_partial++;
1476	if (tail == DEACTIVATE_TO_TAIL)
1477	list_add_tail(&page->lru, &n->partial);
1478	else
1479	list_add(&page->lru, &n->partial);
1480	}
1481
1482	/*
1483	* list_lock must be held.
1484	*/
1485	static inline void remove_partial(struct kmem_cache_node *n,
1486	struct page *page)
1487	{
1488	list_del(&page->lru);
1489	n->nr_partial--;
1490	}
1491
1492	/*
1493	* Lock slab, remove from the partial list and put the object into the
1494	* per cpu freelist.
1495	*
1496	* Returns a list of objects or NULL if it fails.
1497	*
1498	* Must hold list_lock.
1499	*/
1500	static inline void acquire_slab(struct kmem_cache s,
1501	struct kmem_cache_node n, struct page page,
1502	int mode)
1503	{
1504	void *freelist;
1505	unsigned long counters;
1506	struct page new;
1507
1508	/*
1509	* Zap the freelist and set the frozen bit.
1510	* The old freelist is the list of objects for the
1511	* per cpu allocation list.
1512	*/
1513	do {
1514	freelist = page->freelist;
1515	counters = page->counters;
1516	new.counters = counters;
1517	if (mode)
1518	new.inuse = page->objects;
1519
1520	VM_BUG_ON(new.frozen);
1521	new.frozen = 1;
1522
1523	} while (!__cmpxchg_double_slab(s, page,
1524	freelist, counters,
1525	NULL, new.counters,
1526	"lock and freeze"));
1527
1528	remove_partial(n, page);
1529	return freelist;
1530	}
1531
1532	static int put_cpu_partial(struct kmem_cache s, struct page page, int drain);
1533
1534	/*
1535	* Try to allocate a partial slab from a specific node.
1536	*/
1537	static void get_partial_node(struct kmem_cache s,
1538	struct kmem_cache_node n, struct kmem_cache_cpu c)
1539	{
1540	struct page page, page2;
1541	void *object = NULL;
1542
1543	/*
1544	* Racy check. If we mistakenly see no partial slabs then we
1545	* just allocate an empty slab. If we mistakenly try to get a
1546	* partial slab and there is none available then get_partials()
1547	* will return NULL.
1548	*/
1549	if (!n \|\| !n->nr_partial)
1550	return NULL;
1551
1552	spin_lock(&n->list_lock);
1553	list_for_each_entry_safe(page, page2, &n->partial, lru) {
1554	void *t = acquire_slab(s, n, page, object == NULL);
1555	int available;
1556
1557	if (!t)
1558	break;
1559
1560	if (!object) {
1561	c->page = page;
1562	c->node = page_to_nid(page);
1563	stat(s, ALLOC_FROM_PARTIAL);
1564	object = t;
1565	available = page->objects - page->inuse;
1566	} else {
1567	page->freelist = t;
1568	available = put_cpu_partial(s, page, 0);
1569	stat(s, CPU_PARTIAL_NODE);
1570	}
1571	if (kmem_cache_debug(s) \|\| available > s->cpu_partial / 2)
1572	break;
1573
1574	}
1575	spin_unlock(&n->list_lock);
1576	return object;
1577	}
1578
1579	/*
1580	* Get a page from somewhere. Search in increasing NUMA distances.
1581	*/
1582	static struct page get_any_partial(struct kmem_cache s, gfp_t flags,
1583	struct kmem_cache_cpu *c)
1584	{
1585	#ifdef CONFIG_NUMA
1586	struct zonelist *zonelist;
1587	struct zoneref *z;
1588	struct zone *zone;
1589	enum zone_type high_zoneidx = gfp_zone(flags);
1590	void *object;
1591	unsigned int cpuset_mems_cookie;
1592
1593	/*
1594	* The defrag ratio allows a configuration of the tradeoffs between
1595	* inter node defragmentation and node local allocations. A lower
1596	* defrag_ratio increases the tendency to do local allocations
1597	* instead of attempting to obtain partial slabs from other nodes.
1598	*
1599	* If the defrag_ratio is set to 0 then kmalloc() always
1600	* returns node local objects. If the ratio is higher then kmalloc()
1601	* may return off node objects because partial slabs are obtained
1602	* from other nodes and filled up.
1603	*
1604	* If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
1605	* defrag_ratio = 1000) then every (well almost) allocation will
1606	* first attempt to defrag slab caches on other nodes. This means
1607	* scanning over all nodes to look for partial slabs which may be
1608	* expensive if we do it every time we are trying to find a slab
1609	* with available objects.
1610	*/
1611	if (!s->remote_node_defrag_ratio \|\|
1612	get_cycles() % 1024 > s->remote_node_defrag_ratio)
1613	return NULL;
1614
1615	do {
1616	cpuset_mems_cookie = get_mems_allowed();
1617	zonelist = node_zonelist(slab_node(current->mempolicy), flags);
1618	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1619	struct kmem_cache_node *n;
1620
1621	n = get_node(s, zone_to_nid(zone));
1622
1623	if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
1624	n->nr_partial > s->min_partial) {
1625	object = get_partial_node(s, n, c);
1626	if (object) {
1627	/*
1628	* Return the object even if
1629	* put_mems_allowed indicated that
1630	* the cpuset mems_allowed was
1631	* updated in parallel. It's a
1632	* harmless race between the alloc
1633	* and the cpuset update.
1634	*/
1635	put_mems_allowed(cpuset_mems_cookie);
1636	return object;
1637	}
1638	}
1639	}
1640	} while (!put_mems_allowed(cpuset_mems_cookie));
1641	#endif
1642	return NULL;
1643	}
1644
1645	/*
1646	* Get a partial page, lock it and return it.
1647	*/
1648	static void get_partial(struct kmem_cache s, gfp_t flags, int node,
1649	struct kmem_cache_cpu *c)
1650	{
1651	void *object;
1652	int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node;
1653
1654	object = get_partial_node(s, get_node(s, searchnode), c);
1655	if (object \|\| node != NUMA_NO_NODE)
1656	return object;
1657
1658	return get_any_partial(s, flags, c);
1659	}
1660
1661	#ifdef CONFIG_PREEMPT
1662	/*
1663	* Calculate the next globally unique transaction for disambiguiation
1664	* during cmpxchg. The transactions start with the cpu number and are then
1665	* incremented by CONFIG_NR_CPUS.
1666	*/
1667	#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
1668	#else
1669	/*
1670	* No preemption supported therefore also no need to check for
1671	* different cpus.
1672	*/
1673	#define TID_STEP 1
1674	#endif
1675
1676	static inline unsigned long next_tid(unsigned long tid)
1677	{
1678	return tid + TID_STEP;
1679	}
1680
1681	static inline unsigned int tid_to_cpu(unsigned long tid)
1682	{
1683	return tid % TID_STEP;
1684	}
1685
1686	static inline unsigned long tid_to_event(unsigned long tid)
1687	{
1688	return tid / TID_STEP;
1689	}
1690
1691	static inline unsigned int init_tid(int cpu)
1692	{
1693	return cpu;
1694	}
1695
1696	static inline void note_cmpxchg_failure(const char *n,
1697	const struct kmem_cache *s, unsigned long tid)
1698	{
1699	#ifdef SLUB_DEBUG_CMPXCHG
1700	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
1701
1702	printk(KERN_INFO "%s %s: cmpxchg redo ", n, s->name);
1703
1704	#ifdef CONFIG_PREEMPT
1705	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
1706	printk("due to cpu change %d -> %d\n",
1707	tid_to_cpu(tid), tid_to_cpu(actual_tid));
1708	else
1709	#endif
1710	if (tid_to_event(tid) != tid_to_event(actual_tid))
1711	printk("due to cpu running other code. Event %ld->%ld\n",
1712	tid_to_event(tid), tid_to_event(actual_tid));
1713	else
1714	printk("for unknown reason: actual=%lx was=%lx target=%lx\n",
1715	actual_tid, tid, next_tid(tid));
1716	#endif
1717	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
1718	}
1719
1720	void init_kmem_cache_cpus(struct kmem_cache *s)
1721	{
1722	int cpu;
1723
1724	for_each_possible_cpu(cpu)
1725	per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
1726	}
1727
1728	/*
1729	* Remove the cpu slab
1730	*/
1731	static void deactivate_slab(struct kmem_cache s, struct kmem_cache_cpu c)
1732	{
1733	enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
1734	struct page *page = c->page;
1735	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1736	int lock = 0;
1737	enum slab_modes l = M_NONE, m = M_NONE;
1738	void *freelist;
1739	void *nextfree;
1740	int tail = DEACTIVATE_TO_HEAD;
1741	struct page new;
1742	struct page old;
1743
1744	if (page->freelist) {
1745	stat(s, DEACTIVATE_REMOTE_FREES);
1746	tail = DEACTIVATE_TO_TAIL;
1747	}
1748
1749	c->tid = next_tid(c->tid);
1750	c->page = NULL;
1751	freelist = c->freelist;
1752	c->freelist = NULL;
1753
1754	/*
1755	* Stage one: Free all available per cpu objects back
1756	* to the page freelist while it is still frozen. Leave the
1757	* last one.
1758	*
1759	* There is no need to take the list->lock because the page
1760	* is still frozen.
1761	*/
1762	while (freelist && (nextfree = get_freepointer(s, freelist))) {
1763	void *prior;
1764	unsigned long counters;
1765
1766	do {
1767	prior = page->freelist;
1768	counters = page->counters;
1769	set_freepointer(s, freelist, prior);
1770	new.counters = counters;
1771	new.inuse--;
1772	VM_BUG_ON(!new.frozen);
1773
1774	} while (!__cmpxchg_double_slab(s, page,
1775	prior, counters,
1776	freelist, new.counters,
1777	"drain percpu freelist"));
1778
1779	freelist = nextfree;
1780	}
1781
1782	/*
1783	* Stage two: Ensure that the page is unfrozen while the
1784	* list presence reflects the actual number of objects
1785	* during unfreeze.
1786	*
1787	* We setup the list membership and then perform a cmpxchg
1788	* with the count. If there is a mismatch then the page
1789	* is not unfrozen but the page is on the wrong list.
1790	*
1791	* Then we restart the process which may have to remove
1792	* the page from the list that we just put it on again
1793	* because the number of objects in the slab may have
1794	* changed.
1795	*/
1796	redo:
1797
1798	old.freelist = page->freelist;
1799	old.counters = page->counters;
1800	VM_BUG_ON(!old.frozen);
1801
1802	/* Determine target state of the slab */
1803	new.counters = old.counters;
1804	if (freelist) {
1805	new.inuse--;
1806	set_freepointer(s, freelist, old.freelist);
1807	new.freelist = freelist;
1808	} else
1809	new.freelist = old.freelist;
1810
1811	new.frozen = 0;
1812
1813	if (!new.inuse && n->nr_partial > s->min_partial)
1814	m = M_FREE;
1815	else if (new.freelist) {
1816	m = M_PARTIAL;
1817	if (!lock) {
1818	lock = 1;
1819	/*
1820	* Taking the spinlock removes the possiblity
1821	* that acquire_slab() will see a slab page that
1822	* is frozen
1823	*/
1824	spin_lock(&n->list_lock);
1825	}
1826	} else {
1827	m = M_FULL;
1828	if (kmem_cache_debug(s) && !lock) {
1829	lock = 1;
1830	/*
1831	* This also ensures that the scanning of full
1832	* slabs from diagnostic functions will not see
1833	* any frozen slabs.
1834	*/
1835	spin_lock(&n->list_lock);
1836	}
1837	}
1838
1839	if (l != m) {
1840
1841	if (l == M_PARTIAL)
1842
1843	remove_partial(n, page);
1844
1845	else if (l == M_FULL)
1846
1847	remove_full(s, page);
1848
1849	if (m == M_PARTIAL) {
1850
1851	add_partial(n, page, tail);
1852	stat(s, tail);
1853
1854	} else if (m == M_FULL) {
1855
1856	stat(s, DEACTIVATE_FULL);
1857	add_full(s, n, page);
1858
1859	}
1860	}
1861
1862	l = m;
1863	if (!__cmpxchg_double_slab(s, page,
1864	old.freelist, old.counters,
1865	new.freelist, new.counters,
1866	"unfreezing slab"))
1867	goto redo;
1868
1869	if (lock)
1870	spin_unlock(&n->list_lock);
1871
1872	if (m == M_FREE) {
1873	stat(s, DEACTIVATE_EMPTY);
1874	discard_slab(s, page);
1875	stat(s, FREE_SLAB);
1876	}
1877	}
1878
1879	/* Unfreeze all the cpu partial slabs */
1880	static void unfreeze_partials(struct kmem_cache *s)
1881	{
1882	struct kmem_cache_node *n = NULL;
1883	struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
1884	struct page page, discard_page = NULL;
1885
1886	while ((page = c->partial)) {
1887	enum slab_modes { M_PARTIAL, M_FREE };
1888	enum slab_modes l, m;
1889	struct page new;
1890	struct page old;
1891
1892	c->partial = page->next;
1893	l = M_FREE;
1894
1895	do {
1896
1897	old.freelist = page->freelist;
1898	old.counters = page->counters;
1899	VM_BUG_ON(!old.frozen);
1900
1901	new.counters = old.counters;
1902	new.freelist = old.freelist;
1903
1904	new.frozen = 0;
1905
1906	if (!new.inuse && (!n \|\| n->nr_partial > s->min_partial))
1907	m = M_FREE;
1908	else {
1909	struct kmem_cache_node *n2 = get_node(s,
1910	page_to_nid(page));
1911
1912	m = M_PARTIAL;
1913	if (n != n2) {
1914	if (n)
1915	spin_unlock(&n->list_lock);
1916
1917	n = n2;
1918	spin_lock(&n->list_lock);
1919	}
1920	}
1921
1922	if (l != m) {
1923	if (l == M_PARTIAL) {
1924	remove_partial(n, page);
1925	stat(s, FREE_REMOVE_PARTIAL);
1926	} else {
1927	add_partial(n, page,
1928	DEACTIVATE_TO_TAIL);
1929	stat(s, FREE_ADD_PARTIAL);
1930	}
1931
1932	l = m;
1933	}
1934
1935	} while (!cmpxchg_double_slab(s, page,
1936	old.freelist, old.counters,
1937	new.freelist, new.counters,
1938	"unfreezing slab"));
1939
1940	if (m == M_FREE) {
1941	page->next = discard_page;
1942	discard_page = page;
1943	}
1944	}
1945
1946	if (n)
1947	spin_unlock(&n->list_lock);
1948
1949	while (discard_page) {
1950	page = discard_page;
1951	discard_page = discard_page->next;
1952
1953	stat(s, DEACTIVATE_EMPTY);
1954	discard_slab(s, page);
1955	stat(s, FREE_SLAB);
1956	}
1957	}
1958
1959	/*
1960	* Put a page that was just frozen (in __slab_free) into a partial page
1961	* slot if available. This is done without interrupts disabled and without
1962	* preemption disabled. The cmpxchg is racy and may put the partial page
1963	* onto a random cpus partial slot.
1964	*
1965	* If we did not find a slot then simply move all the partials to the
1966	* per node partial list.
1967	*/
1968	int put_cpu_partial(struct kmem_cache s, struct page page, int drain)
1969	{
1970	struct page *oldpage;
1971	int pages;
1972	int pobjects;
1973
1974	do {
1975	pages = 0;
1976	pobjects = 0;
1977	oldpage = this_cpu_read(s->cpu_slab->partial);
1978
1979	if (oldpage) {
1980	pobjects = oldpage->pobjects;
1981	pages = oldpage->pages;
1982	if (drain && pobjects > s->cpu_partial) {
1983	unsigned long flags;
1984	/*
1985	* partial array is full. Move the existing
1986	* set to the per node partial list.
1987	*/
1988	local_irq_save(flags);
1989	unfreeze_partials(s);
1990	local_irq_restore(flags);
1991	pobjects = 0;
1992	pages = 0;
1993	stat(s, CPU_PARTIAL_DRAIN);
1994	}
1995	}
1996
1997	pages++;
1998	pobjects += page->objects - page->inuse;
1999
2000	page->pages = pages;
2001	page->pobjects = pobjects;
2002	page->next = oldpage;
2003
2004	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
2005	return pobjects;
2006	}
2007
2008	static inline void flush_slab(struct kmem_cache s, struct kmem_cache_cpu c)
2009	{
2010	stat(s, CPUSLAB_FLUSH);
2011	deactivate_slab(s, c);
2012	}
2013
2014	/*
2015	* Flush cpu slab.
2016	*
2017	* Called from IPI handler with interrupts disabled.
2018	*/
2019	static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
2020	{
2021	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2022
2023	if (likely(c)) {
2024	if (c->page)
2025	flush_slab(s, c);
2026
2027	unfreeze_partials(s);
2028	}
2029	}
2030
2031	static void flush_cpu_slab(void *d)
2032	{
2033	struct kmem_cache *s = d;
2034
2035	__flush_cpu_slab(s, smp_processor_id());
2036	}
2037
2038	static bool has_cpu_slab(int cpu, void *info)
2039	{
2040	struct kmem_cache *s = info;
2041	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
2042
2043	return c->page \|\| c->partial;
2044	}
2045
2046	static void flush_all(struct kmem_cache *s)
2047	{
2048	on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
2049	}
2050
2051	/*
2052	* Check if the objects in a per cpu structure fit numa
2053	* locality expectations.
2054	*/
2055	static inline int node_match(struct kmem_cache_cpu *c, int node)
2056	{
2057	#ifdef CONFIG_NUMA
2058	if (node != NUMA_NO_NODE && c->node != node)
2059	return 0;
2060	#endif
2061	return 1;
2062	}
2063
2064	static int count_free(struct page *page)
2065	{
2066	return page->objects - page->inuse;
2067	}
2068
2069	static unsigned long count_partial(struct kmem_cache_node *n,
2070	int (get_count)(struct page ))
2071	{
2072	unsigned long flags;
2073	unsigned long x = 0;
2074	struct page *page;
2075
2076	spin_lock_irqsave(&n->list_lock, flags);
2077	list_for_each_entry(page, &n->partial, lru)
2078	x += get_count(page);
2079	spin_unlock_irqrestore(&n->list_lock, flags);
2080	return x;
2081	}
2082
2083	static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
2084	{
2085	#ifdef CONFIG_SLUB_DEBUG
2086	return atomic_long_read(&n->total_objects);
2087	#else
2088	return 0;
2089	#endif
2090	}
2091
2092	static noinline void
2093	slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
2094	{
2095	int node;
2096
2097	printk(KERN_WARNING
2098	"SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
2099	nid, gfpflags);
2100	printk(KERN_WARNING " cache: %s, object size: %d, buffer size: %d, "
2101	"default order: %d, min order: %d\n", s->name, s->objsize,
2102	s->size, oo_order(s->oo), oo_order(s->min));
2103
2104	if (oo_order(s->min) > get_order(s->objsize))
2105	printk(KERN_WARNING " %s debugging increased min order, use "
2106	"slub_debug=O to disable.\n", s->name);
2107
2108	for_each_online_node(node) {
2109	struct kmem_cache_node *n = get_node(s, node);
2110	unsigned long nr_slabs;
2111	unsigned long nr_objs;
2112	unsigned long nr_free;
2113
2114	if (!n)
2115	continue;
2116
2117	nr_free = count_partial(n, count_free);
2118	nr_slabs = node_nr_slabs(n);
2119	nr_objs = node_nr_objs(n);
2120
2121	printk(KERN_WARNING
2122	" node %d: slabs: %ld, objs: %ld, free: %ld\n",
2123	node, nr_slabs, nr_objs, nr_free);
2124	}
2125	}
2126
2127	static inline void new_slab_objects(struct kmem_cache s, gfp_t flags,
2128	int node, struct kmem_cache_cpu **pc)
2129	{
2130	void *object;
2131	struct kmem_cache_cpu *c;
2132	struct page *page = new_slab(s, flags, node);
2133
2134	if (page) {
2135	c = __this_cpu_ptr(s->cpu_slab);
2136	if (c->page)
2137	flush_slab(s, c);
2138
2139	/*
2140	* No other reference to the page yet so we can
2141	* muck around with it freely without cmpxchg
2142	*/
2143	object = page->freelist;
2144	page->freelist = NULL;
2145
2146	stat(s, ALLOC_SLAB);
2147	c->node = page_to_nid(page);
2148	c->page = page;
2149	*pc = c;
2150	} else
2151	object = NULL;
2152
2153	return object;
2154	}
2155
2156	/*
2157	* Check the page->freelist of a page and either transfer the freelist to the per cpu freelist
2158	* or deactivate the page.
2159	*
2160	* The page is still frozen if the return value is not NULL.
2161	*
2162	* If this function returns NULL then the page has been unfrozen.
2163	*/
2164	static inline void get_freelist(struct kmem_cache s, struct page *page)
2165	{
2166	struct page new;
2167	unsigned long counters;
2168	void *freelist;
2169
2170	do {
2171	freelist = page->freelist;
2172	counters = page->counters;
2173	new.counters = counters;
2174	VM_BUG_ON(!new.frozen);
2175
2176	new.inuse = page->objects;
2177	new.frozen = freelist != NULL;
2178
2179	} while (!cmpxchg_double_slab(s, page,
2180	freelist, counters,
2181	NULL, new.counters,
2182	"get_freelist"));
2183
2184	return freelist;
2185	}
2186
2187	/*
2188	* Slow path. The lockless freelist is empty or we need to perform
2189	* debugging duties.
2190	*
2191	* Processing is still very fast if new objects have been freed to the
2192	* regular freelist. In that case we simply take over the regular freelist
2193	* as the lockless freelist and zap the regular freelist.
2194	*
2195	* If that is not working then we fall back to the partial lists. We take the
2196	* first element of the freelist as the object to allocate now and move the
2197	* rest of the freelist to the lockless freelist.
2198	*
2199	* And if we were unable to get a new slab from the partial slab lists then
2200	* we need to allocate a new slab. This is the slowest path since it involves
2201	* a call to the page allocator and the setup of a new slab.
2202	*/
2203	static void __slab_alloc(struct kmem_cache s, gfp_t gfpflags, int node,
2204	unsigned long addr, struct kmem_cache_cpu *c)
2205	{
2206	void **object;
2207	unsigned long flags;
2208
2209	local_irq_save(flags);
2210	#ifdef CONFIG_PREEMPT
2211	/*
2212	* We may have been preempted and rescheduled on a different
2213	* cpu before disabling interrupts. Need to reload cpu area
2214	* pointer.
2215	*/
2216	c = this_cpu_ptr(s->cpu_slab);
2217	#endif
2218
2219	if (!c->page)
2220	goto new_slab;
2221	redo:
2222	if (unlikely(!node_match(c, node))) {
2223	stat(s, ALLOC_NODE_MISMATCH);
2224	deactivate_slab(s, c);
2225	goto new_slab;
2226	}
2227
2228	/* must check again c->freelist in case of cpu migration or IRQ */
2229	object = c->freelist;
2230	if (object)
2231	goto load_freelist;
2232
2233	stat(s, ALLOC_SLOWPATH);
2234
2235	object = get_freelist(s, c->page);
2236
2237	if (!object) {
2238	c->page = NULL;
2239	stat(s, DEACTIVATE_BYPASS);
2240	goto new_slab;
2241	}
2242
2243	stat(s, ALLOC_REFILL);
2244
2245	load_freelist:
2246	c->freelist = get_freepointer(s, object);
2247	c->tid = next_tid(c->tid);
2248	local_irq_restore(flags);
2249	return object;
2250
2251	new_slab:
2252
2253	if (c->partial) {
2254	c->page = c->partial;
2255	c->partial = c->page->next;
2256	c->node = page_to_nid(c->page);
2257	stat(s, CPU_PARTIAL_ALLOC);
2258	c->freelist = NULL;
2259	goto redo;
2260	}
2261
2262	/* Then do expensive stuff like retrieving pages from the partial lists */
2263	object = get_partial(s, gfpflags, node, c);
2264
2265	if (unlikely(!object)) {
2266
2267	object = new_slab_objects(s, gfpflags, node, &c);
2268
2269	if (unlikely(!object)) {
2270	if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
2271	slab_out_of_memory(s, gfpflags, node);
2272
2273	local_irq_restore(flags);
2274	return NULL;
2275	}
2276	}
2277
2278	if (likely(!kmem_cache_debug(s)))
2279	goto load_freelist;
2280
2281	/* Only entered in the debug case */
2282	if (!alloc_debug_processing(s, c->page, object, addr))
2283	goto new_slab; /* Slab failed checks. Next slab needed */
2284
2285	c->freelist = get_freepointer(s, object);
2286	deactivate_slab(s, c);
2287	c->node = NUMA_NO_NODE;
2288	local_irq_restore(flags);
2289	return object;
2290	}
2291
2292	/*
2293	* Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
2294	* have the fastpath folded into their functions. So no function call
2295	* overhead for requests that can be satisfied on the fastpath.
2296	*
2297	* The fastpath works by first checking if the lockless freelist can be used.
2298	* If not then __slab_alloc is called for slow processing.
2299	*
2300	* Otherwise we can simply pick the next object from the lockless free list.
2301	*/
2302	static __always_inline void slab_alloc(struct kmem_cache s,
2303	gfp_t gfpflags, int node, unsigned long addr)
2304	{
2305	void **object;
2306	struct kmem_cache_cpu *c;
2307	unsigned long tid;
2308
2309	if (slab_pre_alloc_hook(s, gfpflags))
2310	return NULL;
2311
2312	redo:
2313
2314	/*
2315	* Must read kmem_cache cpu data via this cpu ptr. Preemption is
2316	* enabled. We may switch back and forth between cpus while
2317	* reading from one cpu area. That does not matter as long
2318	* as we end up on the original cpu again when doing the cmpxchg.
2319	*/
2320	c = __this_cpu_ptr(s->cpu_slab);
2321
2322	/*
2323	* The transaction ids are globally unique per cpu and per operation on
2324	* a per cpu queue. Thus they can be guarantee that the cmpxchg_double
2325	* occurs on the right processor and that there was no operation on the
2326	* linked list in between.
2327	*/
2328	tid = c->tid;
2329	barrier();
2330
2331	object = c->freelist;
2332	if (unlikely(!object \|\| !node_match(c, node)))
2333
2334	object = __slab_alloc(s, gfpflags, node, addr, c);
2335
2336	else {
2337	void *next_object = get_freepointer_safe(s, object);
2338
2339	/*
2340	* The cmpxchg will only match if there was no additional
2341	* operation and if we are on the right processor.
2342	*
2343	* The cmpxchg does the following atomically (without lock semantics!)
2344	* 1. Relocate first pointer to the current per cpu area.
2345	* 2. Verify that tid and freelist have not been changed
2346	* 3. If they were not changed replace tid and freelist
2347	*
2348	* Since this is without lock semantics the protection is only against
2349	* code executing on this cpu not from access by other cpus.
2350	*/
2351	if (unlikely(!this_cpu_cmpxchg_double(
2352	s->cpu_slab->freelist, s->cpu_slab->tid,
2353	object, tid,
2354	next_object, next_tid(tid)))) {
2355
2356	note_cmpxchg_failure("slab_alloc", s, tid);
2357	goto redo;
2358	}
2359	prefetch_freepointer(s, next_object);
2360	stat(s, ALLOC_FASTPATH);
2361	}
2362
2363	if (unlikely(gfpflags & __GFP_ZERO) && object)
2364	memset(object, 0, s->objsize);
2365
2366	slab_post_alloc_hook(s, gfpflags, object);
2367
2368	return object;
2369	}
2370
2371	void kmem_cache_alloc(struct kmem_cache s, gfp_t gfpflags)
2372	{
2373	void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
2374
2375	trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
2376
2377	return ret;
2378	}
2379	EXPORT_SYMBOL(kmem_cache_alloc);
2380
2381	#ifdef CONFIG_TRACING
2382	void kmem_cache_alloc_trace(struct kmem_cache s, gfp_t gfpflags, size_t size)
2383	{
2384	void *ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, _RET_IP_);
2385	trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
2386	return ret;
2387	}
2388	EXPORT_SYMBOL(kmem_cache_alloc_trace);
2389
2390	void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
2391	{
2392	void *ret = kmalloc_order(size, flags, order);
2393	trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
2394	return ret;
2395	}
2396	EXPORT_SYMBOL(kmalloc_order_trace);
2397	#endif
2398
2399	#ifdef CONFIG_NUMA
2400	void kmem_cache_alloc_node(struct kmem_cache s, gfp_t gfpflags, int node)
2401	{
2402	void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
2403
2404	trace_kmem_cache_alloc_node(_RET_IP_, ret,
2405	s->objsize, s->size, gfpflags, node);
2406
2407	return ret;
2408	}
2409	EXPORT_SYMBOL(kmem_cache_alloc_node);
2410
2411	#ifdef CONFIG_TRACING
2412	void kmem_cache_alloc_node_trace(struct kmem_cache s,
2413	gfp_t gfpflags,
2414	int node, size_t size)
2415	{
2416	void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
2417
2418	trace_kmalloc_node(_RET_IP_, ret,
2419	size, s->size, gfpflags, node);
2420	return ret;
2421	}
2422	EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
2423	#endif
2424	#endif
2425
2426	/*
2427	* Slow patch handling. This may still be called frequently since objects
2428	* have a longer lifetime than the cpu slabs in most processing loads.
2429	*
2430	* So we still attempt to reduce cache line usage. Just take the slab
2431	* lock and free the item. If there is no additional partial page
2432	* handling required then we can return immediately.
2433	*/
2434	static void __slab_free(struct kmem_cache s, struct page page,
2435	void *x, unsigned long addr)
2436	{
2437	void *prior;
2438	void *object = (void )x;
2439	int was_frozen;
2440	int inuse;
2441	struct page new;
2442	unsigned long counters;
2443	struct kmem_cache_node *n = NULL;
2444	unsigned long uninitialized_var(flags);
2445
2446	stat(s, FREE_SLOWPATH);
2447
2448	if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr))
2449	return;
2450
2451	do {
2452	prior = page->freelist;
2453	counters = page->counters;
2454	set_freepointer(s, object, prior);
2455	new.counters = counters;
2456	was_frozen = new.frozen;
2457	new.inuse--;
2458	if ((!new.inuse \|\| !prior) && !was_frozen && !n) {
2459
2460	if (!kmem_cache_debug(s) && !prior)
2461
2462	/*
2463	* Slab was on no list before and will be partially empty
2464	* We can defer the list move and instead freeze it.
2465	*/
2466	new.frozen = 1;
2467
2468	else { /* Needs to be taken off a list */
2469
2470	n = get_node(s, page_to_nid(page));
2471	/*
2472	* Speculatively acquire the list_lock.
2473	* If the cmpxchg does not succeed then we may
2474	* drop the list_lock without any processing.
2475	*
2476	* Otherwise the list_lock will synchronize with
2477	* other processors updating the list of slabs.
2478	*/
2479	spin_lock_irqsave(&n->list_lock, flags);
2480
2481	}
2482	}
2483	inuse = new.inuse;
2484
2485	} while (!cmpxchg_double_slab(s, page,
2486	prior, counters,
2487	object, new.counters,
2488	"__slab_free"));
2489
2490	if (likely(!n)) {
2491
2492	/*
2493	* If we just froze the page then put it onto the
2494	* per cpu partial list.
2495	*/
2496	if (new.frozen && !was_frozen) {
2497	put_cpu_partial(s, page, 1);
2498	stat(s, CPU_PARTIAL_FREE);
2499	}
2500	/*
2501	* The list lock was not taken therefore no list
2502	* activity can be necessary.
2503	*/
2504	if (was_frozen)
2505	stat(s, FREE_FROZEN);
2506	return;
2507	}
2508
2509	/*
2510	* was_frozen may have been set after we acquired the list_lock in
2511	* an earlier loop. So we need to check it here again.
2512	*/
2513	if (was_frozen)
2514	stat(s, FREE_FROZEN);
2515	else {
2516	if (unlikely(!inuse && n->nr_partial > s->min_partial))
2517	goto slab_empty;
2518
2519	/*
2520	* Objects left in the slab. If it was not on the partial list before
2521	* then add it.
2522	*/
2523	if (unlikely(!prior)) {
2524	remove_full(s, page);
2525	add_partial(n, page, DEACTIVATE_TO_TAIL);
2526	stat(s, FREE_ADD_PARTIAL);
2527	}
2528	}
2529	spin_unlock_irqrestore(&n->list_lock, flags);
2530	return;
2531
2532	slab_empty:
2533	if (prior) {
2534	/*
2535	* Slab on the partial list.
2536	*/
2537	remove_partial(n, page);
2538	stat(s, FREE_REMOVE_PARTIAL);
2539	} else
2540	/* Slab must be on the full list */
2541	remove_full(s, page);
2542
2543	spin_unlock_irqrestore(&n->list_lock, flags);
2544	stat(s, FREE_SLAB);
2545	discard_slab(s, page);
2546	}
2547
2548	/*
2549	* Fastpath with forced inlining to produce a kfree and kmem_cache_free that
2550	* can perform fastpath freeing without additional function calls.
2551	*
2552	* The fastpath is only possible if we are freeing to the current cpu slab
2553	* of this processor. This typically the case if we have just allocated
2554	* the item before.
2555	*
2556	* If fastpath is not possible then fall back to __slab_free where we deal
2557	* with all sorts of special processing.
2558	*/
2559	static __always_inline void slab_free(struct kmem_cache *s,
2560	struct page page, void x, unsigned long addr)
2561	{
2562	void *object = (void )x;
2563	struct kmem_cache_cpu *c;
2564	unsigned long tid;
2565
2566	slab_free_hook(s, x);
2567
2568	redo:
2569	/*
2570	* Determine the currently cpus per cpu slab.
2571	* The cpu may change afterward. However that does not matter since
2572	* data is retrieved via this pointer. If we are on the same cpu
2573	* during the cmpxchg then the free will succedd.
2574	*/
2575	c = __this_cpu_ptr(s->cpu_slab);
2576
2577	tid = c->tid;
2578	barrier();
2579
2580	if (likely(page == c->page)) {
2581	set_freepointer(s, object, c->freelist);
2582
2583	if (unlikely(!this_cpu_cmpxchg_double(
2584	s->cpu_slab->freelist, s->cpu_slab->tid,
2585	c->freelist, tid,
2586	object, next_tid(tid)))) {
2587
2588	note_cmpxchg_failure("slab_free", s, tid);
2589	goto redo;
2590	}
2591	stat(s, FREE_FASTPATH);
2592	} else
2593	__slab_free(s, page, x, addr);
2594
2595	}
2596
2597	void kmem_cache_free(struct kmem_cache s, void x)
2598	{
2599	struct page *page;
2600
2601	page = virt_to_head_page(x);
2602
2603	slab_free(s, page, x, _RET_IP_);
2604
2605	trace_kmem_cache_free(_RET_IP_, x);
2606	}
2607	EXPORT_SYMBOL(kmem_cache_free);
2608
2609	/*
2610	* Object placement in a slab is made very easy because we always start at
2611	* offset 0. If we tune the size of the object to the alignment then we can
2612	* get the required alignment by putting one properly sized object after
2613	* another.
2614	*
2615	* Notice that the allocation order determines the sizes of the per cpu
2616	* caches. Each processor has always one slab available for allocations.
2617	* Increasing the allocation order reduces the number of times that slabs
2618	* must be moved on and off the partial lists and is therefore a factor in
2619	* locking overhead.
2620	*/
2621
2622	/*
2623	* Mininum / Maximum order of slab pages. This influences locking overhead
2624	* and slab fragmentation. A higher order reduces the number of partial slabs
2625	* and increases the number of allocations possible without having to
2626	* take the list_lock.
2627	*/
2628	static int slub_min_order;
2629	static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
2630	static int slub_min_objects;
2631
2632	/*
2633	* Merge control. If this is set then no merging of slab caches will occur.
2634	* (Could be removed. This was introduced to pacify the merge skeptics.)
2635	*/
2636	static int slub_nomerge;
2637
2638	/*
2639	* Calculate the order of allocation given an slab object size.
2640	*
2641	* The order of allocation has significant impact on performance and other
2642	* system components. Generally order 0 allocations should be preferred since
2643	* order 0 does not cause fragmentation in the page allocator. Larger objects
2644	* be problematic to put into order 0 slabs because there may be too much
2645	* unused space left. We go to a higher order if more than 1/16th of the slab
2646	* would be wasted.
2647	*
2648	* In order to reach satisfactory performance we must ensure that a minimum
2649	* number of objects is in one slab. Otherwise we may generate too much
2650	* activity on the partial lists which requires taking the list_lock. This is
2651	* less a concern for large slabs though which are rarely used.
2652	*
2653	* slub_max_order specifies the order where we begin to stop considering the
2654	* number of objects in a slab as critical. If we reach slub_max_order then
2655	* we try to keep the page order as low as possible. So we accept more waste
2656	* of space in favor of a small page order.
2657	*
2658	* Higher order allocations also allow the placement of more objects in a
2659	* slab and thereby reduce object handling overhead. If the user has
2660	* requested a higher mininum order then we start with that one instead of
2661	* the smallest order which will fit the object.
2662	*/
2663	static inline int slab_order(int size, int min_objects,
2664	int max_order, int fract_leftover, int reserved)
2665	{
2666	int order;
2667	int rem;
2668	int min_order = slub_min_order;
2669
2670	if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
2671	return get_order(size * MAX_OBJS_PER_PAGE) - 1;
2672
2673	for (order = max(min_order,
2674	fls(min_objects * size - 1) - PAGE_SHIFT);
2675	order <= max_order; order++) {
2676
2677	unsigned long slab_size = PAGE_SIZE << order;
2678
2679	if (slab_size < min_objects * size + reserved)
2680	continue;
2681
2682	rem = (slab_size - reserved) % size;
2683
2684	if (rem <= slab_size / fract_leftover)
2685	break;
2686
2687	}
2688
2689	return order;
2690	}
2691
2692	static inline int calculate_order(int size, int reserved)
2693	{
2694	int order;
2695	int min_objects;
2696	int fraction;
2697	int max_objects;
2698
2699	/*
2700	* Attempt to find best configuration for a slab. This
2701	* works by first attempting to generate a layout with
2702	* the best configuration and backing off gradually.
2703	*
2704	* First we reduce the acceptable waste in a slab. Then
2705	* we reduce the minimum objects required in a slab.
2706	*/
2707	min_objects = slub_min_objects;
2708	if (!min_objects)
2709	min_objects = 4 * (fls(nr_cpu_ids) + 1);
2710	max_objects = order_objects(slub_max_order, size, reserved);
2711	min_objects = min(min_objects, max_objects);
2712
2713	while (min_objects > 1) {
2714	fraction = 16;
2715	while (fraction >= 4) {
2716	order = slab_order(size, min_objects,
2717	slub_max_order, fraction, reserved);
2718	if (order <= slub_max_order)
2719	return order;
2720	fraction /= 2;
2721	}
2722	min_objects--;
2723	}
2724
2725	/*
2726	* We were unable to place multiple objects in a slab. Now
2727	* lets see if we can place a single object there.
2728	*/
2729	order = slab_order(size, 1, slub_max_order, 1, reserved);
2730	if (order <= slub_max_order)
2731	return order;
2732
2733	/*
2734	* Doh this slab cannot be placed using slub_max_order.
2735	*/
2736	order = slab_order(size, 1, MAX_ORDER, 1, reserved);
2737	if (order < MAX_ORDER)
2738	return order;
2739	return -ENOSYS;
2740	}
2741
2742	/*
2743	* Figure out what the alignment of the objects will be.
2744	*/
2745	static unsigned long calculate_alignment(unsigned long flags,
2746	unsigned long align, unsigned long size)
2747	{
2748	/*
2749	* If the user wants hardware cache aligned objects then follow that
2750	* suggestion if the object is sufficiently large.
2751	*
2752	* The hardware cache alignment cannot override the specified
2753	* alignment though. If that is greater then use it.
2754	*/
2755	if (flags & SLAB_HWCACHE_ALIGN) {
2756	unsigned long ralign = cache_line_size();
2757	while (size <= ralign / 2)
2758	ralign /= 2;
2759	align = max(align, ralign);
2760	}
2761
2762	if (align < ARCH_SLAB_MINALIGN)
2763	align = ARCH_SLAB_MINALIGN;
2764
2765	return ALIGN(align, sizeof(void *));
2766	}
2767
2768	static void
2769	init_kmem_cache_node(struct kmem_cache_node n, struct kmem_cache s)
2770	{
2771	n->nr_partial = 0;
2772	spin_lock_init(&n->list_lock);
2773	INIT_LIST_HEAD(&n->partial);
2774	#ifdef CONFIG_SLUB_DEBUG
2775	atomic_long_set(&n->nr_slabs, 0);
2776	atomic_long_set(&n->total_objects, 0);
2777	INIT_LIST_HEAD(&n->full);
2778	#endif
2779	}
2780
2781	static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
2782	{
2783	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
2784	SLUB_PAGE_SHIFT * sizeof(struct kmem_cache_cpu));
2785
2786	/*
2787	* Must align to double word boundary for the double cmpxchg
2788	* instructions to work; see __pcpu_double_call_return_bool().
2789	*/
2790	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
2791	2 * sizeof(void *));
2792
2793	if (!s->cpu_slab)
2794	return 0;
2795
2796	init_kmem_cache_cpus(s);
2797
2798	return 1;
2799	}
2800
2801	static struct kmem_cache *kmem_cache_node;
2802
2803	/*
2804	* No kmalloc_node yet so do it by hand. We know that this is the first
2805	* slab on the node for this slabcache. There are no concurrent accesses
2806	* possible.
2807	*
2808	* Note that this function only works on the kmalloc_node_cache
2809	* when allocating for the kmalloc_node_cache. This is used for bootstrapping
2810	* memory on a fresh node that has no slab structures yet.
2811	*/
2812	static void early_kmem_cache_node_alloc(int node)
2813	{
2814	struct page *page;
2815	struct kmem_cache_node *n;
2816
2817	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
2818
2819	page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
2820
2821	BUG_ON(!page);
2822	if (page_to_nid(page) != node) {
2823	printk(KERN_ERR "SLUB: Unable to allocate memory from "
2824	"node %d\n", node);
2825	printk(KERN_ERR "SLUB: Allocating a useless per node structure "
2826	"in order to be able to continue\n");
2827	}
2828
2829	n = page->freelist;
2830	BUG_ON(!n);
2831	page->freelist = get_freepointer(kmem_cache_node, n);
2832	page->inuse = 1;
2833	page->frozen = 0;
2834	kmem_cache_node->node[node] = n;
2835	#ifdef CONFIG_SLUB_DEBUG
2836	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
2837	init_tracking(kmem_cache_node, n);
2838	#endif
2839	init_kmem_cache_node(n, kmem_cache_node);
2840	inc_slabs_node(kmem_cache_node, node, page->objects);
2841
2842	add_partial(n, page, DEACTIVATE_TO_HEAD);
2843	}
2844
2845	static void free_kmem_cache_nodes(struct kmem_cache *s)
2846	{
2847	int node;
2848
2849	for_each_node_state(node, N_NORMAL_MEMORY) {
2850	struct kmem_cache_node *n = s->node[node];
2851
2852	if (n)
2853	kmem_cache_free(kmem_cache_node, n);
2854
2855	s->node[node] = NULL;
2856	}
2857	}
2858
2859	static int init_kmem_cache_nodes(struct kmem_cache *s)
2860	{
2861	int node;
2862
2863	for_each_node_state(node, N_NORMAL_MEMORY) {
2864	struct kmem_cache_node *n;
2865
2866	if (slab_state == DOWN) {
2867	early_kmem_cache_node_alloc(node);
2868	continue;
2869	}
2870	n = kmem_cache_alloc_node(kmem_cache_node,
2871	GFP_KERNEL, node);
2872
2873	if (!n) {
2874	free_kmem_cache_nodes(s);
2875	return 0;
2876	}
2877
2878	s->node[node] = n;
2879	init_kmem_cache_node(n, s);
2880	}
2881	return 1;
2882	}
2883
2884	static void set_min_partial(struct kmem_cache *s, unsigned long min)
2885	{
2886	if (min < MIN_PARTIAL)
2887	min = MIN_PARTIAL;
2888	else if (min > MAX_PARTIAL)
2889	min = MAX_PARTIAL;
2890	s->min_partial = min;
2891	}
2892
2893	/*
2894	* calculate_sizes() determines the order and the distribution of data within
2895	* a slab object.
2896	*/
2897	static int calculate_sizes(struct kmem_cache *s, int forced_order)
2898	{
2899	unsigned long flags = s->flags;
2900	unsigned long size = s->objsize;
2901	unsigned long align = s->align;
2902	int order;
2903
2904	/*
2905	* Round up object size to the next word boundary. We can only
2906	* place the free pointer at word boundaries and this determines
2907	* the possible location of the free pointer.
2908	*/
2909	size = ALIGN(size, sizeof(void *));
2910
2911	#ifdef CONFIG_SLUB_DEBUG
2912	/*
2913	* Determine if we can poison the object itself. If the user of
2914	* the slab may touch the object after free or before allocation
2915	* then we should never poison the object itself.
2916	*/
2917	if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
2918	!s->ctor)
2919	s->flags \|= __OBJECT_POISON;
2920	else
2921	s->flags &= ~__OBJECT_POISON;
2922
2923
2924	/*
2925	* If we are Redzoning then check if there is some space between the
2926	* end of the object and the free pointer. If not then add an
2927	* additional word to have some bytes to store Redzone information.
2928	*/
2929	if ((flags & SLAB_RED_ZONE) && size == s->objsize)
2930	size += sizeof(void *);
2931	#endif
2932
2933	/*
2934	* With that we have determined the number of bytes in actual use
2935	* by the object. This is the potential offset to the free pointer.
2936	*/
2937	s->inuse = size;
2938
2939	if (((flags & (SLAB_DESTROY_BY_RCU \| SLAB_POISON)) \|\|
2940	s->ctor)) {
2941	/*
2942	* Relocate free pointer after the object if it is not
2943	* permitted to overwrite the first word of the object on
2944	* kmem_cache_free.
2945	*
2946	* This is the case if we do RCU, have a constructor or
2947	* destructor or are poisoning the objects.
2948	*/
2949	s->offset = size;
2950	size += sizeof(void *);
2951	}
2952
2953	#ifdef CONFIG_SLUB_DEBUG
2954	if (flags & SLAB_STORE_USER)
2955	/*
2956	* Need to store information about allocs and frees after
2957	* the object.
2958	*/
2959	size += 2 * sizeof(struct track);
2960
2961	if (flags & SLAB_RED_ZONE)
2962	/*
2963	* Add some empty padding so that we can catch
2964	* overwrites from earlier objects rather than let
2965	* tracking information or the free pointer be
2966	* corrupted if a user writes before the start
2967	* of the object.
2968	*/
2969	size += sizeof(void *);
2970	#endif
2971
2972	/*
2973	* Determine the alignment based on various parameters that the
2974	* user specified and the dynamic determination of cache line size
2975	* on bootup.
2976	*/
2977	align = calculate_alignment(flags, align, s->objsize);
2978	s->align = align;
2979
2980	/*
2981	* SLUB stores one object immediately after another beginning from
2982	* offset 0. In order to align the objects we have to simply size
2983	* each object to conform to the alignment.
2984	*/
2985	size = ALIGN(size, align);
2986	s->size = size;
2987	if (forced_order >= 0)
2988	order = forced_order;
2989	else
2990	order = calculate_order(size, s->reserved);
2991
2992	if (order < 0)
2993	return 0;
2994
2995	s->allocflags = 0;
2996	if (order)
2997	s->allocflags \|= __GFP_COMP;
2998
2999	if (s->flags & SLAB_CACHE_DMA)
3000	s->allocflags \|= SLUB_DMA;
3001
3002	if (s->flags & SLAB_RECLAIM_ACCOUNT)
3003	s->allocflags \|= __GFP_RECLAIMABLE;
3004
3005	/*
3006	* Determine the number of objects per slab
3007	*/
3008	s->oo = oo_make(order, size, s->reserved);
3009	s->min = oo_make(get_order(size), size, s->reserved);
3010	if (oo_objects(s->oo) > oo_objects(s->max))
3011	s->max = s->oo;
3012
3013	return !!oo_objects(s->oo);
3014
3015	}
3016
3017	static int kmem_cache_open(struct kmem_cache *s,
3018	const char *name, size_t size,
3019	size_t align, unsigned long flags,
3020	void (ctor)(void ))
3021	{
3022	memset(s, 0, kmem_size);
3023	s->name = name;
3024	s->ctor = ctor;
3025	s->objsize = size;
3026	s->align = align;
3027	s->flags = kmem_cache_flags(size, flags, name, ctor);
3028	s->reserved = 0;
3029
3030	if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU))
3031	s->reserved = sizeof(struct rcu_head);
3032
3033	if (!calculate_sizes(s, -1))
3034	goto error;
3035	if (disable_higher_order_debug) {
3036	/*
3037	* Disable debugging flags that store metadata if the min slab
3038	* order increased.
3039	*/
3040	if (get_order(s->size) > get_order(s->objsize)) {
3041	s->flags &= ~DEBUG_METADATA_FLAGS;
3042	s->offset = 0;
3043	if (!calculate_sizes(s, -1))
3044	goto error;
3045	}
3046	}
3047
3048	#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
3049	defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
3050	if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0)
3051	/* Enable fast mode */
3052	s->flags \|= __CMPXCHG_DOUBLE;
3053	#endif
3054
3055	/*
3056	* The larger the object size is, the more pages we want on the partial
3057	* list to avoid pounding the page allocator excessively.
3058	*/
3059	set_min_partial(s, ilog2(s->size) / 2);
3060
3061	/*
3062	* cpu_partial determined the maximum number of objects kept in the
3063	* per cpu partial lists of a processor.
3064	*
3065	* Per cpu partial lists mainly contain slabs that just have one
3066	* object freed. If they are used for allocation then they can be
3067	* filled up again with minimal effort. The slab will never hit the
3068	* per node partial lists and therefore no locking will be required.
3069	*
3070	* This setting also determines
3071	*
3072	* A) The number of objects from per cpu partial slabs dumped to the
3073	* per node list when we reach the limit.
3074	* B) The number of objects in cpu partial slabs to extract from the
3075	* per node list when we run out of per cpu objects. We only fetch 50%
3076	* to keep some capacity around for frees.
3077	*/
3078	if (kmem_cache_debug(s))
3079	s->cpu_partial = 0;
3080	else if (s->size >= PAGE_SIZE)
3081	s->cpu_partial = 2;
3082	else if (s->size >= 1024)
3083	s->cpu_partial = 6;
3084	else if (s->size >= 256)
3085	s->cpu_partial = 13;
3086	else
3087	s->cpu_partial = 30;
3088
3089	s->refcount = 1;
3090	#ifdef CONFIG_NUMA
3091	s->remote_node_defrag_ratio = 1000;
3092	#endif
3093	if (!init_kmem_cache_nodes(s))
3094	goto error;
3095
3096	if (alloc_kmem_cache_cpus(s))
3097	return 1;
3098
3099	free_kmem_cache_nodes(s);
3100	error:
3101	if (flags & SLAB_PANIC)
3102	panic("Cannot create slab %s size=%lu realsize=%u "
3103	"order=%u offset=%u flags=%lx\n",
3104	s->name, (unsigned long)size, s->size, oo_order(s->oo),
3105	s->offset, flags);
3106	return 0;
3107	}
3108
3109	/*
3110	* Determine the size of a slab object
3111	*/
3112	unsigned int kmem_cache_size(struct kmem_cache *s)
3113	{
3114	return s->objsize;
3115	}
3116	EXPORT_SYMBOL(kmem_cache_size);
3117
3118	static void list_slab_objects(struct kmem_cache s, struct page page,
3119	const char *text)
3120	{
3121	#ifdef CONFIG_SLUB_DEBUG
3122	void *addr = page_address(page);
3123	void *p;
3124	unsigned long map = kzalloc(BITS_TO_LONGS(page->objects)
3125	sizeof(long), GFP_ATOMIC);
3126	if (!map)
3127	return;
3128	slab_err(s, page, "%s", text);
3129	slab_lock(page);
3130
3131	get_map(s, page, map);
3132	for_each_object(p, s, addr, page->objects) {
3133
3134	if (!test_bit(slab_index(p, s, addr), map)) {
3135	printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n",
3136	p, p - addr);
3137	print_tracking(s, p);
3138	}
3139	}
3140	slab_unlock(page);
3141	kfree(map);
3142	#endif
3143	}
3144
3145	/*
3146	* Attempt to free all partial slabs on a node.
3147	* This is called from kmem_cache_close(). We must be the last thread
3148	* using the cache and therefore we do not need to lock anymore.
3149	*/
3150	static void free_partial(struct kmem_cache s, struct kmem_cache_node n)
3151	{
3152	struct page page, h;
3153
3154	list_for_each_entry_safe(page, h, &n->partial, lru) {
3155	if (!page->inuse) {
3156	remove_partial(n, page);
3157	discard_slab(s, page);
3158	} else {
3159	list_slab_objects(s, page,
3160	"Objects remaining on kmem_cache_close()");
3161	}
3162	}
3163	}
3164
3165	/*
3166	* Release all resources used by a slab cache.
3167	*/
3168	static inline int kmem_cache_close(struct kmem_cache *s)
3169	{
3170	int node;
3171
3172	flush_all(s);
3173	free_percpu(s->cpu_slab);
3174	/* Attempt to free all objects */
3175	for_each_node_state(node, N_NORMAL_MEMORY) {
3176	struct kmem_cache_node *n = get_node(s, node);
3177
3178	free_partial(s, n);
3179	if (n->nr_partial \|\| slabs_node(s, node))
3180	return 1;
3181	}
3182	free_kmem_cache_nodes(s);
3183	return 0;
3184	}
3185
3186	/*
3187	* Close a cache and release the kmem_cache structure
3188	* (must be used for caches created using kmem_cache_create)
3189	*/
3190	void kmem_cache_destroy(struct kmem_cache *s)
3191	{
3192	down_write(&slub_lock);
3193	s->refcount--;
3194	if (!s->refcount) {
3195	list_del(&s->list);
3196	up_write(&slub_lock);
3197	if (kmem_cache_close(s)) {
3198	printk(KERN_ERR "SLUB %s: %s called for cache that "
3199	"still has objects.\n", s->name, __func__);
3200	dump_stack();
3201	}
3202	if (s->flags & SLAB_DESTROY_BY_RCU)
3203	rcu_barrier();
3204	sysfs_slab_remove(s);
3205	} else
3206	up_write(&slub_lock);
3207	}
3208	EXPORT_SYMBOL(kmem_cache_destroy);
3209
3210	/********************************************************************
3211	* Kmalloc subsystem
3212	*******************************************************************/
3213
3214	struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
3215	EXPORT_SYMBOL(kmalloc_caches);
3216
3217	static struct kmem_cache *kmem_cache;
3218
3219	#ifdef CONFIG_ZONE_DMA
3220	static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
3221	#endif
3222
3223	static int __init setup_slub_min_order(char *str)
3224	{
3225	get_option(&str, &slub_min_order);
3226
3227	return 1;
3228	}
3229
3230	__setup("slub_min_order=", setup_slub_min_order);
3231
3232	static int __init setup_slub_max_order(char *str)
3233	{
3234	get_option(&str, &slub_max_order);
3235	slub_max_order = min(slub_max_order, MAX_ORDER - 1);
3236
3237	return 1;
3238	}
3239
3240	__setup("slub_max_order=", setup_slub_max_order);
3241
3242	static int __init setup_slub_min_objects(char *str)
3243	{
3244	get_option(&str, &slub_min_objects);
3245
3246	return 1;
3247	}
3248
3249	__setup("slub_min_objects=", setup_slub_min_objects);
3250
3251	static int __init setup_slub_nomerge(char *str)
3252	{
3253	slub_nomerge = 1;
3254	return 1;
3255	}
3256
3257	__setup("slub_nomerge", setup_slub_nomerge);
3258
3259	static struct kmem_cache __init create_kmalloc_cache(const char name,
3260	int size, unsigned int flags)
3261	{
3262	struct kmem_cache *s;
3263
3264	s = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3265
3266	/*
3267	* This function is called with IRQs disabled during early-boot on
3268	* single CPU so there's no need to take slub_lock here.
3269	*/
3270	if (!kmem_cache_open(s, name, size, ARCH_KMALLOC_MINALIGN,
3271	flags, NULL))
3272	goto panic;
3273
3274	list_add(&s->list, &slab_caches);
3275	return s;
3276
3277	panic:
3278	panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
3279	return NULL;
3280	}
3281
3282	/*
3283	* Conversion table for small slabs sizes / 8 to the index in the
3284	* kmalloc array. This is necessary for slabs < 192 since we have non power
3285	* of two cache sizes there. The size of larger slabs can be determined using
3286	* fls.
3287	*/
3288	static s8 size_index[24] = {
3289	3, /* 8 */
3290	4, /* 16 */
3291	5, /* 24 */
3292	5, /* 32 */
3293	6, /* 40 */
3294	6, /* 48 */
3295	6, /* 56 */
3296	6, /* 64 */
3297	1, /* 72 */
3298	1, /* 80 */
3299	1, /* 88 */
3300	1, /* 96 */
3301	7, /* 104 */
3302	7, /* 112 */
3303	7, /* 120 */
3304	7, /* 128 */
3305	2, /* 136 */
3306	2, /* 144 */
3307	2, /* 152 */
3308	2, /* 160 */
3309	2, /* 168 */
3310	2, /* 176 */
3311	2, /* 184 */
3312	2 /* 192 */
3313	};
3314
3315	static inline int size_index_elem(size_t bytes)
3316	{
3317	return (bytes - 1) / 8;
3318	}
3319
3320	static struct kmem_cache *get_slab(size_t size, gfp_t flags)
3321	{
3322	int index;
3323
3324	if (size <= 192) {
3325	if (!size)
3326	return ZERO_SIZE_PTR;
3327
3328	index = size_index[size_index_elem(size)];
3329	} else
3330	index = fls(size - 1);
3331
3332	#ifdef CONFIG_ZONE_DMA
3333	if (unlikely((flags & SLUB_DMA)))
3334	return kmalloc_dma_caches[index];
3335
3336	#endif
3337	return kmalloc_caches[index];
3338	}
3339
3340	void *__kmalloc(size_t size, gfp_t flags)
3341	{
3342	struct kmem_cache *s;
3343	void *ret;
3344
3345	if (unlikely(size > SLUB_MAX_SIZE))
3346	return kmalloc_large(size, flags);
3347
3348	s = get_slab(size, flags);
3349
3350	if (unlikely(ZERO_OR_NULL_PTR(s)))
3351	return s;
3352
3353	ret = slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_);
3354
3355	trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
3356
3357	return ret;
3358	}
3359	EXPORT_SYMBOL(__kmalloc);
3360
3361	#ifdef CONFIG_NUMA
3362	static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
3363	{
3364	struct page *page;
3365	void *ptr = NULL;
3366
3367	flags \|= __GFP_COMP \| __GFP_NOTRACK;
3368	page = alloc_pages_node(node, flags, get_order(size));
3369	if (page)
3370	ptr = page_address(page);
3371
3372	kmemleak_alloc(ptr, size, 1, flags);
3373	return ptr;
3374	}
3375
3376	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3377	{
3378	struct kmem_cache *s;
3379	void *ret;
3380
3381	if (unlikely(size > SLUB_MAX_SIZE)) {
3382	ret = kmalloc_large_node(size, flags, node);
3383
3384	trace_kmalloc_node(_RET_IP_, ret,
3385	size, PAGE_SIZE << get_order(size),
3386	flags, node);
3387
3388	return ret;
3389	}
3390
3391	s = get_slab(size, flags);
3392
3393	if (unlikely(ZERO_OR_NULL_PTR(s)))
3394	return s;
3395
3396	ret = slab_alloc(s, flags, node, _RET_IP_);
3397
3398	trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
3399
3400	return ret;
3401	}
3402	EXPORT_SYMBOL(__kmalloc_node);
3403	#endif
3404
3405	size_t ksize(const void *object)
3406	{
3407	struct page *page;
3408
3409	if (unlikely(object == ZERO_SIZE_PTR))
3410	return 0;
3411
3412	page = virt_to_head_page(object);
3413
3414	if (unlikely(!PageSlab(page))) {
3415	WARN_ON(!PageCompound(page));
3416	return PAGE_SIZE << compound_order(page);
3417	}
3418
3419	return slab_ksize(page->slab);
3420	}
3421	EXPORT_SYMBOL(ksize);
3422
3423	#ifdef CONFIG_SLUB_DEBUG
3424	bool verify_mem_not_deleted(const void *x)
3425	{
3426	struct page *page;
3427	void object = (void )x;
3428	unsigned long flags;
3429	bool rv;
3430
3431	if (unlikely(ZERO_OR_NULL_PTR(x)))
3432	return false;
3433
3434	local_irq_save(flags);
3435
3436	page = virt_to_head_page(x);
3437	if (unlikely(!PageSlab(page))) {
3438	/* maybe it was from stack? */
3439	rv = true;
3440	goto out_unlock;
3441	}
3442
3443	slab_lock(page);
3444	if (on_freelist(page->slab, page, object)) {
3445	object_err(page->slab, page, object, "Object is on free-list");
3446	rv = false;
3447	} else {
3448	rv = true;
3449	}
3450	slab_unlock(page);
3451
3452	out_unlock:
3453	local_irq_restore(flags);
3454	return rv;
3455	}
3456	EXPORT_SYMBOL(verify_mem_not_deleted);
3457	#endif
3458
3459	void kfree(const void *x)
3460	{
3461	struct page *page;
3462	void object = (void )x;
3463
3464	trace_kfree(_RET_IP_, x);
3465
3466	if (unlikely(ZERO_OR_NULL_PTR(x)))
3467	return;
3468
3469	page = virt_to_head_page(x);
3470	if (unlikely(!PageSlab(page))) {
3471	BUG_ON(!PageCompound(page));
3472	kmemleak_free(x);
3473	put_page(page);
3474	return;
3475	}
3476	slab_free(page->slab, page, object, _RET_IP_);
3477	}
3478	EXPORT_SYMBOL(kfree);
3479
3480	/*
3481	* kmem_cache_shrink removes empty slabs from the partial lists and sorts
3482	* the remaining slabs by the number of items in use. The slabs with the
3483	* most items in use come first. New allocations will then fill those up
3484	* and thus they can be removed from the partial lists.
3485	*
3486	* The slabs with the least items are placed last. This results in them
3487	* being allocated from last increasing the chance that the last objects
3488	* are freed in them.
3489	*/
3490	int kmem_cache_shrink(struct kmem_cache *s)
3491	{
3492	int node;
3493	int i;
3494	struct kmem_cache_node *n;
3495	struct page *page;
3496	struct page *t;
3497	int objects = oo_objects(s->max);
3498	struct list_head *slabs_by_inuse =
3499	kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
3500	unsigned long flags;
3501
3502	if (!slabs_by_inuse)
3503	return -ENOMEM;
3504
3505	flush_all(s);
3506	for_each_node_state(node, N_NORMAL_MEMORY) {
3507	n = get_node(s, node);
3508
3509	if (!n->nr_partial)
3510	continue;
3511
3512	for (i = 0; i < objects; i++)
3513	INIT_LIST_HEAD(slabs_by_inuse + i);
3514
3515	spin_lock_irqsave(&n->list_lock, flags);
3516
3517	/*
3518	* Build lists indexed by the items in use in each slab.
3519	*
3520	* Note that concurrent frees may occur while we hold the
3521	* list_lock. page->inuse here is the upper limit.
3522	*/
3523	list_for_each_entry_safe(page, t, &n->partial, lru) {
3524	list_move(&page->lru, slabs_by_inuse + page->inuse);
3525	if (!page->inuse)
3526	n->nr_partial--;
3527	}
3528
3529	/*
3530	* Rebuild the partial list with the slabs filled up most
3531	* first and the least used slabs at the end.
3532	*/
3533	for (i = objects - 1; i > 0; i--)
3534	list_splice(slabs_by_inuse + i, n->partial.prev);
3535
3536	spin_unlock_irqrestore(&n->list_lock, flags);
3537
3538	/* Release empty slabs */
3539	list_for_each_entry_safe(page, t, slabs_by_inuse, lru)
3540	discard_slab(s, page);
3541	}
3542
3543	kfree(slabs_by_inuse);
3544	return 0;
3545	}
3546	EXPORT_SYMBOL(kmem_cache_shrink);
3547
3548	#if defined(CONFIG_MEMORY_HOTPLUG)
3549	static int slab_mem_going_offline_callback(void *arg)
3550	{
3551	struct kmem_cache *s;
3552
3553	down_read(&slub_lock);
3554	list_for_each_entry(s, &slab_caches, list)
3555	kmem_cache_shrink(s);
3556	up_read(&slub_lock);
3557
3558	return 0;
3559	}
3560
3561	static void slab_mem_offline_callback(void *arg)
3562	{
3563	struct kmem_cache_node *n;
3564	struct kmem_cache *s;
3565	struct memory_notify *marg = arg;
3566	int offline_node;
3567
3568	offline_node = marg->status_change_nid;
3569
3570	/*
3571	* If the node still has available memory. we need kmem_cache_node
3572	* for it yet.
3573	*/
3574	if (offline_node < 0)
3575	return;
3576
3577	down_read(&slub_lock);
3578	list_for_each_entry(s, &slab_caches, list) {
3579	n = get_node(s, offline_node);
3580	if (n) {
3581	/*
3582	* if n->nr_slabs > 0, slabs still exist on the node
3583	* that is going down. We were unable to free them,
3584	* and offline_pages() function shouldn't call this
3585	* callback. So, we must fail.
3586	*/
3587	BUG_ON(slabs_node(s, offline_node));
3588
3589	s->node[offline_node] = NULL;
3590	kmem_cache_free(kmem_cache_node, n);
3591	}
3592	}
3593	up_read(&slub_lock);
3594	}
3595
3596	static int slab_mem_going_online_callback(void *arg)
3597	{
3598	struct kmem_cache_node *n;
3599	struct kmem_cache *s;
3600	struct memory_notify *marg = arg;
3601	int nid = marg->status_change_nid;
3602	int ret = 0;
3603
3604	/*
3605	* If the node's memory is already available, then kmem_cache_node is
3606	* already created. Nothing to do.
3607	*/
3608	if (nid < 0)
3609	return 0;
3610
3611	/*
3612	* We are bringing a node online. No memory is available yet. We must
3613	* allocate a kmem_cache_node structure in order to bring the node
3614	* online.
3615	*/
3616	down_read(&slub_lock);
3617	list_for_each_entry(s, &slab_caches, list) {
3618	/*
3619	* XXX: kmem_cache_alloc_node will fallback to other nodes
3620	* since memory is not yet available from the node that
3621	* is brought up.
3622	*/
3623	n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
3624	if (!n) {
3625	ret = -ENOMEM;
3626	goto out;
3627	}
3628	init_kmem_cache_node(n, s);
3629	s->node[nid] = n;
3630	}
3631	out:
3632	up_read(&slub_lock);
3633	return ret;
3634	}
3635
3636	static int slab_memory_callback(struct notifier_block *self,
3637	unsigned long action, void *arg)
3638	{
3639	int ret = 0;
3640
3641	switch (action) {
3642	case MEM_GOING_ONLINE:
3643	ret = slab_mem_going_online_callback(arg);
3644	break;
3645	case MEM_GOING_OFFLINE:
3646	ret = slab_mem_going_offline_callback(arg);
3647	break;
3648	case MEM_OFFLINE:
3649	case MEM_CANCEL_ONLINE:
3650	slab_mem_offline_callback(arg);
3651	break;
3652	case MEM_ONLINE:
3653	case MEM_CANCEL_OFFLINE:
3654	break;
3655	}
3656	if (ret)
3657	ret = notifier_from_errno(ret);
3658	else
3659	ret = NOTIFY_OK;
3660	return ret;
3661	}
3662
3663	#endif /* CONFIG_MEMORY_HOTPLUG */
3664
3665	/********************************************************************
3666	* Basic setup of slabs
3667	*******************************************************************/
3668
3669	/*
3670	* Used for early kmem_cache structures that were allocated using
3671	* the page allocator
3672	*/
3673
3674	static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
3675	{
3676	int node;
3677
3678	list_add(&s->list, &slab_caches);
3679	s->refcount = -1;
3680
3681	for_each_node_state(node, N_NORMAL_MEMORY) {
3682	struct kmem_cache_node *n = get_node(s, node);
3683	struct page *p;
3684
3685	if (n) {
3686	list_for_each_entry(p, &n->partial, lru)
3687	p->slab = s;
3688
3689	#ifdef CONFIG_SLUB_DEBUG
3690	list_for_each_entry(p, &n->full, lru)
3691	p->slab = s;
3692	#endif
3693	}
3694	}
3695	}
3696
3697	void __init kmem_cache_init(void)
3698	{
3699	int i;
3700	int caches = 0;
3701	struct kmem_cache *temp_kmem_cache;
3702	int order;
3703	struct kmem_cache *temp_kmem_cache_node;
3704	unsigned long kmalloc_size;
3705
3706	if (debug_guardpage_minorder())
3707	slub_max_order = 0;
3708
3709	kmem_size = offsetof(struct kmem_cache, node) +
3710	nr_node_ids * sizeof(struct kmem_cache_node *);
3711
3712	/* Allocate two kmem_caches from the page allocator */
3713	kmalloc_size = ALIGN(kmem_size, cache_line_size());
3714	order = get_order(2 * kmalloc_size);
3715	kmem_cache = (void *)__get_free_pages(GFP_NOWAIT, order);
3716
3717	/*
3718	* Must first have the slab cache available for the allocations of the
3719	* struct kmem_cache_node's. There is special bootstrap code in
3720	* kmem_cache_open for slab_state == DOWN.
3721	*/
3722	kmem_cache_node = (void *)kmem_cache + kmalloc_size;
3723
3724	kmem_cache_open(kmem_cache_node, "kmem_cache_node",
3725	sizeof(struct kmem_cache_node),
3726	0, SLAB_HWCACHE_ALIGN \| SLAB_PANIC, NULL);
3727
3728	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
3729
3730	/* Able to allocate the per node structures */
3731	slab_state = PARTIAL;
3732
3733	temp_kmem_cache = kmem_cache;
3734	kmem_cache_open(kmem_cache, "kmem_cache", kmem_size,
3735	0, SLAB_HWCACHE_ALIGN \| SLAB_PANIC, NULL);
3736	kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3737	memcpy(kmem_cache, temp_kmem_cache, kmem_size);
3738
3739	/*
3740	* Allocate kmem_cache_node properly from the kmem_cache slab.
3741	* kmem_cache_node is separately allocated so no need to
3742	* update any list pointers.
3743	*/
3744	temp_kmem_cache_node = kmem_cache_node;
3745
3746	kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3747	memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
3748
3749	kmem_cache_bootstrap_fixup(kmem_cache_node);
3750
3751	caches++;
3752	kmem_cache_bootstrap_fixup(kmem_cache);
3753	caches++;
3754	/* Free temporary boot structure */
3755	free_pages((unsigned long)temp_kmem_cache, order);
3756
3757	/* Now we can use the kmem_cache to allocate kmalloc slabs */
3758
3759	/*
3760	* Patch up the size_index table if we have strange large alignment
3761	* requirements for the kmalloc array. This is only the case for
3762	* MIPS it seems. The standard arches will not generate any code here.
3763	*
3764	* Largest permitted alignment is 256 bytes due to the way we
3765	* handle the index determination for the smaller caches.
3766	*
3767	* Make sure that nothing crazy happens if someone starts tinkering
3768	* around with ARCH_KMALLOC_MINALIGN
3769	*/
3770	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 \|\|
3771	(KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
3772
3773	for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
3774	int elem = size_index_elem(i);
3775	if (elem >= ARRAY_SIZE(size_index))
3776	break;
3777	size_index[elem] = KMALLOC_SHIFT_LOW;
3778	}
3779
3780	if (KMALLOC_MIN_SIZE == 64) {
3781	/*
3782	* The 96 byte size cache is not used if the alignment
3783	* is 64 byte.
3784	*/
3785	for (i = 64 + 8; i <= 96; i += 8)
3786	size_index[size_index_elem(i)] = 7;
3787	} else if (KMALLOC_MIN_SIZE == 128) {
3788	/*
3789	* The 192 byte sized cache is not used if the alignment
3790	* is 128 byte. Redirect kmalloc to use the 256 byte cache
3791	* instead.
3792	*/
3793	for (i = 128 + 8; i <= 192; i += 8)
3794	size_index[size_index_elem(i)] = 8;
3795	}
3796
3797	/* Caches that are not of the two-to-the-power-of size */
3798	if (KMALLOC_MIN_SIZE <= 32) {
3799	kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
3800	caches++;
3801	}
3802
3803	if (KMALLOC_MIN_SIZE <= 64) {
3804	kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
3805	caches++;
3806	}
3807
3808	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
3809	kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
3810	caches++;
3811	}
3812
3813	slab_state = UP;
3814
3815	/* Provide the correct kmalloc names now that the caches are up */
3816	if (KMALLOC_MIN_SIZE <= 32) {
3817	kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
3818	BUG_ON(!kmalloc_caches[1]->name);
3819	}
3820
3821	if (KMALLOC_MIN_SIZE <= 64) {
3822	kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
3823	BUG_ON(!kmalloc_caches[2]->name);
3824	}
3825
3826	for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
3827	char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
3828
3829	BUG_ON(!s);
3830	kmalloc_caches[i]->name = s;
3831	}
3832
3833	#ifdef CONFIG_SMP
3834	register_cpu_notifier(&slab_notifier);
3835	#endif
3836
3837	#ifdef CONFIG_ZONE_DMA
3838	for (i = 0; i < SLUB_PAGE_SHIFT; i++) {
3839	struct kmem_cache *s = kmalloc_caches[i];
3840
3841	if (s && s->size) {
3842	char *name = kasprintf(GFP_NOWAIT,
3843	"dma-kmalloc-%d", s->objsize);
3844
3845	BUG_ON(!name);
3846	kmalloc_dma_caches[i] = create_kmalloc_cache(name,
3847	s->objsize, SLAB_CACHE_DMA);
3848	}
3849	}
3850	#endif
3851	printk(KERN_INFO
3852	"SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
3853	" CPUs=%d, Nodes=%d\n",
3854	caches, cache_line_size(),
3855	slub_min_order, slub_max_order, slub_min_objects,
3856	nr_cpu_ids, nr_node_ids);
3857	}
3858
3859	void __init kmem_cache_init_late(void)
3860	{
3861	}
3862
3863	/*
3864	* Find a mergeable slab cache
3865	*/
3866	static int slab_unmergeable(struct kmem_cache *s)
3867	{
3868	if (slub_nomerge \|\| (s->flags & SLUB_NEVER_MERGE))
3869	return 1;
3870
3871	if (s->ctor)
3872	return 1;
3873
3874	/*
3875	* We may have set a slab to be unmergeable during bootstrap.
3876	*/
3877	if (s->refcount < 0)
3878	return 1;
3879
3880	return 0;
3881	}
3882
3883	static struct kmem_cache *find_mergeable(size_t size,
3884	size_t align, unsigned long flags, const char *name,
3885	void (ctor)(void ))
3886	{
3887	struct kmem_cache *s;
3888
3889	if (slub_nomerge \|\| (flags & SLUB_NEVER_MERGE))
3890	return NULL;
3891
3892	if (ctor)
3893	return NULL;
3894
3895	size = ALIGN(size, sizeof(void *));
3896	align = calculate_alignment(flags, align, size);
3897	size = ALIGN(size, align);
3898	flags = kmem_cache_flags(size, flags, name, NULL);
3899
3900	list_for_each_entry(s, &slab_caches, list) {
3901	if (slab_unmergeable(s))
3902	continue;
3903
3904	if (size > s->size)
3905	continue;
3906
3907	if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
3908	continue;
3909	/*
3910	* Check if alignment is compatible.
3911	* Courtesy of Adrian Drzewiecki
3912	*/
3913	if ((s->size & ~(align - 1)) != s->size)
3914	continue;
3915
3916	if (s->size - size >= sizeof(void *))
3917	continue;
3918
3919	return s;
3920	}
3921	return NULL;
3922	}
3923
3924	struct kmem_cache kmem_cache_create(const char name, size_t size,
3925	size_t align, unsigned long flags, void (ctor)(void ))
3926	{
3927	struct kmem_cache *s;
3928	char *n;
3929
3930	if (WARN_ON(!name))
3931	return NULL;
3932
3933	down_write(&slub_lock);
3934	s = find_mergeable(size, align, flags, name, ctor);
3935	if (s) {
3936	s->refcount++;
3937	/*
3938	* Adjust the object sizes so that we clear
3939	* the complete object on kzalloc.
3940	*/
3941	s->objsize = max(s->objsize, (int)size);
3942	s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
3943
3944	if (sysfs_slab_alias(s, name)) {
3945	s->refcount--;
3946	goto err;
3947	}
3948	up_write(&slub_lock);
3949	return s;
3950	}
3951
3952	n = kstrdup(name, GFP_KERNEL);
3953	if (!n)
3954	goto err;
3955
3956	s = kmalloc(kmem_size, GFP_KERNEL);
3957	if (s) {
3958	if (kmem_cache_open(s, n,
3959	size, align, flags, ctor)) {
3960	list_add(&s->list, &slab_caches);
3961	up_write(&slub_lock);
3962	if (sysfs_slab_add(s)) {
3963	down_write(&slub_lock);
3964	list_del(&s->list);
3965	kfree(n);
3966	kfree(s);
3967	goto err;
3968	}
3969	return s;
3970	}
3971	kfree(n);
3972	kfree(s);
3973	}
3974	err:
3975	up_write(&slub_lock);
3976
3977	if (flags & SLAB_PANIC)
3978	panic("Cannot create slabcache %s\n", name);
3979	else
3980	s = NULL;
3981	return s;
3982	}
3983	EXPORT_SYMBOL(kmem_cache_create);
3984
3985	#ifdef CONFIG_SMP
3986	/*
3987	* Use the cpu notifier to insure that the cpu slabs are flushed when
3988	* necessary.
3989	*/
3990	static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
3991	unsigned long action, void *hcpu)
3992	{
3993	long cpu = (long)hcpu;
3994	struct kmem_cache *s;
3995	unsigned long flags;
3996
3997	switch (action) {
3998	case CPU_UP_CANCELED:
3999	case CPU_UP_CANCELED_FROZEN:
4000	case CPU_DEAD:
4001	case CPU_DEAD_FROZEN:
4002	down_read(&slub_lock);
4003	list_for_each_entry(s, &slab_caches, list) {
4004	local_irq_save(flags);
4005	__flush_cpu_slab(s, cpu);
4006	local_irq_restore(flags);
4007	}
4008	up_read(&slub_lock);
4009	break;
4010	default:
4011	break;
4012	}
4013	return NOTIFY_OK;
4014	}
4015
4016	static struct notifier_block __cpuinitdata slab_notifier = {
4017	.notifier_call = slab_cpuup_callback
4018	};
4019
4020	#endif
4021
4022	void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
4023	{
4024	struct kmem_cache *s;
4025	void *ret;
4026
4027	if (unlikely(size > SLUB_MAX_SIZE))
4028	return kmalloc_large(size, gfpflags);
4029
4030	s = get_slab(size, gfpflags);
4031
4032	if (unlikely(ZERO_OR_NULL_PTR(s)))
4033	return s;
4034
4035	ret = slab_alloc(s, gfpflags, NUMA_NO_NODE, caller);
4036
4037	/* Honor the call site pointer we received. */
4038	trace_kmalloc(caller, ret, size, s->size, gfpflags);
4039
4040	return ret;
4041	}
4042
4043	#ifdef CONFIG_NUMA
4044	void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
4045	int node, unsigned long caller)
4046	{
4047	struct kmem_cache *s;
4048	void *ret;
4049
4050	if (unlikely(size > SLUB_MAX_SIZE)) {
4051	ret = kmalloc_large_node(size, gfpflags, node);
4052
4053	trace_kmalloc_node(caller, ret,
4054	size, PAGE_SIZE << get_order(size),
4055	gfpflags, node);
4056
4057	return ret;
4058	}
4059
4060	s = get_slab(size, gfpflags);
4061
4062	if (unlikely(ZERO_OR_NULL_PTR(s)))
4063	return s;
4064
4065	ret = slab_alloc(s, gfpflags, node, caller);
4066
4067	/* Honor the call site pointer we received. */
4068	trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
4069
4070	return ret;
4071	}
4072	#endif
4073
4074	#ifdef CONFIG_SYSFS
4075	static int count_inuse(struct page *page)
4076	{
4077	return page->inuse;
4078	}
4079
4080	static int count_total(struct page *page)
4081	{
4082	return page->objects;
4083	}
4084	#endif
4085
4086	#ifdef CONFIG_SLUB_DEBUG
4087	static int validate_slab(struct kmem_cache s, struct page page,
4088	unsigned long *map)
4089	{
4090	void *p;
4091	void *addr = page_address(page);
4092
4093	if (!check_slab(s, page) \|\|
4094	!on_freelist(s, page, NULL))
4095	return 0;
4096
4097	/* Now we know that a valid freelist exists */
4098	bitmap_zero(map, page->objects);
4099
4100	get_map(s, page, map);
4101	for_each_object(p, s, addr, page->objects) {
4102	if (test_bit(slab_index(p, s, addr), map))
4103	if (!check_object(s, page, p, SLUB_RED_INACTIVE))
4104	return 0;
4105	}
4106
4107	for_each_object(p, s, addr, page->objects)
4108	if (!test_bit(slab_index(p, s, addr), map))
4109	if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4110	return 0;
4111	return 1;
4112	}
4113
4114	static void validate_slab_slab(struct kmem_cache s, struct page page,
4115	unsigned long *map)
4116	{
4117	slab_lock(page);
4118	validate_slab(s, page, map);
4119	slab_unlock(page);
4120	}
4121
4122	static int validate_slab_node(struct kmem_cache *s,
4123	struct kmem_cache_node n, unsigned long map)
4124	{
4125	unsigned long count = 0;
4126	struct page *page;
4127	unsigned long flags;
4128
4129	spin_lock_irqsave(&n->list_lock, flags);
4130
4131	list_for_each_entry(page, &n->partial, lru) {
4132	validate_slab_slab(s, page, map);
4133	count++;
4134	}
4135	if (count != n->nr_partial)
4136	printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
4137	"counter=%ld\n", s->name, count, n->nr_partial);
4138
4139	if (!(s->flags & SLAB_STORE_USER))
4140	goto out;
4141
4142	list_for_each_entry(page, &n->full, lru) {
4143	validate_slab_slab(s, page, map);
4144	count++;
4145	}
4146	if (count != atomic_long_read(&n->nr_slabs))
4147	printk(KERN_ERR "SLUB: %s %ld slabs counted but "
4148	"counter=%ld\n", s->name, count,
4149	atomic_long_read(&n->nr_slabs));
4150
4151	out:
4152	spin_unlock_irqrestore(&n->list_lock, flags);
4153	return count;
4154	}
4155
4156	static long validate_slab_cache(struct kmem_cache *s)
4157	{
4158	int node;
4159	unsigned long count = 0;
4160	unsigned long map = kmalloc(BITS_TO_LONGS(oo_objects(s->max))
4161	sizeof(unsigned long), GFP_KERNEL);
4162
4163	if (!map)
4164	return -ENOMEM;
4165
4166	flush_all(s);
4167	for_each_node_state(node, N_NORMAL_MEMORY) {
4168	struct kmem_cache_node *n = get_node(s, node);
4169
4170	count += validate_slab_node(s, n, map);
4171	}
4172	kfree(map);
4173	return count;
4174	}
4175	/*
4176	* Generate lists of code addresses where slabcache objects are allocated
4177	* and freed.
4178	*/
4179
4180	struct location {
4181	unsigned long count;
4182	unsigned long addr;
4183	long long sum_time;
4184	long min_time;
4185	long max_time;
4186	long min_pid;
4187	long max_pid;
4188	DECLARE_BITMAP(cpus, NR_CPUS);
4189	nodemask_t nodes;
4190	};
4191
4192	struct loc_track {
4193	unsigned long max;
4194	unsigned long count;
4195	struct location *loc;
4196	};
4197
4198	static void free_loc_track(struct loc_track *t)
4199	{
4200	if (t->max)
4201	free_pages((unsigned long)t->loc,
4202	get_order(sizeof(struct location) * t->max));
4203	}
4204
4205	static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
4206	{
4207	struct location *l;
4208	int order;
4209
4210	order = get_order(sizeof(struct location) * max);
4211
4212	l = (void *)__get_free_pages(flags, order);
4213	if (!l)
4214	return 0;
4215
4216	if (t->count) {
4217	memcpy(l, t->loc, sizeof(struct location) * t->count);
4218	free_loc_track(t);
4219	}
4220	t->max = max;
4221	t->loc = l;
4222	return 1;
4223	}
4224
4225	static int add_location(struct loc_track t, struct kmem_cache s,
4226	const struct track *track)
4227	{
4228	long start, end, pos;
4229	struct location *l;
4230	unsigned long caddr;
4231	unsigned long age = jiffies - track->when;
4232
4233	start = -1;
4234	end = t->count;
4235
4236	for ( ; ; ) {
4237	pos = start + (end - start + 1) / 2;
4238
4239	/*
4240	* There is nothing at "end". If we end up there
4241	* we need to add something to before end.
4242	*/
4243	if (pos == end)
4244	break;
4245
4246	caddr = t->loc[pos].addr;
4247	if (track->addr == caddr) {
4248
4249	l = &t->loc[pos];
4250	l->count++;
4251	if (track->when) {
4252	l->sum_time += age;
4253	if (age < l->min_time)
4254	l->min_time = age;
4255	if (age > l->max_time)
4256	l->max_time = age;
4257
4258	if (track->pid < l->min_pid)
4259	l->min_pid = track->pid;
4260	if (track->pid > l->max_pid)
4261	l->max_pid = track->pid;
4262
4263	cpumask_set_cpu(track->cpu,
4264	to_cpumask(l->cpus));
4265	}
4266	node_set(page_to_nid(virt_to_page(track)), l->nodes);
4267	return 1;
4268	}
4269
4270	if (track->addr < caddr)
4271	end = pos;
4272	else
4273	start = pos;
4274	}
4275
4276	/*
4277	* Not found. Insert new tracking element.
4278	*/
4279	if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
4280	return 0;
4281
4282	l = t->loc + pos;
4283	if (pos < t->count)
4284	memmove(l + 1, l,
4285	(t->count - pos) * sizeof(struct location));
4286	t->count++;
4287	l->count = 1;
4288	l->addr = track->addr;
4289	l->sum_time = age;
4290	l->min_time = age;
4291	l->max_time = age;
4292	l->min_pid = track->pid;
4293	l->max_pid = track->pid;
4294	cpumask_clear(to_cpumask(l->cpus));
4295	cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
4296	nodes_clear(l->nodes);
4297	node_set(page_to_nid(virt_to_page(track)), l->nodes);
4298	return 1;
4299	}
4300
4301	static void process_slab(struct loc_track t, struct kmem_cache s,
4302	struct page *page, enum track_item alloc,
4303	unsigned long *map)
4304	{
4305	void *addr = page_address(page);
4306	void *p;
4307
4308	bitmap_zero(map, page->objects);
4309	get_map(s, page, map);
4310
4311	for_each_object(p, s, addr, page->objects)
4312	if (!test_bit(slab_index(p, s, addr), map))
4313	add_location(t, s, get_track(s, p, alloc));
4314	}
4315
4316	static int list_locations(struct kmem_cache s, char buf,
4317	enum track_item alloc)
4318	{
4319	int len = 0;
4320	unsigned long i;
4321	struct loc_track t = { 0, 0, NULL };
4322	int node;
4323	unsigned long map = kmalloc(BITS_TO_LONGS(oo_objects(s->max))
4324	sizeof(unsigned long), GFP_KERNEL);
4325
4326	if (!map \|\| !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
4327	GFP_TEMPORARY)) {
4328	kfree(map);
4329	return sprintf(buf, "Out of memory\n");
4330	}
4331	/* Push back cpu slabs */
4332	flush_all(s);
4333
4334	for_each_node_state(node, N_NORMAL_MEMORY) {
4335	struct kmem_cache_node *n = get_node(s, node);
4336	unsigned long flags;
4337	struct page *page;
4338
4339	if (!atomic_long_read(&n->nr_slabs))
4340	continue;
4341
4342	spin_lock_irqsave(&n->list_lock, flags);
4343	list_for_each_entry(page, &n->partial, lru)
4344	process_slab(&t, s, page, alloc, map);
4345	list_for_each_entry(page, &n->full, lru)
4346	process_slab(&t, s, page, alloc, map);
4347	spin_unlock_irqrestore(&n->list_lock, flags);
4348	}
4349
4350	for (i = 0; i < t.count; i++) {
4351	struct location *l = &t.loc[i];
4352
4353	if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
4354	break;
4355	len += sprintf(buf + len, "%7ld ", l->count);
4356
4357	if (l->addr)
4358	len += sprintf(buf + len, "%pS", (void *)l->addr);
4359	else
4360	len += sprintf(buf + len, "<not-available>");
4361
4362	if (l->sum_time != l->min_time) {
4363	len += sprintf(buf + len, " age=%ld/%ld/%ld",
4364	l->min_time,
4365	(long)div_u64(l->sum_time, l->count),
4366	l->max_time);
4367	} else
4368	len += sprintf(buf + len, " age=%ld",
4369	l->min_time);
4370
4371	if (l->min_pid != l->max_pid)
4372	len += sprintf(buf + len, " pid=%ld-%ld",
4373	l->min_pid, l->max_pid);
4374	else
4375	len += sprintf(buf + len, " pid=%ld",
4376	l->min_pid);
4377
4378	if (num_online_cpus() > 1 &&
4379	!cpumask_empty(to_cpumask(l->cpus)) &&
4380	len < PAGE_SIZE - 60) {
4381	len += sprintf(buf + len, " cpus=");
4382	len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50,
4383	to_cpumask(l->cpus));
4384	}
4385
4386	if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
4387	len < PAGE_SIZE - 60) {
4388	len += sprintf(buf + len, " nodes=");
4389	len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
4390	l->nodes);
4391	}
4392
4393	len += sprintf(buf + len, "\n");
4394	}
4395
4396	free_loc_track(&t);
4397	kfree(map);
4398	if (!t.count)
4399	len += sprintf(buf, "No data\n");
4400	return len;
4401	}
4402	#endif
4403
4404	#ifdef SLUB_RESILIENCY_TEST
4405	static void resiliency_test(void)
4406	{
4407	u8 *p;
4408
4409	BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 \|\| SLUB_PAGE_SHIFT < 10);
4410
4411	printk(KERN_ERR "SLUB resiliency testing\n");
4412	printk(KERN_ERR "-----------------------\n");
4413	printk(KERN_ERR "A. Corruption after allocation\n");
4414
4415	p = kzalloc(16, GFP_KERNEL);
4416	p[16] = 0x12;
4417	printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
4418	" 0x12->0x%p\n\n", p + 16);
4419
4420	validate_slab_cache(kmalloc_caches[4]);
4421
4422	/* Hmmm... The next two are dangerous */
4423	p = kzalloc(32, GFP_KERNEL);
4424	p[32 + sizeof(void *)] = 0x34;
4425	printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
4426	" 0x34 -> -0x%p\n", p);
4427	printk(KERN_ERR
4428	"If allocated object is overwritten then not detectable\n\n");
4429
4430	validate_slab_cache(kmalloc_caches[5]);
4431	p = kzalloc(64, GFP_KERNEL);
4432	p += 64 + (get_cycles() & 0xff) * sizeof(void *);
4433	*p = 0x56;
4434	printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
4435	p);
4436	printk(KERN_ERR
4437	"If allocated object is overwritten then not detectable\n\n");
4438	validate_slab_cache(kmalloc_caches[6]);
4439
4440	printk(KERN_ERR "\nB. Corruption after free\n");
4441	p = kzalloc(128, GFP_KERNEL);
4442	kfree(p);
4443	*p = 0x78;
4444	printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
4445	validate_slab_cache(kmalloc_caches[7]);
4446
4447	p = kzalloc(256, GFP_KERNEL);
4448	kfree(p);
4449	p[50] = 0x9a;
4450	printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n",
4451	p);
4452	validate_slab_cache(kmalloc_caches[8]);
4453
4454	p = kzalloc(512, GFP_KERNEL);
4455	kfree(p);
4456	p[512] = 0xab;
4457	printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
4458	validate_slab_cache(kmalloc_caches[9]);
4459	}
4460	#else
4461	#ifdef CONFIG_SYSFS
4462	static void resiliency_test(void) {};
4463	#endif
4464	#endif
4465
4466	#ifdef CONFIG_SYSFS
4467	enum slab_stat_type {
4468	SL_ALL, /* All slabs */
4469	SL_PARTIAL, /* Only partially allocated slabs */
4470	SL_CPU, /* Only slabs used for cpu caches */
4471	SL_OBJECTS, /* Determine allocated objects not slabs */
4472	SL_TOTAL /* Determine object capacity not slabs */
4473	};
4474
4475	#define SO_ALL (1 << SL_ALL)
4476	#define SO_PARTIAL (1 << SL_PARTIAL)
4477	#define SO_CPU (1 << SL_CPU)
4478	#define SO_OBJECTS (1 << SL_OBJECTS)
4479	#define SO_TOTAL (1 << SL_TOTAL)
4480
4481	static ssize_t show_slab_objects(struct kmem_cache *s,
4482	char *buf, unsigned long flags)
4483	{
4484	unsigned long total = 0;
4485	int node;
4486	int x;
4487	unsigned long *nodes;
4488	unsigned long *per_cpu;
4489
4490	nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
4491	if (!nodes)
4492	return -ENOMEM;
4493	per_cpu = nodes + nr_node_ids;
4494
4495	if (flags & SO_CPU) {
4496	int cpu;
4497
4498	for_each_possible_cpu(cpu) {
4499	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
4500	int node = ACCESS_ONCE(c->node);
4501	struct page *page;
4502
4503	if (node < 0)
4504	continue;
4505	page = ACCESS_ONCE(c->page);
4506	if (page) {
4507	if (flags & SO_TOTAL)
4508	x = page->objects;
4509	else if (flags & SO_OBJECTS)
4510	x = page->inuse;
4511	else
4512	x = 1;
4513
4514	total += x;
4515	nodes[node] += x;
4516	}
4517	page = c->partial;
4518
4519	if (page) {
4520	x = page->pobjects;
4521	total += x;
4522	nodes[node] += x;
4523	}
4524	per_cpu[node]++;
4525	}
4526	}
4527
4528	lock_memory_hotplug();
4529	#ifdef CONFIG_SLUB_DEBUG
4530	if (flags & SO_ALL) {
4531	for_each_node_state(node, N_NORMAL_MEMORY) {
4532	struct kmem_cache_node *n = get_node(s, node);
4533
4534	if (flags & SO_TOTAL)
4535	x = atomic_long_read(&n->total_objects);
4536	else if (flags & SO_OBJECTS)
4537	x = atomic_long_read(&n->total_objects) -
4538	count_partial(n, count_free);
4539
4540	else
4541	x = atomic_long_read(&n->nr_slabs);
4542	total += x;
4543	nodes[node] += x;
4544	}
4545
4546	} else
4547	#endif
4548	if (flags & SO_PARTIAL) {
4549	for_each_node_state(node, N_NORMAL_MEMORY) {
4550	struct kmem_cache_node *n = get_node(s, node);
4551
4552	if (flags & SO_TOTAL)
4553	x = count_partial(n, count_total);
4554	else if (flags & SO_OBJECTS)
4555	x = count_partial(n, count_inuse);
4556	else
4557	x = n->nr_partial;
4558	total += x;
4559	nodes[node] += x;
4560	}
4561	}
4562	x = sprintf(buf, "%lu", total);
4563	#ifdef CONFIG_NUMA
4564	for_each_node_state(node, N_NORMAL_MEMORY)
4565	if (nodes[node])
4566	x += sprintf(buf + x, " N%d=%lu",
4567	node, nodes[node]);
4568	#endif
4569	unlock_memory_hotplug();
4570	kfree(nodes);
4571	return x + sprintf(buf + x, "\n");
4572	}
4573
4574	#ifdef CONFIG_SLUB_DEBUG
4575	static int any_slab_objects(struct kmem_cache *s)
4576	{
4577	int node;
4578
4579	for_each_online_node(node) {
4580	struct kmem_cache_node *n = get_node(s, node);
4581
4582	if (!n)
4583	continue;
4584
4585	if (atomic_long_read(&n->total_objects))
4586	return 1;
4587	}
4588	return 0;
4589	}
4590	#endif
4591
4592	#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
4593	#define to_slab(n) container_of(n, struct kmem_cache, kobj)
4594
4595	struct slab_attribute {
4596	struct attribute attr;
4597	ssize_t (show)(struct kmem_cache s, char *buf);
4598	ssize_t (store)(struct kmem_cache s, const char *x, size_t count);
4599	};
4600
4601	#define SLAB_ATTR_RO(_name) \
4602	static struct slab_attribute _name##_attr = \
4603	__ATTR(_name, 0400, _name##_show, NULL)
4604
4605	#define SLAB_ATTR(_name) \
4606	static struct slab_attribute _name##_attr = \
4607	__ATTR(_name, 0600, _name##_show, _name##_store)
4608
4609	static ssize_t slab_size_show(struct kmem_cache s, char buf)
4610	{
4611	return sprintf(buf, "%d\n", s->size);
4612	}
4613	SLAB_ATTR_RO(slab_size);
4614
4615	static ssize_t align_show(struct kmem_cache s, char buf)
4616	{
4617	return sprintf(buf, "%d\n", s->align);
4618	}
4619	SLAB_ATTR_RO(align);
4620
4621	static ssize_t object_size_show(struct kmem_cache s, char buf)
4622	{
4623	return sprintf(buf, "%d\n", s->objsize);
4624	}
4625	SLAB_ATTR_RO(object_size);
4626
4627	static ssize_t objs_per_slab_show(struct kmem_cache s, char buf)
4628	{
4629	return sprintf(buf, "%d\n", oo_objects(s->oo));
4630	}
4631	SLAB_ATTR_RO(objs_per_slab);
4632
4633	static ssize_t order_store(struct kmem_cache *s,
4634	const char *buf, size_t length)
4635	{
4636	unsigned long order;
4637	int err;
4638
4639	err = strict_strtoul(buf, 10, &order);
4640	if (err)
4641	return err;
4642
4643	if (order > slub_max_order \|\| order < slub_min_order)
4644	return -EINVAL;
4645
4646	calculate_sizes(s, order);
4647	return length;
4648	}
4649
4650	static ssize_t order_show(struct kmem_cache s, char buf)
4651	{
4652	return sprintf(buf, "%d\n", oo_order(s->oo));
4653	}
4654	SLAB_ATTR(order);
4655
4656	static ssize_t min_partial_show(struct kmem_cache s, char buf)
4657	{
4658	return sprintf(buf, "%lu\n", s->min_partial);
4659	}
4660
4661	static ssize_t min_partial_store(struct kmem_cache s, const char buf,
4662	size_t length)
4663	{
4664	unsigned long min;
4665	int err;
4666
4667	err = strict_strtoul(buf, 10, &min);
4668	if (err)
4669	return err;
4670
4671	set_min_partial(s, min);
4672	return length;
4673	}
4674	SLAB_ATTR(min_partial);
4675
4676	static ssize_t cpu_partial_show(struct kmem_cache s, char buf)
4677	{
4678	return sprintf(buf, "%u\n", s->cpu_partial);
4679	}
4680
4681	static ssize_t cpu_partial_store(struct kmem_cache s, const char buf,
4682	size_t length)
4683	{
4684	unsigned long objects;
4685	int err;
4686
4687	err = strict_strtoul(buf, 10, &objects);
4688	if (err)
4689	return err;
4690	if (objects && kmem_cache_debug(s))
4691	return -EINVAL;
4692
4693	s->cpu_partial = objects;
4694	flush_all(s);
4695	return length;
4696	}
4697	SLAB_ATTR(cpu_partial);
4698
4699	static ssize_t ctor_show(struct kmem_cache s, char buf)
4700	{
4701	if (!s->ctor)
4702	return 0;
4703	return sprintf(buf, "%pS\n", s->ctor);
4704	}
4705	SLAB_ATTR_RO(ctor);
4706
4707	static ssize_t aliases_show(struct kmem_cache s, char buf)
4708	{
4709	return sprintf(buf, "%d\n", s->refcount - 1);
4710	}
4711	SLAB_ATTR_RO(aliases);
4712
4713	static ssize_t partial_show(struct kmem_cache s, char buf)
4714	{
4715	return show_slab_objects(s, buf, SO_PARTIAL);
4716	}
4717	SLAB_ATTR_RO(partial);
4718
4719	static ssize_t cpu_slabs_show(struct kmem_cache s, char buf)
4720	{
4721	return show_slab_objects(s, buf, SO_CPU);
4722	}
4723	SLAB_ATTR_RO(cpu_slabs);
4724
4725	static ssize_t objects_show(struct kmem_cache s, char buf)
4726	{
4727	return show_slab_objects(s, buf, SO_ALL\|SO_OBJECTS);
4728	}
4729	SLAB_ATTR_RO(objects);
4730
4731	static ssize_t objects_partial_show(struct kmem_cache s, char buf)
4732	{
4733	return show_slab_objects(s, buf, SO_PARTIAL\|SO_OBJECTS);
4734	}
4735	SLAB_ATTR_RO(objects_partial);
4736
4737	static ssize_t slabs_cpu_partial_show(struct kmem_cache s, char buf)
4738	{
4739	int objects = 0;
4740	int pages = 0;
4741	int cpu;
4742	int len;
4743
4744	for_each_online_cpu(cpu) {
4745	struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial;
4746
4747	if (page) {
4748	pages += page->pages;
4749	objects += page->pobjects;
4750	}
4751	}
4752
4753	len = sprintf(buf, "%d(%d)", objects, pages);
4754
4755	#ifdef CONFIG_SMP
4756	for_each_online_cpu(cpu) {
4757	struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial;
4758
4759	if (page && len < PAGE_SIZE - 20)
4760	len += sprintf(buf + len, " C%d=%d(%d)", cpu,
4761	page->pobjects, page->pages);
4762	}
4763	#endif
4764	return len + sprintf(buf + len, "\n");
4765	}
4766	SLAB_ATTR_RO(slabs_cpu_partial);
4767
4768	static ssize_t reclaim_account_show(struct kmem_cache s, char buf)
4769	{
4770	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4771	}
4772
4773	static ssize_t reclaim_account_store(struct kmem_cache *s,
4774	const char *buf, size_t length)
4775	{
4776	s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4777	if (buf[0] == '1')
4778	s->flags \|= SLAB_RECLAIM_ACCOUNT;
4779	return length;
4780	}
4781	SLAB_ATTR(reclaim_account);
4782
4783	static ssize_t hwcache_align_show(struct kmem_cache s, char buf)
4784	{
4785	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
4786	}
4787	SLAB_ATTR_RO(hwcache_align);
4788
4789	#ifdef CONFIG_ZONE_DMA
4790	static ssize_t cache_dma_show(struct kmem_cache s, char buf)
4791	{
4792	return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4793	}
4794	SLAB_ATTR_RO(cache_dma);
4795	#endif
4796
4797	static ssize_t destroy_by_rcu_show(struct kmem_cache s, char buf)
4798	{
4799	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4800	}
4801	SLAB_ATTR_RO(destroy_by_rcu);
4802
4803	static ssize_t reserved_show(struct kmem_cache s, char buf)
4804	{
4805	return sprintf(buf, "%d\n", s->reserved);
4806	}
4807	SLAB_ATTR_RO(reserved);
4808
4809	#ifdef CONFIG_SLUB_DEBUG
4810	static ssize_t slabs_show(struct kmem_cache s, char buf)
4811	{
4812	return show_slab_objects(s, buf, SO_ALL);
4813	}
4814	SLAB_ATTR_RO(slabs);
4815
4816	static ssize_t total_objects_show(struct kmem_cache s, char buf)
4817	{
4818	return show_slab_objects(s, buf, SO_ALL\|SO_TOTAL);
4819	}
4820	SLAB_ATTR_RO(total_objects);
4821
4822	static ssize_t sanity_checks_show(struct kmem_cache s, char buf)
4823	{
4824	return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
4825	}
4826
4827	static ssize_t sanity_checks_store(struct kmem_cache *s,
4828	const char *buf, size_t length)
4829	{
4830	s->flags &= ~SLAB_DEBUG_FREE;
4831	if (buf[0] == '1') {
4832	s->flags &= ~__CMPXCHG_DOUBLE;
4833	s->flags \|= SLAB_DEBUG_FREE;
4834	}
4835	return length;
4836	}
4837	SLAB_ATTR(sanity_checks);
4838
4839	static ssize_t trace_show(struct kmem_cache s, char buf)
4840	{
4841	return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
4842	}
4843
4844	static ssize_t trace_store(struct kmem_cache s, const char buf,
4845	size_t length)
4846	{
4847	s->flags &= ~SLAB_TRACE;
4848	if (buf[0] == '1') {
4849	s->flags &= ~__CMPXCHG_DOUBLE;
4850	s->flags \|= SLAB_TRACE;
4851	}
4852	return length;
4853	}
4854	SLAB_ATTR(trace);
4855
4856	static ssize_t red_zone_show(struct kmem_cache s, char buf)
4857	{
4858	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
4859	}
4860
4861	static ssize_t red_zone_store(struct kmem_cache *s,
4862	const char *buf, size_t length)
4863	{
4864	if (any_slab_objects(s))
4865	return -EBUSY;
4866
4867	s->flags &= ~SLAB_RED_ZONE;
4868	if (buf[0] == '1') {
4869	s->flags &= ~__CMPXCHG_DOUBLE;
4870	s->flags \|= SLAB_RED_ZONE;
4871	}
4872	calculate_sizes(s, -1);
4873	return length;
4874	}
4875	SLAB_ATTR(red_zone);
4876
4877	static ssize_t poison_show(struct kmem_cache s, char buf)
4878	{
4879	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
4880	}
4881
4882	static ssize_t poison_store(struct kmem_cache *s,
4883	const char *buf, size_t length)
4884	{
4885	if (any_slab_objects(s))
4886	return -EBUSY;
4887
4888	s->flags &= ~SLAB_POISON;
4889	if (buf[0] == '1') {
4890	s->flags &= ~__CMPXCHG_DOUBLE;
4891	s->flags \|= SLAB_POISON;
4892	}
4893	calculate_sizes(s, -1);
4894	return length;
4895	}
4896	SLAB_ATTR(poison);
4897
4898	static ssize_t store_user_show(struct kmem_cache s, char buf)
4899	{
4900	return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
4901	}
4902
4903	static ssize_t store_user_store(struct kmem_cache *s,
4904	const char *buf, size_t length)
4905	{
4906	if (any_slab_objects(s))
4907	return -EBUSY;
4908
4909	s->flags &= ~SLAB_STORE_USER;
4910	if (buf[0] == '1') {
4911	s->flags &= ~__CMPXCHG_DOUBLE;
4912	s->flags \|= SLAB_STORE_USER;
4913	}
4914	calculate_sizes(s, -1);
4915	return length;
4916	}
4917	SLAB_ATTR(store_user);
4918
4919	static ssize_t validate_show(struct kmem_cache s, char buf)
4920	{
4921	return 0;
4922	}
4923
4924	static ssize_t validate_store(struct kmem_cache *s,
4925	const char *buf, size_t length)
4926	{
4927	int ret = -EINVAL;
4928
4929	if (buf[0] == '1') {
4930	ret = validate_slab_cache(s);
4931	if (ret >= 0)
4932	ret = length;
4933	}
4934	return ret;
4935	}
4936	SLAB_ATTR(validate);
4937
4938	static ssize_t alloc_calls_show(struct kmem_cache s, char buf)
4939	{
4940	if (!(s->flags & SLAB_STORE_USER))
4941	return -ENOSYS;
4942	return list_locations(s, buf, TRACK_ALLOC);
4943	}
4944	SLAB_ATTR_RO(alloc_calls);
4945
4946	static ssize_t free_calls_show(struct kmem_cache s, char buf)
4947	{
4948	if (!(s->flags & SLAB_STORE_USER))
4949	return -ENOSYS;
4950	return list_locations(s, buf, TRACK_FREE);
4951	}
4952	SLAB_ATTR_RO(free_calls);
4953	#endif /* CONFIG_SLUB_DEBUG */
4954
4955	#ifdef CONFIG_FAILSLAB
4956	static ssize_t failslab_show(struct kmem_cache s, char buf)
4957	{
4958	return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
4959	}
4960
4961	static ssize_t failslab_store(struct kmem_cache s, const char buf,
4962	size_t length)
4963	{
4964	s->flags &= ~SLAB_FAILSLAB;
4965	if (buf[0] == '1')
4966	s->flags \|= SLAB_FAILSLAB;
4967	return length;
4968	}
4969	SLAB_ATTR(failslab);
4970	#endif
4971
4972	static ssize_t shrink_show(struct kmem_cache s, char buf)
4973	{
4974	return 0;
4975	}
4976
4977	static ssize_t shrink_store(struct kmem_cache *s,
4978	const char *buf, size_t length)
4979	{
4980	if (buf[0] == '1') {
4981	int rc = kmem_cache_shrink(s);
4982
4983	if (rc)
4984	return rc;
4985	} else
4986	return -EINVAL;
4987	return length;
4988	}
4989	SLAB_ATTR(shrink);
4990
4991	#ifdef CONFIG_NUMA
4992	static ssize_t remote_node_defrag_ratio_show(struct kmem_cache s, char buf)
4993	{
4994	return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
4995	}
4996
4997	static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
4998	const char *buf, size_t length)
4999	{
5000	unsigned long ratio;
5001	int err;
5002
5003	err = strict_strtoul(buf, 10, &ratio);
5004	if (err)
5005	return err;
5006
5007	if (ratio <= 100)
5008	s->remote_node_defrag_ratio = ratio * 10;
5009
5010	return length;
5011	}
5012	SLAB_ATTR(remote_node_defrag_ratio);
5013	#endif
5014
5015	#ifdef CONFIG_SLUB_STATS
5016	static int show_stat(struct kmem_cache s, char buf, enum stat_item si)
5017	{
5018	unsigned long sum = 0;
5019	int cpu;
5020	int len;
5021	int data = kmalloc(nr_cpu_ids sizeof(int), GFP_KERNEL);
5022
5023	if (!data)
5024	return -ENOMEM;
5025
5026	for_each_online_cpu(cpu) {
5027	unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
5028
5029	data[cpu] = x;
5030	sum += x;
5031	}
5032
5033	len = sprintf(buf, "%lu", sum);
5034
5035	#ifdef CONFIG_SMP
5036	for_each_online_cpu(cpu) {
5037	if (data[cpu] && len < PAGE_SIZE - 20)
5038	len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
5039	}
5040	#endif
5041	kfree(data);
5042	return len + sprintf(buf + len, "\n");
5043	}
5044
5045	static void clear_stat(struct kmem_cache *s, enum stat_item si)
5046	{
5047	int cpu;
5048
5049	for_each_online_cpu(cpu)
5050	per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
5051	}
5052
5053	#define STAT_ATTR(si, text) \
5054	static ssize_t text##_show(struct kmem_cache s, char buf) \
5055	{ \
5056	return show_stat(s, buf, si); \
5057	} \
5058	static ssize_t text##_store(struct kmem_cache *s, \
5059	const char *buf, size_t length) \
5060	{ \
5061	if (buf[0] != '0') \
5062	return -EINVAL; \
5063	clear_stat(s, si); \
5064	return length; \
5065	} \
5066	SLAB_ATTR(text); \
5067
5068	STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
5069	STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
5070	STAT_ATTR(FREE_FASTPATH, free_fastpath);
5071	STAT_ATTR(FREE_SLOWPATH, free_slowpath);
5072	STAT_ATTR(FREE_FROZEN, free_frozen);
5073	STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
5074	STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
5075	STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
5076	STAT_ATTR(ALLOC_SLAB, alloc_slab);
5077	STAT_ATTR(ALLOC_REFILL, alloc_refill);
5078	STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
5079	STAT_ATTR(FREE_SLAB, free_slab);
5080	STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
5081	STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
5082	STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
5083	STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
5084	STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
5085	STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
5086	STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
5087	STAT_ATTR(ORDER_FALLBACK, order_fallback);
5088	STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
5089	STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
5090	STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
5091	STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
5092	STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
5093	STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
5094	#endif
5095
5096	static struct attribute *slab_attrs[] = {
5097	&slab_size_attr.attr,
5098	&object_size_attr.attr,
5099	&objs_per_slab_attr.attr,
5100	&order_attr.attr,
5101	&min_partial_attr.attr,
5102	&cpu_partial_attr.attr,
5103	&objects_attr.attr,
5104	&objects_partial_attr.attr,
5105	&partial_attr.attr,
5106	&cpu_slabs_attr.attr,
5107	&ctor_attr.attr,
5108	&aliases_attr.attr,
5109	&align_attr.attr,
5110	&hwcache_align_attr.attr,
5111	&reclaim_account_attr.attr,
5112	&destroy_by_rcu_attr.attr,
5113	&shrink_attr.attr,
5114	&reserved_attr.attr,
5115	&slabs_cpu_partial_attr.attr,
5116	#ifdef CONFIG_SLUB_DEBUG
5117	&total_objects_attr.attr,
5118	&slabs_attr.attr,
5119	&sanity_checks_attr.attr,
5120	&trace_attr.attr,
5121	&red_zone_attr.attr,
5122	&poison_attr.attr,
5123	&store_user_attr.attr,
5124	&validate_attr.attr,
5125	&alloc_calls_attr.attr,
5126	&free_calls_attr.attr,
5127	#endif
5128	#ifdef CONFIG_ZONE_DMA
5129	&cache_dma_attr.attr,
5130	#endif
5131	#ifdef CONFIG_NUMA
5132	&remote_node_defrag_ratio_attr.attr,
5133	#endif
5134	#ifdef CONFIG_SLUB_STATS
5135	&alloc_fastpath_attr.attr,
5136	&alloc_slowpath_attr.attr,
5137	&free_fastpath_attr.attr,
5138	&free_slowpath_attr.attr,
5139	&free_frozen_attr.attr,
5140	&free_add_partial_attr.attr,
5141	&free_remove_partial_attr.attr,
5142	&alloc_from_partial_attr.attr,
5143	&alloc_slab_attr.attr,
5144	&alloc_refill_attr.attr,
5145	&alloc_node_mismatch_attr.attr,
5146	&free_slab_attr.attr,
5147	&cpuslab_flush_attr.attr,
5148	&deactivate_full_attr.attr,
5149	&deactivate_empty_attr.attr,
5150	&deactivate_to_head_attr.attr,
5151	&deactivate_to_tail_attr.attr,
5152	&deactivate_remote_frees_attr.attr,
5153	&deactivate_bypass_attr.attr,
5154	&order_fallback_attr.attr,
5155	&cmpxchg_double_fail_attr.attr,
5156	&cmpxchg_double_cpu_fail_attr.attr,
5157	&cpu_partial_alloc_attr.attr,
5158	&cpu_partial_free_attr.attr,
5159	&cpu_partial_node_attr.attr,
5160	&cpu_partial_drain_attr.attr,
5161	#endif
5162	#ifdef CONFIG_FAILSLAB
5163	&failslab_attr.attr,
5164	#endif
5165
5166	NULL
5167	};
5168
5169	static struct attribute_group slab_attr_group = {
5170	.attrs = slab_attrs,
5171	};
5172
5173	static ssize_t slab_attr_show(struct kobject *kobj,
5174	struct attribute *attr,
5175	char *buf)
5176	{
5177	struct slab_attribute *attribute;
5178	struct kmem_cache *s;
5179	int err;
5180
5181	attribute = to_slab_attr(attr);
5182	s = to_slab(kobj);
5183
5184	if (!attribute->show)
5185	return -EIO;
5186
5187	err = attribute->show(s, buf);
5188
5189	return err;
5190	}
5191
5192	static ssize_t slab_attr_store(struct kobject *kobj,
5193	struct attribute *attr,
5194	const char *buf, size_t len)
5195	{
5196	struct slab_attribute *attribute;
5197	struct kmem_cache *s;
5198	int err;
5199
5200	attribute = to_slab_attr(attr);
5201	s = to_slab(kobj);
5202
5203	if (!attribute->store)
5204	return -EIO;
5205
5206	err = attribute->store(s, buf, len);
5207
5208	return err;
5209	}
5210
5211	static void kmem_cache_release(struct kobject *kobj)
5212	{
5213	struct kmem_cache *s = to_slab(kobj);
5214
5215	kfree(s->name);
5216	kfree(s);
5217	}
5218
5219	static const struct sysfs_ops slab_sysfs_ops = {
5220	.show = slab_attr_show,
5221	.store = slab_attr_store,
5222	};
5223
5224	static struct kobj_type slab_ktype = {
5225	.sysfs_ops = &slab_sysfs_ops,
5226	.release = kmem_cache_release
5227	};
5228
5229	static int uevent_filter(struct kset kset, struct kobject kobj)
5230	{
5231	struct kobj_type *ktype = get_ktype(kobj);
5232
5233	if (ktype == &slab_ktype)
5234	return 1;
5235	return 0;
5236	}
5237
5238	static const struct kset_uevent_ops slab_uevent_ops = {
5239	.filter = uevent_filter,
5240	};
5241
5242	static struct kset *slab_kset;
5243
5244	#define ID_STR_LENGTH 64
5245
5246	/* Create a unique string id for a slab cache:
5247	*
5248	* Format :[flags-]size
5249	*/
5250	static char create_unique_id(struct kmem_cache s)
5251	{
5252	char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
5253	char *p = name;
5254
5255	BUG_ON(!name);
5256
5257	*p++ = ':';
5258	/*
5259	* First flags affecting slabcache operations. We will only
5260	* get here for aliasable slabs so we do not need to support
5261	* too many flags. The flags here must cover all flags that
5262	* are matched during merging to guarantee that the id is
5263	* unique.
5264	*/
5265	if (s->flags & SLAB_CACHE_DMA)
5266	*p++ = 'd';
5267	if (s->flags & SLAB_RECLAIM_ACCOUNT)
5268	*p++ = 'a';
5269	if (s->flags & SLAB_DEBUG_FREE)
5270	*p++ = 'F';
5271	if (!(s->flags & SLAB_NOTRACK))
5272	*p++ = 't';
5273	if (p != name + 1)
5274	*p++ = '-';
5275	p += sprintf(p, "%07d", s->size);
5276	BUG_ON(p > name + ID_STR_LENGTH - 1);
5277	return name;
5278	}
5279
5280	static int sysfs_slab_add(struct kmem_cache *s)
5281	{
5282	int err;
5283	const char *name;
5284	int unmergeable;
5285
5286	if (slab_state < SYSFS)
5287	/* Defer until later */
5288	return 0;
5289
5290	unmergeable = slab_unmergeable(s);
5291	if (unmergeable) {
5292	/*
5293	* Slabcache can never be merged so we can use the name proper.
5294	* This is typically the case for debug situations. In that
5295	* case we can catch duplicate names easily.
5296	*/
5297	sysfs_remove_link(&slab_kset->kobj, s->name);
5298	name = s->name;
5299	} else {
5300	/*
5301	* Create a unique name for the slab as a target
5302	* for the symlinks.
5303	*/
5304	name = create_unique_id(s);
5305	}
5306
5307	s->kobj.kset = slab_kset;
5308	err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
5309	if (err) {
5310	kobject_put(&s->kobj);
5311	return err;
5312	}
5313
5314	err = sysfs_create_group(&s->kobj, &slab_attr_group);
5315	if (err) {
5316	kobject_del(&s->kobj);
5317	kobject_put(&s->kobj);
5318	return err;
5319	}
5320	kobject_uevent(&s->kobj, KOBJ_ADD);
5321	if (!unmergeable) {
5322	/* Setup first alias */
5323	sysfs_slab_alias(s, s->name);
5324	kfree(name);
5325	}
5326	return 0;
5327	}
5328
5329	static void sysfs_slab_remove(struct kmem_cache *s)
5330	{
5331	if (slab_state < SYSFS)
5332	/*
5333	* Sysfs has not been setup yet so no need to remove the
5334	* cache from sysfs.
5335	*/
5336	return;
5337
5338	kobject_uevent(&s->kobj, KOBJ_REMOVE);
5339	kobject_del(&s->kobj);
5340	kobject_put(&s->kobj);
5341	}
5342
5343	/*
5344	* Need to buffer aliases during bootup until sysfs becomes
5345	* available lest we lose that information.
5346	*/
5347	struct saved_alias {
5348	struct kmem_cache *s;
5349	const char *name;
5350	struct saved_alias *next;
5351	};
5352
5353	static struct saved_alias *alias_list;
5354
5355	static int sysfs_slab_alias(struct kmem_cache s, const char name)
5356	{
5357	struct saved_alias *al;
5358
5359	if (slab_state == SYSFS) {
5360	/*
5361	* If we have a leftover link then remove it.
5362	*/
5363	sysfs_remove_link(&slab_kset->kobj, name);
5364	return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
5365	}
5366
5367	al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
5368	if (!al)
5369	return -ENOMEM;
5370
5371	al->s = s;
5372	al->name = name;
5373	al->next = alias_list;
5374	alias_list = al;
5375	return 0;
5376	}
5377
5378	static int __init slab_sysfs_init(void)
5379	{
5380	struct kmem_cache *s;
5381	int err;
5382
5383	down_write(&slub_lock);
5384
5385	slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
5386	if (!slab_kset) {
5387	up_write(&slub_lock);
5388	printk(KERN_ERR "Cannot register slab subsystem.\n");
5389	return -ENOSYS;
5390	}
5391
5392	slab_state = SYSFS;
5393
5394	list_for_each_entry(s, &slab_caches, list) {
5395	err = sysfs_slab_add(s);
5396	if (err)
5397	printk(KERN_ERR "SLUB: Unable to add boot slab %s"
5398	" to sysfs\n", s->name);
5399	}
5400
5401	while (alias_list) {
5402	struct saved_alias *al = alias_list;
5403
5404	alias_list = alias_list->next;
5405	err = sysfs_slab_alias(al->s, al->name);
5406	if (err)
5407	printk(KERN_ERR "SLUB: Unable to add boot slab alias"
5408	" %s to sysfs\n", s->name);
5409	kfree(al);
5410	}
5411
5412	up_write(&slub_lock);
5413	resiliency_test();
5414	return 0;
5415	}
5416
5417	__initcall(slab_sysfs_init);
5418	#endif /* CONFIG_SYSFS */
5419
5420	/*
5421	* The /proc/slabinfo ABI
5422	*/
5423	#ifdef CONFIG_SLABINFO
5424	static void print_slabinfo_header(struct seq_file *m)
5425	{
5426	seq_puts(m, "slabinfo - version: 2.1\n");
5427	seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
5428	"<objperslab> <pagesperslab>");
5429	seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
5430	seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
5431	seq_putc(m, '\n');
5432	}
5433
5434	static void s_start(struct seq_file m, loff_t *pos)
5435	{
5436	loff_t n = *pos;
5437
5438	down_read(&slub_lock);
5439	if (!n)
5440	print_slabinfo_header(m);
5441
5442	return seq_list_start(&slab_caches, *pos);
5443	}
5444
5445	static void s_next(struct seq_file m, void p, loff_t pos)
5446	{
5447	return seq_list_next(p, &slab_caches, pos);
5448	}
5449
5450	static void s_stop(struct seq_file m, void p)
5451	{
5452	up_read(&slub_lock);
5453	}
5454
5455	static int s_show(struct seq_file m, void p)
5456	{
5457	unsigned long nr_partials = 0;
5458	unsigned long nr_slabs = 0;
5459	unsigned long nr_inuse = 0;
5460	unsigned long nr_objs = 0;
5461	unsigned long nr_free = 0;
5462	struct kmem_cache *s;
5463	int node;
5464
5465	s = list_entry(p, struct kmem_cache, list);
5466
5467	for_each_online_node(node) {
5468	struct kmem_cache_node *n = get_node(s, node);
5469
5470	if (!n)
5471	continue;
5472
5473	nr_partials += n->nr_partial;
5474	nr_slabs += atomic_long_read(&n->nr_slabs);
5475	nr_objs += atomic_long_read(&n->total_objects);
5476	nr_free += count_partial(n, count_free);
5477	}
5478
5479	nr_inuse = nr_objs - nr_free;
5480
5481	seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
5482	nr_objs, s->size, oo_objects(s->oo),
5483	(1 << oo_order(s->oo)));
5484	seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0);
5485	seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
5486	0UL);
5487	seq_putc(m, '\n');
5488	return 0;
5489	}
5490
5491	static const struct seq_operations slabinfo_op = {
5492	.start = s_start,
5493	.next = s_next,
5494	.stop = s_stop,
5495	.show = s_show,
5496	};
5497
5498	static int slabinfo_open(struct inode inode, struct file file)
5499	{
5500	return seq_open(file, &slabinfo_op);
5501	}
5502
5503	static const struct file_operations proc_slabinfo_operations = {
5504	.open = slabinfo_open,
5505	.read = seq_read,
5506	.llseek = seq_lseek,
5507	.release = seq_release,
5508	};
5509
5510	static int __init slab_proc_init(void)
5511	{
5512	proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations);
5513	return 0;
5514	}
5515	module_init(slab_proc_init);
5516	#endif /* CONFIG_SLABINFO */
5517

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/slub.c

interactive