Kernel

Kernel Git Source Tree

Root/mm/slab.c

1	/*
2	* linux/mm/slab.c
3	* Written by Mark Hemment, 1996/97.
4	* (markhe@nextd.demon.co.uk)
5	*
6	* kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7	*
8	* Major cleanup, different bufctl logic, per-cpu arrays
9	* (c) 2000 Manfred Spraul
10	*
11	* Cleanup, make the head arrays unconditional, preparation for NUMA
12	* (c) 2002 Manfred Spraul
13	*
14	* An implementation of the Slab Allocator as described in outline in;
15	* UNIX Internals: The New Frontiers by Uresh Vahalia
16	* Pub: Prentice Hall ISBN 0-13-101908-2
17	* or with a little more detail in;
18	* The Slab Allocator: An Object-Caching Kernel Memory Allocator
19	* Jeff Bonwick (Sun Microsystems).
20	* Presented at: USENIX Summer 1994 Technical Conference
21	*
22	* The memory is organized in caches, one cache for each object type.
23	* (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24	* Each cache consists out of many slabs (they are small (usually one
25	* page long) and always contiguous), and each slab contains multiple
26	* initialized objects.
27	*
28	* This means, that your constructor is used only for newly allocated
29	* slabs and you must pass objects with the same initializations to
30	* kmem_cache_free.
31	*
32	* Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33	* normal). If you need a special memory type, then must create a new
34	* cache for that memory type.
35	*
36	* In order to reduce fragmentation, the slabs are sorted in 3 groups:
37	* full slabs with 0 free objects
38	* partial slabs
39	* empty slabs with no allocated objects
40	*
41	* If partial slabs exist, then new allocations come from these slabs,
42	* otherwise from empty slabs or new slabs are allocated.
43	*
44	* kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45	* during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46	*
47	* Each cache has a short per-cpu head array, most allocs
48	* and frees go into that array, and if that array overflows, then 1/2
49	* of the entries in the array are given back into the global cache.
50	* The head array is strictly LIFO and should improve the cache hit rates.
51	* On SMP, it additionally reduces the spinlock operations.
52	*
53	* The c_cpuarray may not be read with enabled local interrupts -
54	* it's changed with a smp_call_function().
55	*
56	* SMP synchronization:
57	* constructors and destructors are called without any locking.
58	* Several members in struct kmem_cache and struct slab never change, they
59	* are accessed without any locking.
60	* The per-cpu arrays are never accessed from the wrong cpu, no locking,
61	* and local interrupts are disabled so slab code is preempt-safe.
62	* The non-constant members are protected with a per-cache irq spinlock.
63	*
64	* Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65	* in 2000 - many ideas in the current implementation are derived from
66	* his patch.
67	*
68	* Further notes from the original documentation:
69	*
70	* 11 April '97. Started multi-threading - markhe
71	* The global cache-chain is protected by the mutex 'slab_mutex'.
72	* The sem is only needed when accessing/extending the cache-chain, which
73	* can never happen inside an interrupt (kmem_cache_create(),
74	* kmem_cache_shrink() and kmem_cache_reap()).
75	*
76	* At present, each engine can be growing a cache. This should be blocked.
77	*
78	* 15 March 2005. NUMA slab allocator.
79	* Shai Fultheim <shai@scalex86.org>.
80	* Shobhit Dayal <shobhit@calsoftinc.com>
81	* Alok N Kataria <alokk@calsoftinc.com>
82	* Christoph Lameter <christoph@lameter.com>
83	*
84	* Modified the slab allocator to be node aware on NUMA systems.
85	* Each node has its own list of partial, free and full slabs.
86	* All object allocations for a node occur from node specific slab lists.
87	*/
88
89	#include <linux/slab.h>
90	#include "slab.h"
91	#include <linux/mm.h>
92	#include <linux/poison.h>
93	#include <linux/swap.h>
94	#include <linux/cache.h>
95	#include <linux/interrupt.h>
96	#include <linux/init.h>
97	#include <linux/compiler.h>
98	#include <linux/cpuset.h>
99	#include <linux/proc_fs.h>
100	#include <linux/seq_file.h>
101	#include <linux/notifier.h>
102	#include <linux/kallsyms.h>
103	#include <linux/cpu.h>
104	#include <linux/sysctl.h>
105	#include <linux/module.h>
106	#include <linux/rcupdate.h>
107	#include <linux/string.h>
108	#include <linux/uaccess.h>
109	#include <linux/nodemask.h>
110	#include <linux/kmemleak.h>
111	#include <linux/mempolicy.h>
112	#include <linux/mutex.h>
113	#include <linux/fault-inject.h>
114	#include <linux/rtmutex.h>
115	#include <linux/reciprocal_div.h>
116	#include <linux/debugobjects.h>
117	#include <linux/kmemcheck.h>
118	#include <linux/memory.h>
119	#include <linux/prefetch.h>
120
121	#include <net/sock.h>
122
123	#include <asm/cacheflush.h>
124	#include <asm/tlbflush.h>
125	#include <asm/page.h>
126
127	#include <trace/events/kmem.h>
128
129	#include "internal.h"
130
131	/*
132	* DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
133	* 0 for faster, smaller code (especially in the critical paths).
134	*
135	* STATS - 1 to collect stats for /proc/slabinfo.
136	* 0 for faster, smaller code (especially in the critical paths).
137	*
138	* FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
139	*/
140
141	#ifdef CONFIG_DEBUG_SLAB
142	#define DEBUG 1
143	#define STATS 1
144	#define FORCED_DEBUG 1
145	#else
146	#define DEBUG 0
147	#define STATS 0
148	#define FORCED_DEBUG 0
149	#endif
150
151	/* Shouldn't this be in a header file somewhere? */
152	#define BYTES_PER_WORD sizeof(void *)
153	#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
154
155	#ifndef ARCH_KMALLOC_FLAGS
156	#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
157	#endif
158
159	/*
160	* true if a page was allocated from pfmemalloc reserves for network-based
161	* swap
162	*/
163	static bool pfmemalloc_active __read_mostly;
164
165	/* Legal flag mask for kmem_cache_create(). */
166	#if DEBUG
167	# define CREATE_MASK (SLAB_RED_ZONE \| \
168	SLAB_POISON \| SLAB_HWCACHE_ALIGN \| \
169	SLAB_CACHE_DMA \| \
170	SLAB_STORE_USER \| \
171	SLAB_RECLAIM_ACCOUNT \| SLAB_PANIC \| \
172	SLAB_DESTROY_BY_RCU \| SLAB_MEM_SPREAD \| \
173	SLAB_DEBUG_OBJECTS \| SLAB_NOLEAKTRACE \| SLAB_NOTRACK)
174	#else
175	# define CREATE_MASK (SLAB_HWCACHE_ALIGN \| \
176	SLAB_CACHE_DMA \| \
177	SLAB_RECLAIM_ACCOUNT \| SLAB_PANIC \| \
178	SLAB_DESTROY_BY_RCU \| SLAB_MEM_SPREAD \| \
179	SLAB_DEBUG_OBJECTS \| SLAB_NOLEAKTRACE \| SLAB_NOTRACK)
180	#endif
181
182	/*
183	* kmem_bufctl_t:
184	*
185	* Bufctl's are used for linking objs within a slab
186	* linked offsets.
187	*
188	* This implementation relies on "struct page" for locating the cache &
189	* slab an object belongs to.
190	* This allows the bufctl structure to be small (one int), but limits
191	* the number of objects a slab (not a cache) can contain when off-slab
192	* bufctls are used. The limit is the size of the largest general cache
193	* that does not use off-slab slabs.
194	* For 32bit archs with 4 kB pages, is this 56.
195	* This is not serious, as it is only for large objects, when it is unwise
196	* to have too many per slab.
197	* Note: This limit can be raised by introducing a general cache whose size
198	* is less than 512 (PAGE_SIZE<<3), but greater than 256.
199	*/
200
201	typedef unsigned int kmem_bufctl_t;
202	#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
203	#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
204	#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2)
205	#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3)
206
207	/*
208	* struct slab_rcu
209	*
210	* slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
211	* arrange for kmem_freepages to be called via RCU. This is useful if
212	* we need to approach a kernel structure obliquely, from its address
213	* obtained without the usual locking. We can lock the structure to
214	* stabilize it and check it's still at the given address, only if we
215	* can be sure that the memory has not been meanwhile reused for some
216	* other kind of object (which our subsystem's lock might corrupt).
217	*
218	* rcu_read_lock before reading the address, then rcu_read_unlock after
219	* taking the spinlock within the structure expected at that address.
220	*/
221	struct slab_rcu {
222	struct rcu_head head;
223	struct kmem_cache *cachep;
224	void *addr;
225	};
226
227	/*
228	* struct slab
229	*
230	* Manages the objs in a slab. Placed either at the beginning of mem allocated
231	* for a slab, or allocated from an general cache.
232	* Slabs are chained into three list: fully used, partial, fully free slabs.
233	*/
234	struct slab {
235	union {
236	struct {
237	struct list_head list;
238	unsigned long colouroff;
239	void s_mem; / including colour offset */
240	unsigned int inuse; /* num of objs active in slab */
241	kmem_bufctl_t free;
242	unsigned short nodeid;
243	};
244	struct slab_rcu __slab_cover_slab_rcu;
245	};
246	};
247
248	/*
249	* struct array_cache
250	*
251	* Purpose:
252	* - LIFO ordering, to hand out cache-warm objects from _alloc
253	* - reduce the number of linked list operations
254	* - reduce spinlock operations
255	*
256	* The limit is stored in the per-cpu structure to reduce the data cache
257	* footprint.
258	*
259	*/
260	struct array_cache {
261	unsigned int avail;
262	unsigned int limit;
263	unsigned int batchcount;
264	unsigned int touched;
265	spinlock_t lock;
266	void entry[]; /
267	* Must have this definition in here for the proper
268	* alignment of array_cache. Also simplifies accessing
269	* the entries.
270	*
271	* Entries should not be directly dereferenced as
272	* entries belonging to slabs marked pfmemalloc will
273	* have the lower bits set SLAB_OBJ_PFMEMALLOC
274	*/
275	};
276
277	#define SLAB_OBJ_PFMEMALLOC 1
278	static inline bool is_obj_pfmemalloc(void *objp)
279	{
280	return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
281	}
282
283	static inline void set_obj_pfmemalloc(void **objp)
284	{
285	objp = (void )((unsigned long)*objp \| SLAB_OBJ_PFMEMALLOC);
286	return;
287	}
288
289	static inline void clear_obj_pfmemalloc(void **objp)
290	{
291	objp = (void )((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
292	}
293
294	/*
295	* bootstrap: The caches do not work without cpuarrays anymore, but the
296	* cpuarrays are allocated from the generic caches...
297	*/
298	#define BOOT_CPUCACHE_ENTRIES 1
299	struct arraycache_init {
300	struct array_cache cache;
301	void *entries[BOOT_CPUCACHE_ENTRIES];
302	};
303
304	/*
305	* The slab lists for all objects.
306	*/
307	struct kmem_list3 {
308	struct list_head slabs_partial; /* partial list first, better asm code */
309	struct list_head slabs_full;
310	struct list_head slabs_free;
311	unsigned long free_objects;
312	unsigned int free_limit;
313	unsigned int colour_next; /* Per-node cache coloring */
314	spinlock_t list_lock;
315	struct array_cache shared; / shared per node */
316	struct array_cache *alien; / on other nodes */
317	unsigned long next_reap; /* updated without locking */
318	int free_touched; /* updated without locking */
319	};
320
321	/*
322	* Need this for bootstrapping a per node allocator.
323	*/
324	#define NUM_INIT_LISTS (3 * MAX_NUMNODES)
325	static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
326	#define CACHE_CACHE 0
327	#define SIZE_AC MAX_NUMNODES
328	#define SIZE_L3 (2 * MAX_NUMNODES)
329
330	static int drain_freelist(struct kmem_cache *cache,
331	struct kmem_list3 *l3, int tofree);
332	static void free_block(struct kmem_cache cachep, void *objpp, int len,
333	int node);
334	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
335	static void cache_reap(struct work_struct *unused);
336
337	/*
338	* This function must be completely optimized away if a constant is passed to
339	* it. Mostly the same as what is in linux/slab.h except it returns an index.
340	*/
341	static __always_inline int index_of(const size_t size)
342	{
343	extern void __bad_size(void);
344
345	if (__builtin_constant_p(size)) {
346	int i = 0;
347
348	#define CACHE(x) \
349	if (size <=x) \
350	return i; \
351	else \
352	i++;
353	#include <linux/kmalloc_sizes.h>
354	#undef CACHE
355	__bad_size();
356	} else
357	__bad_size();
358	return 0;
359	}
360
361	static int slab_early_init = 1;
362
363	#define INDEX_AC index_of(sizeof(struct arraycache_init))
364	#define INDEX_L3 index_of(sizeof(struct kmem_list3))
365
366	static void kmem_list3_init(struct kmem_list3 *parent)
367	{
368	INIT_LIST_HEAD(&parent->slabs_full);
369	INIT_LIST_HEAD(&parent->slabs_partial);
370	INIT_LIST_HEAD(&parent->slabs_free);
371	parent->shared = NULL;
372	parent->alien = NULL;
373	parent->colour_next = 0;
374	spin_lock_init(&parent->list_lock);
375	parent->free_objects = 0;
376	parent->free_touched = 0;
377	}
378
379	#define MAKE_LIST(cachep, listp, slab, nodeid) \
380	do { \
381	INIT_LIST_HEAD(listp); \
382	list_splice(&(cachep->nodelists[nodeid]->slab), listp); \
383	} while (0)
384
385	#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
386	do { \
387	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
388	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
389	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
390	} while (0)
391
392	#define CFLGS_OFF_SLAB (0x80000000UL)
393	#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
394
395	#define BATCHREFILL_LIMIT 16
396	/*
397	* Optimization question: fewer reaps means less probability for unnessary
398	* cpucache drain/refill cycles.
399	*
400	* OTOH the cpuarrays can contain lots of objects,
401	* which could lock up otherwise freeable slabs.
402	*/
403	#define REAPTIMEOUT_CPUC (2*HZ)
404	#define REAPTIMEOUT_LIST3 (4*HZ)
405
406	#if STATS
407	#define STATS_INC_ACTIVE(x) ((x)->num_active++)
408	#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
409	#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
410	#define STATS_INC_GROWN(x) ((x)->grown++)
411	#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
412	#define STATS_SET_HIGH(x) \
413	do { \
414	if ((x)->num_active > (x)->high_mark) \
415	(x)->high_mark = (x)->num_active; \
416	} while (0)
417	#define STATS_INC_ERR(x) ((x)->errors++)
418	#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
419	#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
420	#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
421	#define STATS_SET_FREEABLE(x, i) \
422	do { \
423	if ((x)->max_freeable < i) \
424	(x)->max_freeable = i; \
425	} while (0)
426	#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
427	#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
428	#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
429	#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
430	#else
431	#define STATS_INC_ACTIVE(x) do { } while (0)
432	#define STATS_DEC_ACTIVE(x) do { } while (0)
433	#define STATS_INC_ALLOCED(x) do { } while (0)
434	#define STATS_INC_GROWN(x) do { } while (0)
435	#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
436	#define STATS_SET_HIGH(x) do { } while (0)
437	#define STATS_INC_ERR(x) do { } while (0)
438	#define STATS_INC_NODEALLOCS(x) do { } while (0)
439	#define STATS_INC_NODEFREES(x) do { } while (0)
440	#define STATS_INC_ACOVERFLOW(x) do { } while (0)
441	#define STATS_SET_FREEABLE(x, i) do { } while (0)
442	#define STATS_INC_ALLOCHIT(x) do { } while (0)
443	#define STATS_INC_ALLOCMISS(x) do { } while (0)
444	#define STATS_INC_FREEHIT(x) do { } while (0)
445	#define STATS_INC_FREEMISS(x) do { } while (0)
446	#endif
447
448	#if DEBUG
449
450	/*
451	* memory layout of objects:
452	* 0 : objp
453	* 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
454	* the end of an object is aligned with the end of the real
455	* allocation. Catches writes behind the end of the allocation.
456	* cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
457	* redzone word.
458	* cachep->obj_offset: The real object.
459	* cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
460	* cachep->size - 1* BYTES_PER_WORD: last caller address
461	* [BYTES_PER_WORD long]
462	*/
463	static int obj_offset(struct kmem_cache *cachep)
464	{
465	return cachep->obj_offset;
466	}
467
468	static unsigned long long dbg_redzone1(struct kmem_cache cachep, void *objp)
469	{
470	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
471	return (unsigned long long*) (objp + obj_offset(cachep) -
472	sizeof(unsigned long long));
473	}
474
475	static unsigned long long dbg_redzone2(struct kmem_cache cachep, void *objp)
476	{
477	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
478	if (cachep->flags & SLAB_STORE_USER)
479	return (unsigned long long *)(objp + cachep->size -
480	sizeof(unsigned long long) -
481	REDZONE_ALIGN);
482	return (unsigned long long *) (objp + cachep->size -
483	sizeof(unsigned long long));
484	}
485
486	static void *dbg_userword(struct kmem_cache cachep, void *objp)
487	{
488	BUG_ON(!(cachep->flags & SLAB_STORE_USER));
489	return (void **)(objp + cachep->size - BYTES_PER_WORD);
490	}
491
492	#else
493
494	#define obj_offset(x) 0
495	#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
496	#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
497	#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
498
499	#endif
500
501	#ifdef CONFIG_TRACING
502	size_t slab_buffer_size(struct kmem_cache *cachep)
503	{
504	return cachep->size;
505	}
506	EXPORT_SYMBOL(slab_buffer_size);
507	#endif
508
509	/*
510	* Do not go above this order unless 0 objects fit into the slab or
511	* overridden on the command line.
512	*/
513	#define SLAB_MAX_ORDER_HI 1
514	#define SLAB_MAX_ORDER_LO 0
515	static int slab_max_order = SLAB_MAX_ORDER_LO;
516	static bool slab_max_order_set __initdata;
517
518	static inline struct kmem_cache page_get_cache(struct page page)
519	{
520	page = compound_head(page);
521	BUG_ON(!PageSlab(page));
522	return page->slab_cache;
523	}
524
525	static inline struct kmem_cache virt_to_cache(const void obj)
526	{
527	struct page *page = virt_to_head_page(obj);
528	return page->slab_cache;
529	}
530
531	static inline struct slab virt_to_slab(const void obj)
532	{
533	struct page *page = virt_to_head_page(obj);
534
535	VM_BUG_ON(!PageSlab(page));
536	return page->slab_page;
537	}
538
539	static inline void index_to_obj(struct kmem_cache cache, struct slab *slab,
540	unsigned int idx)
541	{
542	return slab->s_mem + cache->size * idx;
543	}
544
545	/*
546	* We want to avoid an expensive divide : (offset / cache->size)
547	* Using the fact that size is a constant for a particular cache,
548	* we can replace (offset / cache->size) by
549	* reciprocal_divide(offset, cache->reciprocal_buffer_size)
550	*/
551	static inline unsigned int obj_to_index(const struct kmem_cache *cache,
552	const struct slab slab, void obj)
553	{
554	u32 offset = (obj - slab->s_mem);
555	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
556	}
557
558	/*
559	* These are the default caches for kmalloc. Custom caches can have other sizes.
560	*/
561	struct cache_sizes malloc_sizes[] = {
562	#define CACHE(x) { .cs_size = (x) },
563	#include <linux/kmalloc_sizes.h>
564	CACHE(ULONG_MAX)
565	#undef CACHE
566	};
567	EXPORT_SYMBOL(malloc_sizes);
568
569	/* Must match cache_sizes above. Out of line to keep cache footprint low. */
570	struct cache_names {
571	char *name;
572	char *name_dma;
573	};
574
575	static struct cache_names __initdata cache_names[] = {
576	#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
577	#include <linux/kmalloc_sizes.h>
578	{NULL,}
579	#undef CACHE
580	};
581
582	static struct arraycache_init initarray_cache __initdata =
583	{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
584	static struct arraycache_init initarray_generic =
585	{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
586
587	/* internal cache of cache description objs */
588	static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES];
589	static struct kmem_cache cache_cache = {
590	.nodelists = cache_cache_nodelists,
591	.batchcount = 1,
592	.limit = BOOT_CPUCACHE_ENTRIES,
593	.shared = 1,
594	.size = sizeof(struct kmem_cache),
595	.name = "kmem_cache",
596	};
597
598	#define BAD_ALIEN_MAGIC 0x01020304ul
599
600	#ifdef CONFIG_LOCKDEP
601
602	/*
603	* Slab sometimes uses the kmalloc slabs to store the slab headers
604	* for other slabs "off slab".
605	* The locking for this is tricky in that it nests within the locks
606	* of all other slabs in a few places; to deal with this special
607	* locking we put on-slab caches into a separate lock-class.
608	*
609	* We set lock class for alien array caches which are up during init.
610	* The lock annotation will be lost if all cpus of a node goes down and
611	* then comes back up during hotplug
612	*/
613	static struct lock_class_key on_slab_l3_key;
614	static struct lock_class_key on_slab_alc_key;
615
616	static struct lock_class_key debugobj_l3_key;
617	static struct lock_class_key debugobj_alc_key;
618
619	static void slab_set_lock_classes(struct kmem_cache *cachep,
620	struct lock_class_key l3_key, struct lock_class_key alc_key,
621	int q)
622	{
623	struct array_cache **alc;
624	struct kmem_list3 *l3;
625	int r;
626
627	l3 = cachep->nodelists[q];
628	if (!l3)
629	return;
630
631	lockdep_set_class(&l3->list_lock, l3_key);
632	alc = l3->alien;
633	/*
634	* FIXME: This check for BAD_ALIEN_MAGIC
635	* should go away when common slab code is taught to
636	* work even without alien caches.
637	* Currently, non NUMA code returns BAD_ALIEN_MAGIC
638	* for alloc_alien_cache,
639	*/
640	if (!alc \|\| (unsigned long)alc == BAD_ALIEN_MAGIC)
641	return;
642	for_each_node(r) {
643	if (alc[r])
644	lockdep_set_class(&alc[r]->lock, alc_key);
645	}
646	}
647
648	static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
649	{
650	slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
651	}
652
653	static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
654	{
655	int node;
656
657	for_each_online_node(node)
658	slab_set_debugobj_lock_classes_node(cachep, node);
659	}
660
661	static void init_node_lock_keys(int q)
662	{
663	struct cache_sizes *s = malloc_sizes;
664
665	if (slab_state < UP)
666	return;
667
668	for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
669	struct kmem_list3 *l3;
670
671	l3 = s->cs_cachep->nodelists[q];
672	if (!l3 \|\| OFF_SLAB(s->cs_cachep))
673	continue;
674
675	slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
676	&on_slab_alc_key, q);
677	}
678	}
679
680	static inline void init_lock_keys(void)
681	{
682	int node;
683
684	for_each_node(node)
685	init_node_lock_keys(node);
686	}
687	#else
688	static void init_node_lock_keys(int q)
689	{
690	}
691
692	static inline void init_lock_keys(void)
693	{
694	}
695
696	static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
697	{
698	}
699
700	static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
701	{
702	}
703	#endif
704
705	static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
706
707	static inline struct array_cache cpu_cache_get(struct kmem_cache cachep)
708	{
709	return cachep->array[smp_processor_id()];
710	}
711
712	static inline struct kmem_cache *__find_general_cachep(size_t size,
713	gfp_t gfpflags)
714	{
715	struct cache_sizes *csizep = malloc_sizes;
716
717	#if DEBUG
718	/* This happens if someone tries to call
719	* kmem_cache_create(), or __kmalloc(), before
720	* the generic caches are initialized.
721	*/
722	BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
723	#endif
724	if (!size)
725	return ZERO_SIZE_PTR;
726
727	while (size > csizep->cs_size)
728	csizep++;
729
730	/*
731	* Really subtle: The last entry with cs->cs_size==ULONG_MAX
732	* has cs_{dma,}cachep==NULL. Thus no special case
733	* for large kmalloc calls required.
734	*/
735	#ifdef CONFIG_ZONE_DMA
736	if (unlikely(gfpflags & GFP_DMA))
737	return csizep->cs_dmacachep;
738	#endif
739	return csizep->cs_cachep;
740	}
741
742	static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
743	{
744	return __find_general_cachep(size, gfpflags);
745	}
746
747	static size_t slab_mgmt_size(size_t nr_objs, size_t align)
748	{
749	return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
750	}
751
752	/*
753	* Calculate the number of objects and left-over bytes for a given buffer size.
754	*/
755	static void cache_estimate(unsigned long gfporder, size_t buffer_size,
756	size_t align, int flags, size_t *left_over,
757	unsigned int *num)
758	{
759	int nr_objs;
760	size_t mgmt_size;
761	size_t slab_size = PAGE_SIZE << gfporder;
762
763	/*
764	* The slab management structure can be either off the slab or
765	* on it. For the latter case, the memory allocated for a
766	* slab is used for:
767	*
768	* - The struct slab
769	* - One kmem_bufctl_t for each object
770	* - Padding to respect alignment of @align
771	* - @buffer_size bytes for each object
772	*
773	* If the slab management structure is off the slab, then the
774	* alignment will already be calculated into the size. Because
775	* the slabs are all pages aligned, the objects will be at the
776	* correct alignment when allocated.
777	*/
778	if (flags & CFLGS_OFF_SLAB) {
779	mgmt_size = 0;
780	nr_objs = slab_size / buffer_size;
781
782	if (nr_objs > SLAB_LIMIT)
783	nr_objs = SLAB_LIMIT;
784	} else {
785	/*
786	* Ignore padding for the initial guess. The padding
787	* is at most @align-1 bytes, and @buffer_size is at
788	* least @align. In the worst case, this result will
789	* be one greater than the number of objects that fit
790	* into the memory allocation when taking the padding
791	* into account.
792	*/
793	nr_objs = (slab_size - sizeof(struct slab)) /
794	(buffer_size + sizeof(kmem_bufctl_t));
795
796	/*
797	* This calculated number will be either the right
798	* amount, or one greater than what we want.
799	*/
800	if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
801	> slab_size)
802	nr_objs--;
803
804	if (nr_objs > SLAB_LIMIT)
805	nr_objs = SLAB_LIMIT;
806
807	mgmt_size = slab_mgmt_size(nr_objs, align);
808	}
809	*num = nr_objs;
810	left_over = slab_size - nr_objsbuffer_size - mgmt_size;
811	}
812
813	#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
814
815	static void __slab_error(const char function, struct kmem_cache cachep,
816	char *msg)
817	{
818	printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
819	function, cachep->name, msg);
820	dump_stack();
821	}
822
823	/*
824	* By default on NUMA we use alien caches to stage the freeing of
825	* objects allocated from other nodes. This causes massive memory
826	* inefficiencies when using fake NUMA setup to split memory into a
827	* large number of small nodes, so it can be disabled on the command
828	* line
829	*/
830
831	static int use_alien_caches __read_mostly = 1;
832	static int __init noaliencache_setup(char *s)
833	{
834	use_alien_caches = 0;
835	return 1;
836	}
837	__setup("noaliencache", noaliencache_setup);
838
839	static int __init slab_max_order_setup(char *str)
840	{
841	get_option(&str, &slab_max_order);
842	slab_max_order = slab_max_order < 0 ? 0 :
843	min(slab_max_order, MAX_ORDER - 1);
844	slab_max_order_set = true;
845
846	return 1;
847	}
848	__setup("slab_max_order=", slab_max_order_setup);
849
850	#ifdef CONFIG_NUMA
851	/*
852	* Special reaping functions for NUMA systems called from cache_reap().
853	* These take care of doing round robin flushing of alien caches (containing
854	* objects freed on different nodes from which they were allocated) and the
855	* flushing of remote pcps by calling drain_node_pages.
856	*/
857	static DEFINE_PER_CPU(unsigned long, slab_reap_node);
858
859	static void init_reap_node(int cpu)
860	{
861	int node;
862
863	node = next_node(cpu_to_mem(cpu), node_online_map);
864	if (node == MAX_NUMNODES)
865	node = first_node(node_online_map);
866
867	per_cpu(slab_reap_node, cpu) = node;
868	}
869
870	static void next_reap_node(void)
871	{
872	int node = __this_cpu_read(slab_reap_node);
873
874	node = next_node(node, node_online_map);
875	if (unlikely(node >= MAX_NUMNODES))
876	node = first_node(node_online_map);
877	__this_cpu_write(slab_reap_node, node);
878	}
879
880	#else
881	#define init_reap_node(cpu) do { } while (0)
882	#define next_reap_node(void) do { } while (0)
883	#endif
884
885	/*
886	* Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
887	* via the workqueue/eventd.
888	* Add the CPU number into the expiration time to minimize the possibility of
889	* the CPUs getting into lockstep and contending for the global cache chain
890	* lock.
891	*/
892	static void __cpuinit start_cpu_timer(int cpu)
893	{
894	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
895
896	/*
897	* When this gets called from do_initcalls via cpucache_init(),
898	* init_workqueues() has already run, so keventd will be setup
899	* at that time.
900	*/
901	if (keventd_up() && reap_work->work.func == NULL) {
902	init_reap_node(cpu);
903	INIT_DELAYED_WORK_DEFERRABLE(reap_work, cache_reap);
904	schedule_delayed_work_on(cpu, reap_work,
905	__round_jiffies_relative(HZ, cpu));
906	}
907	}
908
909	static struct array_cache *alloc_arraycache(int node, int entries,
910	int batchcount, gfp_t gfp)
911	{
912	int memsize = sizeof(void ) entries + sizeof(struct array_cache);
913	struct array_cache *nc = NULL;
914
915	nc = kmalloc_node(memsize, gfp, node);
916	/*
917	* The array_cache structures contain pointers to free object.
918	* However, when such objects are allocated or transferred to another
919	* cache the pointers are not cleared and they could be counted as
920	* valid references during a kmemleak scan. Therefore, kmemleak must
921	* not scan such objects.
922	*/
923	kmemleak_no_scan(nc);
924	if (nc) {
925	nc->avail = 0;
926	nc->limit = entries;
927	nc->batchcount = batchcount;
928	nc->touched = 0;
929	spin_lock_init(&nc->lock);
930	}
931	return nc;
932	}
933
934	static inline bool is_slab_pfmemalloc(struct slab *slabp)
935	{
936	struct page *page = virt_to_page(slabp->s_mem);
937
938	return PageSlabPfmemalloc(page);
939	}
940
941	/* Clears pfmemalloc_active if no slabs have pfmalloc set */
942	static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
943	struct array_cache *ac)
944	{
945	struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()];
946	struct slab *slabp;
947	unsigned long flags;
948
949	if (!pfmemalloc_active)
950	return;
951
952	spin_lock_irqsave(&l3->list_lock, flags);
953	list_for_each_entry(slabp, &l3->slabs_full, list)
954	if (is_slab_pfmemalloc(slabp))
955	goto out;
956
957	list_for_each_entry(slabp, &l3->slabs_partial, list)
958	if (is_slab_pfmemalloc(slabp))
959	goto out;
960
961	list_for_each_entry(slabp, &l3->slabs_free, list)
962	if (is_slab_pfmemalloc(slabp))
963	goto out;
964
965	pfmemalloc_active = false;
966	out:
967	spin_unlock_irqrestore(&l3->list_lock, flags);
968	}
969
970	static void __ac_get_obj(struct kmem_cache cachep, struct array_cache *ac,
971	gfp_t flags, bool force_refill)
972	{
973	int i;
974	void *objp = ac->entry[--ac->avail];
975
976	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
977	if (unlikely(is_obj_pfmemalloc(objp))) {
978	struct kmem_list3 *l3;
979
980	if (gfp_pfmemalloc_allowed(flags)) {
981	clear_obj_pfmemalloc(&objp);
982	return objp;
983	}
984
985	/* The caller cannot use PFMEMALLOC objects, find another one */
986	for (i = 1; i < ac->avail; i++) {
987	/* If a !PFMEMALLOC object is found, swap them */
988	if (!is_obj_pfmemalloc(ac->entry[i])) {
989	objp = ac->entry[i];
990	ac->entry[i] = ac->entry[ac->avail];
991	ac->entry[ac->avail] = objp;
992	return objp;
993	}
994	}
995
996	/*
997	* If there are empty slabs on the slabs_free list and we are
998	* being forced to refill the cache, mark this one !pfmemalloc.
999	*/
1000	l3 = cachep->nodelists[numa_mem_id()];
1001	if (!list_empty(&l3->slabs_free) && force_refill) {
1002	struct slab *slabp = virt_to_slab(objp);
1003	ClearPageSlabPfmemalloc(virt_to_page(slabp->s_mem));
1004	clear_obj_pfmemalloc(&objp);
1005	recheck_pfmemalloc_active(cachep, ac);
1006	return objp;
1007	}
1008
1009	/* No !PFMEMALLOC objects available */
1010	ac->avail++;
1011	objp = NULL;
1012	}
1013
1014	return objp;
1015	}
1016
1017	static inline void ac_get_obj(struct kmem_cache cachep,
1018	struct array_cache *ac, gfp_t flags, bool force_refill)
1019	{
1020	void *objp;
1021
1022	if (unlikely(sk_memalloc_socks()))
1023	objp = __ac_get_obj(cachep, ac, flags, force_refill);
1024	else
1025	objp = ac->entry[--ac->avail];
1026
1027	return objp;
1028	}
1029
1030	static void __ac_put_obj(struct kmem_cache cachep, struct array_cache *ac,
1031	void *objp)
1032	{
1033	if (unlikely(pfmemalloc_active)) {
1034	/* Some pfmemalloc slabs exist, check if this is one */
1035	struct page *page = virt_to_page(objp);
1036	if (PageSlabPfmemalloc(page))
1037	set_obj_pfmemalloc(&objp);
1038	}
1039
1040	return objp;
1041	}
1042
1043	static inline void ac_put_obj(struct kmem_cache cachep, struct array_cache ac,
1044	void *objp)
1045	{
1046	if (unlikely(sk_memalloc_socks()))
1047	objp = __ac_put_obj(cachep, ac, objp);
1048
1049	ac->entry[ac->avail++] = objp;
1050	}
1051
1052	/*
1053	* Transfer objects in one arraycache to another.
1054	* Locking must be handled by the caller.
1055	*
1056	* Return the number of entries transferred.
1057	*/
1058	static int transfer_objects(struct array_cache *to,
1059	struct array_cache *from, unsigned int max)
1060	{
1061	/* Figure out how many entries to transfer */
1062	int nr = min3(from->avail, max, to->limit - to->avail);
1063
1064	if (!nr)
1065	return 0;
1066
1067	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
1068	sizeof(void ) nr);
1069
1070	from->avail -= nr;
1071	to->avail += nr;
1072	return nr;
1073	}
1074
1075	#ifndef CONFIG_NUMA
1076
1077	#define drain_alien_cache(cachep, alien) do { } while (0)
1078	#define reap_alien(cachep, l3) do { } while (0)
1079
1080	static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
1081	{
1082	return (struct array_cache **)BAD_ALIEN_MAGIC;
1083	}
1084
1085	static inline void free_alien_cache(struct array_cache **ac_ptr)
1086	{
1087	}
1088
1089	static inline int cache_free_alien(struct kmem_cache cachep, void objp)
1090	{
1091	return 0;
1092	}
1093
1094	static inline void alternate_node_alloc(struct kmem_cache cachep,
1095	gfp_t flags)
1096	{
1097	return NULL;
1098	}
1099
1100	static inline void ____cache_alloc_node(struct kmem_cache cachep,
1101	gfp_t flags, int nodeid)
1102	{
1103	return NULL;
1104	}
1105
1106	#else /* CONFIG_NUMA */
1107
1108	static void ____cache_alloc_node(struct kmem_cache , gfp_t, int);
1109	static void alternate_node_alloc(struct kmem_cache , gfp_t);
1110
1111	static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
1112	{
1113	struct array_cache **ac_ptr;
1114	int memsize = sizeof(void ) nr_node_ids;
1115	int i;
1116
1117	if (limit > 1)
1118	limit = 12;
1119	ac_ptr = kzalloc_node(memsize, gfp, node);
1120	if (ac_ptr) {
1121	for_each_node(i) {
1122	if (i == node \|\| !node_online(i))
1123	continue;
1124	ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);
1125	if (!ac_ptr[i]) {
1126	for (i--; i >= 0; i--)
1127	kfree(ac_ptr[i]);
1128	kfree(ac_ptr);
1129	return NULL;
1130	}
1131	}
1132	}
1133	return ac_ptr;
1134	}
1135
1136	static void free_alien_cache(struct array_cache **ac_ptr)
1137	{
1138	int i;
1139
1140	if (!ac_ptr)
1141	return;
1142	for_each_node(i)
1143	kfree(ac_ptr[i]);
1144	kfree(ac_ptr);
1145	}
1146
1147	static void __drain_alien_cache(struct kmem_cache *cachep,
1148	struct array_cache *ac, int node)
1149	{
1150	struct kmem_list3 *rl3 = cachep->nodelists[node];
1151
1152	if (ac->avail) {
1153	spin_lock(&rl3->list_lock);
1154	/*
1155	* Stuff objects into the remote nodes shared array first.
1156	* That way we could avoid the overhead of putting the objects
1157	* into the free lists and getting them back later.
1158	*/
1159	if (rl3->shared)
1160	transfer_objects(rl3->shared, ac, ac->limit);
1161
1162	free_block(cachep, ac->entry, ac->avail, node);
1163	ac->avail = 0;
1164	spin_unlock(&rl3->list_lock);
1165	}
1166	}
1167
1168	/*
1169	* Called from cache_reap() to regularly drain alien caches round robin.
1170	*/
1171	static void reap_alien(struct kmem_cache cachep, struct kmem_list3 l3)
1172	{
1173	int node = __this_cpu_read(slab_reap_node);
1174
1175	if (l3->alien) {
1176	struct array_cache *ac = l3->alien[node];
1177
1178	if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {
1179	__drain_alien_cache(cachep, ac, node);
1180	spin_unlock_irq(&ac->lock);
1181	}
1182	}
1183	}
1184
1185	static void drain_alien_cache(struct kmem_cache *cachep,
1186	struct array_cache **alien)
1187	{
1188	int i = 0;
1189	struct array_cache *ac;
1190	unsigned long flags;
1191
1192	for_each_online_node(i) {
1193	ac = alien[i];
1194	if (ac) {
1195	spin_lock_irqsave(&ac->lock, flags);
1196	__drain_alien_cache(cachep, ac, i);
1197	spin_unlock_irqrestore(&ac->lock, flags);
1198	}
1199	}
1200	}
1201
1202	static inline int cache_free_alien(struct kmem_cache cachep, void objp)
1203	{
1204	struct slab *slabp = virt_to_slab(objp);
1205	int nodeid = slabp->nodeid;
1206	struct kmem_list3 *l3;
1207	struct array_cache *alien = NULL;
1208	int node;
1209
1210	node = numa_mem_id();
1211
1212	/*
1213	* Make sure we are not freeing a object from another node to the array
1214	* cache on this cpu.
1215	*/
1216	if (likely(slabp->nodeid == node))
1217	return 0;
1218
1219	l3 = cachep->nodelists[node];
1220	STATS_INC_NODEFREES(cachep);
1221	if (l3->alien && l3->alien[nodeid]) {
1222	alien = l3->alien[nodeid];
1223	spin_lock(&alien->lock);
1224	if (unlikely(alien->avail == alien->limit)) {
1225	STATS_INC_ACOVERFLOW(cachep);
1226	__drain_alien_cache(cachep, alien, nodeid);
1227	}
1228	ac_put_obj(cachep, alien, objp);
1229	spin_unlock(&alien->lock);
1230	} else {
1231	spin_lock(&(cachep->nodelists[nodeid])->list_lock);
1232	free_block(cachep, &objp, 1, nodeid);
1233	spin_unlock(&(cachep->nodelists[nodeid])->list_lock);
1234	}
1235	return 1;
1236	}
1237	#endif
1238
1239	/*
1240	* Allocates and initializes nodelists for a node on each slab cache, used for
1241	* either memory or cpu hotplug. If memory is being hot-added, the kmem_list3
1242	* will be allocated off-node since memory is not yet online for the new node.
1243	* When hotplugging memory or a cpu, existing nodelists are not replaced if
1244	* already in use.
1245	*
1246	* Must hold slab_mutex.
1247	*/
1248	static int init_cache_nodelists_node(int node)
1249	{
1250	struct kmem_cache *cachep;
1251	struct kmem_list3 *l3;
1252	const int memsize = sizeof(struct kmem_list3);
1253
1254	list_for_each_entry(cachep, &slab_caches, list) {
1255	/*
1256	* Set up the size64 kmemlist for cpu before we can
1257	* begin anything. Make sure some other cpu on this
1258	* node has not already allocated this
1259	*/
1260	if (!cachep->nodelists[node]) {
1261	l3 = kmalloc_node(memsize, GFP_KERNEL, node);
1262	if (!l3)
1263	return -ENOMEM;
1264	kmem_list3_init(l3);
1265	l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
1266	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1267
1268	/*
1269	* The l3s don't come and go as CPUs come and
1270	* go. slab_mutex is sufficient
1271	* protection here.
1272	*/
1273	cachep->nodelists[node] = l3;
1274	}
1275
1276	spin_lock_irq(&cachep->nodelists[node]->list_lock);
1277	cachep->nodelists[node]->free_limit =
1278	(1 + nr_cpus_node(node)) *
1279	cachep->batchcount + cachep->num;
1280	spin_unlock_irq(&cachep->nodelists[node]->list_lock);
1281	}
1282	return 0;
1283	}
1284
1285	static void __cpuinit cpuup_canceled(long cpu)
1286	{
1287	struct kmem_cache *cachep;
1288	struct kmem_list3 *l3 = NULL;
1289	int node = cpu_to_mem(cpu);
1290	const struct cpumask *mask = cpumask_of_node(node);
1291
1292	list_for_each_entry(cachep, &slab_caches, list) {
1293	struct array_cache *nc;
1294	struct array_cache *shared;
1295	struct array_cache **alien;
1296
1297	/* cpu is dead; no one can alloc from it. */
1298	nc = cachep->array[cpu];
1299	cachep->array[cpu] = NULL;
1300	l3 = cachep->nodelists[node];
1301
1302	if (!l3)
1303	goto free_array_cache;
1304
1305	spin_lock_irq(&l3->list_lock);
1306
1307	/* Free limit for this kmem_list3 */
1308	l3->free_limit -= cachep->batchcount;
1309	if (nc)
1310	free_block(cachep, nc->entry, nc->avail, node);
1311
1312	if (!cpumask_empty(mask)) {
1313	spin_unlock_irq(&l3->list_lock);
1314	goto free_array_cache;
1315	}
1316
1317	shared = l3->shared;
1318	if (shared) {
1319	free_block(cachep, shared->entry,
1320	shared->avail, node);
1321	l3->shared = NULL;
1322	}
1323
1324	alien = l3->alien;
1325	l3->alien = NULL;
1326
1327	spin_unlock_irq(&l3->list_lock);
1328
1329	kfree(shared);
1330	if (alien) {
1331	drain_alien_cache(cachep, alien);
1332	free_alien_cache(alien);
1333	}
1334	free_array_cache:
1335	kfree(nc);
1336	}
1337	/*
1338	* In the previous loop, all the objects were freed to
1339	* the respective cache's slabs, now we can go ahead and
1340	* shrink each nodelist to its limit.
1341	*/
1342	list_for_each_entry(cachep, &slab_caches, list) {
1343	l3 = cachep->nodelists[node];
1344	if (!l3)
1345	continue;
1346	drain_freelist(cachep, l3, l3->free_objects);
1347	}
1348	}
1349
1350	static int __cpuinit cpuup_prepare(long cpu)
1351	{
1352	struct kmem_cache *cachep;
1353	struct kmem_list3 *l3 = NULL;
1354	int node = cpu_to_mem(cpu);
1355	int err;
1356
1357	/*
1358	* We need to do this right in the beginning since
1359	* alloc_arraycache's are going to use this list.
1360	* kmalloc_node allows us to add the slab to the right
1361	* kmem_list3 and not this cpu's kmem_list3
1362	*/
1363	err = init_cache_nodelists_node(node);
1364	if (err < 0)
1365	goto bad;
1366
1367	/*
1368	* Now we can go ahead with allocating the shared arrays and
1369	* array caches
1370	*/
1371	list_for_each_entry(cachep, &slab_caches, list) {
1372	struct array_cache *nc;
1373	struct array_cache *shared = NULL;
1374	struct array_cache **alien = NULL;
1375
1376	nc = alloc_arraycache(node, cachep->limit,
1377	cachep->batchcount, GFP_KERNEL);
1378	if (!nc)
1379	goto bad;
1380	if (cachep->shared) {
1381	shared = alloc_arraycache(node,
1382	cachep->shared * cachep->batchcount,
1383	0xbaadf00d, GFP_KERNEL);
1384	if (!shared) {
1385	kfree(nc);
1386	goto bad;
1387	}
1388	}
1389	if (use_alien_caches) {
1390	alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
1391	if (!alien) {
1392	kfree(shared);
1393	kfree(nc);
1394	goto bad;
1395	}
1396	}
1397	cachep->array[cpu] = nc;
1398	l3 = cachep->nodelists[node];
1399	BUG_ON(!l3);
1400
1401	spin_lock_irq(&l3->list_lock);
1402	if (!l3->shared) {
1403	/*
1404	* We are serialised from CPU_DEAD or
1405	* CPU_UP_CANCELLED by the cpucontrol lock
1406	*/
1407	l3->shared = shared;
1408	shared = NULL;
1409	}
1410	#ifdef CONFIG_NUMA
1411	if (!l3->alien) {
1412	l3->alien = alien;
1413	alien = NULL;
1414	}
1415	#endif
1416	spin_unlock_irq(&l3->list_lock);
1417	kfree(shared);
1418	free_alien_cache(alien);
1419	if (cachep->flags & SLAB_DEBUG_OBJECTS)
1420	slab_set_debugobj_lock_classes_node(cachep, node);
1421	}
1422	init_node_lock_keys(node);
1423
1424	return 0;
1425	bad:
1426	cpuup_canceled(cpu);
1427	return -ENOMEM;
1428	}
1429
1430	static int __cpuinit cpuup_callback(struct notifier_block *nfb,
1431	unsigned long action, void *hcpu)
1432	{
1433	long cpu = (long)hcpu;
1434	int err = 0;
1435
1436	switch (action) {
1437	case CPU_UP_PREPARE:
1438	case CPU_UP_PREPARE_FROZEN:
1439	mutex_lock(&slab_mutex);
1440	err = cpuup_prepare(cpu);
1441	mutex_unlock(&slab_mutex);
1442	break;
1443	case CPU_ONLINE:
1444	case CPU_ONLINE_FROZEN:
1445	start_cpu_timer(cpu);
1446	break;
1447	#ifdef CONFIG_HOTPLUG_CPU
1448	case CPU_DOWN_PREPARE:
1449	case CPU_DOWN_PREPARE_FROZEN:
1450	/*
1451	* Shutdown cache reaper. Note that the slab_mutex is
1452	* held so that if cache_reap() is invoked it cannot do
1453	* anything expensive but will only modify reap_work
1454	* and reschedule the timer.
1455	*/
1456	cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1457	/* Now the cache_reaper is guaranteed to be not running. */
1458	per_cpu(slab_reap_work, cpu).work.func = NULL;
1459	break;
1460	case CPU_DOWN_FAILED:
1461	case CPU_DOWN_FAILED_FROZEN:
1462	start_cpu_timer(cpu);
1463	break;
1464	case CPU_DEAD:
1465	case CPU_DEAD_FROZEN:
1466	/*
1467	* Even if all the cpus of a node are down, we don't free the
1468	* kmem_list3 of any cache. This to avoid a race between
1469	* cpu_down, and a kmalloc allocation from another cpu for
1470	* memory from the node of the cpu going down. The list3
1471	* structure is usually allocated from kmem_cache_create() and
1472	* gets destroyed at kmem_cache_destroy().
1473	*/
1474	/* fall through */
1475	#endif
1476	case CPU_UP_CANCELED:
1477	case CPU_UP_CANCELED_FROZEN:
1478	mutex_lock(&slab_mutex);
1479	cpuup_canceled(cpu);
1480	mutex_unlock(&slab_mutex);
1481	break;
1482	}
1483	return notifier_from_errno(err);
1484	}
1485
1486	static struct notifier_block __cpuinitdata cpucache_notifier = {
1487	&cpuup_callback, NULL, 0
1488	};
1489
1490	#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1491	/*
1492	* Drains freelist for a node on each slab cache, used for memory hot-remove.
1493	* Returns -EBUSY if all objects cannot be drained so that the node is not
1494	* removed.
1495	*
1496	* Must hold slab_mutex.
1497	*/
1498	static int __meminit drain_cache_nodelists_node(int node)
1499	{
1500	struct kmem_cache *cachep;
1501	int ret = 0;
1502
1503	list_for_each_entry(cachep, &slab_caches, list) {
1504	struct kmem_list3 *l3;
1505
1506	l3 = cachep->nodelists[node];
1507	if (!l3)
1508	continue;
1509
1510	drain_freelist(cachep, l3, l3->free_objects);
1511
1512	if (!list_empty(&l3->slabs_full) \|\|
1513	!list_empty(&l3->slabs_partial)) {
1514	ret = -EBUSY;
1515	break;
1516	}
1517	}
1518	return ret;
1519	}
1520
1521	static int __meminit slab_memory_callback(struct notifier_block *self,
1522	unsigned long action, void *arg)
1523	{
1524	struct memory_notify *mnb = arg;
1525	int ret = 0;
1526	int nid;
1527
1528	nid = mnb->status_change_nid;
1529	if (nid < 0)
1530	goto out;
1531
1532	switch (action) {
1533	case MEM_GOING_ONLINE:
1534	mutex_lock(&slab_mutex);
1535	ret = init_cache_nodelists_node(nid);
1536	mutex_unlock(&slab_mutex);
1537	break;
1538	case MEM_GOING_OFFLINE:
1539	mutex_lock(&slab_mutex);
1540	ret = drain_cache_nodelists_node(nid);
1541	mutex_unlock(&slab_mutex);
1542	break;
1543	case MEM_ONLINE:
1544	case MEM_OFFLINE:
1545	case MEM_CANCEL_ONLINE:
1546	case MEM_CANCEL_OFFLINE:
1547	break;
1548	}
1549	out:
1550	return notifier_from_errno(ret);
1551	}
1552	#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1553
1554	/*
1555	* swap the static kmem_list3 with kmalloced memory
1556	*/
1557	static void __init init_list(struct kmem_cache cachep, struct kmem_list3 list,
1558	int nodeid)
1559	{
1560	struct kmem_list3 *ptr;
1561
1562	ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid);
1563	BUG_ON(!ptr);
1564
1565	memcpy(ptr, list, sizeof(struct kmem_list3));
1566	/*
1567	* Do not assume that spinlocks can be initialized via memcpy:
1568	*/
1569	spin_lock_init(&ptr->list_lock);
1570
1571	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1572	cachep->nodelists[nodeid] = ptr;
1573	}
1574
1575	/*
1576	* For setting up all the kmem_list3s for cache whose buffer_size is same as
1577	* size of kmem_list3.
1578	*/
1579	static void __init set_up_list3s(struct kmem_cache *cachep, int index)
1580	{
1581	int node;
1582
1583	for_each_online_node(node) {
1584	cachep->nodelists[node] = &initkmem_list3[index + node];
1585	cachep->nodelists[node]->next_reap = jiffies +
1586	REAPTIMEOUT_LIST3 +
1587	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1588	}
1589	}
1590
1591	/*
1592	* Initialisation. Called after the page allocator have been initialised and
1593	* before smp_init().
1594	*/
1595	void __init kmem_cache_init(void)
1596	{
1597	size_t left_over;
1598	struct cache_sizes *sizes;
1599	struct cache_names *names;
1600	int i;
1601	int order;
1602	int node;
1603
1604	if (num_possible_nodes() == 1)
1605	use_alien_caches = 0;
1606
1607	for (i = 0; i < NUM_INIT_LISTS; i++) {
1608	kmem_list3_init(&initkmem_list3[i]);
1609	if (i < MAX_NUMNODES)
1610	cache_cache.nodelists[i] = NULL;
1611	}
1612	set_up_list3s(&cache_cache, CACHE_CACHE);
1613
1614	/*
1615	* Fragmentation resistance on low memory - only use bigger
1616	* page orders on machines with more than 32MB of memory if
1617	* not overridden on the command line.
1618	*/
1619	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
1620	slab_max_order = SLAB_MAX_ORDER_HI;
1621
1622	/* Bootstrap is tricky, because several objects are allocated
1623	* from caches that do not exist yet:
1624	* 1) initialize the cache_cache cache: it contains the struct
1625	* kmem_cache structures of all caches, except cache_cache itself:
1626	* cache_cache is statically allocated.
1627	* Initially an __init data area is used for the head array and the
1628	* kmem_list3 structures, it's replaced with a kmalloc allocated
1629	* array at the end of the bootstrap.
1630	* 2) Create the first kmalloc cache.
1631	* The struct kmem_cache for the new cache is allocated normally.
1632	* An __init data area is used for the head array.
1633	* 3) Create the remaining kmalloc caches, with minimally sized
1634	* head arrays.
1635	* 4) Replace the __init data head arrays for cache_cache and the first
1636	* kmalloc cache with kmalloc allocated arrays.
1637	* 5) Replace the __init data for kmem_list3 for cache_cache and
1638	* the other cache's with kmalloc allocated memory.
1639	* 6) Resize the head arrays of the kmalloc caches to their final sizes.
1640	*/
1641
1642	node = numa_mem_id();
1643
1644	/* 1) create the cache_cache */
1645	INIT_LIST_HEAD(&slab_caches);
1646	list_add(&cache_cache.list, &slab_caches);
1647	cache_cache.colour_off = cache_line_size();
1648	cache_cache.array[smp_processor_id()] = &initarray_cache.cache;
1649	cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
1650
1651	/*
1652	* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1653	*/
1654	cache_cache.size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
1655	nr_node_ids * sizeof(struct kmem_list3 *);
1656	cache_cache.object_size = cache_cache.size;
1657	cache_cache.size = ALIGN(cache_cache.size,
1658	cache_line_size());
1659	cache_cache.reciprocal_buffer_size =
1660	reciprocal_value(cache_cache.size);
1661
1662	for (order = 0; order < MAX_ORDER; order++) {
1663	cache_estimate(order, cache_cache.size,
1664	cache_line_size(), 0, &left_over, &cache_cache.num);
1665	if (cache_cache.num)
1666	break;
1667	}
1668	BUG_ON(!cache_cache.num);
1669	cache_cache.gfporder = order;
1670	cache_cache.colour = left_over / cache_cache.colour_off;
1671	cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
1672	sizeof(struct slab), cache_line_size());
1673
1674	/* 2+3) create the kmalloc caches */
1675	sizes = malloc_sizes;
1676	names = cache_names;
1677
1678	/*
1679	* Initialize the caches that provide memory for the array cache and the
1680	* kmem_list3 structures first. Without this, further allocations will
1681	* bug.
1682	*/
1683
1684	sizes[INDEX_AC].cs_cachep = __kmem_cache_create(names[INDEX_AC].name,
1685	sizes[INDEX_AC].cs_size,
1686	ARCH_KMALLOC_MINALIGN,
1687	ARCH_KMALLOC_FLAGS\|SLAB_PANIC,
1688	NULL);
1689
1690	if (INDEX_AC != INDEX_L3) {
1691	sizes[INDEX_L3].cs_cachep =
1692	__kmem_cache_create(names[INDEX_L3].name,
1693	sizes[INDEX_L3].cs_size,
1694	ARCH_KMALLOC_MINALIGN,
1695	ARCH_KMALLOC_FLAGS\|SLAB_PANIC,
1696	NULL);
1697	}
1698
1699	slab_early_init = 0;
1700
1701	while (sizes->cs_size != ULONG_MAX) {
1702	/*
1703	* For performance, all the general caches are L1 aligned.
1704	* This should be particularly beneficial on SMP boxes, as it
1705	* eliminates "false sharing".
1706	* Note for systems short on memory removing the alignment will
1707	* allow tighter packing of the smaller caches.
1708	*/
1709	if (!sizes->cs_cachep) {
1710	sizes->cs_cachep = __kmem_cache_create(names->name,
1711	sizes->cs_size,
1712	ARCH_KMALLOC_MINALIGN,
1713	ARCH_KMALLOC_FLAGS\|SLAB_PANIC,
1714	NULL);
1715	}
1716	#ifdef CONFIG_ZONE_DMA
1717	sizes->cs_dmacachep = __kmem_cache_create(
1718	names->name_dma,
1719	sizes->cs_size,
1720	ARCH_KMALLOC_MINALIGN,
1721	ARCH_KMALLOC_FLAGS\|SLAB_CACHE_DMA\|
1722	SLAB_PANIC,
1723	NULL);
1724	#endif
1725	sizes++;
1726	names++;
1727	}
1728	/* 4) Replace the bootstrap head arrays */
1729	{
1730	struct array_cache *ptr;
1731
1732	ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1733
1734	BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache);
1735	memcpy(ptr, cpu_cache_get(&cache_cache),
1736	sizeof(struct arraycache_init));
1737	/*
1738	* Do not assume that spinlocks can be initialized via memcpy:
1739	*/
1740	spin_lock_init(&ptr->lock);
1741
1742	cache_cache.array[smp_processor_id()] = ptr;
1743
1744	ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1745
1746	BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
1747	!= &initarray_generic.cache);
1748	memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
1749	sizeof(struct arraycache_init));
1750	/*
1751	* Do not assume that spinlocks can be initialized via memcpy:
1752	*/
1753	spin_lock_init(&ptr->lock);
1754
1755	malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
1756	ptr;
1757	}
1758	/* 5) Replace the bootstrap kmem_list3's */
1759	{
1760	int nid;
1761
1762	for_each_online_node(nid) {
1763	init_list(&cache_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
1764
1765	init_list(malloc_sizes[INDEX_AC].cs_cachep,
1766	&initkmem_list3[SIZE_AC + nid], nid);
1767
1768	if (INDEX_AC != INDEX_L3) {
1769	init_list(malloc_sizes[INDEX_L3].cs_cachep,
1770	&initkmem_list3[SIZE_L3 + nid], nid);
1771	}
1772	}
1773	}
1774
1775	slab_state = UP;
1776	}
1777
1778	void __init kmem_cache_init_late(void)
1779	{
1780	struct kmem_cache *cachep;
1781
1782	slab_state = UP;
1783
1784	/* Annotate slab for lockdep -- annotate the malloc caches */
1785	init_lock_keys();
1786
1787	/* 6) resize the head arrays to their final sizes */
1788	mutex_lock(&slab_mutex);
1789	list_for_each_entry(cachep, &slab_caches, list)
1790	if (enable_cpucache(cachep, GFP_NOWAIT))
1791	BUG();
1792	mutex_unlock(&slab_mutex);
1793
1794	/* Done! */
1795	slab_state = FULL;
1796
1797	/*
1798	* Register a cpu startup notifier callback that initializes
1799	* cpu_cache_get for all new cpus
1800	*/
1801	register_cpu_notifier(&cpucache_notifier);
1802
1803	#ifdef CONFIG_NUMA
1804	/*
1805	* Register a memory hotplug callback that initializes and frees
1806	* nodelists.
1807	*/
1808	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1809	#endif
1810
1811	/*
1812	* The reap timers are started later, with a module init call: That part
1813	* of the kernel is not yet operational.
1814	*/
1815	}
1816
1817	static int __init cpucache_init(void)
1818	{
1819	int cpu;
1820
1821	/*
1822	* Register the timers that return unneeded pages to the page allocator
1823	*/
1824	for_each_online_cpu(cpu)
1825	start_cpu_timer(cpu);
1826
1827	/* Done! */
1828	slab_state = FULL;
1829	return 0;
1830	}
1831	__initcall(cpucache_init);
1832
1833	static noinline void
1834	slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
1835	{
1836	struct kmem_list3 *l3;
1837	struct slab *slabp;
1838	unsigned long flags;
1839	int node;
1840
1841	printk(KERN_WARNING
1842	"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
1843	nodeid, gfpflags);
1844	printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n",
1845	cachep->name, cachep->size, cachep->gfporder);
1846
1847	for_each_online_node(node) {
1848	unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
1849	unsigned long active_slabs = 0, num_slabs = 0;
1850
1851	l3 = cachep->nodelists[node];
1852	if (!l3)
1853	continue;
1854
1855	spin_lock_irqsave(&l3->list_lock, flags);
1856	list_for_each_entry(slabp, &l3->slabs_full, list) {
1857	active_objs += cachep->num;
1858	active_slabs++;
1859	}
1860	list_for_each_entry(slabp, &l3->slabs_partial, list) {
1861	active_objs += slabp->inuse;
1862	active_slabs++;
1863	}
1864	list_for_each_entry(slabp, &l3->slabs_free, list)
1865	num_slabs++;
1866
1867	free_objects += l3->free_objects;
1868	spin_unlock_irqrestore(&l3->list_lock, flags);
1869
1870	num_slabs += active_slabs;
1871	num_objs = num_slabs * cachep->num;
1872	printk(KERN_WARNING
1873	" node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
1874	node, active_slabs, num_slabs, active_objs, num_objs,
1875	free_objects);
1876	}
1877	}
1878
1879	/*
1880	* Interface to system's page allocator. No need to hold the cache-lock.
1881	*
1882	* If we requested dmaable memory, we will get it. Even if we
1883	* did not request dmaable memory, we might get it, but that
1884	* would be relatively rare and ignorable.
1885	*/
1886	static void kmem_getpages(struct kmem_cache cachep, gfp_t flags, int nodeid)
1887	{
1888	struct page *page;
1889	int nr_pages;
1890	int i;
1891
1892	#ifndef CONFIG_MMU
1893	/*
1894	* Nommu uses slab's for process anonymous memory allocations, and thus
1895	* requires __GFP_COMP to properly refcount higher order allocations
1896	*/
1897	flags \|= __GFP_COMP;
1898	#endif
1899
1900	flags \|= cachep->allocflags;
1901	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1902	flags \|= __GFP_RECLAIMABLE;
1903
1904	page = alloc_pages_exact_node(nodeid, flags \| __GFP_NOTRACK, cachep->gfporder);
1905	if (!page) {
1906	if (!(flags & __GFP_NOWARN) && printk_ratelimit())
1907	slab_out_of_memory(cachep, flags, nodeid);
1908	return NULL;
1909	}
1910
1911	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1912	if (unlikely(page->pfmemalloc))
1913	pfmemalloc_active = true;
1914
1915	nr_pages = (1 << cachep->gfporder);
1916	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1917	add_zone_page_state(page_zone(page),
1918	NR_SLAB_RECLAIMABLE, nr_pages);
1919	else
1920	add_zone_page_state(page_zone(page),
1921	NR_SLAB_UNRECLAIMABLE, nr_pages);
1922	for (i = 0; i < nr_pages; i++) {
1923	__SetPageSlab(page + i);
1924
1925	if (page->pfmemalloc)
1926	SetPageSlabPfmemalloc(page + i);
1927	}
1928
1929	if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
1930	kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
1931
1932	if (cachep->ctor)
1933	kmemcheck_mark_uninitialized_pages(page, nr_pages);
1934	else
1935	kmemcheck_mark_unallocated_pages(page, nr_pages);
1936	}
1937
1938	return page_address(page);
1939	}
1940
1941	/*
1942	* Interface to system's page release.
1943	*/
1944	static void kmem_freepages(struct kmem_cache cachep, void addr)
1945	{
1946	unsigned long i = (1 << cachep->gfporder);
1947	struct page *page = virt_to_page(addr);
1948	const unsigned long nr_freed = i;
1949
1950	kmemcheck_free_shadow(page, cachep->gfporder);
1951
1952	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1953	sub_zone_page_state(page_zone(page),
1954	NR_SLAB_RECLAIMABLE, nr_freed);
1955	else
1956	sub_zone_page_state(page_zone(page),
1957	NR_SLAB_UNRECLAIMABLE, nr_freed);
1958	while (i--) {
1959	BUG_ON(!PageSlab(page));
1960	__ClearPageSlabPfmemalloc(page);
1961	__ClearPageSlab(page);
1962	page++;
1963	}
1964	if (current->reclaim_state)
1965	current->reclaim_state->reclaimed_slab += nr_freed;
1966	free_pages((unsigned long)addr, cachep->gfporder);
1967	}
1968
1969	static void kmem_rcu_free(struct rcu_head *head)
1970	{
1971	struct slab_rcu slab_rcu = (struct slab_rcu )head;
1972	struct kmem_cache *cachep = slab_rcu->cachep;
1973
1974	kmem_freepages(cachep, slab_rcu->addr);
1975	if (OFF_SLAB(cachep))
1976	kmem_cache_free(cachep->slabp_cache, slab_rcu);
1977	}
1978
1979	#if DEBUG
1980
1981	#ifdef CONFIG_DEBUG_PAGEALLOC
1982	static void store_stackinfo(struct kmem_cache cachep, unsigned long addr,
1983	unsigned long caller)
1984	{
1985	int size = cachep->object_size;
1986
1987	addr = (unsigned long )&((char )addr)[obj_offset(cachep)];
1988
1989	if (size < 5 * sizeof(unsigned long))
1990	return;
1991
1992	*addr++ = 0x12345678;
1993	*addr++ = caller;
1994	*addr++ = smp_processor_id();
1995	size -= 3 * sizeof(unsigned long);
1996	{
1997	unsigned long *sptr = &caller;
1998	unsigned long svalue;
1999
2000	while (!kstack_end(sptr)) {
2001	svalue = *sptr++;
2002	if (kernel_text_address(svalue)) {
2003	*addr++ = svalue;
2004	size -= sizeof(unsigned long);
2005	if (size <= sizeof(unsigned long))
2006	break;
2007	}
2008	}
2009
2010	}
2011	*addr++ = 0x87654321;
2012	}
2013	#endif
2014
2015	static void poison_obj(struct kmem_cache cachep, void addr, unsigned char val)
2016	{
2017	int size = cachep->object_size;
2018	addr = &((char *)addr)[obj_offset(cachep)];
2019
2020	memset(addr, val, size);
2021	(unsigned char )(addr + size - 1) = POISON_END;
2022	}
2023
2024	static void dump_line(char *data, int offset, int limit)
2025	{
2026	int i;
2027	unsigned char error = 0;
2028	int bad_count = 0;
2029
2030	printk(KERN_ERR "%03x: ", offset);
2031	for (i = 0; i < limit; i++) {
2032	if (data[offset + i] != POISON_FREE) {
2033	error = data[offset + i];
2034	bad_count++;
2035	}
2036	}
2037	print_hex_dump(KERN_CONT, "", 0, 16, 1,
2038	&data[offset], limit, 1);
2039
2040	if (bad_count == 1) {
2041	error ^= POISON_FREE;
2042	if (!(error & (error - 1))) {
2043	printk(KERN_ERR "Single bit error detected. Probably "
2044	"bad RAM.\n");
2045	#ifdef CONFIG_X86
2046	printk(KERN_ERR "Run memtest86+ or a similar memory "
2047	"test tool.\n");
2048	#else
2049	printk(KERN_ERR "Run a memory test tool.\n");
2050	#endif
2051	}
2052	}
2053	}
2054	#endif
2055
2056	#if DEBUG
2057
2058	static void print_objinfo(struct kmem_cache cachep, void objp, int lines)
2059	{
2060	int i, size;
2061	char *realobj;
2062
2063	if (cachep->flags & SLAB_RED_ZONE) {
2064	printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
2065	*dbg_redzone1(cachep, objp),
2066	*dbg_redzone2(cachep, objp));
2067	}
2068
2069	if (cachep->flags & SLAB_STORE_USER) {
2070	printk(KERN_ERR "Last user: [<%p>]",
2071	*dbg_userword(cachep, objp));
2072	print_symbol("(%s)",
2073	(unsigned long)*dbg_userword(cachep, objp));
2074	printk("\n");
2075	}
2076	realobj = (char *)objp + obj_offset(cachep);
2077	size = cachep->object_size;
2078	for (i = 0; i < size && lines; i += 16, lines--) {
2079	int limit;
2080	limit = 16;
2081	if (i + limit > size)
2082	limit = size - i;
2083	dump_line(realobj, i, limit);
2084	}
2085	}
2086
2087	static void check_poison_obj(struct kmem_cache cachep, void objp)
2088	{
2089	char *realobj;
2090	int size, i;
2091	int lines = 0;
2092
2093	realobj = (char *)objp + obj_offset(cachep);
2094	size = cachep->object_size;
2095
2096	for (i = 0; i < size; i++) {
2097	char exp = POISON_FREE;
2098	if (i == size - 1)
2099	exp = POISON_END;
2100	if (realobj[i] != exp) {
2101	int limit;
2102	/* Mismatch ! */
2103	/* Print header */
2104	if (lines == 0) {
2105	printk(KERN_ERR
2106	"Slab corruption (%s): %s start=%p, len=%d\n",
2107	print_tainted(), cachep->name, realobj, size);
2108	print_objinfo(cachep, objp, 0);
2109	}
2110	/* Hexdump the affected line */
2111	i = (i / 16) * 16;
2112	limit = 16;
2113	if (i + limit > size)
2114	limit = size - i;
2115	dump_line(realobj, i, limit);
2116	i += 16;
2117	lines++;
2118	/* Limit to 5 lines */
2119	if (lines > 5)
2120	break;
2121	}
2122	}
2123	if (lines != 0) {
2124	/* Print some data about the neighboring objects, if they
2125	* exist:
2126	*/
2127	struct slab *slabp = virt_to_slab(objp);
2128	unsigned int objnr;
2129
2130	objnr = obj_to_index(cachep, slabp, objp);
2131	if (objnr) {
2132	objp = index_to_obj(cachep, slabp, objnr - 1);
2133	realobj = (char *)objp + obj_offset(cachep);
2134	printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
2135	realobj, size);
2136	print_objinfo(cachep, objp, 2);
2137	}
2138	if (objnr + 1 < cachep->num) {
2139	objp = index_to_obj(cachep, slabp, objnr + 1);
2140	realobj = (char *)objp + obj_offset(cachep);
2141	printk(KERN_ERR "Next obj: start=%p, len=%d\n",
2142	realobj, size);
2143	print_objinfo(cachep, objp, 2);
2144	}
2145	}
2146	}
2147	#endif
2148
2149	#if DEBUG
2150	static void slab_destroy_debugcheck(struct kmem_cache cachep, struct slab slabp)
2151	{
2152	int i;
2153	for (i = 0; i < cachep->num; i++) {
2154	void *objp = index_to_obj(cachep, slabp, i);
2155
2156	if (cachep->flags & SLAB_POISON) {
2157	#ifdef CONFIG_DEBUG_PAGEALLOC
2158	if (cachep->size % PAGE_SIZE == 0 &&
2159	OFF_SLAB(cachep))
2160	kernel_map_pages(virt_to_page(objp),
2161	cachep->size / PAGE_SIZE, 1);
2162	else
2163	check_poison_obj(cachep, objp);
2164	#else
2165	check_poison_obj(cachep, objp);
2166	#endif
2167	}
2168	if (cachep->flags & SLAB_RED_ZONE) {
2169	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2170	slab_error(cachep, "start of a freed object "
2171	"was overwritten");
2172	if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2173	slab_error(cachep, "end of a freed object "
2174	"was overwritten");
2175	}
2176	}
2177	}
2178	#else
2179	static void slab_destroy_debugcheck(struct kmem_cache cachep, struct slab slabp)
2180	{
2181	}
2182	#endif
2183
2184	/**
2185	* slab_destroy - destroy and release all objects in a slab
2186	* @cachep: cache pointer being destroyed
2187	* @slabp: slab pointer being destroyed
2188	*
2189	* Destroy all the objs in a slab, and release the mem back to the system.
2190	* Before calling the slab must have been unlinked from the cache. The
2191	* cache-lock is not held/needed.
2192	*/
2193	static void slab_destroy(struct kmem_cache cachep, struct slab slabp)
2194	{
2195	void *addr = slabp->s_mem - slabp->colouroff;
2196
2197	slab_destroy_debugcheck(cachep, slabp);
2198	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
2199	struct slab_rcu *slab_rcu;
2200
2201	slab_rcu = (struct slab_rcu *)slabp;
2202	slab_rcu->cachep = cachep;
2203	slab_rcu->addr = addr;
2204	call_rcu(&slab_rcu->head, kmem_rcu_free);
2205	} else {
2206	kmem_freepages(cachep, addr);
2207	if (OFF_SLAB(cachep))
2208	kmem_cache_free(cachep->slabp_cache, slabp);
2209	}
2210	}
2211
2212	static void __kmem_cache_destroy(struct kmem_cache *cachep)
2213	{
2214	int i;
2215	struct kmem_list3 *l3;
2216
2217	for_each_online_cpu(i)
2218	kfree(cachep->array[i]);
2219
2220	/* NUMA: free the list3 structures */
2221	for_each_online_node(i) {
2222	l3 = cachep->nodelists[i];
2223	if (l3) {
2224	kfree(l3->shared);
2225	free_alien_cache(l3->alien);
2226	kfree(l3);
2227	}
2228	}
2229	kmem_cache_free(&cache_cache, cachep);
2230	}
2231
2232
2233	/**
2234	* calculate_slab_order - calculate size (page order) of slabs
2235	* @cachep: pointer to the cache that is being created
2236	* @size: size of objects to be created in this cache.
2237	* @align: required alignment for the objects.
2238	* @flags: slab allocation flags
2239	*
2240	* Also calculates the number of objects per slab.
2241	*
2242	* This could be made much more intelligent. For now, try to avoid using
2243	* high order pages for slabs. When the gfp() functions are more friendly
2244	* towards high-order requests, this should be changed.
2245	*/
2246	static size_t calculate_slab_order(struct kmem_cache *cachep,
2247	size_t size, size_t align, unsigned long flags)
2248	{
2249	unsigned long offslab_limit;
2250	size_t left_over = 0;
2251	int gfporder;
2252
2253	for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
2254	unsigned int num;
2255	size_t remainder;
2256
2257	cache_estimate(gfporder, size, align, flags, &remainder, &num);
2258	if (!num)
2259	continue;
2260
2261	if (flags & CFLGS_OFF_SLAB) {
2262	/*
2263	* Max number of objs-per-slab for caches which
2264	* use off-slab slabs. Needed to avoid a possible
2265	* looping condition in cache_grow().
2266	*/
2267	offslab_limit = size - sizeof(struct slab);
2268	offslab_limit /= sizeof(kmem_bufctl_t);
2269
2270	if (num > offslab_limit)
2271	break;
2272	}
2273
2274	/* Found something acceptable - save it away */
2275	cachep->num = num;
2276	cachep->gfporder = gfporder;
2277	left_over = remainder;
2278
2279	/*
2280	* A VFS-reclaimable slab tends to have most allocations
2281	* as GFP_NOFS and we really don't want to have to be allocating
2282	* higher-order pages when we are unable to shrink dcache.
2283	*/
2284	if (flags & SLAB_RECLAIM_ACCOUNT)
2285	break;
2286
2287	/*
2288	* Large number of objects is good, but very large slabs are
2289	* currently bad for the gfp()s.
2290	*/
2291	if (gfporder >= slab_max_order)
2292	break;
2293
2294	/*
2295	* Acceptable internal fragmentation?
2296	*/
2297	if (left_over * 8 <= (PAGE_SIZE << gfporder))
2298	break;
2299	}
2300	return left_over;
2301	}
2302
2303	static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2304	{
2305	if (slab_state >= FULL)
2306	return enable_cpucache(cachep, gfp);
2307
2308	if (slab_state == DOWN) {
2309	/*
2310	* Note: the first kmem_cache_create must create the cache
2311	* that's used by kmalloc(24), otherwise the creation of
2312	* further caches will BUG().
2313	*/
2314	cachep->array[smp_processor_id()] = &initarray_generic.cache;
2315
2316	/*
2317	* If the cache that's used by kmalloc(sizeof(kmem_list3)) is
2318	* the first cache, then we need to set up all its list3s,
2319	* otherwise the creation of further caches will BUG().
2320	*/
2321	set_up_list3s(cachep, SIZE_AC);
2322	if (INDEX_AC == INDEX_L3)
2323	slab_state = PARTIAL_L3;
2324	else
2325	slab_state = PARTIAL_ARRAYCACHE;
2326	} else {
2327	cachep->array[smp_processor_id()] =
2328	kmalloc(sizeof(struct arraycache_init), gfp);
2329
2330	if (slab_state == PARTIAL_ARRAYCACHE) {
2331	set_up_list3s(cachep, SIZE_L3);
2332	slab_state = PARTIAL_L3;
2333	} else {
2334	int node;
2335	for_each_online_node(node) {
2336	cachep->nodelists[node] =
2337	kmalloc_node(sizeof(struct kmem_list3),
2338	gfp, node);
2339	BUG_ON(!cachep->nodelists[node]);
2340	kmem_list3_init(cachep->nodelists[node]);
2341	}
2342	}
2343	}
2344	cachep->nodelists[numa_mem_id()]->next_reap =
2345	jiffies + REAPTIMEOUT_LIST3 +
2346	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
2347
2348	cpu_cache_get(cachep)->avail = 0;
2349	cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
2350	cpu_cache_get(cachep)->batchcount = 1;
2351	cpu_cache_get(cachep)->touched = 0;
2352	cachep->batchcount = 1;
2353	cachep->limit = BOOT_CPUCACHE_ENTRIES;
2354	return 0;
2355	}
2356
2357	/**
2358	* __kmem_cache_create - Create a cache.
2359	* @name: A string which is used in /proc/slabinfo to identify this cache.
2360	* @size: The size of objects to be created in this cache.
2361	* @align: The required alignment for the objects.
2362	* @flags: SLAB flags
2363	* @ctor: A constructor for the objects.
2364	*
2365	* Returns a ptr to the cache on success, NULL on failure.
2366	* Cannot be called within a int, but can be interrupted.
2367	* The @ctor is run when new pages are allocated by the cache.
2368	*
2369	* @name must be valid until the cache is destroyed. This implies that
2370	* the module calling this has to destroy the cache before getting unloaded.
2371	*
2372	* The flags are
2373	*
2374	* %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
2375	* to catch references to uninitialised memory.
2376	*
2377	* %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
2378	* for buffer overruns.
2379	*
2380	* %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
2381	* cacheline. This can be beneficial if you're counting cycles as closely
2382	* as davem.
2383	*/
2384	struct kmem_cache *
2385	__kmem_cache_create (const char *name, size_t size, size_t align,
2386	unsigned long flags, void (ctor)(void ))
2387	{
2388	size_t left_over, slab_size, ralign;
2389	struct kmem_cache *cachep = NULL;
2390	gfp_t gfp;
2391
2392	#if DEBUG
2393	#if FORCED_DEBUG
2394	/*
2395	* Enable redzoning and last user accounting, except for caches with
2396	* large objects, if the increased size would increase the object size
2397	* above the next power of two: caches with object sizes just above a
2398	* power of two have a significant amount of internal fragmentation.
2399	*/
2400	if (size < 4096 \|\| fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
2401	2 * sizeof(unsigned long long)))
2402	flags \|= SLAB_RED_ZONE \| SLAB_STORE_USER;
2403	if (!(flags & SLAB_DESTROY_BY_RCU))
2404	flags \|= SLAB_POISON;
2405	#endif
2406	if (flags & SLAB_DESTROY_BY_RCU)
2407	BUG_ON(flags & SLAB_POISON);
2408	#endif
2409	/*
2410	* Always checks flags, a caller might be expecting debug support which
2411	* isn't available.
2412	*/
2413	BUG_ON(flags & ~CREATE_MASK);
2414
2415	/*
2416	* Check that size is in terms of words. This is needed to avoid
2417	* unaligned accesses for some archs when redzoning is used, and makes
2418	* sure any on-slab bufctl's are also correctly aligned.
2419	*/
2420	if (size & (BYTES_PER_WORD - 1)) {
2421	size += (BYTES_PER_WORD - 1);
2422	size &= ~(BYTES_PER_WORD - 1);
2423	}
2424
2425	/* calculate the final buffer alignment: */
2426
2427	/* 1) arch recommendation: can be overridden for debug */
2428	if (flags & SLAB_HWCACHE_ALIGN) {
2429	/*
2430	* Default alignment: as specified by the arch code. Except if
2431	* an object is really small, then squeeze multiple objects into
2432	* one cacheline.
2433	*/
2434	ralign = cache_line_size();
2435	while (size <= ralign / 2)
2436	ralign /= 2;
2437	} else {
2438	ralign = BYTES_PER_WORD;
2439	}
2440
2441	/*
2442	* Redzoning and user store require word alignment or possibly larger.
2443	* Note this will be overridden by architecture or caller mandated
2444	* alignment if either is greater than BYTES_PER_WORD.
2445	*/
2446	if (flags & SLAB_STORE_USER)
2447	ralign = BYTES_PER_WORD;
2448
2449	if (flags & SLAB_RED_ZONE) {
2450	ralign = REDZONE_ALIGN;
2451	/* If redzoning, ensure that the second redzone is suitably
2452	* aligned, by adjusting the object size accordingly. */
2453	size += REDZONE_ALIGN - 1;
2454	size &= ~(REDZONE_ALIGN - 1);
2455	}
2456
2457	/* 2) arch mandated alignment */
2458	if (ralign < ARCH_SLAB_MINALIGN) {
2459	ralign = ARCH_SLAB_MINALIGN;
2460	}
2461	/* 3) caller mandated alignment */
2462	if (ralign < align) {
2463	ralign = align;
2464	}
2465	/* disable debug if necessary */
2466	if (ralign > __alignof__(unsigned long long))
2467	flags &= ~(SLAB_RED_ZONE \| SLAB_STORE_USER);
2468	/*
2469	* 4) Store it.
2470	*/
2471	align = ralign;
2472
2473	if (slab_is_available())
2474	gfp = GFP_KERNEL;
2475	else
2476	gfp = GFP_NOWAIT;
2477
2478	/* Get cache's description obj. */
2479	cachep = kmem_cache_zalloc(&cache_cache, gfp);
2480	if (!cachep)
2481	return NULL;
2482
2483	cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
2484	cachep->object_size = size;
2485	cachep->align = align;
2486	#if DEBUG
2487
2488	/*
2489	* Both debugging options require word-alignment which is calculated
2490	* into align above.
2491	*/
2492	if (flags & SLAB_RED_ZONE) {
2493	/* add space for red zone words */
2494	cachep->obj_offset += sizeof(unsigned long long);
2495	size += 2 * sizeof(unsigned long long);
2496	}
2497	if (flags & SLAB_STORE_USER) {
2498	/* user store requires one word storage behind the end of
2499	* the real object. But if the second red zone needs to be
2500	* aligned to 64 bits, we must allow that much space.
2501	*/
2502	if (flags & SLAB_RED_ZONE)
2503	size += REDZONE_ALIGN;
2504	else
2505	size += BYTES_PER_WORD;
2506	}
2507	#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
2508	if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
2509	&& cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) {
2510	cachep->obj_offset += PAGE_SIZE - ALIGN(size, align);
2511	size = PAGE_SIZE;
2512	}
2513	#endif
2514	#endif
2515
2516	/*
2517	* Determine if the slab management is 'on' or 'off' slab.
2518	* (bootstrapping cannot cope with offslab caches so don't do
2519	* it too early on. Always use on-slab management when
2520	* SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
2521	*/
2522	if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init &&
2523	!(flags & SLAB_NOLEAKTRACE))
2524	/*
2525	* Size is large, assume best to place the slab management obj
2526	* off-slab (should allow better packing of objs).
2527	*/
2528	flags \|= CFLGS_OFF_SLAB;
2529
2530	size = ALIGN(size, align);
2531
2532	left_over = calculate_slab_order(cachep, size, align, flags);
2533
2534	if (!cachep->num) {
2535	printk(KERN_ERR
2536	"kmem_cache_create: couldn't create cache %s.\n", name);
2537	kmem_cache_free(&cache_cache, cachep);
2538	return NULL;
2539	}
2540	slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
2541	+ sizeof(struct slab), align);
2542
2543	/*
2544	* If the slab has been placed off-slab, and we have enough space then
2545	* move it on-slab. This is at the expense of any extra colouring.
2546	*/
2547	if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
2548	flags &= ~CFLGS_OFF_SLAB;
2549	left_over -= slab_size;
2550	}
2551
2552	if (flags & CFLGS_OFF_SLAB) {
2553	/* really off slab. No need for manual alignment */
2554	slab_size =
2555	cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
2556
2557	#ifdef CONFIG_PAGE_POISONING
2558	/* If we're going to use the generic kernel_map_pages()
2559	* poisoning, then it's going to smash the contents of
2560	* the redzone and userword anyhow, so switch them off.
2561	*/
2562	if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
2563	flags &= ~(SLAB_RED_ZONE \| SLAB_STORE_USER);
2564	#endif
2565	}
2566
2567	cachep->colour_off = cache_line_size();
2568	/* Offset must be a multiple of the alignment. */
2569	if (cachep->colour_off < align)
2570	cachep->colour_off = align;
2571	cachep->colour = left_over / cachep->colour_off;
2572	cachep->slab_size = slab_size;
2573	cachep->flags = flags;
2574	cachep->allocflags = 0;
2575	if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
2576	cachep->allocflags \|= GFP_DMA;
2577	cachep->size = size;
2578	cachep->reciprocal_buffer_size = reciprocal_value(size);
2579
2580	if (flags & CFLGS_OFF_SLAB) {
2581	cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
2582	/*
2583	* This is a possibility for one of the malloc_sizes caches.
2584	* But since we go off slab only for object size greater than
2585	* PAGE_SIZE/8, and malloc_sizes gets created in ascending order,
2586	* this should not happen at all.
2587	* But leave a BUG_ON for some lucky dude.
2588	*/
2589	BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));
2590	}
2591	cachep->ctor = ctor;
2592	cachep->name = name;
2593
2594	if (setup_cpu_cache(cachep, gfp)) {
2595	__kmem_cache_destroy(cachep);
2596	return NULL;
2597	}
2598
2599	if (flags & SLAB_DEBUG_OBJECTS) {
2600	/*
2601	* Would deadlock through slab_destroy()->call_rcu()->
2602	* debug_object_activate()->kmem_cache_alloc().
2603	*/
2604	WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
2605
2606	slab_set_debugobj_lock_classes(cachep);
2607	}
2608
2609	/* cache setup completed, link it into the list */
2610	list_add(&cachep->list, &slab_caches);
2611	return cachep;
2612	}
2613
2614	#if DEBUG
2615	static void check_irq_off(void)
2616	{
2617	BUG_ON(!irqs_disabled());
2618	}
2619
2620	static void check_irq_on(void)
2621	{
2622	BUG_ON(irqs_disabled());
2623	}
2624
2625	static void check_spinlock_acquired(struct kmem_cache *cachep)
2626	{
2627	#ifdef CONFIG_SMP
2628	check_irq_off();
2629	assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock);
2630	#endif
2631	}
2632
2633	static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2634	{
2635	#ifdef CONFIG_SMP
2636	check_irq_off();
2637	assert_spin_locked(&cachep->nodelists[node]->list_lock);
2638	#endif
2639	}
2640
2641	#else
2642	#define check_irq_off() do { } while(0)
2643	#define check_irq_on() do { } while(0)
2644	#define check_spinlock_acquired(x) do { } while(0)
2645	#define check_spinlock_acquired_node(x, y) do { } while(0)
2646	#endif
2647
2648	static void drain_array(struct kmem_cache cachep, struct kmem_list3 l3,
2649	struct array_cache *ac,
2650	int force, int node);
2651
2652	static void do_drain(void *arg)
2653	{
2654	struct kmem_cache *cachep = arg;
2655	struct array_cache *ac;
2656	int node = numa_mem_id();
2657
2658	check_irq_off();
2659	ac = cpu_cache_get(cachep);
2660	spin_lock(&cachep->nodelists[node]->list_lock);
2661	free_block(cachep, ac->entry, ac->avail, node);
2662	spin_unlock(&cachep->nodelists[node]->list_lock);
2663	ac->avail = 0;
2664	}
2665
2666	static void drain_cpu_caches(struct kmem_cache *cachep)
2667	{
2668	struct kmem_list3 *l3;
2669	int node;
2670
2671	on_each_cpu(do_drain, cachep, 1);
2672	check_irq_on();
2673	for_each_online_node(node) {
2674	l3 = cachep->nodelists[node];
2675	if (l3 && l3->alien)
2676	drain_alien_cache(cachep, l3->alien);
2677	}
2678
2679	for_each_online_node(node) {
2680	l3 = cachep->nodelists[node];
2681	if (l3)
2682	drain_array(cachep, l3, l3->shared, 1, node);
2683	}
2684	}
2685
2686	/*
2687	* Remove slabs from the list of free slabs.
2688	* Specify the number of slabs to drain in tofree.
2689	*
2690	* Returns the actual number of slabs released.
2691	*/
2692	static int drain_freelist(struct kmem_cache *cache,
2693	struct kmem_list3 *l3, int tofree)
2694	{
2695	struct list_head *p;
2696	int nr_freed;
2697	struct slab *slabp;
2698
2699	nr_freed = 0;
2700	while (nr_freed < tofree && !list_empty(&l3->slabs_free)) {
2701
2702	spin_lock_irq(&l3->list_lock);
2703	p = l3->slabs_free.prev;
2704	if (p == &l3->slabs_free) {
2705	spin_unlock_irq(&l3->list_lock);
2706	goto out;
2707	}
2708
2709	slabp = list_entry(p, struct slab, list);
2710	#if DEBUG
2711	BUG_ON(slabp->inuse);
2712	#endif
2713	list_del(&slabp->list);
2714	/*
2715	* Safe to drop the lock. The slab is no longer linked
2716	* to the cache.
2717	*/
2718	l3->free_objects -= cache->num;
2719	spin_unlock_irq(&l3->list_lock);
2720	slab_destroy(cache, slabp);
2721	nr_freed++;
2722	}
2723	out:
2724	return nr_freed;
2725	}
2726
2727	/* Called with slab_mutex held to protect against cpu hotplug */
2728	static int __cache_shrink(struct kmem_cache *cachep)
2729	{
2730	int ret = 0, i = 0;
2731	struct kmem_list3 *l3;
2732
2733	drain_cpu_caches(cachep);
2734
2735	check_irq_on();
2736	for_each_online_node(i) {
2737	l3 = cachep->nodelists[i];
2738	if (!l3)
2739	continue;
2740
2741	drain_freelist(cachep, l3, l3->free_objects);
2742
2743	ret += !list_empty(&l3->slabs_full) \|\|
2744	!list_empty(&l3->slabs_partial);
2745	}
2746	return (ret ? 1 : 0);
2747	}
2748
2749	/**
2750	* kmem_cache_shrink - Shrink a cache.
2751	* @cachep: The cache to shrink.
2752	*
2753	* Releases as many slabs as possible for a cache.
2754	* To help debugging, a zero exit status indicates all slabs were released.
2755	*/
2756	int kmem_cache_shrink(struct kmem_cache *cachep)
2757	{
2758	int ret;
2759	BUG_ON(!cachep \|\| in_interrupt());
2760
2761	get_online_cpus();
2762	mutex_lock(&slab_mutex);
2763	ret = __cache_shrink(cachep);
2764	mutex_unlock(&slab_mutex);
2765	put_online_cpus();
2766	return ret;
2767	}
2768	EXPORT_SYMBOL(kmem_cache_shrink);
2769
2770	/**
2771	* kmem_cache_destroy - delete a cache
2772	* @cachep: the cache to destroy
2773	*
2774	* Remove a &struct kmem_cache object from the slab cache.
2775	*
2776	* It is expected this function will be called by a module when it is
2777	* unloaded. This will remove the cache completely, and avoid a duplicate
2778	* cache being allocated each time a module is loaded and unloaded, if the
2779	* module doesn't have persistent in-kernel storage across loads and unloads.
2780	*
2781	* The cache must be empty before calling this function.
2782	*
2783	* The caller must guarantee that no one will allocate memory from the cache
2784	* during the kmem_cache_destroy().
2785	*/
2786	void kmem_cache_destroy(struct kmem_cache *cachep)
2787	{
2788	BUG_ON(!cachep \|\| in_interrupt());
2789
2790	/* Find the cache in the chain of caches. */
2791	get_online_cpus();
2792	mutex_lock(&slab_mutex);
2793	/*
2794	* the chain is never empty, cache_cache is never destroyed
2795	*/
2796	list_del(&cachep->list);
2797	if (__cache_shrink(cachep)) {
2798	slab_error(cachep, "Can't free all objects");
2799	list_add(&cachep->list, &slab_caches);
2800	mutex_unlock(&slab_mutex);
2801	put_online_cpus();
2802	return;
2803	}
2804
2805	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
2806	rcu_barrier();
2807
2808	__kmem_cache_destroy(cachep);
2809	mutex_unlock(&slab_mutex);
2810	put_online_cpus();
2811	}
2812	EXPORT_SYMBOL(kmem_cache_destroy);
2813
2814	/*
2815	* Get the memory for a slab management obj.
2816	* For a slab cache when the slab descriptor is off-slab, slab descriptors
2817	* always come from malloc_sizes caches. The slab descriptor cannot
2818	* come from the same cache which is getting created because,
2819	* when we are searching for an appropriate cache for these
2820	* descriptors in kmem_cache_create, we search through the malloc_sizes array.
2821	* If we are creating a malloc_sizes cache here it would not be visible to
2822	* kmem_find_general_cachep till the initialization is complete.
2823	* Hence we cannot have slabp_cache same as the original cache.
2824	*/
2825	static struct slab alloc_slabmgmt(struct kmem_cache cachep, void *objp,
2826	int colour_off, gfp_t local_flags,
2827	int nodeid)
2828	{
2829	struct slab *slabp;
2830
2831	if (OFF_SLAB(cachep)) {
2832	/* Slab management obj is off-slab. */
2833	slabp = kmem_cache_alloc_node(cachep->slabp_cache,
2834	local_flags, nodeid);
2835	/*
2836	* If the first object in the slab is leaked (it's allocated
2837	* but no one has a reference to it), we want to make sure
2838	* kmemleak does not treat the ->s_mem pointer as a reference
2839	* to the object. Otherwise we will not report the leak.
2840	*/
2841	kmemleak_scan_area(&slabp->list, sizeof(struct list_head),
2842	local_flags);
2843	if (!slabp)
2844	return NULL;
2845	} else {
2846	slabp = objp + colour_off;
2847	colour_off += cachep->slab_size;
2848	}
2849	slabp->inuse = 0;
2850	slabp->colouroff = colour_off;
2851	slabp->s_mem = objp + colour_off;
2852	slabp->nodeid = nodeid;
2853	slabp->free = 0;
2854	return slabp;
2855	}
2856
2857	static inline kmem_bufctl_t slab_bufctl(struct slab slabp)
2858	{
2859	return (kmem_bufctl_t *) (slabp + 1);
2860	}
2861
2862	static void cache_init_objs(struct kmem_cache *cachep,
2863	struct slab *slabp)
2864	{
2865	int i;
2866
2867	for (i = 0; i < cachep->num; i++) {
2868	void *objp = index_to_obj(cachep, slabp, i);
2869	#if DEBUG
2870	/* need to poison the objs? */
2871	if (cachep->flags & SLAB_POISON)
2872	poison_obj(cachep, objp, POISON_FREE);
2873	if (cachep->flags & SLAB_STORE_USER)
2874	*dbg_userword(cachep, objp) = NULL;
2875
2876	if (cachep->flags & SLAB_RED_ZONE) {
2877	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2878	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2879	}
2880	/*
2881	* Constructors are not allowed to allocate memory from the same
2882	* cache which they are a constructor for. Otherwise, deadlock.
2883	* They must also be threaded.
2884	*/
2885	if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2886	cachep->ctor(objp + obj_offset(cachep));
2887
2888	if (cachep->flags & SLAB_RED_ZONE) {
2889	if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2890	slab_error(cachep, "constructor overwrote the"
2891	" end of an object");
2892	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2893	slab_error(cachep, "constructor overwrote the"
2894	" start of an object");
2895	}
2896	if ((cachep->size % PAGE_SIZE) == 0 &&
2897	OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
2898	kernel_map_pages(virt_to_page(objp),
2899	cachep->size / PAGE_SIZE, 0);
2900	#else
2901	if (cachep->ctor)
2902	cachep->ctor(objp);
2903	#endif
2904	slab_bufctl(slabp)[i] = i + 1;
2905	}
2906	slab_bufctl(slabp)[i - 1] = BUFCTL_END;
2907	}
2908
2909	static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
2910	{
2911	if (CONFIG_ZONE_DMA_FLAG) {
2912	if (flags & GFP_DMA)
2913	BUG_ON(!(cachep->allocflags & GFP_DMA));
2914	else
2915	BUG_ON(cachep->allocflags & GFP_DMA);
2916	}
2917	}
2918
2919	static void slab_get_obj(struct kmem_cache cachep, struct slab *slabp,
2920	int nodeid)
2921	{
2922	void *objp = index_to_obj(cachep, slabp, slabp->free);
2923	kmem_bufctl_t next;
2924
2925	slabp->inuse++;
2926	next = slab_bufctl(slabp)[slabp->free];
2927	#if DEBUG
2928	slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2929	WARN_ON(slabp->nodeid != nodeid);
2930	#endif
2931	slabp->free = next;
2932
2933	return objp;
2934	}
2935
2936	static void slab_put_obj(struct kmem_cache cachep, struct slab slabp,
2937	void *objp, int nodeid)
2938	{
2939	unsigned int objnr = obj_to_index(cachep, slabp, objp);
2940
2941	#if DEBUG
2942	/* Verify that the slab belongs to the intended node */
2943	WARN_ON(slabp->nodeid != nodeid);
2944
2945	if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
2946	printk(KERN_ERR "slab: double free detected in cache "
2947	"'%s', objp %p\n", cachep->name, objp);
2948	BUG();
2949	}
2950	#endif
2951	slab_bufctl(slabp)[objnr] = slabp->free;
2952	slabp->free = objnr;
2953	slabp->inuse--;
2954	}
2955
2956	/*
2957	* Map pages beginning at addr to the given cache and slab. This is required
2958	* for the slab allocator to be able to lookup the cache and slab of a
2959	* virtual address for kfree, ksize, and slab debugging.
2960	*/
2961	static void slab_map_pages(struct kmem_cache cache, struct slab slab,
2962	void *addr)
2963	{
2964	int nr_pages;
2965	struct page *page;
2966
2967	page = virt_to_page(addr);
2968
2969	nr_pages = 1;
2970	if (likely(!PageCompound(page)))
2971	nr_pages <<= cache->gfporder;
2972
2973	do {
2974	page->slab_cache = cache;
2975	page->slab_page = slab;
2976	page++;
2977	} while (--nr_pages);
2978	}
2979
2980	/*
2981	* Grow (by 1) the number of slabs within a cache. This is called by
2982	* kmem_cache_alloc() when there are no active objs left in a cache.
2983	*/
2984	static int cache_grow(struct kmem_cache *cachep,
2985	gfp_t flags, int nodeid, void *objp)
2986	{
2987	struct slab *slabp;
2988	size_t offset;
2989	gfp_t local_flags;
2990	struct kmem_list3 *l3;
2991
2992	/*
2993	* Be lazy and only check for valid flags here, keeping it out of the
2994	* critical path in kmem_cache_alloc().
2995	*/
2996	BUG_ON(flags & GFP_SLAB_BUG_MASK);
2997	local_flags = flags & (GFP_CONSTRAINT_MASK\|GFP_RECLAIM_MASK);
2998
2999	/* Take the l3 list lock to change the colour_next on this node */
3000	check_irq_off();
3001	l3 = cachep->nodelists[nodeid];
3002	spin_lock(&l3->list_lock);
3003
3004	/* Get colour for the slab, and cal the next value. */
3005	offset = l3->colour_next;
3006	l3->colour_next++;
3007	if (l3->colour_next >= cachep->colour)
3008	l3->colour_next = 0;
3009	spin_unlock(&l3->list_lock);
3010
3011	offset *= cachep->colour_off;
3012
3013	if (local_flags & __GFP_WAIT)
3014	local_irq_enable();
3015
3016	/*
3017	* The test for missing atomic flag is performed here, rather than
3018	* the more obvious place, simply to reduce the critical path length
3019	* in kmem_cache_alloc(). If a caller is seriously mis-behaving they
3020	* will eventually be caught here (where it matters).
3021	*/
3022	kmem_flagcheck(cachep, flags);
3023
3024	/*
3025	* Get mem for the objs. Attempt to allocate a physical page from
3026	* 'nodeid'.
3027	*/
3028	if (!objp)
3029	objp = kmem_getpages(cachep, local_flags, nodeid);
3030	if (!objp)
3031	goto failed;
3032
3033	/* Get slab management. */
3034	slabp = alloc_slabmgmt(cachep, objp, offset,
3035	local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
3036	if (!slabp)
3037	goto opps1;
3038
3039	slab_map_pages(cachep, slabp, objp);
3040
3041	cache_init_objs(cachep, slabp);
3042
3043	if (local_flags & __GFP_WAIT)
3044	local_irq_disable();
3045	check_irq_off();
3046	spin_lock(&l3->list_lock);
3047
3048	/* Make slab active. */
3049	list_add_tail(&slabp->list, &(l3->slabs_free));
3050	STATS_INC_GROWN(cachep);
3051	l3->free_objects += cachep->num;
3052	spin_unlock(&l3->list_lock);
3053	return 1;
3054	opps1:
3055	kmem_freepages(cachep, objp);
3056	failed:
3057	if (local_flags & __GFP_WAIT)
3058	local_irq_disable();
3059	return 0;
3060	}
3061
3062	#if DEBUG
3063
3064	/*
3065	* Perform extra freeing checks:
3066	* - detect bad pointers.
3067	* - POISON/RED_ZONE checking
3068	*/
3069	static void kfree_debugcheck(const void *objp)
3070	{
3071	if (!virt_addr_valid(objp)) {
3072	printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
3073	(unsigned long)objp);
3074	BUG();
3075	}
3076	}
3077
3078	static inline void verify_redzone_free(struct kmem_cache cache, void obj)
3079	{
3080	unsigned long long redzone1, redzone2;
3081
3082	redzone1 = *dbg_redzone1(cache, obj);
3083	redzone2 = *dbg_redzone2(cache, obj);
3084
3085	/*
3086	* Redzone is ok.
3087	*/
3088	if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
3089	return;
3090
3091	if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
3092	slab_error(cache, "double free detected");
3093	else
3094	slab_error(cache, "memory outside object was overwritten");
3095
3096	printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
3097	obj, redzone1, redzone2);
3098	}
3099
3100	static void cache_free_debugcheck(struct kmem_cache cachep, void *objp,
3101	void *caller)
3102	{
3103	struct page *page;
3104	unsigned int objnr;
3105	struct slab *slabp;
3106
3107	BUG_ON(virt_to_cache(objp) != cachep);
3108
3109	objp -= obj_offset(cachep);
3110	kfree_debugcheck(objp);
3111	page = virt_to_head_page(objp);
3112
3113	slabp = page->slab_page;
3114
3115	if (cachep->flags & SLAB_RED_ZONE) {
3116	verify_redzone_free(cachep, objp);
3117	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
3118	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
3119	}
3120	if (cachep->flags & SLAB_STORE_USER)
3121	*dbg_userword(cachep, objp) = caller;
3122
3123	objnr = obj_to_index(cachep, slabp, objp);
3124
3125	BUG_ON(objnr >= cachep->num);
3126	BUG_ON(objp != index_to_obj(cachep, slabp, objnr));
3127
3128	#ifdef CONFIG_DEBUG_SLAB_LEAK
3129	slab_bufctl(slabp)[objnr] = BUFCTL_FREE;
3130	#endif
3131	if (cachep->flags & SLAB_POISON) {
3132	#ifdef CONFIG_DEBUG_PAGEALLOC
3133	if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
3134	store_stackinfo(cachep, objp, (unsigned long)caller);
3135	kernel_map_pages(virt_to_page(objp),
3136	cachep->size / PAGE_SIZE, 0);
3137	} else {
3138	poison_obj(cachep, objp, POISON_FREE);
3139	}
3140	#else
3141	poison_obj(cachep, objp, POISON_FREE);
3142	#endif
3143	}
3144	return objp;
3145	}
3146
3147	static void check_slabp(struct kmem_cache cachep, struct slab slabp)
3148	{
3149	kmem_bufctl_t i;
3150	int entries = 0;
3151
3152	/* Check slab's freelist to see if this obj is there. */
3153	for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
3154	entries++;
3155	if (entries > cachep->num \|\| i >= cachep->num)
3156	goto bad;
3157	}
3158	if (entries != cachep->num - slabp->inuse) {
3159	bad:
3160	printk(KERN_ERR "slab: Internal list corruption detected in "
3161	"cache '%s'(%d), slabp %p(%d). Tainted(%s). Hexdump:\n",
3162	cachep->name, cachep->num, slabp, slabp->inuse,
3163	print_tainted());
3164	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp,
3165	sizeof(slabp) + cachep->num sizeof(kmem_bufctl_t),
3166	1);
3167	BUG();
3168	}
3169	}
3170	#else
3171	#define kfree_debugcheck(x) do { } while(0)
3172	#define cache_free_debugcheck(x,objp,z) (objp)
3173	#define check_slabp(x,y) do { } while(0)
3174	#endif
3175
3176	static void cache_alloc_refill(struct kmem_cache cachep, gfp_t flags,
3177	bool force_refill)
3178	{
3179	int batchcount;
3180	struct kmem_list3 *l3;
3181	struct array_cache *ac;
3182	int node;
3183
3184	check_irq_off();
3185	node = numa_mem_id();
3186	if (unlikely(force_refill))
3187	goto force_grow;
3188	retry:
3189	ac = cpu_cache_get(cachep);
3190	batchcount = ac->batchcount;
3191	if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
3192	/*
3193	* If there was little recent activity on this cache, then
3194	* perform only a partial refill. Otherwise we could generate
3195	* refill bouncing.
3196	*/
3197	batchcount = BATCHREFILL_LIMIT;
3198	}
3199	l3 = cachep->nodelists[node];
3200
3201	BUG_ON(ac->avail > 0 \|\| !l3);
3202	spin_lock(&l3->list_lock);
3203
3204	/* See if we can refill from the shared array */
3205	if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) {
3206	l3->shared->touched = 1;
3207	goto alloc_done;
3208	}
3209
3210	while (batchcount > 0) {
3211	struct list_head *entry;
3212	struct slab *slabp;
3213	/* Get slab alloc is to come from. */
3214	entry = l3->slabs_partial.next;
3215	if (entry == &l3->slabs_partial) {
3216	l3->free_touched = 1;
3217	entry = l3->slabs_free.next;
3218	if (entry == &l3->slabs_free)
3219	goto must_grow;
3220	}
3221
3222	slabp = list_entry(entry, struct slab, list);
3223	check_slabp(cachep, slabp);
3224	check_spinlock_acquired(cachep);
3225
3226	/*
3227	* The slab was either on partial or free list so
3228	* there must be at least one object available for
3229	* allocation.
3230	*/
3231	BUG_ON(slabp->inuse >= cachep->num);
3232
3233	while (slabp->inuse < cachep->num && batchcount--) {
3234	STATS_INC_ALLOCED(cachep);
3235	STATS_INC_ACTIVE(cachep);
3236	STATS_SET_HIGH(cachep);
3237
3238	ac_put_obj(cachep, ac, slab_get_obj(cachep, slabp,
3239	node));
3240	}
3241	check_slabp(cachep, slabp);
3242
3243	/* move slabp to correct slabp list: */
3244	list_del(&slabp->list);
3245	if (slabp->free == BUFCTL_END)
3246	list_add(&slabp->list, &l3->slabs_full);
3247	else
3248	list_add(&slabp->list, &l3->slabs_partial);
3249	}
3250
3251	must_grow:
3252	l3->free_objects -= ac->avail;
3253	alloc_done:
3254	spin_unlock(&l3->list_lock);
3255
3256	if (unlikely(!ac->avail)) {
3257	int x;
3258	force_grow:
3259	x = cache_grow(cachep, flags \| GFP_THISNODE, node, NULL);
3260
3261	/* cache_grow can reenable interrupts, then ac could change. */
3262	ac = cpu_cache_get(cachep);
3263
3264	/* no objects in sight? abort */
3265	if (!x && (ac->avail == 0 \|\| force_refill))
3266	return NULL;
3267
3268	if (!ac->avail) /* objects refilled by interrupt? */
3269	goto retry;
3270	}
3271	ac->touched = 1;
3272
3273	return ac_get_obj(cachep, ac, flags, force_refill);
3274	}
3275
3276	static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
3277	gfp_t flags)
3278	{
3279	might_sleep_if(flags & __GFP_WAIT);
3280	#if DEBUG
3281	kmem_flagcheck(cachep, flags);
3282	#endif
3283	}
3284
3285	#if DEBUG
3286	static void cache_alloc_debugcheck_after(struct kmem_cache cachep,
3287	gfp_t flags, void objp, void caller)
3288	{
3289	if (!objp)
3290	return objp;
3291	if (cachep->flags & SLAB_POISON) {
3292	#ifdef CONFIG_DEBUG_PAGEALLOC
3293	if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
3294	kernel_map_pages(virt_to_page(objp),
3295	cachep->size / PAGE_SIZE, 1);
3296	else
3297	check_poison_obj(cachep, objp);
3298	#else
3299	check_poison_obj(cachep, objp);
3300	#endif
3301	poison_obj(cachep, objp, POISON_INUSE);
3302	}
3303	if (cachep->flags & SLAB_STORE_USER)
3304	*dbg_userword(cachep, objp) = caller;
3305
3306	if (cachep->flags & SLAB_RED_ZONE) {
3307	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE \|\|
3308	*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
3309	slab_error(cachep, "double free, or memory outside"
3310	" object was overwritten");
3311	printk(KERN_ERR
3312	"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
3313	objp, *dbg_redzone1(cachep, objp),
3314	*dbg_redzone2(cachep, objp));
3315	}
3316	*dbg_redzone1(cachep, objp) = RED_ACTIVE;
3317	*dbg_redzone2(cachep, objp) = RED_ACTIVE;
3318	}
3319	#ifdef CONFIG_DEBUG_SLAB_LEAK
3320	{
3321	struct slab *slabp;
3322	unsigned objnr;
3323
3324	slabp = virt_to_head_page(objp)->slab_page;
3325	objnr = (unsigned)(objp - slabp->s_mem) / cachep->size;
3326	slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
3327	}
3328	#endif
3329	objp += obj_offset(cachep);
3330	if (cachep->ctor && cachep->flags & SLAB_POISON)
3331	cachep->ctor(objp);
3332	if (ARCH_SLAB_MINALIGN &&
3333	((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
3334	printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
3335	objp, (int)ARCH_SLAB_MINALIGN);
3336	}
3337	return objp;
3338	}
3339	#else
3340	#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
3341	#endif
3342
3343	static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
3344	{
3345	if (cachep == &cache_cache)
3346	return false;
3347
3348	return should_failslab(cachep->object_size, flags, cachep->flags);
3349	}
3350
3351	static inline void ____cache_alloc(struct kmem_cache cachep, gfp_t flags)
3352	{
3353	void *objp;
3354	struct array_cache *ac;
3355	bool force_refill = false;
3356
3357	check_irq_off();
3358
3359	ac = cpu_cache_get(cachep);
3360	if (likely(ac->avail)) {
3361	ac->touched = 1;
3362	objp = ac_get_obj(cachep, ac, flags, false);
3363
3364	/*
3365	* Allow for the possibility all avail objects are not allowed
3366	* by the current flags
3367	*/
3368	if (objp) {
3369	STATS_INC_ALLOCHIT(cachep);
3370	goto out;
3371	}
3372	force_refill = true;
3373	}
3374
3375	STATS_INC_ALLOCMISS(cachep);
3376	objp = cache_alloc_refill(cachep, flags, force_refill);
3377	/*
3378	* the 'ac' may be updated by cache_alloc_refill(),
3379	* and kmemleak_erase() requires its correct value.
3380	*/
3381	ac = cpu_cache_get(cachep);
3382
3383	out:
3384	/*
3385	* To avoid a false negative, if an object that is in one of the
3386	* per-CPU caches is leaked, we need to make sure kmemleak doesn't
3387	* treat the array pointers as a reference to the object.
3388	*/
3389	if (objp)
3390	kmemleak_erase(&ac->entry[ac->avail]);
3391	return objp;
3392	}
3393
3394	#ifdef CONFIG_NUMA
3395	/*
3396	* Try allocating on another node if PF_SPREAD_SLAB\|PF_MEMPOLICY.
3397	*
3398	* If we are in_interrupt, then process context, including cpusets and
3399	* mempolicy, may not apply and should not be used for allocation policy.
3400	*/
3401	static void alternate_node_alloc(struct kmem_cache cachep, gfp_t flags)
3402	{
3403	int nid_alloc, nid_here;
3404
3405	if (in_interrupt() \|\| (flags & __GFP_THISNODE))
3406	return NULL;
3407	nid_alloc = nid_here = numa_mem_id();
3408	if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
3409	nid_alloc = cpuset_slab_spread_node();
3410	else if (current->mempolicy)
3411	nid_alloc = slab_node();
3412	if (nid_alloc != nid_here)
3413	return ____cache_alloc_node(cachep, flags, nid_alloc);
3414	return NULL;
3415	}
3416
3417	/*
3418	* Fallback function if there was no memory available and no objects on a
3419	* certain node and fall back is permitted. First we scan all the
3420	* available nodelists for available objects. If that fails then we
3421	* perform an allocation without specifying a node. This allows the page
3422	* allocator to do its reclaim / fallback magic. We then insert the
3423	* slab into the proper nodelist and then allocate from it.
3424	*/
3425	static void fallback_alloc(struct kmem_cache cache, gfp_t flags)
3426	{
3427	struct zonelist *zonelist;
3428	gfp_t local_flags;
3429	struct zoneref *z;
3430	struct zone *zone;
3431	enum zone_type high_zoneidx = gfp_zone(flags);
3432	void *obj = NULL;
3433	int nid;
3434	unsigned int cpuset_mems_cookie;
3435
3436	if (flags & __GFP_THISNODE)
3437	return NULL;
3438
3439	local_flags = flags & (GFP_CONSTRAINT_MASK\|GFP_RECLAIM_MASK);
3440
3441	retry_cpuset:
3442	cpuset_mems_cookie = get_mems_allowed();
3443	zonelist = node_zonelist(slab_node(), flags);
3444
3445	retry:
3446	/*
3447	* Look through allowed nodes for objects available
3448	* from existing per node queues.
3449	*/
3450	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3451	nid = zone_to_nid(zone);
3452
3453	if (cpuset_zone_allowed_hardwall(zone, flags) &&
3454	cache->nodelists[nid] &&
3455	cache->nodelists[nid]->free_objects) {
3456	obj = ____cache_alloc_node(cache,
3457	flags \| GFP_THISNODE, nid);
3458	if (obj)
3459	break;
3460	}
3461	}
3462
3463	if (!obj) {
3464	/*
3465	* This allocation will be performed within the constraints
3466	* of the current cpuset / memory policy requirements.
3467	* We may trigger various forms of reclaim on the allowed
3468	* set and go into memory reserves if necessary.
3469	*/
3470	if (local_flags & __GFP_WAIT)
3471	local_irq_enable();
3472	kmem_flagcheck(cache, flags);
3473	obj = kmem_getpages(cache, local_flags, numa_mem_id());
3474	if (local_flags & __GFP_WAIT)
3475	local_irq_disable();
3476	if (obj) {
3477	/*
3478	* Insert into the appropriate per node queues
3479	*/
3480	nid = page_to_nid(virt_to_page(obj));
3481	if (cache_grow(cache, flags, nid, obj)) {
3482	obj = ____cache_alloc_node(cache,
3483	flags \| GFP_THISNODE, nid);
3484	if (!obj)
3485	/*
3486	* Another processor may allocate the
3487	* objects in the slab since we are
3488	* not holding any locks.
3489	*/
3490	goto retry;
3491	} else {
3492	/* cache_grow already freed obj */
3493	obj = NULL;
3494	}
3495	}
3496	}
3497
3498	if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !obj))
3499	goto retry_cpuset;
3500	return obj;
3501	}
3502
3503	/*
3504	* A interface to enable slab creation on nodeid
3505	*/
3506	static void ____cache_alloc_node(struct kmem_cache cachep, gfp_t flags,
3507	int nodeid)
3508	{
3509	struct list_head *entry;
3510	struct slab *slabp;
3511	struct kmem_list3 *l3;
3512	void *obj;
3513	int x;
3514
3515	l3 = cachep->nodelists[nodeid];
3516	BUG_ON(!l3);
3517
3518	retry:
3519	check_irq_off();
3520	spin_lock(&l3->list_lock);
3521	entry = l3->slabs_partial.next;
3522	if (entry == &l3->slabs_partial) {
3523	l3->free_touched = 1;
3524	entry = l3->slabs_free.next;
3525	if (entry == &l3->slabs_free)
3526	goto must_grow;
3527	}
3528
3529	slabp = list_entry(entry, struct slab, list);
3530	check_spinlock_acquired_node(cachep, nodeid);
3531	check_slabp(cachep, slabp);
3532
3533	STATS_INC_NODEALLOCS(cachep);
3534	STATS_INC_ACTIVE(cachep);
3535	STATS_SET_HIGH(cachep);
3536
3537	BUG_ON(slabp->inuse == cachep->num);
3538
3539	obj = slab_get_obj(cachep, slabp, nodeid);
3540	check_slabp(cachep, slabp);
3541	l3->free_objects--;
3542	/* move slabp to correct slabp list: */
3543	list_del(&slabp->list);
3544
3545	if (slabp->free == BUFCTL_END)
3546	list_add(&slabp->list, &l3->slabs_full);
3547	else
3548	list_add(&slabp->list, &l3->slabs_partial);
3549
3550	spin_unlock(&l3->list_lock);
3551	goto done;
3552
3553	must_grow:
3554	spin_unlock(&l3->list_lock);
3555	x = cache_grow(cachep, flags \| GFP_THISNODE, nodeid, NULL);
3556	if (x)
3557	goto retry;
3558
3559	return fallback_alloc(cachep, flags);
3560
3561	done:
3562	return obj;
3563	}
3564
3565	/**
3566	* kmem_cache_alloc_node - Allocate an object on the specified node
3567	* @cachep: The cache to allocate from.
3568	* @flags: See kmalloc().
3569	* @nodeid: node number of the target node.
3570	* @caller: return address of caller, used for debug information
3571	*
3572	* Identical to kmem_cache_alloc but it will allocate memory on the given
3573	* node, which can improve the performance for cpu bound structures.
3574	*
3575	* Fallback to other node is possible if __GFP_THISNODE is not set.
3576	*/
3577	static __always_inline void *
3578	__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3579	void *caller)
3580	{
3581	unsigned long save_flags;
3582	void *ptr;
3583	int slab_node = numa_mem_id();
3584
3585	flags &= gfp_allowed_mask;
3586
3587	lockdep_trace_alloc(flags);
3588
3589	if (slab_should_failslab(cachep, flags))
3590	return NULL;
3591
3592	cache_alloc_debugcheck_before(cachep, flags);
3593	local_irq_save(save_flags);
3594
3595	if (nodeid == NUMA_NO_NODE)
3596	nodeid = slab_node;
3597
3598	if (unlikely(!cachep->nodelists[nodeid])) {
3599	/* Node not bootstrapped yet */
3600	ptr = fallback_alloc(cachep, flags);
3601	goto out;
3602	}
3603
3604	if (nodeid == slab_node) {
3605	/*
3606	* Use the locally cached objects if possible.
3607	* However ____cache_alloc does not allow fallback
3608	* to other nodes. It may fail while we still have
3609	* objects on other nodes available.
3610	*/
3611	ptr = ____cache_alloc(cachep, flags);
3612	if (ptr)
3613	goto out;
3614	}
3615	/* ___cache_alloc_node can fall back to other nodes */
3616	ptr = ____cache_alloc_node(cachep, flags, nodeid);
3617	out:
3618	local_irq_restore(save_flags);
3619	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
3620	kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags,
3621	flags);
3622
3623	if (likely(ptr))
3624	kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size);
3625
3626	if (unlikely((flags & __GFP_ZERO) && ptr))
3627	memset(ptr, 0, cachep->object_size);
3628
3629	return ptr;
3630	}
3631
3632	static __always_inline void *
3633	__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3634	{
3635	void *objp;
3636
3637	if (unlikely(current->flags & (PF_SPREAD_SLAB \| PF_MEMPOLICY))) {
3638	objp = alternate_node_alloc(cache, flags);
3639	if (objp)
3640	goto out;
3641	}
3642	objp = ____cache_alloc(cache, flags);
3643
3644	/*
3645	* We may just have run out of memory on the local node.
3646	* ____cache_alloc_node() knows how to locate memory on other nodes
3647	*/
3648	if (!objp)
3649	objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3650
3651	out:
3652	return objp;
3653	}
3654	#else
3655
3656	static __always_inline void *
3657	__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3658	{
3659	return ____cache_alloc(cachep, flags);
3660	}
3661
3662	#endif /* CONFIG_NUMA */
3663
3664	static __always_inline void *
3665	__cache_alloc(struct kmem_cache cachep, gfp_t flags, void caller)
3666	{
3667	unsigned long save_flags;
3668	void *objp;
3669
3670	flags &= gfp_allowed_mask;
3671
3672	lockdep_trace_alloc(flags);
3673
3674	if (slab_should_failslab(cachep, flags))
3675	return NULL;
3676
3677	cache_alloc_debugcheck_before(cachep, flags);
3678	local_irq_save(save_flags);
3679	objp = __do_cache_alloc(cachep, flags);
3680	local_irq_restore(save_flags);
3681	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3682	kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags,
3683	flags);
3684	prefetchw(objp);
3685
3686	if (likely(objp))
3687	kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size);
3688
3689	if (unlikely((flags & __GFP_ZERO) && objp))
3690	memset(objp, 0, cachep->object_size);
3691
3692	return objp;
3693	}
3694
3695	/*
3696	* Caller needs to acquire correct kmem_list's list_lock
3697	*/
3698	static void free_block(struct kmem_cache cachep, void *objpp, int nr_objects,
3699	int node)
3700	{
3701	int i;
3702	struct kmem_list3 *l3;
3703
3704	for (i = 0; i < nr_objects; i++) {
3705	void *objp;
3706	struct slab *slabp;
3707
3708	clear_obj_pfmemalloc(&objpp[i]);
3709	objp = objpp[i];
3710
3711	slabp = virt_to_slab(objp);
3712	l3 = cachep->nodelists[node];
3713	list_del(&slabp->list);
3714	check_spinlock_acquired_node(cachep, node);
3715	check_slabp(cachep, slabp);
3716	slab_put_obj(cachep, slabp, objp, node);
3717	STATS_DEC_ACTIVE(cachep);
3718	l3->free_objects++;
3719	check_slabp(cachep, slabp);
3720
3721	/* fixup slab chains */
3722	if (slabp->inuse == 0) {
3723	if (l3->free_objects > l3->free_limit) {
3724	l3->free_objects -= cachep->num;
3725	/* No need to drop any previously held
3726	* lock here, even if we have a off-slab slab
3727	* descriptor it is guaranteed to come from
3728	* a different cache, refer to comments before
3729	* alloc_slabmgmt.
3730	*/
3731	slab_destroy(cachep, slabp);
3732	} else {
3733	list_add(&slabp->list, &l3->slabs_free);
3734	}
3735	} else {
3736	/* Unconditionally move a slab to the end of the
3737	* partial list on free - maximum time for the
3738	* other objects to be freed, too.
3739	*/
3740	list_add_tail(&slabp->list, &l3->slabs_partial);
3741	}
3742	}
3743	}
3744
3745	static void cache_flusharray(struct kmem_cache cachep, struct array_cache ac)
3746	{
3747	int batchcount;
3748	struct kmem_list3 *l3;
3749	int node = numa_mem_id();
3750
3751	batchcount = ac->batchcount;
3752	#if DEBUG
3753	BUG_ON(!batchcount \|\| batchcount > ac->avail);
3754	#endif
3755	check_irq_off();
3756	l3 = cachep->nodelists[node];
3757	spin_lock(&l3->list_lock);
3758	if (l3->shared) {
3759	struct array_cache *shared_array = l3->shared;
3760	int max = shared_array->limit - shared_array->avail;
3761	if (max) {
3762	if (batchcount > max)
3763	batchcount = max;
3764	memcpy(&(shared_array->entry[shared_array->avail]),
3765	ac->entry, sizeof(void ) batchcount);
3766	shared_array->avail += batchcount;
3767	goto free_done;
3768	}
3769	}
3770
3771	free_block(cachep, ac->entry, batchcount, node);
3772	free_done:
3773	#if STATS
3774	{
3775	int i = 0;
3776	struct list_head *p;
3777
3778	p = l3->slabs_free.next;
3779	while (p != &(l3->slabs_free)) {
3780	struct slab *slabp;
3781
3782	slabp = list_entry(p, struct slab, list);
3783	BUG_ON(slabp->inuse);
3784
3785	i++;
3786	p = p->next;
3787	}
3788	STATS_SET_FREEABLE(cachep, i);
3789	}
3790	#endif
3791	spin_unlock(&l3->list_lock);
3792	ac->avail -= batchcount;
3793	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void )ac->avail);
3794	}
3795
3796	/*
3797	* Release an obj back to its cache. If the obj has a constructed state, it must
3798	* be in this state _before_ it is released. Called with disabled ints.
3799	*/
3800	static inline void __cache_free(struct kmem_cache cachep, void objp,
3801	void *caller)
3802	{
3803	struct array_cache *ac = cpu_cache_get(cachep);
3804
3805	check_irq_off();
3806	kmemleak_free_recursive(objp, cachep->flags);
3807	objp = cache_free_debugcheck(cachep, objp, caller);
3808
3809	kmemcheck_slab_free(cachep, objp, cachep->object_size);
3810
3811	/*
3812	* Skip calling cache_free_alien() when the platform is not numa.
3813	* This will avoid cache misses that happen while accessing slabp (which
3814	* is per page memory reference) to get nodeid. Instead use a global
3815	* variable to skip the call, which is mostly likely to be present in
3816	* the cache.
3817	*/
3818	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3819	return;
3820
3821	if (likely(ac->avail < ac->limit)) {
3822	STATS_INC_FREEHIT(cachep);
3823	} else {
3824	STATS_INC_FREEMISS(cachep);
3825	cache_flusharray(cachep, ac);
3826	}
3827
3828	ac_put_obj(cachep, ac, objp);
3829	}
3830
3831	/**
3832	* kmem_cache_alloc - Allocate an object
3833	* @cachep: The cache to allocate from.
3834	* @flags: See kmalloc().
3835	*
3836	* Allocate an object from this cache. The flags are only relevant
3837	* if the cache has no available objects.
3838	*/
3839	void kmem_cache_alloc(struct kmem_cache cachep, gfp_t flags)
3840	{
3841	void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
3842
3843	trace_kmem_cache_alloc(_RET_IP_, ret,
3844	cachep->object_size, cachep->size, flags);
3845
3846	return ret;
3847	}
3848	EXPORT_SYMBOL(kmem_cache_alloc);
3849
3850	#ifdef CONFIG_TRACING
3851	void *
3852	kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags)
3853	{
3854	void *ret;
3855
3856	ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
3857
3858	trace_kmalloc(_RET_IP_, ret,
3859	size, slab_buffer_size(cachep), flags);
3860	return ret;
3861	}
3862	EXPORT_SYMBOL(kmem_cache_alloc_trace);
3863	#endif
3864
3865	#ifdef CONFIG_NUMA
3866	void kmem_cache_alloc_node(struct kmem_cache cachep, gfp_t flags, int nodeid)
3867	{
3868	void *ret = __cache_alloc_node(cachep, flags, nodeid,
3869	__builtin_return_address(0));
3870
3871	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3872	cachep->object_size, cachep->size,
3873	flags, nodeid);
3874
3875	return ret;
3876	}
3877	EXPORT_SYMBOL(kmem_cache_alloc_node);
3878
3879	#ifdef CONFIG_TRACING
3880	void *kmem_cache_alloc_node_trace(size_t size,
3881	struct kmem_cache *cachep,
3882	gfp_t flags,
3883	int nodeid)
3884	{
3885	void *ret;
3886
3887	ret = __cache_alloc_node(cachep, flags, nodeid,
3888	__builtin_return_address(0));
3889	trace_kmalloc_node(_RET_IP_, ret,
3890	size, slab_buffer_size(cachep),
3891	flags, nodeid);
3892	return ret;
3893	}
3894	EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
3895	#endif
3896
3897	static __always_inline void *
3898	__do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
3899	{
3900	struct kmem_cache *cachep;
3901
3902	cachep = kmem_find_general_cachep(size, flags);
3903	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3904	return cachep;
3905	return kmem_cache_alloc_node_trace(size, cachep, flags, node);
3906	}
3907
3908	#if defined(CONFIG_DEBUG_SLAB) \|\| defined(CONFIG_TRACING)
3909	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3910	{
3911	return __do_kmalloc_node(size, flags, node,
3912	__builtin_return_address(0));
3913	}
3914	EXPORT_SYMBOL(__kmalloc_node);
3915
3916	void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3917	int node, unsigned long caller)
3918	{
3919	return __do_kmalloc_node(size, flags, node, (void *)caller);
3920	}
3921	EXPORT_SYMBOL(__kmalloc_node_track_caller);
3922	#else
3923	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3924	{
3925	return __do_kmalloc_node(size, flags, node, NULL);
3926	}
3927	EXPORT_SYMBOL(__kmalloc_node);
3928	#endif /* CONFIG_DEBUG_SLAB \|\| CONFIG_TRACING */
3929	#endif /* CONFIG_NUMA */
3930
3931	/**
3932	* __do_kmalloc - allocate memory
3933	* @size: how many bytes of memory are required.
3934	* @flags: the type of memory to allocate (see kmalloc).
3935	* @caller: function caller for debug tracking of the caller
3936	*/
3937	static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3938	void *caller)
3939	{
3940	struct kmem_cache *cachep;
3941	void *ret;
3942
3943	/* If you want to save a few bytes .text space: replace
3944	* __ with kmem_.
3945	* Then kmalloc uses the uninlined functions instead of the inline
3946	* functions.
3947	*/
3948	cachep = __find_general_cachep(size, flags);
3949	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3950	return cachep;
3951	ret = __cache_alloc(cachep, flags, caller);
3952
3953	trace_kmalloc((unsigned long) caller, ret,
3954	size, cachep->size, flags);
3955
3956	return ret;
3957	}
3958
3959
3960	#if defined(CONFIG_DEBUG_SLAB) \|\| defined(CONFIG_TRACING)
3961	void *__kmalloc(size_t size, gfp_t flags)
3962	{
3963	return __do_kmalloc(size, flags, __builtin_return_address(0));
3964	}
3965	EXPORT_SYMBOL(__kmalloc);
3966
3967	void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3968	{
3969	return __do_kmalloc(size, flags, (void *)caller);
3970	}
3971	EXPORT_SYMBOL(__kmalloc_track_caller);
3972
3973	#else
3974	void *__kmalloc(size_t size, gfp_t flags)
3975	{
3976	return __do_kmalloc(size, flags, NULL);
3977	}
3978	EXPORT_SYMBOL(__kmalloc);
3979	#endif
3980
3981	/**
3982	* kmem_cache_free - Deallocate an object
3983	* @cachep: The cache the allocation was from.
3984	* @objp: The previously allocated object.
3985	*
3986	* Free an object which was previously allocated from this
3987	* cache.
3988	*/
3989	void kmem_cache_free(struct kmem_cache cachep, void objp)
3990	{
3991	unsigned long flags;
3992
3993	local_irq_save(flags);
3994	debug_check_no_locks_freed(objp, cachep->object_size);
3995	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3996	debug_check_no_obj_freed(objp, cachep->object_size);
3997	__cache_free(cachep, objp, __builtin_return_address(0));
3998	local_irq_restore(flags);
3999
4000	trace_kmem_cache_free(_RET_IP_, objp);
4001	}
4002	EXPORT_SYMBOL(kmem_cache_free);
4003
4004	/**
4005	* kfree - free previously allocated memory
4006	* @objp: pointer returned by kmalloc.
4007	*
4008	* If @objp is NULL, no operation is performed.
4009	*
4010	* Don't free memory not originally allocated by kmalloc()
4011	* or you will run into trouble.
4012	*/
4013	void kfree(const void *objp)
4014	{
4015	struct kmem_cache *c;
4016	unsigned long flags;
4017
4018	trace_kfree(_RET_IP_, objp);
4019
4020	if (unlikely(ZERO_OR_NULL_PTR(objp)))
4021	return;
4022	local_irq_save(flags);
4023	kfree_debugcheck(objp);
4024	c = virt_to_cache(objp);
4025	debug_check_no_locks_freed(objp, c->object_size);
4026
4027	debug_check_no_obj_freed(objp, c->object_size);
4028	__cache_free(c, (void *)objp, __builtin_return_address(0));
4029	local_irq_restore(flags);
4030	}
4031	EXPORT_SYMBOL(kfree);
4032
4033	unsigned int kmem_cache_size(struct kmem_cache *cachep)
4034	{
4035	return cachep->object_size;
4036	}
4037	EXPORT_SYMBOL(kmem_cache_size);
4038
4039	/*
4040	* This initializes kmem_list3 or resizes various caches for all nodes.
4041	*/
4042	static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
4043	{
4044	int node;
4045	struct kmem_list3 *l3;
4046	struct array_cache *new_shared;
4047	struct array_cache **new_alien = NULL;
4048
4049	for_each_online_node(node) {
4050
4051	if (use_alien_caches) {
4052	new_alien = alloc_alien_cache(node, cachep->limit, gfp);
4053	if (!new_alien)
4054	goto fail;
4055	}
4056
4057	new_shared = NULL;
4058	if (cachep->shared) {
4059	new_shared = alloc_arraycache(node,
4060	cachep->shared*cachep->batchcount,
4061	0xbaadf00d, gfp);
4062	if (!new_shared) {
4063	free_alien_cache(new_alien);
4064	goto fail;
4065	}
4066	}
4067
4068	l3 = cachep->nodelists[node];
4069	if (l3) {
4070	struct array_cache *shared = l3->shared;
4071
4072	spin_lock_irq(&l3->list_lock);
4073
4074	if (shared)
4075	free_block(cachep, shared->entry,
4076	shared->avail, node);
4077
4078	l3->shared = new_shared;
4079	if (!l3->alien) {
4080	l3->alien = new_alien;
4081	new_alien = NULL;
4082	}
4083	l3->free_limit = (1 + nr_cpus_node(node)) *
4084	cachep->batchcount + cachep->num;
4085	spin_unlock_irq(&l3->list_lock);
4086	kfree(shared);
4087	free_alien_cache(new_alien);
4088	continue;
4089	}
4090	l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node);
4091	if (!l3) {
4092	free_alien_cache(new_alien);
4093	kfree(new_shared);
4094	goto fail;
4095	}
4096
4097	kmem_list3_init(l3);
4098	l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
4099	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
4100	l3->shared = new_shared;
4101	l3->alien = new_alien;
4102	l3->free_limit = (1 + nr_cpus_node(node)) *
4103	cachep->batchcount + cachep->num;
4104	cachep->nodelists[node] = l3;
4105	}
4106	return 0;
4107
4108	fail:
4109	if (!cachep->list.next) {
4110	/* Cache is not active yet. Roll back what we did */
4111	node--;
4112	while (node >= 0) {
4113	if (cachep->nodelists[node]) {
4114	l3 = cachep->nodelists[node];
4115
4116	kfree(l3->shared);
4117	free_alien_cache(l3->alien);
4118	kfree(l3);
4119	cachep->nodelists[node] = NULL;
4120	}
4121	node--;
4122	}
4123	}
4124	return -ENOMEM;
4125	}
4126
4127	struct ccupdate_struct {
4128	struct kmem_cache *cachep;
4129	struct array_cache *new[0];
4130	};
4131
4132	static void do_ccupdate_local(void *info)
4133	{
4134	struct ccupdate_struct *new = info;
4135	struct array_cache *old;
4136
4137	check_irq_off();
4138	old = cpu_cache_get(new->cachep);
4139
4140	new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
4141	new->new[smp_processor_id()] = old;
4142	}
4143
4144	/* Always called with the slab_mutex held */
4145	static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
4146	int batchcount, int shared, gfp_t gfp)
4147	{
4148	struct ccupdate_struct *new;
4149	int i;
4150
4151	new = kzalloc(sizeof(new) + nr_cpu_ids sizeof(struct array_cache *),
4152	gfp);
4153	if (!new)
4154	return -ENOMEM;
4155
4156	for_each_online_cpu(i) {
4157	new->new[i] = alloc_arraycache(cpu_to_mem(i), limit,
4158	batchcount, gfp);
4159	if (!new->new[i]) {
4160	for (i--; i >= 0; i--)
4161	kfree(new->new[i]);
4162	kfree(new);
4163	return -ENOMEM;
4164	}
4165	}
4166	new->cachep = cachep;
4167
4168	on_each_cpu(do_ccupdate_local, (void *)new, 1);
4169
4170	check_irq_on();
4171	cachep->batchcount = batchcount;
4172	cachep->limit = limit;
4173	cachep->shared = shared;
4174
4175	for_each_online_cpu(i) {
4176	struct array_cache *ccold = new->new[i];
4177	if (!ccold)
4178	continue;
4179	spin_lock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
4180	free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i));
4181	spin_unlock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
4182	kfree(ccold);
4183	}
4184	kfree(new);
4185	return alloc_kmemlist(cachep, gfp);
4186	}
4187
4188	/* Called with slab_mutex held always */
4189	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
4190	{
4191	int err;
4192	int limit, shared;
4193
4194	/*
4195	* The head array serves three purposes:
4196	* - create a LIFO ordering, i.e. return objects that are cache-warm
4197	* - reduce the number of spinlock operations.
4198	* - reduce the number of linked list operations on the slab and
4199	* bufctl chains: array operations are cheaper.
4200	* The numbers are guessed, we should auto-tune as described by
4201	* Bonwick.
4202	*/
4203	if (cachep->size > 131072)
4204	limit = 1;
4205	else if (cachep->size > PAGE_SIZE)
4206	limit = 8;
4207	else if (cachep->size > 1024)
4208	limit = 24;
4209	else if (cachep->size > 256)
4210	limit = 54;
4211	else
4212	limit = 120;
4213
4214	/*
4215	* CPU bound tasks (e.g. network routing) can exhibit cpu bound
4216	* allocation behaviour: Most allocs on one cpu, most free operations
4217	* on another cpu. For these cases, an efficient object passing between
4218	* cpus is necessary. This is provided by a shared array. The array
4219	* replaces Bonwick's magazine layer.
4220	* On uniprocessor, it's functionally equivalent (but less efficient)
4221	* to a larger limit. Thus disabled by default.
4222	*/
4223	shared = 0;
4224	if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
4225	shared = 8;
4226
4227	#if DEBUG
4228	/*
4229	* With debugging enabled, large batchcount lead to excessively long
4230	* periods with disabled local interrupts. Limit the batchcount
4231	*/
4232	if (limit > 32)
4233	limit = 32;
4234	#endif
4235	err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared, gfp);
4236	if (err)
4237	printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
4238	cachep->name, -err);
4239	return err;
4240	}
4241
4242	/*
4243	* Drain an array if it contains any elements taking the l3 lock only if
4244	* necessary. Note that the l3 listlock also protects the array_cache
4245	* if drain_array() is used on the shared array.
4246	*/
4247	static void drain_array(struct kmem_cache cachep, struct kmem_list3 l3,
4248	struct array_cache *ac, int force, int node)
4249	{
4250	int tofree;
4251
4252	if (!ac \|\| !ac->avail)
4253	return;
4254	if (ac->touched && !force) {
4255	ac->touched = 0;
4256	} else {
4257	spin_lock_irq(&l3->list_lock);
4258	if (ac->avail) {
4259	tofree = force ? ac->avail : (ac->limit + 4) / 5;
4260	if (tofree > ac->avail)
4261	tofree = (ac->avail + 1) / 2;
4262	free_block(cachep, ac->entry, tofree, node);
4263	ac->avail -= tofree;
4264	memmove(ac->entry, &(ac->entry[tofree]),
4265	sizeof(void ) ac->avail);
4266	}
4267	spin_unlock_irq(&l3->list_lock);
4268	}
4269	}
4270
4271	/**
4272	* cache_reap - Reclaim memory from caches.
4273	* @w: work descriptor
4274	*
4275	* Called from workqueue/eventd every few seconds.
4276	* Purpose:
4277	* - clear the per-cpu caches for this CPU.
4278	* - return freeable pages to the main free memory pool.
4279	*
4280	* If we cannot acquire the cache chain mutex then just give up - we'll try
4281	* again on the next iteration.
4282	*/
4283	static void cache_reap(struct work_struct *w)
4284	{
4285	struct kmem_cache *searchp;
4286	struct kmem_list3 *l3;
4287	int node = numa_mem_id();
4288	struct delayed_work *work = to_delayed_work(w);
4289
4290	if (!mutex_trylock(&slab_mutex))
4291	/* Give up. Setup the next iteration. */
4292	goto out;
4293
4294	list_for_each_entry(searchp, &slab_caches, list) {
4295	check_irq_on();
4296
4297	/*
4298	* We only take the l3 lock if absolutely necessary and we
4299	* have established with reasonable certainty that
4300	* we can do some work if the lock was obtained.
4301	*/
4302	l3 = searchp->nodelists[node];
4303
4304	reap_alien(searchp, l3);
4305
4306	drain_array(searchp, l3, cpu_cache_get(searchp), 0, node);
4307
4308	/*
4309	* These are racy checks but it does not matter
4310	* if we skip one check or scan twice.
4311	*/
4312	if (time_after(l3->next_reap, jiffies))
4313	goto next;
4314
4315	l3->next_reap = jiffies + REAPTIMEOUT_LIST3;
4316
4317	drain_array(searchp, l3, l3->shared, 0, node);
4318
4319	if (l3->free_touched)
4320	l3->free_touched = 0;
4321	else {
4322	int freed;
4323
4324	freed = drain_freelist(searchp, l3, (l3->free_limit +
4325	5 * searchp->num - 1) / (5 * searchp->num));
4326	STATS_ADD_REAPED(searchp, freed);
4327	}
4328	next:
4329	cond_resched();
4330	}
4331	check_irq_on();
4332	mutex_unlock(&slab_mutex);
4333	next_reap_node();
4334	out:
4335	/* Set up the next iteration */
4336	schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_CPUC));
4337	}
4338
4339	#ifdef CONFIG_SLABINFO
4340
4341	static void print_slabinfo_header(struct seq_file *m)
4342	{
4343	/*
4344	* Output format version, so at least we can change it
4345	* without _too_ many complaints.
4346	*/
4347	#if STATS
4348	seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
4349	#else
4350	seq_puts(m, "slabinfo - version: 2.1\n");
4351	#endif
4352	seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
4353	"<objperslab> <pagesperslab>");
4354	seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
4355	seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
4356	#if STATS
4357	seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
4358	"<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
4359	seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
4360	#endif
4361	seq_putc(m, '\n');
4362	}
4363
4364	static void s_start(struct seq_file m, loff_t *pos)
4365	{
4366	loff_t n = *pos;
4367
4368	mutex_lock(&slab_mutex);
4369	if (!n)
4370	print_slabinfo_header(m);
4371
4372	return seq_list_start(&slab_caches, *pos);
4373	}
4374
4375	static void s_next(struct seq_file m, void p, loff_t pos)
4376	{
4377	return seq_list_next(p, &slab_caches, pos);
4378	}
4379
4380	static void s_stop(struct seq_file m, void p)
4381	{
4382	mutex_unlock(&slab_mutex);
4383	}
4384
4385	static int s_show(struct seq_file m, void p)
4386	{
4387	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4388	struct slab *slabp;
4389	unsigned long active_objs;
4390	unsigned long num_objs;
4391	unsigned long active_slabs = 0;
4392	unsigned long num_slabs, free_objects = 0, shared_avail = 0;
4393	const char *name;
4394	char *error = NULL;
4395	int node;
4396	struct kmem_list3 *l3;
4397
4398	active_objs = 0;
4399	num_slabs = 0;
4400	for_each_online_node(node) {
4401	l3 = cachep->nodelists[node];
4402	if (!l3)
4403	continue;
4404
4405	check_irq_on();
4406	spin_lock_irq(&l3->list_lock);
4407
4408	list_for_each_entry(slabp, &l3->slabs_full, list) {
4409	if (slabp->inuse != cachep->num && !error)
4410	error = "slabs_full accounting error";
4411	active_objs += cachep->num;
4412	active_slabs++;
4413	}
4414	list_for_each_entry(slabp, &l3->slabs_partial, list) {
4415	if (slabp->inuse == cachep->num && !error)
4416	error = "slabs_partial inuse accounting error";
4417	if (!slabp->inuse && !error)
4418	error = "slabs_partial/inuse accounting error";
4419	active_objs += slabp->inuse;
4420	active_slabs++;
4421	}
4422	list_for_each_entry(slabp, &l3->slabs_free, list) {
4423	if (slabp->inuse && !error)
4424	error = "slabs_free/inuse accounting error";
4425	num_slabs++;
4426	}
4427	free_objects += l3->free_objects;
4428	if (l3->shared)
4429	shared_avail += l3->shared->avail;
4430
4431	spin_unlock_irq(&l3->list_lock);
4432	}
4433	num_slabs += active_slabs;
4434	num_objs = num_slabs * cachep->num;
4435	if (num_objs - active_objs != free_objects && !error)
4436	error = "free_objects accounting error";
4437
4438	name = cachep->name;
4439	if (error)
4440	printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
4441
4442	seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
4443	name, active_objs, num_objs, cachep->size,
4444	cachep->num, (1 << cachep->gfporder));
4445	seq_printf(m, " : tunables %4u %4u %4u",
4446	cachep->limit, cachep->batchcount, cachep->shared);
4447	seq_printf(m, " : slabdata %6lu %6lu %6lu",
4448	active_slabs, num_slabs, shared_avail);
4449	#if STATS
4450	{ /* list3 stats */
4451	unsigned long high = cachep->high_mark;
4452	unsigned long allocs = cachep->num_allocations;
4453	unsigned long grown = cachep->grown;
4454	unsigned long reaped = cachep->reaped;
4455	unsigned long errors = cachep->errors;
4456	unsigned long max_freeable = cachep->max_freeable;
4457	unsigned long node_allocs = cachep->node_allocs;
4458	unsigned long node_frees = cachep->node_frees;
4459	unsigned long overflows = cachep->node_overflow;
4460
4461	seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4462	"%4lu %4lu %4lu %4lu %4lu",
4463	allocs, high, grown,
4464	reaped, errors, max_freeable, node_allocs,
4465	node_frees, overflows);
4466	}
4467	/* cpu stats */
4468	{
4469	unsigned long allochit = atomic_read(&cachep->allochit);
4470	unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4471	unsigned long freehit = atomic_read(&cachep->freehit);
4472	unsigned long freemiss = atomic_read(&cachep->freemiss);
4473
4474	seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
4475	allochit, allocmiss, freehit, freemiss);
4476	}
4477	#endif
4478	seq_putc(m, '\n');
4479	return 0;
4480	}
4481
4482	/*
4483	* slabinfo_op - iterator that generates /proc/slabinfo
4484	*
4485	* Output layout:
4486	* cache-name
4487	* num-active-objs
4488	* total-objs
4489	* object size
4490	* num-active-slabs
4491	* total-slabs
4492	* num-pages-per-slab
4493	* + further values on SMP and with statistics enabled
4494	*/
4495
4496	static const struct seq_operations slabinfo_op = {
4497	.start = s_start,
4498	.next = s_next,
4499	.stop = s_stop,
4500	.show = s_show,
4501	};
4502
4503	#define MAX_SLABINFO_WRITE 128
4504	/**
4505	* slabinfo_write - Tuning for the slab allocator
4506	* @file: unused
4507	* @buffer: user buffer
4508	* @count: data length
4509	* @ppos: unused
4510	*/
4511	static ssize_t slabinfo_write(struct file file, const char __user buffer,
4512	size_t count, loff_t *ppos)
4513	{
4514	char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
4515	int limit, batchcount, shared, res;
4516	struct kmem_cache *cachep;
4517
4518	if (count > MAX_SLABINFO_WRITE)
4519	return -EINVAL;
4520	if (copy_from_user(&kbuf, buffer, count))
4521	return -EFAULT;
4522	kbuf[MAX_SLABINFO_WRITE] = '\0';
4523
4524	tmp = strchr(kbuf, ' ');
4525	if (!tmp)
4526	return -EINVAL;
4527	*tmp = '\0';
4528	tmp++;
4529	if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4530	return -EINVAL;
4531
4532	/* Find the cache in the chain of caches. */
4533	mutex_lock(&slab_mutex);
4534	res = -EINVAL;
4535	list_for_each_entry(cachep, &slab_caches, list) {
4536	if (!strcmp(cachep->name, kbuf)) {
4537	if (limit < 1 \|\| batchcount < 1 \|\|
4538	batchcount > limit \|\| shared < 0) {
4539	res = 0;
4540	} else {
4541	res = do_tune_cpucache(cachep, limit,
4542	batchcount, shared,
4543	GFP_KERNEL);
4544	}
4545	break;
4546	}
4547	}
4548	mutex_unlock(&slab_mutex);
4549	if (res >= 0)
4550	res = count;
4551	return res;
4552	}
4553
4554	static int slabinfo_open(struct inode inode, struct file file)
4555	{
4556	return seq_open(file, &slabinfo_op);
4557	}
4558
4559	static const struct file_operations proc_slabinfo_operations = {
4560	.open = slabinfo_open,
4561	.read = seq_read,
4562	.write = slabinfo_write,
4563	.llseek = seq_lseek,
4564	.release = seq_release,
4565	};
4566
4567	#ifdef CONFIG_DEBUG_SLAB_LEAK
4568
4569	static void leaks_start(struct seq_file m, loff_t *pos)
4570	{
4571	mutex_lock(&slab_mutex);
4572	return seq_list_start(&slab_caches, *pos);
4573	}
4574
4575	static inline int add_caller(unsigned long *n, unsigned long v)
4576	{
4577	unsigned long *p;
4578	int l;
4579	if (!v)
4580	return 1;
4581	l = n[1];
4582	p = n + 2;
4583	while (l) {
4584	int i = l/2;
4585	unsigned long q = p + 2 i;
4586	if (*q == v) {
4587	q[1]++;
4588	return 1;
4589	}
4590	if (*q > v) {
4591	l = i;
4592	} else {
4593	p = q + 2;
4594	l -= i + 1;
4595	}
4596	}
4597	if (++n[1] == n[0])
4598	return 0;
4599	memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void )p - (void )n));
4600	p[0] = v;
4601	p[1] = 1;
4602	return 1;
4603	}
4604
4605	static void handle_slab(unsigned long n, struct kmem_cache c, struct slab *s)
4606	{
4607	void *p;
4608	int i;
4609	if (n[0] == n[1])
4610	return;
4611	for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) {
4612	if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
4613	continue;
4614	if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4615	return;
4616	}
4617	}
4618
4619	static void show_symbol(struct seq_file *m, unsigned long address)
4620	{
4621	#ifdef CONFIG_KALLSYMS
4622	unsigned long offset, size;
4623	char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4624
4625	if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4626	seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4627	if (modname[0])
4628	seq_printf(m, " [%s]", modname);
4629	return;
4630	}
4631	#endif
4632	seq_printf(m, "%p", (void *)address);
4633	}
4634
4635	static int leaks_show(struct seq_file m, void p)
4636	{
4637	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4638	struct slab *slabp;
4639	struct kmem_list3 *l3;
4640	const char *name;
4641	unsigned long *n = m->private;
4642	int node;
4643	int i;
4644
4645	if (!(cachep->flags & SLAB_STORE_USER))
4646	return 0;
4647	if (!(cachep->flags & SLAB_RED_ZONE))
4648	return 0;
4649
4650	/* OK, we can do it */
4651
4652	n[1] = 0;
4653
4654	for_each_online_node(node) {
4655	l3 = cachep->nodelists[node];
4656	if (!l3)
4657	continue;
4658
4659	check_irq_on();
4660	spin_lock_irq(&l3->list_lock);
4661
4662	list_for_each_entry(slabp, &l3->slabs_full, list)
4663	handle_slab(n, cachep, slabp);
4664	list_for_each_entry(slabp, &l3->slabs_partial, list)
4665	handle_slab(n, cachep, slabp);
4666	spin_unlock_irq(&l3->list_lock);
4667	}
4668	name = cachep->name;
4669	if (n[0] == n[1]) {
4670	/* Increase the buffer size */
4671	mutex_unlock(&slab_mutex);
4672	m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4673	if (!m->private) {
4674	/* Too bad, we are really out */
4675	m->private = n;
4676	mutex_lock(&slab_mutex);
4677	return -ENOMEM;
4678	}
4679	(unsigned long )m->private = n[0] * 2;
4680	kfree(n);
4681	mutex_lock(&slab_mutex);
4682	/* Now make sure this entry will be retried */
4683	m->count = m->size;
4684	return 0;
4685	}
4686	for (i = 0; i < n[1]; i++) {
4687	seq_printf(m, "%s: %lu ", name, n[2*i+3]);
4688	show_symbol(m, n[2*i+2]);
4689	seq_putc(m, '\n');
4690	}
4691
4692	return 0;
4693	}
4694
4695	static const struct seq_operations slabstats_op = {
4696	.start = leaks_start,
4697	.next = s_next,
4698	.stop = s_stop,
4699	.show = leaks_show,
4700	};
4701
4702	static int slabstats_open(struct inode inode, struct file file)
4703	{
4704	unsigned long *n = kzalloc(PAGE_SIZE, GFP_KERNEL);
4705	int ret = -ENOMEM;
4706	if (n) {
4707	ret = seq_open(file, &slabstats_op);
4708	if (!ret) {
4709	struct seq_file *m = file->private_data;
4710	n = PAGE_SIZE / (2 sizeof(unsigned long));
4711	m->private = n;
4712	n = NULL;
4713	}
4714	kfree(n);
4715	}
4716	return ret;
4717	}
4718
4719	static const struct file_operations proc_slabstats_operations = {
4720	.open = slabstats_open,
4721	.read = seq_read,
4722	.llseek = seq_lseek,
4723	.release = seq_release_private,
4724	};
4725	#endif
4726
4727	static int __init slab_proc_init(void)
4728	{
4729	proc_create("slabinfo",S_IWUSR\|S_IRUSR,NULL,&proc_slabinfo_operations);
4730	#ifdef CONFIG_DEBUG_SLAB_LEAK
4731	proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4732	#endif
4733	return 0;
4734	}
4735	module_init(slab_proc_init);
4736	#endif
4737
4738	/**
4739	* ksize - get the actual amount of memory allocated for a given object
4740	* @objp: Pointer to the object
4741	*
4742	* kmalloc may internally round up allocations and return more memory
4743	* than requested. ksize() can be used to determine the actual amount of
4744	* memory allocated. The caller may use this additional memory, even though
4745	* a smaller amount of memory was initially specified with the kmalloc call.
4746	* The caller must guarantee that objp points to a valid object previously
4747	* allocated with either kmalloc() or kmem_cache_alloc(). The object
4748	* must not be freed during the duration of the call.
4749	*/
4750	size_t ksize(const void *objp)
4751	{
4752	BUG_ON(!objp);
4753	if (unlikely(objp == ZERO_SIZE_PTR))
4754	return 0;
4755
4756	return virt_to_cache(objp)->object_size;
4757	}
4758	EXPORT_SYMBOL(ksize);
4759

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/slab.c

interactive