Kernel

Kernel Git Source Tree

Root/mm/slab.c

1	/*
2	* linux/mm/slab.c
3	* Written by Mark Hemment, 1996/97.
4	* (markhe@nextd.demon.co.uk)
5	*
6	* kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7	*
8	* Major cleanup, different bufctl logic, per-cpu arrays
9	* (c) 2000 Manfred Spraul
10	*
11	* Cleanup, make the head arrays unconditional, preparation for NUMA
12	* (c) 2002 Manfred Spraul
13	*
14	* An implementation of the Slab Allocator as described in outline in;
15	* UNIX Internals: The New Frontiers by Uresh Vahalia
16	* Pub: Prentice Hall ISBN 0-13-101908-2
17	* or with a little more detail in;
18	* The Slab Allocator: An Object-Caching Kernel Memory Allocator
19	* Jeff Bonwick (Sun Microsystems).
20	* Presented at: USENIX Summer 1994 Technical Conference
21	*
22	* The memory is organized in caches, one cache for each object type.
23	* (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24	* Each cache consists out of many slabs (they are small (usually one
25	* page long) and always contiguous), and each slab contains multiple
26	* initialized objects.
27	*
28	* This means, that your constructor is used only for newly allocated
29	* slabs and you must pass objects with the same initializations to
30	* kmem_cache_free.
31	*
32	* Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33	* normal). If you need a special memory type, then must create a new
34	* cache for that memory type.
35	*
36	* In order to reduce fragmentation, the slabs are sorted in 3 groups:
37	* full slabs with 0 free objects
38	* partial slabs
39	* empty slabs with no allocated objects
40	*
41	* If partial slabs exist, then new allocations come from these slabs,
42	* otherwise from empty slabs or new slabs are allocated.
43	*
44	* kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45	* during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46	*
47	* Each cache has a short per-cpu head array, most allocs
48	* and frees go into that array, and if that array overflows, then 1/2
49	* of the entries in the array are given back into the global cache.
50	* The head array is strictly LIFO and should improve the cache hit rates.
51	* On SMP, it additionally reduces the spinlock operations.
52	*
53	* The c_cpuarray may not be read with enabled local interrupts -
54	* it's changed with a smp_call_function().
55	*
56	* SMP synchronization:
57	* constructors and destructors are called without any locking.
58	* Several members in struct kmem_cache and struct slab never change, they
59	* are accessed without any locking.
60	* The per-cpu arrays are never accessed from the wrong cpu, no locking,
61	* and local interrupts are disabled so slab code is preempt-safe.
62	* The non-constant members are protected with a per-cache irq spinlock.
63	*
64	* Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65	* in 2000 - many ideas in the current implementation are derived from
66	* his patch.
67	*
68	* Further notes from the original documentation:
69	*
70	* 11 April '97. Started multi-threading - markhe
71	* The global cache-chain is protected by the mutex 'slab_mutex'.
72	* The sem is only needed when accessing/extending the cache-chain, which
73	* can never happen inside an interrupt (kmem_cache_create(),
74	* kmem_cache_shrink() and kmem_cache_reap()).
75	*
76	* At present, each engine can be growing a cache. This should be blocked.
77	*
78	* 15 March 2005. NUMA slab allocator.
79	* Shai Fultheim <shai@scalex86.org>.
80	* Shobhit Dayal <shobhit@calsoftinc.com>
81	* Alok N Kataria <alokk@calsoftinc.com>
82	* Christoph Lameter <christoph@lameter.com>
83	*
84	* Modified the slab allocator to be node aware on NUMA systems.
85	* Each node has its own list of partial, free and full slabs.
86	* All object allocations for a node occur from node specific slab lists.
87	*/
88
89	#include <linux/slab.h>
90	#include <linux/mm.h>
91	#include <linux/poison.h>
92	#include <linux/swap.h>
93	#include <linux/cache.h>
94	#include <linux/interrupt.h>
95	#include <linux/init.h>
96	#include <linux/compiler.h>
97	#include <linux/cpuset.h>
98	#include <linux/proc_fs.h>
99	#include <linux/seq_file.h>
100	#include <linux/notifier.h>
101	#include <linux/kallsyms.h>
102	#include <linux/cpu.h>
103	#include <linux/sysctl.h>
104	#include <linux/module.h>
105	#include <linux/rcupdate.h>
106	#include <linux/string.h>
107	#include <linux/uaccess.h>
108	#include <linux/nodemask.h>
109	#include <linux/kmemleak.h>
110	#include <linux/mempolicy.h>
111	#include <linux/mutex.h>
112	#include <linux/fault-inject.h>
113	#include <linux/rtmutex.h>
114	#include <linux/reciprocal_div.h>
115	#include <linux/debugobjects.h>
116	#include <linux/kmemcheck.h>
117	#include <linux/memory.h>
118	#include <linux/prefetch.h>
119
120	#include <net/sock.h>
121
122	#include <asm/cacheflush.h>
123	#include <asm/tlbflush.h>
124	#include <asm/page.h>
125
126	#include <trace/events/kmem.h>
127
128	#include "internal.h"
129
130	#include "slab.h"
131
132	/*
133	* DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
134	* 0 for faster, smaller code (especially in the critical paths).
135	*
136	* STATS - 1 to collect stats for /proc/slabinfo.
137	* 0 for faster, smaller code (especially in the critical paths).
138	*
139	* FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
140	*/
141
142	#ifdef CONFIG_DEBUG_SLAB
143	#define DEBUG 1
144	#define STATS 1
145	#define FORCED_DEBUG 1
146	#else
147	#define DEBUG 0
148	#define STATS 0
149	#define FORCED_DEBUG 0
150	#endif
151
152	/* Shouldn't this be in a header file somewhere? */
153	#define BYTES_PER_WORD sizeof(void *)
154	#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
155
156	#ifndef ARCH_KMALLOC_FLAGS
157	#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
158	#endif
159
160	/*
161	* true if a page was allocated from pfmemalloc reserves for network-based
162	* swap
163	*/
164	static bool pfmemalloc_active __read_mostly;
165
166	/*
167	* kmem_bufctl_t:
168	*
169	* Bufctl's are used for linking objs within a slab
170	* linked offsets.
171	*
172	* This implementation relies on "struct page" for locating the cache &
173	* slab an object belongs to.
174	* This allows the bufctl structure to be small (one int), but limits
175	* the number of objects a slab (not a cache) can contain when off-slab
176	* bufctls are used. The limit is the size of the largest general cache
177	* that does not use off-slab slabs.
178	* For 32bit archs with 4 kB pages, is this 56.
179	* This is not serious, as it is only for large objects, when it is unwise
180	* to have too many per slab.
181	* Note: This limit can be raised by introducing a general cache whose size
182	* is less than 512 (PAGE_SIZE<<3), but greater than 256.
183	*/
184
185	typedef unsigned int kmem_bufctl_t;
186	#define BUFCTL_END (((kmem_bufctl_t)(~0U))-0)
187	#define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1)
188	#define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2)
189	#define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3)
190
191	/*
192	* struct slab_rcu
193	*
194	* slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
195	* arrange for kmem_freepages to be called via RCU. This is useful if
196	* we need to approach a kernel structure obliquely, from its address
197	* obtained without the usual locking. We can lock the structure to
198	* stabilize it and check it's still at the given address, only if we
199	* can be sure that the memory has not been meanwhile reused for some
200	* other kind of object (which our subsystem's lock might corrupt).
201	*
202	* rcu_read_lock before reading the address, then rcu_read_unlock after
203	* taking the spinlock within the structure expected at that address.
204	*/
205	struct slab_rcu {
206	struct rcu_head head;
207	struct kmem_cache *cachep;
208	void *addr;
209	};
210
211	/*
212	* struct slab
213	*
214	* Manages the objs in a slab. Placed either at the beginning of mem allocated
215	* for a slab, or allocated from an general cache.
216	* Slabs are chained into three list: fully used, partial, fully free slabs.
217	*/
218	struct slab {
219	union {
220	struct {
221	struct list_head list;
222	unsigned long colouroff;
223	void s_mem; / including colour offset */
224	unsigned int inuse; /* num of objs active in slab */
225	kmem_bufctl_t free;
226	unsigned short nodeid;
227	};
228	struct slab_rcu __slab_cover_slab_rcu;
229	};
230	};
231
232	/*
233	* struct array_cache
234	*
235	* Purpose:
236	* - LIFO ordering, to hand out cache-warm objects from _alloc
237	* - reduce the number of linked list operations
238	* - reduce spinlock operations
239	*
240	* The limit is stored in the per-cpu structure to reduce the data cache
241	* footprint.
242	*
243	*/
244	struct array_cache {
245	unsigned int avail;
246	unsigned int limit;
247	unsigned int batchcount;
248	unsigned int touched;
249	spinlock_t lock;
250	void entry[]; /
251	* Must have this definition in here for the proper
252	* alignment of array_cache. Also simplifies accessing
253	* the entries.
254	*
255	* Entries should not be directly dereferenced as
256	* entries belonging to slabs marked pfmemalloc will
257	* have the lower bits set SLAB_OBJ_PFMEMALLOC
258	*/
259	};
260
261	#define SLAB_OBJ_PFMEMALLOC 1
262	static inline bool is_obj_pfmemalloc(void *objp)
263	{
264	return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
265	}
266
267	static inline void set_obj_pfmemalloc(void **objp)
268	{
269	objp = (void )((unsigned long)*objp \| SLAB_OBJ_PFMEMALLOC);
270	return;
271	}
272
273	static inline void clear_obj_pfmemalloc(void **objp)
274	{
275	objp = (void )((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
276	}
277
278	/*
279	* bootstrap: The caches do not work without cpuarrays anymore, but the
280	* cpuarrays are allocated from the generic caches...
281	*/
282	#define BOOT_CPUCACHE_ENTRIES 1
283	struct arraycache_init {
284	struct array_cache cache;
285	void *entries[BOOT_CPUCACHE_ENTRIES];
286	};
287
288	/*
289	* Need this for bootstrapping a per node allocator.
290	*/
291	#define NUM_INIT_LISTS (3 * MAX_NUMNODES)
292	static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
293	#define CACHE_CACHE 0
294	#define SIZE_AC MAX_NUMNODES
295	#define SIZE_NODE (2 * MAX_NUMNODES)
296
297	static int drain_freelist(struct kmem_cache *cache,
298	struct kmem_cache_node *n, int tofree);
299	static void free_block(struct kmem_cache cachep, void *objpp, int len,
300	int node);
301	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
302	static void cache_reap(struct work_struct *unused);
303
304	static int slab_early_init = 1;
305
306	#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
307	#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
308
309	static void kmem_cache_node_init(struct kmem_cache_node *parent)
310	{
311	INIT_LIST_HEAD(&parent->slabs_full);
312	INIT_LIST_HEAD(&parent->slabs_partial);
313	INIT_LIST_HEAD(&parent->slabs_free);
314	parent->shared = NULL;
315	parent->alien = NULL;
316	parent->colour_next = 0;
317	spin_lock_init(&parent->list_lock);
318	parent->free_objects = 0;
319	parent->free_touched = 0;
320	}
321
322	#define MAKE_LIST(cachep, listp, slab, nodeid) \
323	do { \
324	INIT_LIST_HEAD(listp); \
325	list_splice(&(cachep->node[nodeid]->slab), listp); \
326	} while (0)
327
328	#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
329	do { \
330	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
331	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
332	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
333	} while (0)
334
335	#define CFLGS_OFF_SLAB (0x80000000UL)
336	#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
337
338	#define BATCHREFILL_LIMIT 16
339	/*
340	* Optimization question: fewer reaps means less probability for unnessary
341	* cpucache drain/refill cycles.
342	*
343	* OTOH the cpuarrays can contain lots of objects,
344	* which could lock up otherwise freeable slabs.
345	*/
346	#define REAPTIMEOUT_CPUC (2*HZ)
347	#define REAPTIMEOUT_LIST3 (4*HZ)
348
349	#if STATS
350	#define STATS_INC_ACTIVE(x) ((x)->num_active++)
351	#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
352	#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
353	#define STATS_INC_GROWN(x) ((x)->grown++)
354	#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
355	#define STATS_SET_HIGH(x) \
356	do { \
357	if ((x)->num_active > (x)->high_mark) \
358	(x)->high_mark = (x)->num_active; \
359	} while (0)
360	#define STATS_INC_ERR(x) ((x)->errors++)
361	#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
362	#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
363	#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
364	#define STATS_SET_FREEABLE(x, i) \
365	do { \
366	if ((x)->max_freeable < i) \
367	(x)->max_freeable = i; \
368	} while (0)
369	#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
370	#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
371	#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
372	#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
373	#else
374	#define STATS_INC_ACTIVE(x) do { } while (0)
375	#define STATS_DEC_ACTIVE(x) do { } while (0)
376	#define STATS_INC_ALLOCED(x) do { } while (0)
377	#define STATS_INC_GROWN(x) do { } while (0)
378	#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
379	#define STATS_SET_HIGH(x) do { } while (0)
380	#define STATS_INC_ERR(x) do { } while (0)
381	#define STATS_INC_NODEALLOCS(x) do { } while (0)
382	#define STATS_INC_NODEFREES(x) do { } while (0)
383	#define STATS_INC_ACOVERFLOW(x) do { } while (0)
384	#define STATS_SET_FREEABLE(x, i) do { } while (0)
385	#define STATS_INC_ALLOCHIT(x) do { } while (0)
386	#define STATS_INC_ALLOCMISS(x) do { } while (0)
387	#define STATS_INC_FREEHIT(x) do { } while (0)
388	#define STATS_INC_FREEMISS(x) do { } while (0)
389	#endif
390
391	#if DEBUG
392
393	/*
394	* memory layout of objects:
395	* 0 : objp
396	* 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
397	* the end of an object is aligned with the end of the real
398	* allocation. Catches writes behind the end of the allocation.
399	* cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
400	* redzone word.
401	* cachep->obj_offset: The real object.
402	* cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
403	* cachep->size - 1* BYTES_PER_WORD: last caller address
404	* [BYTES_PER_WORD long]
405	*/
406	static int obj_offset(struct kmem_cache *cachep)
407	{
408	return cachep->obj_offset;
409	}
410
411	static unsigned long long dbg_redzone1(struct kmem_cache cachep, void *objp)
412	{
413	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
414	return (unsigned long long*) (objp + obj_offset(cachep) -
415	sizeof(unsigned long long));
416	}
417
418	static unsigned long long dbg_redzone2(struct kmem_cache cachep, void *objp)
419	{
420	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
421	if (cachep->flags & SLAB_STORE_USER)
422	return (unsigned long long *)(objp + cachep->size -
423	sizeof(unsigned long long) -
424	REDZONE_ALIGN);
425	return (unsigned long long *) (objp + cachep->size -
426	sizeof(unsigned long long));
427	}
428
429	static void *dbg_userword(struct kmem_cache cachep, void *objp)
430	{
431	BUG_ON(!(cachep->flags & SLAB_STORE_USER));
432	return (void **)(objp + cachep->size - BYTES_PER_WORD);
433	}
434
435	#else
436
437	#define obj_offset(x) 0
438	#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
439	#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
440	#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
441
442	#endif
443
444	/*
445	* Do not go above this order unless 0 objects fit into the slab or
446	* overridden on the command line.
447	*/
448	#define SLAB_MAX_ORDER_HI 1
449	#define SLAB_MAX_ORDER_LO 0
450	static int slab_max_order = SLAB_MAX_ORDER_LO;
451	static bool slab_max_order_set __initdata;
452
453	static inline struct kmem_cache virt_to_cache(const void obj)
454	{
455	struct page *page = virt_to_head_page(obj);
456	return page->slab_cache;
457	}
458
459	static inline struct slab virt_to_slab(const void obj)
460	{
461	struct page *page = virt_to_head_page(obj);
462
463	VM_BUG_ON(!PageSlab(page));
464	return page->slab_page;
465	}
466
467	static inline void index_to_obj(struct kmem_cache cache, struct slab *slab,
468	unsigned int idx)
469	{
470	return slab->s_mem + cache->size * idx;
471	}
472
473	/*
474	* We want to avoid an expensive divide : (offset / cache->size)
475	* Using the fact that size is a constant for a particular cache,
476	* we can replace (offset / cache->size) by
477	* reciprocal_divide(offset, cache->reciprocal_buffer_size)
478	*/
479	static inline unsigned int obj_to_index(const struct kmem_cache *cache,
480	const struct slab slab, void obj)
481	{
482	u32 offset = (obj - slab->s_mem);
483	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
484	}
485
486	static struct arraycache_init initarray_generic =
487	{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
488
489	/* internal cache of cache description objs */
490	static struct kmem_cache kmem_cache_boot = {
491	.batchcount = 1,
492	.limit = BOOT_CPUCACHE_ENTRIES,
493	.shared = 1,
494	.size = sizeof(struct kmem_cache),
495	.name = "kmem_cache",
496	};
497
498	#define BAD_ALIEN_MAGIC 0x01020304ul
499
500	#ifdef CONFIG_LOCKDEP
501
502	/*
503	* Slab sometimes uses the kmalloc slabs to store the slab headers
504	* for other slabs "off slab".
505	* The locking for this is tricky in that it nests within the locks
506	* of all other slabs in a few places; to deal with this special
507	* locking we put on-slab caches into a separate lock-class.
508	*
509	* We set lock class for alien array caches which are up during init.
510	* The lock annotation will be lost if all cpus of a node goes down and
511	* then comes back up during hotplug
512	*/
513	static struct lock_class_key on_slab_l3_key;
514	static struct lock_class_key on_slab_alc_key;
515
516	static struct lock_class_key debugobj_l3_key;
517	static struct lock_class_key debugobj_alc_key;
518
519	static void slab_set_lock_classes(struct kmem_cache *cachep,
520	struct lock_class_key l3_key, struct lock_class_key alc_key,
521	int q)
522	{
523	struct array_cache **alc;
524	struct kmem_cache_node *n;
525	int r;
526
527	n = cachep->node[q];
528	if (!n)
529	return;
530
531	lockdep_set_class(&n->list_lock, l3_key);
532	alc = n->alien;
533	/*
534	* FIXME: This check for BAD_ALIEN_MAGIC
535	* should go away when common slab code is taught to
536	* work even without alien caches.
537	* Currently, non NUMA code returns BAD_ALIEN_MAGIC
538	* for alloc_alien_cache,
539	*/
540	if (!alc \|\| (unsigned long)alc == BAD_ALIEN_MAGIC)
541	return;
542	for_each_node(r) {
543	if (alc[r])
544	lockdep_set_class(&alc[r]->lock, alc_key);
545	}
546	}
547
548	static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
549	{
550	slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
551	}
552
553	static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
554	{
555	int node;
556
557	for_each_online_node(node)
558	slab_set_debugobj_lock_classes_node(cachep, node);
559	}
560
561	static void init_node_lock_keys(int q)
562	{
563	int i;
564
565	if (slab_state < UP)
566	return;
567
568	for (i = 1; i <= KMALLOC_SHIFT_HIGH; i++) {
569	struct kmem_cache_node *n;
570	struct kmem_cache *cache = kmalloc_caches[i];
571
572	if (!cache)
573	continue;
574
575	n = cache->node[q];
576	if (!n \|\| OFF_SLAB(cache))
577	continue;
578
579	slab_set_lock_classes(cache, &on_slab_l3_key,
580	&on_slab_alc_key, q);
581	}
582	}
583
584	static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q)
585	{
586	if (!cachep->node[q])
587	return;
588
589	slab_set_lock_classes(cachep, &on_slab_l3_key,
590	&on_slab_alc_key, q);
591	}
592
593	static inline void on_slab_lock_classes(struct kmem_cache *cachep)
594	{
595	int node;
596
597	VM_BUG_ON(OFF_SLAB(cachep));
598	for_each_node(node)
599	on_slab_lock_classes_node(cachep, node);
600	}
601
602	static inline void init_lock_keys(void)
603	{
604	int node;
605
606	for_each_node(node)
607	init_node_lock_keys(node);
608	}
609	#else
610	static void init_node_lock_keys(int q)
611	{
612	}
613
614	static inline void init_lock_keys(void)
615	{
616	}
617
618	static inline void on_slab_lock_classes(struct kmem_cache *cachep)
619	{
620	}
621
622	static inline void on_slab_lock_classes_node(struct kmem_cache *cachep, int node)
623	{
624	}
625
626	static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
627	{
628	}
629
630	static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
631	{
632	}
633	#endif
634
635	static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
636
637	static inline struct array_cache cpu_cache_get(struct kmem_cache cachep)
638	{
639	return cachep->array[smp_processor_id()];
640	}
641
642	static size_t slab_mgmt_size(size_t nr_objs, size_t align)
643	{
644	return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
645	}
646
647	/*
648	* Calculate the number of objects and left-over bytes for a given buffer size.
649	*/
650	static void cache_estimate(unsigned long gfporder, size_t buffer_size,
651	size_t align, int flags, size_t *left_over,
652	unsigned int *num)
653	{
654	int nr_objs;
655	size_t mgmt_size;
656	size_t slab_size = PAGE_SIZE << gfporder;
657
658	/*
659	* The slab management structure can be either off the slab or
660	* on it. For the latter case, the memory allocated for a
661	* slab is used for:
662	*
663	* - The struct slab
664	* - One kmem_bufctl_t for each object
665	* - Padding to respect alignment of @align
666	* - @buffer_size bytes for each object
667	*
668	* If the slab management structure is off the slab, then the
669	* alignment will already be calculated into the size. Because
670	* the slabs are all pages aligned, the objects will be at the
671	* correct alignment when allocated.
672	*/
673	if (flags & CFLGS_OFF_SLAB) {
674	mgmt_size = 0;
675	nr_objs = slab_size / buffer_size;
676
677	if (nr_objs > SLAB_LIMIT)
678	nr_objs = SLAB_LIMIT;
679	} else {
680	/*
681	* Ignore padding for the initial guess. The padding
682	* is at most @align-1 bytes, and @buffer_size is at
683	* least @align. In the worst case, this result will
684	* be one greater than the number of objects that fit
685	* into the memory allocation when taking the padding
686	* into account.
687	*/
688	nr_objs = (slab_size - sizeof(struct slab)) /
689	(buffer_size + sizeof(kmem_bufctl_t));
690
691	/*
692	* This calculated number will be either the right
693	* amount, or one greater than what we want.
694	*/
695	if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
696	> slab_size)
697	nr_objs--;
698
699	if (nr_objs > SLAB_LIMIT)
700	nr_objs = SLAB_LIMIT;
701
702	mgmt_size = slab_mgmt_size(nr_objs, align);
703	}
704	*num = nr_objs;
705	left_over = slab_size - nr_objsbuffer_size - mgmt_size;
706	}
707
708	#if DEBUG
709	#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
710
711	static void __slab_error(const char function, struct kmem_cache cachep,
712	char *msg)
713	{
714	printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
715	function, cachep->name, msg);
716	dump_stack();
717	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
718	}
719	#endif
720
721	/*
722	* By default on NUMA we use alien caches to stage the freeing of
723	* objects allocated from other nodes. This causes massive memory
724	* inefficiencies when using fake NUMA setup to split memory into a
725	* large number of small nodes, so it can be disabled on the command
726	* line
727	*/
728
729	static int use_alien_caches __read_mostly = 1;
730	static int __init noaliencache_setup(char *s)
731	{
732	use_alien_caches = 0;
733	return 1;
734	}
735	__setup("noaliencache", noaliencache_setup);
736
737	static int __init slab_max_order_setup(char *str)
738	{
739	get_option(&str, &slab_max_order);
740	slab_max_order = slab_max_order < 0 ? 0 :
741	min(slab_max_order, MAX_ORDER - 1);
742	slab_max_order_set = true;
743
744	return 1;
745	}
746	__setup("slab_max_order=", slab_max_order_setup);
747
748	#ifdef CONFIG_NUMA
749	/*
750	* Special reaping functions for NUMA systems called from cache_reap().
751	* These take care of doing round robin flushing of alien caches (containing
752	* objects freed on different nodes from which they were allocated) and the
753	* flushing of remote pcps by calling drain_node_pages.
754	*/
755	static DEFINE_PER_CPU(unsigned long, slab_reap_node);
756
757	static void init_reap_node(int cpu)
758	{
759	int node;
760
761	node = next_node(cpu_to_mem(cpu), node_online_map);
762	if (node == MAX_NUMNODES)
763	node = first_node(node_online_map);
764
765	per_cpu(slab_reap_node, cpu) = node;
766	}
767
768	static void next_reap_node(void)
769	{
770	int node = __this_cpu_read(slab_reap_node);
771
772	node = next_node(node, node_online_map);
773	if (unlikely(node >= MAX_NUMNODES))
774	node = first_node(node_online_map);
775	__this_cpu_write(slab_reap_node, node);
776	}
777
778	#else
779	#define init_reap_node(cpu) do { } while (0)
780	#define next_reap_node(void) do { } while (0)
781	#endif
782
783	/*
784	* Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
785	* via the workqueue/eventd.
786	* Add the CPU number into the expiration time to minimize the possibility of
787	* the CPUs getting into lockstep and contending for the global cache chain
788	* lock.
789	*/
790	static void start_cpu_timer(int cpu)
791	{
792	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
793
794	/*
795	* When this gets called from do_initcalls via cpucache_init(),
796	* init_workqueues() has already run, so keventd will be setup
797	* at that time.
798	*/
799	if (keventd_up() && reap_work->work.func == NULL) {
800	init_reap_node(cpu);
801	INIT_DEFERRABLE_WORK(reap_work, cache_reap);
802	schedule_delayed_work_on(cpu, reap_work,
803	__round_jiffies_relative(HZ, cpu));
804	}
805	}
806
807	static struct array_cache *alloc_arraycache(int node, int entries,
808	int batchcount, gfp_t gfp)
809	{
810	int memsize = sizeof(void ) entries + sizeof(struct array_cache);
811	struct array_cache *nc = NULL;
812
813	nc = kmalloc_node(memsize, gfp, node);
814	/*
815	* The array_cache structures contain pointers to free object.
816	* However, when such objects are allocated or transferred to another
817	* cache the pointers are not cleared and they could be counted as
818	* valid references during a kmemleak scan. Therefore, kmemleak must
819	* not scan such objects.
820	*/
821	kmemleak_no_scan(nc);
822	if (nc) {
823	nc->avail = 0;
824	nc->limit = entries;
825	nc->batchcount = batchcount;
826	nc->touched = 0;
827	spin_lock_init(&nc->lock);
828	}
829	return nc;
830	}
831
832	static inline bool is_slab_pfmemalloc(struct slab *slabp)
833	{
834	struct page *page = virt_to_page(slabp->s_mem);
835
836	return PageSlabPfmemalloc(page);
837	}
838
839	/* Clears pfmemalloc_active if no slabs have pfmalloc set */
840	static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
841	struct array_cache *ac)
842	{
843	struct kmem_cache_node *n = cachep->node[numa_mem_id()];
844	struct slab *slabp;
845	unsigned long flags;
846
847	if (!pfmemalloc_active)
848	return;
849
850	spin_lock_irqsave(&n->list_lock, flags);
851	list_for_each_entry(slabp, &n->slabs_full, list)
852	if (is_slab_pfmemalloc(slabp))
853	goto out;
854
855	list_for_each_entry(slabp, &n->slabs_partial, list)
856	if (is_slab_pfmemalloc(slabp))
857	goto out;
858
859	list_for_each_entry(slabp, &n->slabs_free, list)
860	if (is_slab_pfmemalloc(slabp))
861	goto out;
862
863	pfmemalloc_active = false;
864	out:
865	spin_unlock_irqrestore(&n->list_lock, flags);
866	}
867
868	static void __ac_get_obj(struct kmem_cache cachep, struct array_cache *ac,
869	gfp_t flags, bool force_refill)
870	{
871	int i;
872	void *objp = ac->entry[--ac->avail];
873
874	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
875	if (unlikely(is_obj_pfmemalloc(objp))) {
876	struct kmem_cache_node *n;
877
878	if (gfp_pfmemalloc_allowed(flags)) {
879	clear_obj_pfmemalloc(&objp);
880	return objp;
881	}
882
883	/* The caller cannot use PFMEMALLOC objects, find another one */
884	for (i = 0; i < ac->avail; i++) {
885	/* If a !PFMEMALLOC object is found, swap them */
886	if (!is_obj_pfmemalloc(ac->entry[i])) {
887	objp = ac->entry[i];
888	ac->entry[i] = ac->entry[ac->avail];
889	ac->entry[ac->avail] = objp;
890	return objp;
891	}
892	}
893
894	/*
895	* If there are empty slabs on the slabs_free list and we are
896	* being forced to refill the cache, mark this one !pfmemalloc.
897	*/
898	n = cachep->node[numa_mem_id()];
899	if (!list_empty(&n->slabs_free) && force_refill) {
900	struct slab *slabp = virt_to_slab(objp);
901	ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem));
902	clear_obj_pfmemalloc(&objp);
903	recheck_pfmemalloc_active(cachep, ac);
904	return objp;
905	}
906
907	/* No !PFMEMALLOC objects available */
908	ac->avail++;
909	objp = NULL;
910	}
911
912	return objp;
913	}
914
915	static inline void ac_get_obj(struct kmem_cache cachep,
916	struct array_cache *ac, gfp_t flags, bool force_refill)
917	{
918	void *objp;
919
920	if (unlikely(sk_memalloc_socks()))
921	objp = __ac_get_obj(cachep, ac, flags, force_refill);
922	else
923	objp = ac->entry[--ac->avail];
924
925	return objp;
926	}
927
928	static void __ac_put_obj(struct kmem_cache cachep, struct array_cache *ac,
929	void *objp)
930	{
931	if (unlikely(pfmemalloc_active)) {
932	/* Some pfmemalloc slabs exist, check if this is one */
933	struct page *page = virt_to_head_page(objp);
934	if (PageSlabPfmemalloc(page))
935	set_obj_pfmemalloc(&objp);
936	}
937
938	return objp;
939	}
940
941	static inline void ac_put_obj(struct kmem_cache cachep, struct array_cache ac,
942	void *objp)
943	{
944	if (unlikely(sk_memalloc_socks()))
945	objp = __ac_put_obj(cachep, ac, objp);
946
947	ac->entry[ac->avail++] = objp;
948	}
949
950	/*
951	* Transfer objects in one arraycache to another.
952	* Locking must be handled by the caller.
953	*
954	* Return the number of entries transferred.
955	*/
956	static int transfer_objects(struct array_cache *to,
957	struct array_cache *from, unsigned int max)
958	{
959	/* Figure out how many entries to transfer */
960	int nr = min3(from->avail, max, to->limit - to->avail);
961
962	if (!nr)
963	return 0;
964
965	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
966	sizeof(void ) nr);
967
968	from->avail -= nr;
969	to->avail += nr;
970	return nr;
971	}
972
973	#ifndef CONFIG_NUMA
974
975	#define drain_alien_cache(cachep, alien) do { } while (0)
976	#define reap_alien(cachep, n) do { } while (0)
977
978	static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
979	{
980	return (struct array_cache **)BAD_ALIEN_MAGIC;
981	}
982
983	static inline void free_alien_cache(struct array_cache **ac_ptr)
984	{
985	}
986
987	static inline int cache_free_alien(struct kmem_cache cachep, void objp)
988	{
989	return 0;
990	}
991
992	static inline void alternate_node_alloc(struct kmem_cache cachep,
993	gfp_t flags)
994	{
995	return NULL;
996	}
997
998	static inline void ____cache_alloc_node(struct kmem_cache cachep,
999	gfp_t flags, int nodeid)
1000	{
1001	return NULL;
1002	}
1003
1004	#else /* CONFIG_NUMA */
1005
1006	static void ____cache_alloc_node(struct kmem_cache , gfp_t, int);
1007	static void alternate_node_alloc(struct kmem_cache , gfp_t);
1008
1009	static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
1010	{
1011	struct array_cache **ac_ptr;
1012	int memsize = sizeof(void ) nr_node_ids;
1013	int i;
1014
1015	if (limit > 1)
1016	limit = 12;
1017	ac_ptr = kzalloc_node(memsize, gfp, node);
1018	if (ac_ptr) {
1019	for_each_node(i) {
1020	if (i == node \|\| !node_online(i))
1021	continue;
1022	ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);
1023	if (!ac_ptr[i]) {
1024	for (i--; i >= 0; i--)
1025	kfree(ac_ptr[i]);
1026	kfree(ac_ptr);
1027	return NULL;
1028	}
1029	}
1030	}
1031	return ac_ptr;
1032	}
1033
1034	static void free_alien_cache(struct array_cache **ac_ptr)
1035	{
1036	int i;
1037
1038	if (!ac_ptr)
1039	return;
1040	for_each_node(i)
1041	kfree(ac_ptr[i]);
1042	kfree(ac_ptr);
1043	}
1044
1045	static void __drain_alien_cache(struct kmem_cache *cachep,
1046	struct array_cache *ac, int node)
1047	{
1048	struct kmem_cache_node *n = cachep->node[node];
1049
1050	if (ac->avail) {
1051	spin_lock(&n->list_lock);
1052	/*
1053	* Stuff objects into the remote nodes shared array first.
1054	* That way we could avoid the overhead of putting the objects
1055	* into the free lists and getting them back later.
1056	*/
1057	if (n->shared)
1058	transfer_objects(n->shared, ac, ac->limit);
1059
1060	free_block(cachep, ac->entry, ac->avail, node);
1061	ac->avail = 0;
1062	spin_unlock(&n->list_lock);
1063	}
1064	}
1065
1066	/*
1067	* Called from cache_reap() to regularly drain alien caches round robin.
1068	*/
1069	static void reap_alien(struct kmem_cache cachep, struct kmem_cache_node n)
1070	{
1071	int node = __this_cpu_read(slab_reap_node);
1072
1073	if (n->alien) {
1074	struct array_cache *ac = n->alien[node];
1075
1076	if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {
1077	__drain_alien_cache(cachep, ac, node);
1078	spin_unlock_irq(&ac->lock);
1079	}
1080	}
1081	}
1082
1083	static void drain_alien_cache(struct kmem_cache *cachep,
1084	struct array_cache **alien)
1085	{
1086	int i = 0;
1087	struct array_cache *ac;
1088	unsigned long flags;
1089
1090	for_each_online_node(i) {
1091	ac = alien[i];
1092	if (ac) {
1093	spin_lock_irqsave(&ac->lock, flags);
1094	__drain_alien_cache(cachep, ac, i);
1095	spin_unlock_irqrestore(&ac->lock, flags);
1096	}
1097	}
1098	}
1099
1100	static inline int cache_free_alien(struct kmem_cache cachep, void objp)
1101	{
1102	struct slab *slabp = virt_to_slab(objp);
1103	int nodeid = slabp->nodeid;
1104	struct kmem_cache_node *n;
1105	struct array_cache *alien = NULL;
1106	int node;
1107
1108	node = numa_mem_id();
1109
1110	/*
1111	* Make sure we are not freeing a object from another node to the array
1112	* cache on this cpu.
1113	*/
1114	if (likely(slabp->nodeid == node))
1115	return 0;
1116
1117	n = cachep->node[node];
1118	STATS_INC_NODEFREES(cachep);
1119	if (n->alien && n->alien[nodeid]) {
1120	alien = n->alien[nodeid];
1121	spin_lock(&alien->lock);
1122	if (unlikely(alien->avail == alien->limit)) {
1123	STATS_INC_ACOVERFLOW(cachep);
1124	__drain_alien_cache(cachep, alien, nodeid);
1125	}
1126	ac_put_obj(cachep, alien, objp);
1127	spin_unlock(&alien->lock);
1128	} else {
1129	spin_lock(&(cachep->node[nodeid])->list_lock);
1130	free_block(cachep, &objp, 1, nodeid);
1131	spin_unlock(&(cachep->node[nodeid])->list_lock);
1132	}
1133	return 1;
1134	}
1135	#endif
1136
1137	/*
1138	* Allocates and initializes node for a node on each slab cache, used for
1139	* either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
1140	* will be allocated off-node since memory is not yet online for the new node.
1141	* When hotplugging memory or a cpu, existing node are not replaced if
1142	* already in use.
1143	*
1144	* Must hold slab_mutex.
1145	*/
1146	static int init_cache_node_node(int node)
1147	{
1148	struct kmem_cache *cachep;
1149	struct kmem_cache_node *n;
1150	const int memsize = sizeof(struct kmem_cache_node);
1151
1152	list_for_each_entry(cachep, &slab_caches, list) {
1153	/*
1154	* Set up the size64 kmemlist for cpu before we can
1155	* begin anything. Make sure some other cpu on this
1156	* node has not already allocated this
1157	*/
1158	if (!cachep->node[node]) {
1159	n = kmalloc_node(memsize, GFP_KERNEL, node);
1160	if (!n)
1161	return -ENOMEM;
1162	kmem_cache_node_init(n);
1163	n->next_reap = jiffies + REAPTIMEOUT_LIST3 +
1164	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1165
1166	/*
1167	* The l3s don't come and go as CPUs come and
1168	* go. slab_mutex is sufficient
1169	* protection here.
1170	*/
1171	cachep->node[node] = n;
1172	}
1173
1174	spin_lock_irq(&cachep->node[node]->list_lock);
1175	cachep->node[node]->free_limit =
1176	(1 + nr_cpus_node(node)) *
1177	cachep->batchcount + cachep->num;
1178	spin_unlock_irq(&cachep->node[node]->list_lock);
1179	}
1180	return 0;
1181	}
1182
1183	static inline int slabs_tofree(struct kmem_cache *cachep,
1184	struct kmem_cache_node *n)
1185	{
1186	return (n->free_objects + cachep->num - 1) / cachep->num;
1187	}
1188
1189	static void cpuup_canceled(long cpu)
1190	{
1191	struct kmem_cache *cachep;
1192	struct kmem_cache_node *n = NULL;
1193	int node = cpu_to_mem(cpu);
1194	const struct cpumask *mask = cpumask_of_node(node);
1195
1196	list_for_each_entry(cachep, &slab_caches, list) {
1197	struct array_cache *nc;
1198	struct array_cache *shared;
1199	struct array_cache **alien;
1200
1201	/* cpu is dead; no one can alloc from it. */
1202	nc = cachep->array[cpu];
1203	cachep->array[cpu] = NULL;
1204	n = cachep->node[node];
1205
1206	if (!n)
1207	goto free_array_cache;
1208
1209	spin_lock_irq(&n->list_lock);
1210
1211	/* Free limit for this kmem_cache_node */
1212	n->free_limit -= cachep->batchcount;
1213	if (nc)
1214	free_block(cachep, nc->entry, nc->avail, node);
1215
1216	if (!cpumask_empty(mask)) {
1217	spin_unlock_irq(&n->list_lock);
1218	goto free_array_cache;
1219	}
1220
1221	shared = n->shared;
1222	if (shared) {
1223	free_block(cachep, shared->entry,
1224	shared->avail, node);
1225	n->shared = NULL;
1226	}
1227
1228	alien = n->alien;
1229	n->alien = NULL;
1230
1231	spin_unlock_irq(&n->list_lock);
1232
1233	kfree(shared);
1234	if (alien) {
1235	drain_alien_cache(cachep, alien);
1236	free_alien_cache(alien);
1237	}
1238	free_array_cache:
1239	kfree(nc);
1240	}
1241	/*
1242	* In the previous loop, all the objects were freed to
1243	* the respective cache's slabs, now we can go ahead and
1244	* shrink each nodelist to its limit.
1245	*/
1246	list_for_each_entry(cachep, &slab_caches, list) {
1247	n = cachep->node[node];
1248	if (!n)
1249	continue;
1250	drain_freelist(cachep, n, slabs_tofree(cachep, n));
1251	}
1252	}
1253
1254	static int cpuup_prepare(long cpu)
1255	{
1256	struct kmem_cache *cachep;
1257	struct kmem_cache_node *n = NULL;
1258	int node = cpu_to_mem(cpu);
1259	int err;
1260
1261	/*
1262	* We need to do this right in the beginning since
1263	* alloc_arraycache's are going to use this list.
1264	* kmalloc_node allows us to add the slab to the right
1265	* kmem_cache_node and not this cpu's kmem_cache_node
1266	*/
1267	err = init_cache_node_node(node);
1268	if (err < 0)
1269	goto bad;
1270
1271	/*
1272	* Now we can go ahead with allocating the shared arrays and
1273	* array caches
1274	*/
1275	list_for_each_entry(cachep, &slab_caches, list) {
1276	struct array_cache *nc;
1277	struct array_cache *shared = NULL;
1278	struct array_cache **alien = NULL;
1279
1280	nc = alloc_arraycache(node, cachep->limit,
1281	cachep->batchcount, GFP_KERNEL);
1282	if (!nc)
1283	goto bad;
1284	if (cachep->shared) {
1285	shared = alloc_arraycache(node,
1286	cachep->shared * cachep->batchcount,
1287	0xbaadf00d, GFP_KERNEL);
1288	if (!shared) {
1289	kfree(nc);
1290	goto bad;
1291	}
1292	}
1293	if (use_alien_caches) {
1294	alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
1295	if (!alien) {
1296	kfree(shared);
1297	kfree(nc);
1298	goto bad;
1299	}
1300	}
1301	cachep->array[cpu] = nc;
1302	n = cachep->node[node];
1303	BUG_ON(!n);
1304
1305	spin_lock_irq(&n->list_lock);
1306	if (!n->shared) {
1307	/*
1308	* We are serialised from CPU_DEAD or
1309	* CPU_UP_CANCELLED by the cpucontrol lock
1310	*/
1311	n->shared = shared;
1312	shared = NULL;
1313	}
1314	#ifdef CONFIG_NUMA
1315	if (!n->alien) {
1316	n->alien = alien;
1317	alien = NULL;
1318	}
1319	#endif
1320	spin_unlock_irq(&n->list_lock);
1321	kfree(shared);
1322	free_alien_cache(alien);
1323	if (cachep->flags & SLAB_DEBUG_OBJECTS)
1324	slab_set_debugobj_lock_classes_node(cachep, node);
1325	else if (!OFF_SLAB(cachep) &&
1326	!(cachep->flags & SLAB_DESTROY_BY_RCU))
1327	on_slab_lock_classes_node(cachep, node);
1328	}
1329	init_node_lock_keys(node);
1330
1331	return 0;
1332	bad:
1333	cpuup_canceled(cpu);
1334	return -ENOMEM;
1335	}
1336
1337	static int cpuup_callback(struct notifier_block *nfb,
1338	unsigned long action, void *hcpu)
1339	{
1340	long cpu = (long)hcpu;
1341	int err = 0;
1342
1343	switch (action) {
1344	case CPU_UP_PREPARE:
1345	case CPU_UP_PREPARE_FROZEN:
1346	mutex_lock(&slab_mutex);
1347	err = cpuup_prepare(cpu);
1348	mutex_unlock(&slab_mutex);
1349	break;
1350	case CPU_ONLINE:
1351	case CPU_ONLINE_FROZEN:
1352	start_cpu_timer(cpu);
1353	break;
1354	#ifdef CONFIG_HOTPLUG_CPU
1355	case CPU_DOWN_PREPARE:
1356	case CPU_DOWN_PREPARE_FROZEN:
1357	/*
1358	* Shutdown cache reaper. Note that the slab_mutex is
1359	* held so that if cache_reap() is invoked it cannot do
1360	* anything expensive but will only modify reap_work
1361	* and reschedule the timer.
1362	*/
1363	cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1364	/* Now the cache_reaper is guaranteed to be not running. */
1365	per_cpu(slab_reap_work, cpu).work.func = NULL;
1366	break;
1367	case CPU_DOWN_FAILED:
1368	case CPU_DOWN_FAILED_FROZEN:
1369	start_cpu_timer(cpu);
1370	break;
1371	case CPU_DEAD:
1372	case CPU_DEAD_FROZEN:
1373	/*
1374	* Even if all the cpus of a node are down, we don't free the
1375	* kmem_cache_node of any cache. This to avoid a race between
1376	* cpu_down, and a kmalloc allocation from another cpu for
1377	* memory from the node of the cpu going down. The node
1378	* structure is usually allocated from kmem_cache_create() and
1379	* gets destroyed at kmem_cache_destroy().
1380	*/
1381	/* fall through */
1382	#endif
1383	case CPU_UP_CANCELED:
1384	case CPU_UP_CANCELED_FROZEN:
1385	mutex_lock(&slab_mutex);
1386	cpuup_canceled(cpu);
1387	mutex_unlock(&slab_mutex);
1388	break;
1389	}
1390	return notifier_from_errno(err);
1391	}
1392
1393	static struct notifier_block cpucache_notifier = {
1394	&cpuup_callback, NULL, 0
1395	};
1396
1397	#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1398	/*
1399	* Drains freelist for a node on each slab cache, used for memory hot-remove.
1400	* Returns -EBUSY if all objects cannot be drained so that the node is not
1401	* removed.
1402	*
1403	* Must hold slab_mutex.
1404	*/
1405	static int __meminit drain_cache_node_node(int node)
1406	{
1407	struct kmem_cache *cachep;
1408	int ret = 0;
1409
1410	list_for_each_entry(cachep, &slab_caches, list) {
1411	struct kmem_cache_node *n;
1412
1413	n = cachep->node[node];
1414	if (!n)
1415	continue;
1416
1417	drain_freelist(cachep, n, slabs_tofree(cachep, n));
1418
1419	if (!list_empty(&n->slabs_full) \|\|
1420	!list_empty(&n->slabs_partial)) {
1421	ret = -EBUSY;
1422	break;
1423	}
1424	}
1425	return ret;
1426	}
1427
1428	static int __meminit slab_memory_callback(struct notifier_block *self,
1429	unsigned long action, void *arg)
1430	{
1431	struct memory_notify *mnb = arg;
1432	int ret = 0;
1433	int nid;
1434
1435	nid = mnb->status_change_nid;
1436	if (nid < 0)
1437	goto out;
1438
1439	switch (action) {
1440	case MEM_GOING_ONLINE:
1441	mutex_lock(&slab_mutex);
1442	ret = init_cache_node_node(nid);
1443	mutex_unlock(&slab_mutex);
1444	break;
1445	case MEM_GOING_OFFLINE:
1446	mutex_lock(&slab_mutex);
1447	ret = drain_cache_node_node(nid);
1448	mutex_unlock(&slab_mutex);
1449	break;
1450	case MEM_ONLINE:
1451	case MEM_OFFLINE:
1452	case MEM_CANCEL_ONLINE:
1453	case MEM_CANCEL_OFFLINE:
1454	break;
1455	}
1456	out:
1457	return notifier_from_errno(ret);
1458	}
1459	#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1460
1461	/*
1462	* swap the static kmem_cache_node with kmalloced memory
1463	*/
1464	static void __init init_list(struct kmem_cache cachep, struct kmem_cache_node list,
1465	int nodeid)
1466	{
1467	struct kmem_cache_node *ptr;
1468
1469	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
1470	BUG_ON(!ptr);
1471
1472	memcpy(ptr, list, sizeof(struct kmem_cache_node));
1473	/*
1474	* Do not assume that spinlocks can be initialized via memcpy:
1475	*/
1476	spin_lock_init(&ptr->list_lock);
1477
1478	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1479	cachep->node[nodeid] = ptr;
1480	}
1481
1482	/*
1483	* For setting up all the kmem_cache_node for cache whose buffer_size is same as
1484	* size of kmem_cache_node.
1485	*/
1486	static void __init set_up_node(struct kmem_cache *cachep, int index)
1487	{
1488	int node;
1489
1490	for_each_online_node(node) {
1491	cachep->node[node] = &init_kmem_cache_node[index + node];
1492	cachep->node[node]->next_reap = jiffies +
1493	REAPTIMEOUT_LIST3 +
1494	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1495	}
1496	}
1497
1498	/*
1499	* The memory after the last cpu cache pointer is used for the
1500	* the node pointer.
1501	*/
1502	static void setup_node_pointer(struct kmem_cache *cachep)
1503	{
1504	cachep->node = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids];
1505	}
1506
1507	/*
1508	* Initialisation. Called after the page allocator have been initialised and
1509	* before smp_init().
1510	*/
1511	void __init kmem_cache_init(void)
1512	{
1513	int i;
1514
1515	kmem_cache = &kmem_cache_boot;
1516	setup_node_pointer(kmem_cache);
1517
1518	if (num_possible_nodes() == 1)
1519	use_alien_caches = 0;
1520
1521	for (i = 0; i < NUM_INIT_LISTS; i++)
1522	kmem_cache_node_init(&init_kmem_cache_node[i]);
1523
1524	set_up_node(kmem_cache, CACHE_CACHE);
1525
1526	/*
1527	* Fragmentation resistance on low memory - only use bigger
1528	* page orders on machines with more than 32MB of memory if
1529	* not overridden on the command line.
1530	*/
1531	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
1532	slab_max_order = SLAB_MAX_ORDER_HI;
1533
1534	/* Bootstrap is tricky, because several objects are allocated
1535	* from caches that do not exist yet:
1536	* 1) initialize the kmem_cache cache: it contains the struct
1537	* kmem_cache structures of all caches, except kmem_cache itself:
1538	* kmem_cache is statically allocated.
1539	* Initially an __init data area is used for the head array and the
1540	* kmem_cache_node structures, it's replaced with a kmalloc allocated
1541	* array at the end of the bootstrap.
1542	* 2) Create the first kmalloc cache.
1543	* The struct kmem_cache for the new cache is allocated normally.
1544	* An __init data area is used for the head array.
1545	* 3) Create the remaining kmalloc caches, with minimally sized
1546	* head arrays.
1547	* 4) Replace the __init data head arrays for kmem_cache and the first
1548	* kmalloc cache with kmalloc allocated arrays.
1549	* 5) Replace the __init data for kmem_cache_node for kmem_cache and
1550	* the other cache's with kmalloc allocated memory.
1551	* 6) Resize the head arrays of the kmalloc caches to their final sizes.
1552	*/
1553
1554	/* 1) create the kmem_cache */
1555
1556	/*
1557	* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1558	*/
1559	create_boot_cache(kmem_cache, "kmem_cache",
1560	offsetof(struct kmem_cache, array[nr_cpu_ids]) +
1561	nr_node_ids * sizeof(struct kmem_cache_node *),
1562	SLAB_HWCACHE_ALIGN);
1563	list_add(&kmem_cache->list, &slab_caches);
1564
1565	/* 2+3) create the kmalloc caches */
1566
1567	/*
1568	* Initialize the caches that provide memory for the array cache and the
1569	* kmem_cache_node structures first. Without this, further allocations will
1570	* bug.
1571	*/
1572
1573	kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
1574	kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);
1575
1576	if (INDEX_AC != INDEX_NODE)
1577	kmalloc_caches[INDEX_NODE] =
1578	create_kmalloc_cache("kmalloc-node",
1579	kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
1580
1581	slab_early_init = 0;
1582
1583	/* 4) Replace the bootstrap head arrays */
1584	{
1585	struct array_cache *ptr;
1586
1587	ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1588
1589	memcpy(ptr, cpu_cache_get(kmem_cache),
1590	sizeof(struct arraycache_init));
1591	/*
1592	* Do not assume that spinlocks can be initialized via memcpy:
1593	*/
1594	spin_lock_init(&ptr->lock);
1595
1596	kmem_cache->array[smp_processor_id()] = ptr;
1597
1598	ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1599
1600	BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC])
1601	!= &initarray_generic.cache);
1602	memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]),
1603	sizeof(struct arraycache_init));
1604	/*
1605	* Do not assume that spinlocks can be initialized via memcpy:
1606	*/
1607	spin_lock_init(&ptr->lock);
1608
1609	kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr;
1610	}
1611	/* 5) Replace the bootstrap kmem_cache_node */
1612	{
1613	int nid;
1614
1615	for_each_online_node(nid) {
1616	init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
1617
1618	init_list(kmalloc_caches[INDEX_AC],
1619	&init_kmem_cache_node[SIZE_AC + nid], nid);
1620
1621	if (INDEX_AC != INDEX_NODE) {
1622	init_list(kmalloc_caches[INDEX_NODE],
1623	&init_kmem_cache_node[SIZE_NODE + nid], nid);
1624	}
1625	}
1626	}
1627
1628	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
1629	}
1630
1631	void __init kmem_cache_init_late(void)
1632	{
1633	struct kmem_cache *cachep;
1634
1635	slab_state = UP;
1636
1637	/* 6) resize the head arrays to their final sizes */
1638	mutex_lock(&slab_mutex);
1639	list_for_each_entry(cachep, &slab_caches, list)
1640	if (enable_cpucache(cachep, GFP_NOWAIT))
1641	BUG();
1642	mutex_unlock(&slab_mutex);
1643
1644	/* Annotate slab for lockdep -- annotate the malloc caches */
1645	init_lock_keys();
1646
1647	/* Done! */
1648	slab_state = FULL;
1649
1650	/*
1651	* Register a cpu startup notifier callback that initializes
1652	* cpu_cache_get for all new cpus
1653	*/
1654	register_cpu_notifier(&cpucache_notifier);
1655
1656	#ifdef CONFIG_NUMA
1657	/*
1658	* Register a memory hotplug callback that initializes and frees
1659	* node.
1660	*/
1661	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1662	#endif
1663
1664	/*
1665	* The reap timers are started later, with a module init call: That part
1666	* of the kernel is not yet operational.
1667	*/
1668	}
1669
1670	static int __init cpucache_init(void)
1671	{
1672	int cpu;
1673
1674	/*
1675	* Register the timers that return unneeded pages to the page allocator
1676	*/
1677	for_each_online_cpu(cpu)
1678	start_cpu_timer(cpu);
1679
1680	/* Done! */
1681	slab_state = FULL;
1682	return 0;
1683	}
1684	__initcall(cpucache_init);
1685
1686	static noinline void
1687	slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
1688	{
1689	struct kmem_cache_node *n;
1690	struct slab *slabp;
1691	unsigned long flags;
1692	int node;
1693
1694	printk(KERN_WARNING
1695	"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
1696	nodeid, gfpflags);
1697	printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n",
1698	cachep->name, cachep->size, cachep->gfporder);
1699
1700	for_each_online_node(node) {
1701	unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
1702	unsigned long active_slabs = 0, num_slabs = 0;
1703
1704	n = cachep->node[node];
1705	if (!n)
1706	continue;
1707
1708	spin_lock_irqsave(&n->list_lock, flags);
1709	list_for_each_entry(slabp, &n->slabs_full, list) {
1710	active_objs += cachep->num;
1711	active_slabs++;
1712	}
1713	list_for_each_entry(slabp, &n->slabs_partial, list) {
1714	active_objs += slabp->inuse;
1715	active_slabs++;
1716	}
1717	list_for_each_entry(slabp, &n->slabs_free, list)
1718	num_slabs++;
1719
1720	free_objects += n->free_objects;
1721	spin_unlock_irqrestore(&n->list_lock, flags);
1722
1723	num_slabs += active_slabs;
1724	num_objs = num_slabs * cachep->num;
1725	printk(KERN_WARNING
1726	" node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
1727	node, active_slabs, num_slabs, active_objs, num_objs,
1728	free_objects);
1729	}
1730	}
1731
1732	/*
1733	* Interface to system's page allocator. No need to hold the cache-lock.
1734	*
1735	* If we requested dmaable memory, we will get it. Even if we
1736	* did not request dmaable memory, we might get it, but that
1737	* would be relatively rare and ignorable.
1738	*/
1739	static void kmem_getpages(struct kmem_cache cachep, gfp_t flags, int nodeid)
1740	{
1741	struct page *page;
1742	int nr_pages;
1743	int i;
1744
1745	#ifndef CONFIG_MMU
1746	/*
1747	* Nommu uses slab's for process anonymous memory allocations, and thus
1748	* requires __GFP_COMP to properly refcount higher order allocations
1749	*/
1750	flags \|= __GFP_COMP;
1751	#endif
1752
1753	flags \|= cachep->allocflags;
1754	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1755	flags \|= __GFP_RECLAIMABLE;
1756
1757	page = alloc_pages_exact_node(nodeid, flags \| __GFP_NOTRACK, cachep->gfporder);
1758	if (!page) {
1759	if (!(flags & __GFP_NOWARN) && printk_ratelimit())
1760	slab_out_of_memory(cachep, flags, nodeid);
1761	return NULL;
1762	}
1763
1764	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1765	if (unlikely(page->pfmemalloc))
1766	pfmemalloc_active = true;
1767
1768	nr_pages = (1 << cachep->gfporder);
1769	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1770	add_zone_page_state(page_zone(page),
1771	NR_SLAB_RECLAIMABLE, nr_pages);
1772	else
1773	add_zone_page_state(page_zone(page),
1774	NR_SLAB_UNRECLAIMABLE, nr_pages);
1775	for (i = 0; i < nr_pages; i++) {
1776	__SetPageSlab(page + i);
1777
1778	if (page->pfmemalloc)
1779	SetPageSlabPfmemalloc(page + i);
1780	}
1781	memcg_bind_pages(cachep, cachep->gfporder);
1782
1783	if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
1784	kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
1785
1786	if (cachep->ctor)
1787	kmemcheck_mark_uninitialized_pages(page, nr_pages);
1788	else
1789	kmemcheck_mark_unallocated_pages(page, nr_pages);
1790	}
1791
1792	return page_address(page);
1793	}
1794
1795	/*
1796	* Interface to system's page release.
1797	*/
1798	static void kmem_freepages(struct kmem_cache cachep, void addr)
1799	{
1800	unsigned long i = (1 << cachep->gfporder);
1801	struct page *page = virt_to_page(addr);
1802	const unsigned long nr_freed = i;
1803
1804	kmemcheck_free_shadow(page, cachep->gfporder);
1805
1806	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1807	sub_zone_page_state(page_zone(page),
1808	NR_SLAB_RECLAIMABLE, nr_freed);
1809	else
1810	sub_zone_page_state(page_zone(page),
1811	NR_SLAB_UNRECLAIMABLE, nr_freed);
1812	while (i--) {
1813	BUG_ON(!PageSlab(page));
1814	__ClearPageSlabPfmemalloc(page);
1815	__ClearPageSlab(page);
1816	page++;
1817	}
1818
1819	memcg_release_pages(cachep, cachep->gfporder);
1820	if (current->reclaim_state)
1821	current->reclaim_state->reclaimed_slab += nr_freed;
1822	free_memcg_kmem_pages((unsigned long)addr, cachep->gfporder);
1823	}
1824
1825	static void kmem_rcu_free(struct rcu_head *head)
1826	{
1827	struct slab_rcu slab_rcu = (struct slab_rcu )head;
1828	struct kmem_cache *cachep = slab_rcu->cachep;
1829
1830	kmem_freepages(cachep, slab_rcu->addr);
1831	if (OFF_SLAB(cachep))
1832	kmem_cache_free(cachep->slabp_cache, slab_rcu);
1833	}
1834
1835	#if DEBUG
1836
1837	#ifdef CONFIG_DEBUG_PAGEALLOC
1838	static void store_stackinfo(struct kmem_cache cachep, unsigned long addr,
1839	unsigned long caller)
1840	{
1841	int size = cachep->object_size;
1842
1843	addr = (unsigned long )&((char )addr)[obj_offset(cachep)];
1844
1845	if (size < 5 * sizeof(unsigned long))
1846	return;
1847
1848	*addr++ = 0x12345678;
1849	*addr++ = caller;
1850	*addr++ = smp_processor_id();
1851	size -= 3 * sizeof(unsigned long);
1852	{
1853	unsigned long *sptr = &caller;
1854	unsigned long svalue;
1855
1856	while (!kstack_end(sptr)) {
1857	svalue = *sptr++;
1858	if (kernel_text_address(svalue)) {
1859	*addr++ = svalue;
1860	size -= sizeof(unsigned long);
1861	if (size <= sizeof(unsigned long))
1862	break;
1863	}
1864	}
1865
1866	}
1867	*addr++ = 0x87654321;
1868	}
1869	#endif
1870
1871	static void poison_obj(struct kmem_cache cachep, void addr, unsigned char val)
1872	{
1873	int size = cachep->object_size;
1874	addr = &((char *)addr)[obj_offset(cachep)];
1875
1876	memset(addr, val, size);
1877	(unsigned char )(addr + size - 1) = POISON_END;
1878	}
1879
1880	static void dump_line(char *data, int offset, int limit)
1881	{
1882	int i;
1883	unsigned char error = 0;
1884	int bad_count = 0;
1885
1886	printk(KERN_ERR "%03x: ", offset);
1887	for (i = 0; i < limit; i++) {
1888	if (data[offset + i] != POISON_FREE) {
1889	error = data[offset + i];
1890	bad_count++;
1891	}
1892	}
1893	print_hex_dump(KERN_CONT, "", 0, 16, 1,
1894	&data[offset], limit, 1);
1895
1896	if (bad_count == 1) {
1897	error ^= POISON_FREE;
1898	if (!(error & (error - 1))) {
1899	printk(KERN_ERR "Single bit error detected. Probably "
1900	"bad RAM.\n");
1901	#ifdef CONFIG_X86
1902	printk(KERN_ERR "Run memtest86+ or a similar memory "
1903	"test tool.\n");
1904	#else
1905	printk(KERN_ERR "Run a memory test tool.\n");
1906	#endif
1907	}
1908	}
1909	}
1910	#endif
1911
1912	#if DEBUG
1913
1914	static void print_objinfo(struct kmem_cache cachep, void objp, int lines)
1915	{
1916	int i, size;
1917	char *realobj;
1918
1919	if (cachep->flags & SLAB_RED_ZONE) {
1920	printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
1921	*dbg_redzone1(cachep, objp),
1922	*dbg_redzone2(cachep, objp));
1923	}
1924
1925	if (cachep->flags & SLAB_STORE_USER) {
1926	printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
1927	*dbg_userword(cachep, objp),
1928	*dbg_userword(cachep, objp));
1929	}
1930	realobj = (char *)objp + obj_offset(cachep);
1931	size = cachep->object_size;
1932	for (i = 0; i < size && lines; i += 16, lines--) {
1933	int limit;
1934	limit = 16;
1935	if (i + limit > size)
1936	limit = size - i;
1937	dump_line(realobj, i, limit);
1938	}
1939	}
1940
1941	static void check_poison_obj(struct kmem_cache cachep, void objp)
1942	{
1943	char *realobj;
1944	int size, i;
1945	int lines = 0;
1946
1947	realobj = (char *)objp + obj_offset(cachep);
1948	size = cachep->object_size;
1949
1950	for (i = 0; i < size; i++) {
1951	char exp = POISON_FREE;
1952	if (i == size - 1)
1953	exp = POISON_END;
1954	if (realobj[i] != exp) {
1955	int limit;
1956	/* Mismatch ! */
1957	/* Print header */
1958	if (lines == 0) {
1959	printk(KERN_ERR
1960	"Slab corruption (%s): %s start=%p, len=%d\n",
1961	print_tainted(), cachep->name, realobj, size);
1962	print_objinfo(cachep, objp, 0);
1963	}
1964	/* Hexdump the affected line */
1965	i = (i / 16) * 16;
1966	limit = 16;
1967	if (i + limit > size)
1968	limit = size - i;
1969	dump_line(realobj, i, limit);
1970	i += 16;
1971	lines++;
1972	/* Limit to 5 lines */
1973	if (lines > 5)
1974	break;
1975	}
1976	}
1977	if (lines != 0) {
1978	/* Print some data about the neighboring objects, if they
1979	* exist:
1980	*/
1981	struct slab *slabp = virt_to_slab(objp);
1982	unsigned int objnr;
1983
1984	objnr = obj_to_index(cachep, slabp, objp);
1985	if (objnr) {
1986	objp = index_to_obj(cachep, slabp, objnr - 1);
1987	realobj = (char *)objp + obj_offset(cachep);
1988	printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
1989	realobj, size);
1990	print_objinfo(cachep, objp, 2);
1991	}
1992	if (objnr + 1 < cachep->num) {
1993	objp = index_to_obj(cachep, slabp, objnr + 1);
1994	realobj = (char *)objp + obj_offset(cachep);
1995	printk(KERN_ERR "Next obj: start=%p, len=%d\n",
1996	realobj, size);
1997	print_objinfo(cachep, objp, 2);
1998	}
1999	}
2000	}
2001	#endif
2002
2003	#if DEBUG
2004	static void slab_destroy_debugcheck(struct kmem_cache cachep, struct slab slabp)
2005	{
2006	int i;
2007	for (i = 0; i < cachep->num; i++) {
2008	void *objp = index_to_obj(cachep, slabp, i);
2009
2010	if (cachep->flags & SLAB_POISON) {
2011	#ifdef CONFIG_DEBUG_PAGEALLOC
2012	if (cachep->size % PAGE_SIZE == 0 &&
2013	OFF_SLAB(cachep))
2014	kernel_map_pages(virt_to_page(objp),
2015	cachep->size / PAGE_SIZE, 1);
2016	else
2017	check_poison_obj(cachep, objp);
2018	#else
2019	check_poison_obj(cachep, objp);
2020	#endif
2021	}
2022	if (cachep->flags & SLAB_RED_ZONE) {
2023	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2024	slab_error(cachep, "start of a freed object "
2025	"was overwritten");
2026	if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2027	slab_error(cachep, "end of a freed object "
2028	"was overwritten");
2029	}
2030	}
2031	}
2032	#else
2033	static void slab_destroy_debugcheck(struct kmem_cache cachep, struct slab slabp)
2034	{
2035	}
2036	#endif
2037
2038	/**
2039	* slab_destroy - destroy and release all objects in a slab
2040	* @cachep: cache pointer being destroyed
2041	* @slabp: slab pointer being destroyed
2042	*
2043	* Destroy all the objs in a slab, and release the mem back to the system.
2044	* Before calling the slab must have been unlinked from the cache. The
2045	* cache-lock is not held/needed.
2046	*/
2047	static void slab_destroy(struct kmem_cache cachep, struct slab slabp)
2048	{
2049	void *addr = slabp->s_mem - slabp->colouroff;
2050
2051	slab_destroy_debugcheck(cachep, slabp);
2052	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
2053	struct slab_rcu *slab_rcu;
2054
2055	slab_rcu = (struct slab_rcu *)slabp;
2056	slab_rcu->cachep = cachep;
2057	slab_rcu->addr = addr;
2058	call_rcu(&slab_rcu->head, kmem_rcu_free);
2059	} else {
2060	kmem_freepages(cachep, addr);
2061	if (OFF_SLAB(cachep))
2062	kmem_cache_free(cachep->slabp_cache, slabp);
2063	}
2064	}
2065
2066	/**
2067	* calculate_slab_order - calculate size (page order) of slabs
2068	* @cachep: pointer to the cache that is being created
2069	* @size: size of objects to be created in this cache.
2070	* @align: required alignment for the objects.
2071	* @flags: slab allocation flags
2072	*
2073	* Also calculates the number of objects per slab.
2074	*
2075	* This could be made much more intelligent. For now, try to avoid using
2076	* high order pages for slabs. When the gfp() functions are more friendly
2077	* towards high-order requests, this should be changed.
2078	*/
2079	static size_t calculate_slab_order(struct kmem_cache *cachep,
2080	size_t size, size_t align, unsigned long flags)
2081	{
2082	unsigned long offslab_limit;
2083	size_t left_over = 0;
2084	int gfporder;
2085
2086	for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
2087	unsigned int num;
2088	size_t remainder;
2089
2090	cache_estimate(gfporder, size, align, flags, &remainder, &num);
2091	if (!num)
2092	continue;
2093
2094	if (flags & CFLGS_OFF_SLAB) {
2095	/*
2096	* Max number of objs-per-slab for caches which
2097	* use off-slab slabs. Needed to avoid a possible
2098	* looping condition in cache_grow().
2099	*/
2100	offslab_limit = size - sizeof(struct slab);
2101	offslab_limit /= sizeof(kmem_bufctl_t);
2102
2103	if (num > offslab_limit)
2104	break;
2105	}
2106
2107	/* Found something acceptable - save it away */
2108	cachep->num = num;
2109	cachep->gfporder = gfporder;
2110	left_over = remainder;
2111
2112	/*
2113	* A VFS-reclaimable slab tends to have most allocations
2114	* as GFP_NOFS and we really don't want to have to be allocating
2115	* higher-order pages when we are unable to shrink dcache.
2116	*/
2117	if (flags & SLAB_RECLAIM_ACCOUNT)
2118	break;
2119
2120	/*
2121	* Large number of objects is good, but very large slabs are
2122	* currently bad for the gfp()s.
2123	*/
2124	if (gfporder >= slab_max_order)
2125	break;
2126
2127	/*
2128	* Acceptable internal fragmentation?
2129	*/
2130	if (left_over * 8 <= (PAGE_SIZE << gfporder))
2131	break;
2132	}
2133	return left_over;
2134	}
2135
2136	static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2137	{
2138	if (slab_state >= FULL)
2139	return enable_cpucache(cachep, gfp);
2140
2141	if (slab_state == DOWN) {
2142	/*
2143	* Note: Creation of first cache (kmem_cache).
2144	* The setup_node is taken care
2145	* of by the caller of __kmem_cache_create
2146	*/
2147	cachep->array[smp_processor_id()] = &initarray_generic.cache;
2148	slab_state = PARTIAL;
2149	} else if (slab_state == PARTIAL) {
2150	/*
2151	* Note: the second kmem_cache_create must create the cache
2152	* that's used by kmalloc(24), otherwise the creation of
2153	* further caches will BUG().
2154	*/
2155	cachep->array[smp_processor_id()] = &initarray_generic.cache;
2156
2157	/*
2158	* If the cache that's used by kmalloc(sizeof(kmem_cache_node)) is
2159	* the second cache, then we need to set up all its node/,
2160	* otherwise the creation of further caches will BUG().
2161	*/
2162	set_up_node(cachep, SIZE_AC);
2163	if (INDEX_AC == INDEX_NODE)
2164	slab_state = PARTIAL_NODE;
2165	else
2166	slab_state = PARTIAL_ARRAYCACHE;
2167	} else {
2168	/* Remaining boot caches */
2169	cachep->array[smp_processor_id()] =
2170	kmalloc(sizeof(struct arraycache_init), gfp);
2171
2172	if (slab_state == PARTIAL_ARRAYCACHE) {
2173	set_up_node(cachep, SIZE_NODE);
2174	slab_state = PARTIAL_NODE;
2175	} else {
2176	int node;
2177	for_each_online_node(node) {
2178	cachep->node[node] =
2179	kmalloc_node(sizeof(struct kmem_cache_node),
2180	gfp, node);
2181	BUG_ON(!cachep->node[node]);
2182	kmem_cache_node_init(cachep->node[node]);
2183	}
2184	}
2185	}
2186	cachep->node[numa_mem_id()]->next_reap =
2187	jiffies + REAPTIMEOUT_LIST3 +
2188	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
2189
2190	cpu_cache_get(cachep)->avail = 0;
2191	cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
2192	cpu_cache_get(cachep)->batchcount = 1;
2193	cpu_cache_get(cachep)->touched = 0;
2194	cachep->batchcount = 1;
2195	cachep->limit = BOOT_CPUCACHE_ENTRIES;
2196	return 0;
2197	}
2198
2199	/**
2200	* __kmem_cache_create - Create a cache.
2201	* @cachep: cache management descriptor
2202	* @flags: SLAB flags
2203	*
2204	* Returns a ptr to the cache on success, NULL on failure.
2205	* Cannot be called within a int, but can be interrupted.
2206	* The @ctor is run when new pages are allocated by the cache.
2207	*
2208	* The flags are
2209	*
2210	* %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
2211	* to catch references to uninitialised memory.
2212	*
2213	* %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
2214	* for buffer overruns.
2215	*
2216	* %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
2217	* cacheline. This can be beneficial if you're counting cycles as closely
2218	* as davem.
2219	*/
2220	int
2221	__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2222	{
2223	size_t left_over, slab_size, ralign;
2224	gfp_t gfp;
2225	int err;
2226	size_t size = cachep->size;
2227
2228	#if DEBUG
2229	#if FORCED_DEBUG
2230	/*
2231	* Enable redzoning and last user accounting, except for caches with
2232	* large objects, if the increased size would increase the object size
2233	* above the next power of two: caches with object sizes just above a
2234	* power of two have a significant amount of internal fragmentation.
2235	*/
2236	if (size < 4096 \|\| fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
2237	2 * sizeof(unsigned long long)))
2238	flags \|= SLAB_RED_ZONE \| SLAB_STORE_USER;
2239	if (!(flags & SLAB_DESTROY_BY_RCU))
2240	flags \|= SLAB_POISON;
2241	#endif
2242	if (flags & SLAB_DESTROY_BY_RCU)
2243	BUG_ON(flags & SLAB_POISON);
2244	#endif
2245
2246	/*
2247	* Check that size is in terms of words. This is needed to avoid
2248	* unaligned accesses for some archs when redzoning is used, and makes
2249	* sure any on-slab bufctl's are also correctly aligned.
2250	*/
2251	if (size & (BYTES_PER_WORD - 1)) {
2252	size += (BYTES_PER_WORD - 1);
2253	size &= ~(BYTES_PER_WORD - 1);
2254	}
2255
2256	/*
2257	* Redzoning and user store require word alignment or possibly larger.
2258	* Note this will be overridden by architecture or caller mandated
2259	* alignment if either is greater than BYTES_PER_WORD.
2260	*/
2261	if (flags & SLAB_STORE_USER)
2262	ralign = BYTES_PER_WORD;
2263
2264	if (flags & SLAB_RED_ZONE) {
2265	ralign = REDZONE_ALIGN;
2266	/* If redzoning, ensure that the second redzone is suitably
2267	* aligned, by adjusting the object size accordingly. */
2268	size += REDZONE_ALIGN - 1;
2269	size &= ~(REDZONE_ALIGN - 1);
2270	}
2271
2272	/* 3) caller mandated alignment */
2273	if (ralign < cachep->align) {
2274	ralign = cachep->align;
2275	}
2276	/* disable debug if necessary */
2277	if (ralign > __alignof__(unsigned long long))
2278	flags &= ~(SLAB_RED_ZONE \| SLAB_STORE_USER);
2279	/*
2280	* 4) Store it.
2281	*/
2282	cachep->align = ralign;
2283
2284	if (slab_is_available())
2285	gfp = GFP_KERNEL;
2286	else
2287	gfp = GFP_NOWAIT;
2288
2289	setup_node_pointer(cachep);
2290	#if DEBUG
2291
2292	/*
2293	* Both debugging options require word-alignment which is calculated
2294	* into align above.
2295	*/
2296	if (flags & SLAB_RED_ZONE) {
2297	/* add space for red zone words */
2298	cachep->obj_offset += sizeof(unsigned long long);
2299	size += 2 * sizeof(unsigned long long);
2300	}
2301	if (flags & SLAB_STORE_USER) {
2302	/* user store requires one word storage behind the end of
2303	* the real object. But if the second red zone needs to be
2304	* aligned to 64 bits, we must allow that much space.
2305	*/
2306	if (flags & SLAB_RED_ZONE)
2307	size += REDZONE_ALIGN;
2308	else
2309	size += BYTES_PER_WORD;
2310	}
2311	#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
2312	if (size >= kmalloc_size(INDEX_NODE + 1)
2313	&& cachep->object_size > cache_line_size()
2314	&& ALIGN(size, cachep->align) < PAGE_SIZE) {
2315	cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align);
2316	size = PAGE_SIZE;
2317	}
2318	#endif
2319	#endif
2320
2321	/*
2322	* Determine if the slab management is 'on' or 'off' slab.
2323	* (bootstrapping cannot cope with offslab caches so don't do
2324	* it too early on. Always use on-slab management when
2325	* SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
2326	*/
2327	if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init &&
2328	!(flags & SLAB_NOLEAKTRACE))
2329	/*
2330	* Size is large, assume best to place the slab management obj
2331	* off-slab (should allow better packing of objs).
2332	*/
2333	flags \|= CFLGS_OFF_SLAB;
2334
2335	size = ALIGN(size, cachep->align);
2336
2337	left_over = calculate_slab_order(cachep, size, cachep->align, flags);
2338
2339	if (!cachep->num)
2340	return -E2BIG;
2341
2342	slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
2343	+ sizeof(struct slab), cachep->align);
2344
2345	/*
2346	* If the slab has been placed off-slab, and we have enough space then
2347	* move it on-slab. This is at the expense of any extra colouring.
2348	*/
2349	if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
2350	flags &= ~CFLGS_OFF_SLAB;
2351	left_over -= slab_size;
2352	}
2353
2354	if (flags & CFLGS_OFF_SLAB) {
2355	/* really off slab. No need for manual alignment */
2356	slab_size =
2357	cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
2358
2359	#ifdef CONFIG_PAGE_POISONING
2360	/* If we're going to use the generic kernel_map_pages()
2361	* poisoning, then it's going to smash the contents of
2362	* the redzone and userword anyhow, so switch them off.
2363	*/
2364	if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
2365	flags &= ~(SLAB_RED_ZONE \| SLAB_STORE_USER);
2366	#endif
2367	}
2368
2369	cachep->colour_off = cache_line_size();
2370	/* Offset must be a multiple of the alignment. */
2371	if (cachep->colour_off < cachep->align)
2372	cachep->colour_off = cachep->align;
2373	cachep->colour = left_over / cachep->colour_off;
2374	cachep->slab_size = slab_size;
2375	cachep->flags = flags;
2376	cachep->allocflags = 0;
2377	if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
2378	cachep->allocflags \|= GFP_DMA;
2379	cachep->size = size;
2380	cachep->reciprocal_buffer_size = reciprocal_value(size);
2381
2382	if (flags & CFLGS_OFF_SLAB) {
2383	cachep->slabp_cache = kmalloc_slab(slab_size, 0u);
2384	/*
2385	* This is a possibility for one of the malloc_sizes caches.
2386	* But since we go off slab only for object size greater than
2387	* PAGE_SIZE/8, and malloc_sizes gets created in ascending order,
2388	* this should not happen at all.
2389	* But leave a BUG_ON for some lucky dude.
2390	*/
2391	BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache));
2392	}
2393
2394	err = setup_cpu_cache(cachep, gfp);
2395	if (err) {
2396	__kmem_cache_shutdown(cachep);
2397	return err;
2398	}
2399
2400	if (flags & SLAB_DEBUG_OBJECTS) {
2401	/*
2402	* Would deadlock through slab_destroy()->call_rcu()->
2403	* debug_object_activate()->kmem_cache_alloc().
2404	*/
2405	WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
2406
2407	slab_set_debugobj_lock_classes(cachep);
2408	} else if (!OFF_SLAB(cachep) && !(flags & SLAB_DESTROY_BY_RCU))
2409	on_slab_lock_classes(cachep);
2410
2411	return 0;
2412	}
2413
2414	#if DEBUG
2415	static void check_irq_off(void)
2416	{
2417	BUG_ON(!irqs_disabled());
2418	}
2419
2420	static void check_irq_on(void)
2421	{
2422	BUG_ON(irqs_disabled());
2423	}
2424
2425	static void check_spinlock_acquired(struct kmem_cache *cachep)
2426	{
2427	#ifdef CONFIG_SMP
2428	check_irq_off();
2429	assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock);
2430	#endif
2431	}
2432
2433	static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2434	{
2435	#ifdef CONFIG_SMP
2436	check_irq_off();
2437	assert_spin_locked(&cachep->node[node]->list_lock);
2438	#endif
2439	}
2440
2441	#else
2442	#define check_irq_off() do { } while(0)
2443	#define check_irq_on() do { } while(0)
2444	#define check_spinlock_acquired(x) do { } while(0)
2445	#define check_spinlock_acquired_node(x, y) do { } while(0)
2446	#endif
2447
2448	static void drain_array(struct kmem_cache cachep, struct kmem_cache_node n,
2449	struct array_cache *ac,
2450	int force, int node);
2451
2452	static void do_drain(void *arg)
2453	{
2454	struct kmem_cache *cachep = arg;
2455	struct array_cache *ac;
2456	int node = numa_mem_id();
2457
2458	check_irq_off();
2459	ac = cpu_cache_get(cachep);
2460	spin_lock(&cachep->node[node]->list_lock);
2461	free_block(cachep, ac->entry, ac->avail, node);
2462	spin_unlock(&cachep->node[node]->list_lock);
2463	ac->avail = 0;
2464	}
2465
2466	static void drain_cpu_caches(struct kmem_cache *cachep)
2467	{
2468	struct kmem_cache_node *n;
2469	int node;
2470
2471	on_each_cpu(do_drain, cachep, 1);
2472	check_irq_on();
2473	for_each_online_node(node) {
2474	n = cachep->node[node];
2475	if (n && n->alien)
2476	drain_alien_cache(cachep, n->alien);
2477	}
2478
2479	for_each_online_node(node) {
2480	n = cachep->node[node];
2481	if (n)
2482	drain_array(cachep, n, n->shared, 1, node);
2483	}
2484	}
2485
2486	/*
2487	* Remove slabs from the list of free slabs.
2488	* Specify the number of slabs to drain in tofree.
2489	*
2490	* Returns the actual number of slabs released.
2491	*/
2492	static int drain_freelist(struct kmem_cache *cache,
2493	struct kmem_cache_node *n, int tofree)
2494	{
2495	struct list_head *p;
2496	int nr_freed;
2497	struct slab *slabp;
2498
2499	nr_freed = 0;
2500	while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
2501
2502	spin_lock_irq(&n->list_lock);
2503	p = n->slabs_free.prev;
2504	if (p == &n->slabs_free) {
2505	spin_unlock_irq(&n->list_lock);
2506	goto out;
2507	}
2508
2509	slabp = list_entry(p, struct slab, list);
2510	#if DEBUG
2511	BUG_ON(slabp->inuse);
2512	#endif
2513	list_del(&slabp->list);
2514	/*
2515	* Safe to drop the lock. The slab is no longer linked
2516	* to the cache.
2517	*/
2518	n->free_objects -= cache->num;
2519	spin_unlock_irq(&n->list_lock);
2520	slab_destroy(cache, slabp);
2521	nr_freed++;
2522	}
2523	out:
2524	return nr_freed;
2525	}
2526
2527	/* Called with slab_mutex held to protect against cpu hotplug */
2528	static int __cache_shrink(struct kmem_cache *cachep)
2529	{
2530	int ret = 0, i = 0;
2531	struct kmem_cache_node *n;
2532
2533	drain_cpu_caches(cachep);
2534
2535	check_irq_on();
2536	for_each_online_node(i) {
2537	n = cachep->node[i];
2538	if (!n)
2539	continue;
2540
2541	drain_freelist(cachep, n, slabs_tofree(cachep, n));
2542
2543	ret += !list_empty(&n->slabs_full) \|\|
2544	!list_empty(&n->slabs_partial);
2545	}
2546	return (ret ? 1 : 0);
2547	}
2548
2549	/**
2550	* kmem_cache_shrink - Shrink a cache.
2551	* @cachep: The cache to shrink.
2552	*
2553	* Releases as many slabs as possible for a cache.
2554	* To help debugging, a zero exit status indicates all slabs were released.
2555	*/
2556	int kmem_cache_shrink(struct kmem_cache *cachep)
2557	{
2558	int ret;
2559	BUG_ON(!cachep \|\| in_interrupt());
2560
2561	get_online_cpus();
2562	mutex_lock(&slab_mutex);
2563	ret = __cache_shrink(cachep);
2564	mutex_unlock(&slab_mutex);
2565	put_online_cpus();
2566	return ret;
2567	}
2568	EXPORT_SYMBOL(kmem_cache_shrink);
2569
2570	int __kmem_cache_shutdown(struct kmem_cache *cachep)
2571	{
2572	int i;
2573	struct kmem_cache_node *n;
2574	int rc = __cache_shrink(cachep);
2575
2576	if (rc)
2577	return rc;
2578
2579	for_each_online_cpu(i)
2580	kfree(cachep->array[i]);
2581
2582	/* NUMA: free the node structures */
2583	for_each_online_node(i) {
2584	n = cachep->node[i];
2585	if (n) {
2586	kfree(n->shared);
2587	free_alien_cache(n->alien);
2588	kfree(n);
2589	}
2590	}
2591	return 0;
2592	}
2593
2594	/*
2595	* Get the memory for a slab management obj.
2596	* For a slab cache when the slab descriptor is off-slab, slab descriptors
2597	* always come from malloc_sizes caches. The slab descriptor cannot
2598	* come from the same cache which is getting created because,
2599	* when we are searching for an appropriate cache for these
2600	* descriptors in kmem_cache_create, we search through the malloc_sizes array.
2601	* If we are creating a malloc_sizes cache here it would not be visible to
2602	* kmem_find_general_cachep till the initialization is complete.
2603	* Hence we cannot have slabp_cache same as the original cache.
2604	*/
2605	static struct slab alloc_slabmgmt(struct kmem_cache cachep, void *objp,
2606	int colour_off, gfp_t local_flags,
2607	int nodeid)
2608	{
2609	struct slab *slabp;
2610
2611	if (OFF_SLAB(cachep)) {
2612	/* Slab management obj is off-slab. */
2613	slabp = kmem_cache_alloc_node(cachep->slabp_cache,
2614	local_flags, nodeid);
2615	/*
2616	* If the first object in the slab is leaked (it's allocated
2617	* but no one has a reference to it), we want to make sure
2618	* kmemleak does not treat the ->s_mem pointer as a reference
2619	* to the object. Otherwise we will not report the leak.
2620	*/
2621	kmemleak_scan_area(&slabp->list, sizeof(struct list_head),
2622	local_flags);
2623	if (!slabp)
2624	return NULL;
2625	} else {
2626	slabp = objp + colour_off;
2627	colour_off += cachep->slab_size;
2628	}
2629	slabp->inuse = 0;
2630	slabp->colouroff = colour_off;
2631	slabp->s_mem = objp + colour_off;
2632	slabp->nodeid = nodeid;
2633	slabp->free = 0;
2634	return slabp;
2635	}
2636
2637	static inline kmem_bufctl_t slab_bufctl(struct slab slabp)
2638	{
2639	return (kmem_bufctl_t *) (slabp + 1);
2640	}
2641
2642	static void cache_init_objs(struct kmem_cache *cachep,
2643	struct slab *slabp)
2644	{
2645	int i;
2646
2647	for (i = 0; i < cachep->num; i++) {
2648	void *objp = index_to_obj(cachep, slabp, i);
2649	#if DEBUG
2650	/* need to poison the objs? */
2651	if (cachep->flags & SLAB_POISON)
2652	poison_obj(cachep, objp, POISON_FREE);
2653	if (cachep->flags & SLAB_STORE_USER)
2654	*dbg_userword(cachep, objp) = NULL;
2655
2656	if (cachep->flags & SLAB_RED_ZONE) {
2657	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2658	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2659	}
2660	/*
2661	* Constructors are not allowed to allocate memory from the same
2662	* cache which they are a constructor for. Otherwise, deadlock.
2663	* They must also be threaded.
2664	*/
2665	if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2666	cachep->ctor(objp + obj_offset(cachep));
2667
2668	if (cachep->flags & SLAB_RED_ZONE) {
2669	if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2670	slab_error(cachep, "constructor overwrote the"
2671	" end of an object");
2672	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2673	slab_error(cachep, "constructor overwrote the"
2674	" start of an object");
2675	}
2676	if ((cachep->size % PAGE_SIZE) == 0 &&
2677	OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
2678	kernel_map_pages(virt_to_page(objp),
2679	cachep->size / PAGE_SIZE, 0);
2680	#else
2681	if (cachep->ctor)
2682	cachep->ctor(objp);
2683	#endif
2684	slab_bufctl(slabp)[i] = i + 1;
2685	}
2686	slab_bufctl(slabp)[i - 1] = BUFCTL_END;
2687	}
2688
2689	static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
2690	{
2691	if (CONFIG_ZONE_DMA_FLAG) {
2692	if (flags & GFP_DMA)
2693	BUG_ON(!(cachep->allocflags & GFP_DMA));
2694	else
2695	BUG_ON(cachep->allocflags & GFP_DMA);
2696	}
2697	}
2698
2699	static void slab_get_obj(struct kmem_cache cachep, struct slab *slabp,
2700	int nodeid)
2701	{
2702	void *objp = index_to_obj(cachep, slabp, slabp->free);
2703	kmem_bufctl_t next;
2704
2705	slabp->inuse++;
2706	next = slab_bufctl(slabp)[slabp->free];
2707	#if DEBUG
2708	slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2709	WARN_ON(slabp->nodeid != nodeid);
2710	#endif
2711	slabp->free = next;
2712
2713	return objp;
2714	}
2715
2716	static void slab_put_obj(struct kmem_cache cachep, struct slab slabp,
2717	void *objp, int nodeid)
2718	{
2719	unsigned int objnr = obj_to_index(cachep, slabp, objp);
2720
2721	#if DEBUG
2722	/* Verify that the slab belongs to the intended node */
2723	WARN_ON(slabp->nodeid != nodeid);
2724
2725	if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) {
2726	printk(KERN_ERR "slab: double free detected in cache "
2727	"'%s', objp %p\n", cachep->name, objp);
2728	BUG();
2729	}
2730	#endif
2731	slab_bufctl(slabp)[objnr] = slabp->free;
2732	slabp->free = objnr;
2733	slabp->inuse--;
2734	}
2735
2736	/*
2737	* Map pages beginning at addr to the given cache and slab. This is required
2738	* for the slab allocator to be able to lookup the cache and slab of a
2739	* virtual address for kfree, ksize, and slab debugging.
2740	*/
2741	static void slab_map_pages(struct kmem_cache cache, struct slab slab,
2742	void *addr)
2743	{
2744	int nr_pages;
2745	struct page *page;
2746
2747	page = virt_to_page(addr);
2748
2749	nr_pages = 1;
2750	if (likely(!PageCompound(page)))
2751	nr_pages <<= cache->gfporder;
2752
2753	do {
2754	page->slab_cache = cache;
2755	page->slab_page = slab;
2756	page++;
2757	} while (--nr_pages);
2758	}
2759
2760	/*
2761	* Grow (by 1) the number of slabs within a cache. This is called by
2762	* kmem_cache_alloc() when there are no active objs left in a cache.
2763	*/
2764	static int cache_grow(struct kmem_cache *cachep,
2765	gfp_t flags, int nodeid, void *objp)
2766	{
2767	struct slab *slabp;
2768	size_t offset;
2769	gfp_t local_flags;
2770	struct kmem_cache_node *n;
2771
2772	/*
2773	* Be lazy and only check for valid flags here, keeping it out of the
2774	* critical path in kmem_cache_alloc().
2775	*/
2776	BUG_ON(flags & GFP_SLAB_BUG_MASK);
2777	local_flags = flags & (GFP_CONSTRAINT_MASK\|GFP_RECLAIM_MASK);
2778
2779	/* Take the node list lock to change the colour_next on this node */
2780	check_irq_off();
2781	n = cachep->node[nodeid];
2782	spin_lock(&n->list_lock);
2783
2784	/* Get colour for the slab, and cal the next value. */
2785	offset = n->colour_next;
2786	n->colour_next++;
2787	if (n->colour_next >= cachep->colour)
2788	n->colour_next = 0;
2789	spin_unlock(&n->list_lock);
2790
2791	offset *= cachep->colour_off;
2792
2793	if (local_flags & __GFP_WAIT)
2794	local_irq_enable();
2795
2796	/*
2797	* The test for missing atomic flag is performed here, rather than
2798	* the more obvious place, simply to reduce the critical path length
2799	* in kmem_cache_alloc(). If a caller is seriously mis-behaving they
2800	* will eventually be caught here (where it matters).
2801	*/
2802	kmem_flagcheck(cachep, flags);
2803
2804	/*
2805	* Get mem for the objs. Attempt to allocate a physical page from
2806	* 'nodeid'.
2807	*/
2808	if (!objp)
2809	objp = kmem_getpages(cachep, local_flags, nodeid);
2810	if (!objp)
2811	goto failed;
2812
2813	/* Get slab management. */
2814	slabp = alloc_slabmgmt(cachep, objp, offset,
2815	local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
2816	if (!slabp)
2817	goto opps1;
2818
2819	slab_map_pages(cachep, slabp, objp);
2820
2821	cache_init_objs(cachep, slabp);
2822
2823	if (local_flags & __GFP_WAIT)
2824	local_irq_disable();
2825	check_irq_off();
2826	spin_lock(&n->list_lock);
2827
2828	/* Make slab active. */
2829	list_add_tail(&slabp->list, &(n->slabs_free));
2830	STATS_INC_GROWN(cachep);
2831	n->free_objects += cachep->num;
2832	spin_unlock(&n->list_lock);
2833	return 1;
2834	opps1:
2835	kmem_freepages(cachep, objp);
2836	failed:
2837	if (local_flags & __GFP_WAIT)
2838	local_irq_disable();
2839	return 0;
2840	}
2841
2842	#if DEBUG
2843
2844	/*
2845	* Perform extra freeing checks:
2846	* - detect bad pointers.
2847	* - POISON/RED_ZONE checking
2848	*/
2849	static void kfree_debugcheck(const void *objp)
2850	{
2851	if (!virt_addr_valid(objp)) {
2852	printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
2853	(unsigned long)objp);
2854	BUG();
2855	}
2856	}
2857
2858	static inline void verify_redzone_free(struct kmem_cache cache, void obj)
2859	{
2860	unsigned long long redzone1, redzone2;
2861
2862	redzone1 = *dbg_redzone1(cache, obj);
2863	redzone2 = *dbg_redzone2(cache, obj);
2864
2865	/*
2866	* Redzone is ok.
2867	*/
2868	if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2869	return;
2870
2871	if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2872	slab_error(cache, "double free detected");
2873	else
2874	slab_error(cache, "memory outside object was overwritten");
2875
2876	printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
2877	obj, redzone1, redzone2);
2878	}
2879
2880	static void cache_free_debugcheck(struct kmem_cache cachep, void *objp,
2881	unsigned long caller)
2882	{
2883	struct page *page;
2884	unsigned int objnr;
2885	struct slab *slabp;
2886
2887	BUG_ON(virt_to_cache(objp) != cachep);
2888
2889	objp -= obj_offset(cachep);
2890	kfree_debugcheck(objp);
2891	page = virt_to_head_page(objp);
2892
2893	slabp = page->slab_page;
2894
2895	if (cachep->flags & SLAB_RED_ZONE) {
2896	verify_redzone_free(cachep, objp);
2897	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2898	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2899	}
2900	if (cachep->flags & SLAB_STORE_USER)
2901	dbg_userword(cachep, objp) = (void )caller;
2902
2903	objnr = obj_to_index(cachep, slabp, objp);
2904
2905	BUG_ON(objnr >= cachep->num);
2906	BUG_ON(objp != index_to_obj(cachep, slabp, objnr));
2907
2908	#ifdef CONFIG_DEBUG_SLAB_LEAK
2909	slab_bufctl(slabp)[objnr] = BUFCTL_FREE;
2910	#endif
2911	if (cachep->flags & SLAB_POISON) {
2912	#ifdef CONFIG_DEBUG_PAGEALLOC
2913	if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
2914	store_stackinfo(cachep, objp, caller);
2915	kernel_map_pages(virt_to_page(objp),
2916	cachep->size / PAGE_SIZE, 0);
2917	} else {
2918	poison_obj(cachep, objp, POISON_FREE);
2919	}
2920	#else
2921	poison_obj(cachep, objp, POISON_FREE);
2922	#endif
2923	}
2924	return objp;
2925	}
2926
2927	static void check_slabp(struct kmem_cache cachep, struct slab slabp)
2928	{
2929	kmem_bufctl_t i;
2930	int entries = 0;
2931
2932	/* Check slab's freelist to see if this obj is there. */
2933	for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
2934	entries++;
2935	if (entries > cachep->num \|\| i >= cachep->num)
2936	goto bad;
2937	}
2938	if (entries != cachep->num - slabp->inuse) {
2939	bad:
2940	printk(KERN_ERR "slab: Internal list corruption detected in "
2941	"cache '%s'(%d), slabp %p(%d). Tainted(%s). Hexdump:\n",
2942	cachep->name, cachep->num, slabp, slabp->inuse,
2943	print_tainted());
2944	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp,
2945	sizeof(slabp) + cachep->num sizeof(kmem_bufctl_t),
2946	1);
2947	BUG();
2948	}
2949	}
2950	#else
2951	#define kfree_debugcheck(x) do { } while(0)
2952	#define cache_free_debugcheck(x,objp,z) (objp)
2953	#define check_slabp(x,y) do { } while(0)
2954	#endif
2955
2956	static void cache_alloc_refill(struct kmem_cache cachep, gfp_t flags,
2957	bool force_refill)
2958	{
2959	int batchcount;
2960	struct kmem_cache_node *n;
2961	struct array_cache *ac;
2962	int node;
2963
2964	check_irq_off();
2965	node = numa_mem_id();
2966	if (unlikely(force_refill))
2967	goto force_grow;
2968	retry:
2969	ac = cpu_cache_get(cachep);
2970	batchcount = ac->batchcount;
2971	if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
2972	/*
2973	* If there was little recent activity on this cache, then
2974	* perform only a partial refill. Otherwise we could generate
2975	* refill bouncing.
2976	*/
2977	batchcount = BATCHREFILL_LIMIT;
2978	}
2979	n = cachep->node[node];
2980
2981	BUG_ON(ac->avail > 0 \|\| !n);
2982	spin_lock(&n->list_lock);
2983
2984	/* See if we can refill from the shared array */
2985	if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
2986	n->shared->touched = 1;
2987	goto alloc_done;
2988	}
2989
2990	while (batchcount > 0) {
2991	struct list_head *entry;
2992	struct slab *slabp;
2993	/* Get slab alloc is to come from. */
2994	entry = n->slabs_partial.next;
2995	if (entry == &n->slabs_partial) {
2996	n->free_touched = 1;
2997	entry = n->slabs_free.next;
2998	if (entry == &n->slabs_free)
2999	goto must_grow;
3000	}
3001
3002	slabp = list_entry(entry, struct slab, list);
3003	check_slabp(cachep, slabp);
3004	check_spinlock_acquired(cachep);
3005
3006	/*
3007	* The slab was either on partial or free list so
3008	* there must be at least one object available for
3009	* allocation.
3010	*/
3011	BUG_ON(slabp->inuse >= cachep->num);
3012
3013	while (slabp->inuse < cachep->num && batchcount--) {
3014	STATS_INC_ALLOCED(cachep);
3015	STATS_INC_ACTIVE(cachep);
3016	STATS_SET_HIGH(cachep);
3017
3018	ac_put_obj(cachep, ac, slab_get_obj(cachep, slabp,
3019	node));
3020	}
3021	check_slabp(cachep, slabp);
3022
3023	/* move slabp to correct slabp list: */
3024	list_del(&slabp->list);
3025	if (slabp->free == BUFCTL_END)
3026	list_add(&slabp->list, &n->slabs_full);
3027	else
3028	list_add(&slabp->list, &n->slabs_partial);
3029	}
3030
3031	must_grow:
3032	n->free_objects -= ac->avail;
3033	alloc_done:
3034	spin_unlock(&n->list_lock);
3035
3036	if (unlikely(!ac->avail)) {
3037	int x;
3038	force_grow:
3039	x = cache_grow(cachep, flags \| GFP_THISNODE, node, NULL);
3040
3041	/* cache_grow can reenable interrupts, then ac could change. */
3042	ac = cpu_cache_get(cachep);
3043	node = numa_mem_id();
3044
3045	/* no objects in sight? abort */
3046	if (!x && (ac->avail == 0 \|\| force_refill))
3047	return NULL;
3048
3049	if (!ac->avail) /* objects refilled by interrupt? */
3050	goto retry;
3051	}
3052	ac->touched = 1;
3053
3054	return ac_get_obj(cachep, ac, flags, force_refill);
3055	}
3056
3057	static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
3058	gfp_t flags)
3059	{
3060	might_sleep_if(flags & __GFP_WAIT);
3061	#if DEBUG
3062	kmem_flagcheck(cachep, flags);
3063	#endif
3064	}
3065
3066	#if DEBUG
3067	static void cache_alloc_debugcheck_after(struct kmem_cache cachep,
3068	gfp_t flags, void *objp, unsigned long caller)
3069	{
3070	if (!objp)
3071	return objp;
3072	if (cachep->flags & SLAB_POISON) {
3073	#ifdef CONFIG_DEBUG_PAGEALLOC
3074	if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
3075	kernel_map_pages(virt_to_page(objp),
3076	cachep->size / PAGE_SIZE, 1);
3077	else
3078	check_poison_obj(cachep, objp);
3079	#else
3080	check_poison_obj(cachep, objp);
3081	#endif
3082	poison_obj(cachep, objp, POISON_INUSE);
3083	}
3084	if (cachep->flags & SLAB_STORE_USER)
3085	dbg_userword(cachep, objp) = (void )caller;
3086
3087	if (cachep->flags & SLAB_RED_ZONE) {
3088	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE \|\|
3089	*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
3090	slab_error(cachep, "double free, or memory outside"
3091	" object was overwritten");
3092	printk(KERN_ERR
3093	"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
3094	objp, *dbg_redzone1(cachep, objp),
3095	*dbg_redzone2(cachep, objp));
3096	}
3097	*dbg_redzone1(cachep, objp) = RED_ACTIVE;
3098	*dbg_redzone2(cachep, objp) = RED_ACTIVE;
3099	}
3100	#ifdef CONFIG_DEBUG_SLAB_LEAK
3101	{
3102	struct slab *slabp;
3103	unsigned objnr;
3104
3105	slabp = virt_to_head_page(objp)->slab_page;
3106	objnr = (unsigned)(objp - slabp->s_mem) / cachep->size;
3107	slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE;
3108	}
3109	#endif
3110	objp += obj_offset(cachep);
3111	if (cachep->ctor && cachep->flags & SLAB_POISON)
3112	cachep->ctor(objp);
3113	if (ARCH_SLAB_MINALIGN &&
3114	((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
3115	printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
3116	objp, (int)ARCH_SLAB_MINALIGN);
3117	}
3118	return objp;
3119	}
3120	#else
3121	#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
3122	#endif
3123
3124	static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
3125	{
3126	if (cachep == kmem_cache)
3127	return false;
3128
3129	return should_failslab(cachep->object_size, flags, cachep->flags);
3130	}
3131
3132	static inline void ____cache_alloc(struct kmem_cache cachep, gfp_t flags)
3133	{
3134	void *objp;
3135	struct array_cache *ac;
3136	bool force_refill = false;
3137
3138	check_irq_off();
3139
3140	ac = cpu_cache_get(cachep);
3141	if (likely(ac->avail)) {
3142	ac->touched = 1;
3143	objp = ac_get_obj(cachep, ac, flags, false);
3144
3145	/*
3146	* Allow for the possibility all avail objects are not allowed
3147	* by the current flags
3148	*/
3149	if (objp) {
3150	STATS_INC_ALLOCHIT(cachep);
3151	goto out;
3152	}
3153	force_refill = true;
3154	}
3155
3156	STATS_INC_ALLOCMISS(cachep);
3157	objp = cache_alloc_refill(cachep, flags, force_refill);
3158	/*
3159	* the 'ac' may be updated by cache_alloc_refill(),
3160	* and kmemleak_erase() requires its correct value.
3161	*/
3162	ac = cpu_cache_get(cachep);
3163
3164	out:
3165	/*
3166	* To avoid a false negative, if an object that is in one of the
3167	* per-CPU caches is leaked, we need to make sure kmemleak doesn't
3168	* treat the array pointers as a reference to the object.
3169	*/
3170	if (objp)
3171	kmemleak_erase(&ac->entry[ac->avail]);
3172	return objp;
3173	}
3174
3175	#ifdef CONFIG_NUMA
3176	/*
3177	* Try allocating on another node if PF_SPREAD_SLAB\|PF_MEMPOLICY.
3178	*
3179	* If we are in_interrupt, then process context, including cpusets and
3180	* mempolicy, may not apply and should not be used for allocation policy.
3181	*/
3182	static void alternate_node_alloc(struct kmem_cache cachep, gfp_t flags)
3183	{
3184	int nid_alloc, nid_here;
3185
3186	if (in_interrupt() \|\| (flags & __GFP_THISNODE))
3187	return NULL;
3188	nid_alloc = nid_here = numa_mem_id();
3189	if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
3190	nid_alloc = cpuset_slab_spread_node();
3191	else if (current->mempolicy)
3192	nid_alloc = slab_node();
3193	if (nid_alloc != nid_here)
3194	return ____cache_alloc_node(cachep, flags, nid_alloc);
3195	return NULL;
3196	}
3197
3198	/*
3199	* Fallback function if there was no memory available and no objects on a
3200	* certain node and fall back is permitted. First we scan all the
3201	* available node for available objects. If that fails then we
3202	* perform an allocation without specifying a node. This allows the page
3203	* allocator to do its reclaim / fallback magic. We then insert the
3204	* slab into the proper nodelist and then allocate from it.
3205	*/
3206	static void fallback_alloc(struct kmem_cache cache, gfp_t flags)
3207	{
3208	struct zonelist *zonelist;
3209	gfp_t local_flags;
3210	struct zoneref *z;
3211	struct zone *zone;
3212	enum zone_type high_zoneidx = gfp_zone(flags);
3213	void *obj = NULL;
3214	int nid;
3215	unsigned int cpuset_mems_cookie;
3216
3217	if (flags & __GFP_THISNODE)
3218	return NULL;
3219
3220	local_flags = flags & (GFP_CONSTRAINT_MASK\|GFP_RECLAIM_MASK);
3221
3222	retry_cpuset:
3223	cpuset_mems_cookie = get_mems_allowed();
3224	zonelist = node_zonelist(slab_node(), flags);
3225
3226	retry:
3227	/*
3228	* Look through allowed nodes for objects available
3229	* from existing per node queues.
3230	*/
3231	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3232	nid = zone_to_nid(zone);
3233
3234	if (cpuset_zone_allowed_hardwall(zone, flags) &&
3235	cache->node[nid] &&
3236	cache->node[nid]->free_objects) {
3237	obj = ____cache_alloc_node(cache,
3238	flags \| GFP_THISNODE, nid);
3239	if (obj)
3240	break;
3241	}
3242	}
3243
3244	if (!obj) {
3245	/*
3246	* This allocation will be performed within the constraints
3247	* of the current cpuset / memory policy requirements.
3248	* We may trigger various forms of reclaim on the allowed
3249	* set and go into memory reserves if necessary.
3250	*/
3251	if (local_flags & __GFP_WAIT)
3252	local_irq_enable();
3253	kmem_flagcheck(cache, flags);
3254	obj = kmem_getpages(cache, local_flags, numa_mem_id());
3255	if (local_flags & __GFP_WAIT)
3256	local_irq_disable();
3257	if (obj) {
3258	/*
3259	* Insert into the appropriate per node queues
3260	*/
3261	nid = page_to_nid(virt_to_page(obj));
3262	if (cache_grow(cache, flags, nid, obj)) {
3263	obj = ____cache_alloc_node(cache,
3264	flags \| GFP_THISNODE, nid);
3265	if (!obj)
3266	/*
3267	* Another processor may allocate the
3268	* objects in the slab since we are
3269	* not holding any locks.
3270	*/
3271	goto retry;
3272	} else {
3273	/* cache_grow already freed obj */
3274	obj = NULL;
3275	}
3276	}
3277	}
3278
3279	if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !obj))
3280	goto retry_cpuset;
3281	return obj;
3282	}
3283
3284	/*
3285	* A interface to enable slab creation on nodeid
3286	*/
3287	static void ____cache_alloc_node(struct kmem_cache cachep, gfp_t flags,
3288	int nodeid)
3289	{
3290	struct list_head *entry;
3291	struct slab *slabp;
3292	struct kmem_cache_node *n;
3293	void *obj;
3294	int x;
3295
3296	VM_BUG_ON(nodeid > num_online_nodes());
3297	n = cachep->node[nodeid];
3298	BUG_ON(!n);
3299
3300	retry:
3301	check_irq_off();
3302	spin_lock(&n->list_lock);
3303	entry = n->slabs_partial.next;
3304	if (entry == &n->slabs_partial) {
3305	n->free_touched = 1;
3306	entry = n->slabs_free.next;
3307	if (entry == &n->slabs_free)
3308	goto must_grow;
3309	}
3310
3311	slabp = list_entry(entry, struct slab, list);
3312	check_spinlock_acquired_node(cachep, nodeid);
3313	check_slabp(cachep, slabp);
3314
3315	STATS_INC_NODEALLOCS(cachep);
3316	STATS_INC_ACTIVE(cachep);
3317	STATS_SET_HIGH(cachep);
3318
3319	BUG_ON(slabp->inuse == cachep->num);
3320
3321	obj = slab_get_obj(cachep, slabp, nodeid);
3322	check_slabp(cachep, slabp);
3323	n->free_objects--;
3324	/* move slabp to correct slabp list: */
3325	list_del(&slabp->list);
3326
3327	if (slabp->free == BUFCTL_END)
3328	list_add(&slabp->list, &n->slabs_full);
3329	else
3330	list_add(&slabp->list, &n->slabs_partial);
3331
3332	spin_unlock(&n->list_lock);
3333	goto done;
3334
3335	must_grow:
3336	spin_unlock(&n->list_lock);
3337	x = cache_grow(cachep, flags \| GFP_THISNODE, nodeid, NULL);
3338	if (x)
3339	goto retry;
3340
3341	return fallback_alloc(cachep, flags);
3342
3343	done:
3344	return obj;
3345	}
3346
3347	static __always_inline void *
3348	slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3349	unsigned long caller)
3350	{
3351	unsigned long save_flags;
3352	void *ptr;
3353	int slab_node = numa_mem_id();
3354
3355	flags &= gfp_allowed_mask;
3356
3357	lockdep_trace_alloc(flags);
3358
3359	if (slab_should_failslab(cachep, flags))
3360	return NULL;
3361
3362	cachep = memcg_kmem_get_cache(cachep, flags);
3363
3364	cache_alloc_debugcheck_before(cachep, flags);
3365	local_irq_save(save_flags);
3366
3367	if (nodeid == NUMA_NO_NODE)
3368	nodeid = slab_node;
3369
3370	if (unlikely(!cachep->node[nodeid])) {
3371	/* Node not bootstrapped yet */
3372	ptr = fallback_alloc(cachep, flags);
3373	goto out;
3374	}
3375
3376	if (nodeid == slab_node) {
3377	/*
3378	* Use the locally cached objects if possible.
3379	* However ____cache_alloc does not allow fallback
3380	* to other nodes. It may fail while we still have
3381	* objects on other nodes available.
3382	*/
3383	ptr = ____cache_alloc(cachep, flags);
3384	if (ptr)
3385	goto out;
3386	}
3387	/* ___cache_alloc_node can fall back to other nodes */
3388	ptr = ____cache_alloc_node(cachep, flags, nodeid);
3389	out:
3390	local_irq_restore(save_flags);
3391	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
3392	kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags,
3393	flags);
3394
3395	if (likely(ptr))
3396	kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size);
3397
3398	if (unlikely((flags & __GFP_ZERO) && ptr))
3399	memset(ptr, 0, cachep->object_size);
3400
3401	return ptr;
3402	}
3403
3404	static __always_inline void *
3405	__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3406	{
3407	void *objp;
3408
3409	if (unlikely(current->flags & (PF_SPREAD_SLAB \| PF_MEMPOLICY))) {
3410	objp = alternate_node_alloc(cache, flags);
3411	if (objp)
3412	goto out;
3413	}
3414	objp = ____cache_alloc(cache, flags);
3415
3416	/*
3417	* We may just have run out of memory on the local node.
3418	* ____cache_alloc_node() knows how to locate memory on other nodes
3419	*/
3420	if (!objp)
3421	objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3422
3423	out:
3424	return objp;
3425	}
3426	#else
3427
3428	static __always_inline void *
3429	__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3430	{
3431	return ____cache_alloc(cachep, flags);
3432	}
3433
3434	#endif /* CONFIG_NUMA */
3435
3436	static __always_inline void *
3437	slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
3438	{
3439	unsigned long save_flags;
3440	void *objp;
3441
3442	flags &= gfp_allowed_mask;
3443
3444	lockdep_trace_alloc(flags);
3445
3446	if (slab_should_failslab(cachep, flags))
3447	return NULL;
3448
3449	cachep = memcg_kmem_get_cache(cachep, flags);
3450
3451	cache_alloc_debugcheck_before(cachep, flags);
3452	local_irq_save(save_flags);
3453	objp = __do_cache_alloc(cachep, flags);
3454	local_irq_restore(save_flags);
3455	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3456	kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags,
3457	flags);
3458	prefetchw(objp);
3459
3460	if (likely(objp))
3461	kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size);
3462
3463	if (unlikely((flags & __GFP_ZERO) && objp))
3464	memset(objp, 0, cachep->object_size);
3465
3466	return objp;
3467	}
3468
3469	/*
3470	* Caller needs to acquire correct kmem_list's list_lock
3471	*/
3472	static void free_block(struct kmem_cache cachep, void *objpp, int nr_objects,
3473	int node)
3474	{
3475	int i;
3476	struct kmem_cache_node *n;
3477
3478	for (i = 0; i < nr_objects; i++) {
3479	void *objp;
3480	struct slab *slabp;
3481
3482	clear_obj_pfmemalloc(&objpp[i]);
3483	objp = objpp[i];
3484
3485	slabp = virt_to_slab(objp);
3486	n = cachep->node[node];
3487	list_del(&slabp->list);
3488	check_spinlock_acquired_node(cachep, node);
3489	check_slabp(cachep, slabp);
3490	slab_put_obj(cachep, slabp, objp, node);
3491	STATS_DEC_ACTIVE(cachep);
3492	n->free_objects++;
3493	check_slabp(cachep, slabp);
3494
3495	/* fixup slab chains */
3496	if (slabp->inuse == 0) {
3497	if (n->free_objects > n->free_limit) {
3498	n->free_objects -= cachep->num;
3499	/* No need to drop any previously held
3500	* lock here, even if we have a off-slab slab
3501	* descriptor it is guaranteed to come from
3502	* a different cache, refer to comments before
3503	* alloc_slabmgmt.
3504	*/
3505	slab_destroy(cachep, slabp);
3506	} else {
3507	list_add(&slabp->list, &n->slabs_free);
3508	}
3509	} else {
3510	/* Unconditionally move a slab to the end of the
3511	* partial list on free - maximum time for the
3512	* other objects to be freed, too.
3513	*/
3514	list_add_tail(&slabp->list, &n->slabs_partial);
3515	}
3516	}
3517	}
3518
3519	static void cache_flusharray(struct kmem_cache cachep, struct array_cache ac)
3520	{
3521	int batchcount;
3522	struct kmem_cache_node *n;
3523	int node = numa_mem_id();
3524
3525	batchcount = ac->batchcount;
3526	#if DEBUG
3527	BUG_ON(!batchcount \|\| batchcount > ac->avail);
3528	#endif
3529	check_irq_off();
3530	n = cachep->node[node];
3531	spin_lock(&n->list_lock);
3532	if (n->shared) {
3533	struct array_cache *shared_array = n->shared;
3534	int max = shared_array->limit - shared_array->avail;
3535	if (max) {
3536	if (batchcount > max)
3537	batchcount = max;
3538	memcpy(&(shared_array->entry[shared_array->avail]),
3539	ac->entry, sizeof(void ) batchcount);
3540	shared_array->avail += batchcount;
3541	goto free_done;
3542	}
3543	}
3544
3545	free_block(cachep, ac->entry, batchcount, node);
3546	free_done:
3547	#if STATS
3548	{
3549	int i = 0;
3550	struct list_head *p;
3551
3552	p = n->slabs_free.next;
3553	while (p != &(n->slabs_free)) {
3554	struct slab *slabp;
3555
3556	slabp = list_entry(p, struct slab, list);
3557	BUG_ON(slabp->inuse);
3558
3559	i++;
3560	p = p->next;
3561	}
3562	STATS_SET_FREEABLE(cachep, i);
3563	}
3564	#endif
3565	spin_unlock(&n->list_lock);
3566	ac->avail -= batchcount;
3567	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void )ac->avail);
3568	}
3569
3570	/*
3571	* Release an obj back to its cache. If the obj has a constructed state, it must
3572	* be in this state _before_ it is released. Called with disabled ints.
3573	*/
3574	static inline void __cache_free(struct kmem_cache cachep, void objp,
3575	unsigned long caller)
3576	{
3577	struct array_cache *ac = cpu_cache_get(cachep);
3578
3579	check_irq_off();
3580	kmemleak_free_recursive(objp, cachep->flags);
3581	objp = cache_free_debugcheck(cachep, objp, caller);
3582
3583	kmemcheck_slab_free(cachep, objp, cachep->object_size);
3584
3585	/*
3586	* Skip calling cache_free_alien() when the platform is not numa.
3587	* This will avoid cache misses that happen while accessing slabp (which
3588	* is per page memory reference) to get nodeid. Instead use a global
3589	* variable to skip the call, which is mostly likely to be present in
3590	* the cache.
3591	*/
3592	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3593	return;
3594
3595	if (likely(ac->avail < ac->limit)) {
3596	STATS_INC_FREEHIT(cachep);
3597	} else {
3598	STATS_INC_FREEMISS(cachep);
3599	cache_flusharray(cachep, ac);
3600	}
3601
3602	ac_put_obj(cachep, ac, objp);
3603	}
3604
3605	/**
3606	* kmem_cache_alloc - Allocate an object
3607	* @cachep: The cache to allocate from.
3608	* @flags: See kmalloc().
3609	*
3610	* Allocate an object from this cache. The flags are only relevant
3611	* if the cache has no available objects.
3612	*/
3613	void kmem_cache_alloc(struct kmem_cache cachep, gfp_t flags)
3614	{
3615	void *ret = slab_alloc(cachep, flags, _RET_IP_);
3616
3617	trace_kmem_cache_alloc(_RET_IP_, ret,
3618	cachep->object_size, cachep->size, flags);
3619
3620	return ret;
3621	}
3622	EXPORT_SYMBOL(kmem_cache_alloc);
3623
3624	#ifdef CONFIG_TRACING
3625	void *
3626	kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
3627	{
3628	void *ret;
3629
3630	ret = slab_alloc(cachep, flags, _RET_IP_);
3631
3632	trace_kmalloc(_RET_IP_, ret,
3633	size, cachep->size, flags);
3634	return ret;
3635	}
3636	EXPORT_SYMBOL(kmem_cache_alloc_trace);
3637	#endif
3638
3639	#ifdef CONFIG_NUMA
3640	/**
3641	* kmem_cache_alloc_node - Allocate an object on the specified node
3642	* @cachep: The cache to allocate from.
3643	* @flags: See kmalloc().
3644	* @nodeid: node number of the target node.
3645	*
3646	* Identical to kmem_cache_alloc but it will allocate memory on the given
3647	* node, which can improve the performance for cpu bound structures.
3648	*
3649	* Fallback to other node is possible if __GFP_THISNODE is not set.
3650	*/
3651	void kmem_cache_alloc_node(struct kmem_cache cachep, gfp_t flags, int nodeid)
3652	{
3653	void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3654
3655	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3656	cachep->object_size, cachep->size,
3657	flags, nodeid);
3658
3659	return ret;
3660	}
3661	EXPORT_SYMBOL(kmem_cache_alloc_node);
3662
3663	#ifdef CONFIG_TRACING
3664	void kmem_cache_alloc_node_trace(struct kmem_cache cachep,
3665	gfp_t flags,
3666	int nodeid,
3667	size_t size)
3668	{
3669	void *ret;
3670
3671	ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3672
3673	trace_kmalloc_node(_RET_IP_, ret,
3674	size, cachep->size,
3675	flags, nodeid);
3676	return ret;
3677	}
3678	EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
3679	#endif
3680
3681	static __always_inline void *
3682	__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
3683	{
3684	struct kmem_cache *cachep;
3685
3686	cachep = kmalloc_slab(size, flags);
3687	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3688	return cachep;
3689	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
3690	}
3691
3692	#if defined(CONFIG_DEBUG_SLAB) \|\| defined(CONFIG_TRACING)
3693	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3694	{
3695	return __do_kmalloc_node(size, flags, node, _RET_IP_);
3696	}
3697	EXPORT_SYMBOL(__kmalloc_node);
3698
3699	void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3700	int node, unsigned long caller)
3701	{
3702	return __do_kmalloc_node(size, flags, node, caller);
3703	}
3704	EXPORT_SYMBOL(__kmalloc_node_track_caller);
3705	#else
3706	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3707	{
3708	return __do_kmalloc_node(size, flags, node, 0);
3709	}
3710	EXPORT_SYMBOL(__kmalloc_node);
3711	#endif /* CONFIG_DEBUG_SLAB \|\| CONFIG_TRACING */
3712	#endif /* CONFIG_NUMA */
3713
3714	/**
3715	* __do_kmalloc - allocate memory
3716	* @size: how many bytes of memory are required.
3717	* @flags: the type of memory to allocate (see kmalloc).
3718	* @caller: function caller for debug tracking of the caller
3719	*/
3720	static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3721	unsigned long caller)
3722	{
3723	struct kmem_cache *cachep;
3724	void *ret;
3725
3726	/* If you want to save a few bytes .text space: replace
3727	* __ with kmem_.
3728	* Then kmalloc uses the uninlined functions instead of the inline
3729	* functions.
3730	*/
3731	cachep = kmalloc_slab(size, flags);
3732	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3733	return cachep;
3734	ret = slab_alloc(cachep, flags, caller);
3735
3736	trace_kmalloc(caller, ret,
3737	size, cachep->size, flags);
3738
3739	return ret;
3740	}
3741
3742
3743	#if defined(CONFIG_DEBUG_SLAB) \|\| defined(CONFIG_TRACING)
3744	void *__kmalloc(size_t size, gfp_t flags)
3745	{
3746	return __do_kmalloc(size, flags, _RET_IP_);
3747	}
3748	EXPORT_SYMBOL(__kmalloc);
3749
3750	void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3751	{
3752	return __do_kmalloc(size, flags, caller);
3753	}
3754	EXPORT_SYMBOL(__kmalloc_track_caller);
3755
3756	#else
3757	void *__kmalloc(size_t size, gfp_t flags)
3758	{
3759	return __do_kmalloc(size, flags, 0);
3760	}
3761	EXPORT_SYMBOL(__kmalloc);
3762	#endif
3763
3764	/**
3765	* kmem_cache_free - Deallocate an object
3766	* @cachep: The cache the allocation was from.
3767	* @objp: The previously allocated object.
3768	*
3769	* Free an object which was previously allocated from this
3770	* cache.
3771	*/
3772	void kmem_cache_free(struct kmem_cache cachep, void objp)
3773	{
3774	unsigned long flags;
3775	cachep = cache_from_obj(cachep, objp);
3776	if (!cachep)
3777	return;
3778
3779	local_irq_save(flags);
3780	debug_check_no_locks_freed(objp, cachep->object_size);
3781	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3782	debug_check_no_obj_freed(objp, cachep->object_size);
3783	__cache_free(cachep, objp, _RET_IP_);
3784	local_irq_restore(flags);
3785
3786	trace_kmem_cache_free(_RET_IP_, objp);
3787	}
3788	EXPORT_SYMBOL(kmem_cache_free);
3789
3790	/**
3791	* kfree - free previously allocated memory
3792	* @objp: pointer returned by kmalloc.
3793	*
3794	* If @objp is NULL, no operation is performed.
3795	*
3796	* Don't free memory not originally allocated by kmalloc()
3797	* or you will run into trouble.
3798	*/
3799	void kfree(const void *objp)
3800	{
3801	struct kmem_cache *c;
3802	unsigned long flags;
3803
3804	trace_kfree(_RET_IP_, objp);
3805
3806	if (unlikely(ZERO_OR_NULL_PTR(objp)))
3807	return;
3808	local_irq_save(flags);
3809	kfree_debugcheck(objp);
3810	c = virt_to_cache(objp);
3811	debug_check_no_locks_freed(objp, c->object_size);
3812
3813	debug_check_no_obj_freed(objp, c->object_size);
3814	__cache_free(c, (void *)objp, _RET_IP_);
3815	local_irq_restore(flags);
3816	}
3817	EXPORT_SYMBOL(kfree);
3818
3819	/*
3820	* This initializes kmem_cache_node or resizes various caches for all nodes.
3821	*/
3822	static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
3823	{
3824	int node;
3825	struct kmem_cache_node *n;
3826	struct array_cache *new_shared;
3827	struct array_cache **new_alien = NULL;
3828
3829	for_each_online_node(node) {
3830
3831	if (use_alien_caches) {
3832	new_alien = alloc_alien_cache(node, cachep->limit, gfp);
3833	if (!new_alien)
3834	goto fail;
3835	}
3836
3837	new_shared = NULL;
3838	if (cachep->shared) {
3839	new_shared = alloc_arraycache(node,
3840	cachep->shared*cachep->batchcount,
3841	0xbaadf00d, gfp);
3842	if (!new_shared) {
3843	free_alien_cache(new_alien);
3844	goto fail;
3845	}
3846	}
3847
3848	n = cachep->node[node];
3849	if (n) {
3850	struct array_cache *shared = n->shared;
3851
3852	spin_lock_irq(&n->list_lock);
3853
3854	if (shared)
3855	free_block(cachep, shared->entry,
3856	shared->avail, node);
3857
3858	n->shared = new_shared;
3859	if (!n->alien) {
3860	n->alien = new_alien;
3861	new_alien = NULL;
3862	}
3863	n->free_limit = (1 + nr_cpus_node(node)) *
3864	cachep->batchcount + cachep->num;
3865	spin_unlock_irq(&n->list_lock);
3866	kfree(shared);
3867	free_alien_cache(new_alien);
3868	continue;
3869	}
3870	n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
3871	if (!n) {
3872	free_alien_cache(new_alien);
3873	kfree(new_shared);
3874	goto fail;
3875	}
3876
3877	kmem_cache_node_init(n);
3878	n->next_reap = jiffies + REAPTIMEOUT_LIST3 +
3879	((unsigned long)cachep) % REAPTIMEOUT_LIST3;
3880	n->shared = new_shared;
3881	n->alien = new_alien;
3882	n->free_limit = (1 + nr_cpus_node(node)) *
3883	cachep->batchcount + cachep->num;
3884	cachep->node[node] = n;
3885	}
3886	return 0;
3887
3888	fail:
3889	if (!cachep->list.next) {
3890	/* Cache is not active yet. Roll back what we did */
3891	node--;
3892	while (node >= 0) {
3893	if (cachep->node[node]) {
3894	n = cachep->node[node];
3895
3896	kfree(n->shared);
3897	free_alien_cache(n->alien);
3898	kfree(n);
3899	cachep->node[node] = NULL;
3900	}
3901	node--;
3902	}
3903	}
3904	return -ENOMEM;
3905	}
3906
3907	struct ccupdate_struct {
3908	struct kmem_cache *cachep;
3909	struct array_cache *new[0];
3910	};
3911
3912	static void do_ccupdate_local(void *info)
3913	{
3914	struct ccupdate_struct *new = info;
3915	struct array_cache *old;
3916
3917	check_irq_off();
3918	old = cpu_cache_get(new->cachep);
3919
3920	new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
3921	new->new[smp_processor_id()] = old;
3922	}
3923
3924	/* Always called with the slab_mutex held */
3925	static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3926	int batchcount, int shared, gfp_t gfp)
3927	{
3928	struct ccupdate_struct *new;
3929	int i;
3930
3931	new = kzalloc(sizeof(new) + nr_cpu_ids sizeof(struct array_cache *),
3932	gfp);
3933	if (!new)
3934	return -ENOMEM;
3935
3936	for_each_online_cpu(i) {
3937	new->new[i] = alloc_arraycache(cpu_to_mem(i), limit,
3938	batchcount, gfp);
3939	if (!new->new[i]) {
3940	for (i--; i >= 0; i--)
3941	kfree(new->new[i]);
3942	kfree(new);
3943	return -ENOMEM;
3944	}
3945	}
3946	new->cachep = cachep;
3947
3948	on_each_cpu(do_ccupdate_local, (void *)new, 1);
3949
3950	check_irq_on();
3951	cachep->batchcount = batchcount;
3952	cachep->limit = limit;
3953	cachep->shared = shared;
3954
3955	for_each_online_cpu(i) {
3956	struct array_cache *ccold = new->new[i];
3957	if (!ccold)
3958	continue;
3959	spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
3960	free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i));
3961	spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
3962	kfree(ccold);
3963	}
3964	kfree(new);
3965	return alloc_kmemlist(cachep, gfp);
3966	}
3967
3968	static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3969	int batchcount, int shared, gfp_t gfp)
3970	{
3971	int ret;
3972	struct kmem_cache *c = NULL;
3973	int i = 0;
3974
3975	ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3976
3977	if (slab_state < FULL)
3978	return ret;
3979
3980	if ((ret < 0) \|\| !is_root_cache(cachep))
3981	return ret;
3982
3983	VM_BUG_ON(!mutex_is_locked(&slab_mutex));
3984	for_each_memcg_cache_index(i) {
3985	c = cache_from_memcg(cachep, i);
3986	if (c)
3987	/* return value determined by the parent cache only */
3988	__do_tune_cpucache(c, limit, batchcount, shared, gfp);
3989	}
3990
3991	return ret;
3992	}
3993
3994	/* Called with slab_mutex held always */
3995	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
3996	{
3997	int err;
3998	int limit = 0;
3999	int shared = 0;
4000	int batchcount = 0;
4001
4002	if (!is_root_cache(cachep)) {
4003	struct kmem_cache *root = memcg_root_cache(cachep);
4004	limit = root->limit;
4005	shared = root->shared;
4006	batchcount = root->batchcount;
4007	}
4008
4009	if (limit && shared && batchcount)
4010	goto skip_setup;
4011	/*
4012	* The head array serves three purposes:
4013	* - create a LIFO ordering, i.e. return objects that are cache-warm
4014	* - reduce the number of spinlock operations.
4015	* - reduce the number of linked list operations on the slab and
4016	* bufctl chains: array operations are cheaper.
4017	* The numbers are guessed, we should auto-tune as described by
4018	* Bonwick.
4019	*/
4020	if (cachep->size > 131072)
4021	limit = 1;
4022	else if (cachep->size > PAGE_SIZE)
4023	limit = 8;
4024	else if (cachep->size > 1024)
4025	limit = 24;
4026	else if (cachep->size > 256)
4027	limit = 54;
4028	else
4029	limit = 120;
4030
4031	/*
4032	* CPU bound tasks (e.g. network routing) can exhibit cpu bound
4033	* allocation behaviour: Most allocs on one cpu, most free operations
4034	* on another cpu. For these cases, an efficient object passing between
4035	* cpus is necessary. This is provided by a shared array. The array
4036	* replaces Bonwick's magazine layer.
4037	* On uniprocessor, it's functionally equivalent (but less efficient)
4038	* to a larger limit. Thus disabled by default.
4039	*/
4040	shared = 0;
4041	if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
4042	shared = 8;
4043
4044	#if DEBUG
4045	/*
4046	* With debugging enabled, large batchcount lead to excessively long
4047	* periods with disabled local interrupts. Limit the batchcount
4048	*/
4049	if (limit > 32)
4050	limit = 32;
4051	#endif
4052	batchcount = (limit + 1) / 2;
4053	skip_setup:
4054	err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
4055	if (err)
4056	printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
4057	cachep->name, -err);
4058	return err;
4059	}
4060
4061	/*
4062	* Drain an array if it contains any elements taking the node lock only if
4063	* necessary. Note that the node listlock also protects the array_cache
4064	* if drain_array() is used on the shared array.
4065	*/
4066	static void drain_array(struct kmem_cache cachep, struct kmem_cache_node n,
4067	struct array_cache *ac, int force, int node)
4068	{
4069	int tofree;
4070
4071	if (!ac \|\| !ac->avail)
4072	return;
4073	if (ac->touched && !force) {
4074	ac->touched = 0;
4075	} else {
4076	spin_lock_irq(&n->list_lock);
4077	if (ac->avail) {
4078	tofree = force ? ac->avail : (ac->limit + 4) / 5;
4079	if (tofree > ac->avail)
4080	tofree = (ac->avail + 1) / 2;
4081	free_block(cachep, ac->entry, tofree, node);
4082	ac->avail -= tofree;
4083	memmove(ac->entry, &(ac->entry[tofree]),
4084	sizeof(void ) ac->avail);
4085	}
4086	spin_unlock_irq(&n->list_lock);
4087	}
4088	}
4089
4090	/**
4091	* cache_reap - Reclaim memory from caches.
4092	* @w: work descriptor
4093	*
4094	* Called from workqueue/eventd every few seconds.
4095	* Purpose:
4096	* - clear the per-cpu caches for this CPU.
4097	* - return freeable pages to the main free memory pool.
4098	*
4099	* If we cannot acquire the cache chain mutex then just give up - we'll try
4100	* again on the next iteration.
4101	*/
4102	static void cache_reap(struct work_struct *w)
4103	{
4104	struct kmem_cache *searchp;
4105	struct kmem_cache_node *n;
4106	int node = numa_mem_id();
4107	struct delayed_work *work = to_delayed_work(w);
4108
4109	if (!mutex_trylock(&slab_mutex))
4110	/* Give up. Setup the next iteration. */
4111	goto out;
4112
4113	list_for_each_entry(searchp, &slab_caches, list) {
4114	check_irq_on();
4115
4116	/*
4117	* We only take the node lock if absolutely necessary and we
4118	* have established with reasonable certainty that
4119	* we can do some work if the lock was obtained.
4120	*/
4121	n = searchp->node[node];
4122
4123	reap_alien(searchp, n);
4124
4125	drain_array(searchp, n, cpu_cache_get(searchp), 0, node);
4126
4127	/*
4128	* These are racy checks but it does not matter
4129	* if we skip one check or scan twice.
4130	*/
4131	if (time_after(n->next_reap, jiffies))
4132	goto next;
4133
4134	n->next_reap = jiffies + REAPTIMEOUT_LIST3;
4135
4136	drain_array(searchp, n, n->shared, 0, node);
4137
4138	if (n->free_touched)
4139	n->free_touched = 0;
4140	else {
4141	int freed;
4142
4143	freed = drain_freelist(searchp, n, (n->free_limit +
4144	5 * searchp->num - 1) / (5 * searchp->num));
4145	STATS_ADD_REAPED(searchp, freed);
4146	}
4147	next:
4148	cond_resched();
4149	}
4150	check_irq_on();
4151	mutex_unlock(&slab_mutex);
4152	next_reap_node();
4153	out:
4154	/* Set up the next iteration */
4155	schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_CPUC));
4156	}
4157
4158	#ifdef CONFIG_SLABINFO
4159	void get_slabinfo(struct kmem_cache cachep, struct slabinfo sinfo)
4160	{
4161	struct slab *slabp;
4162	unsigned long active_objs;
4163	unsigned long num_objs;
4164	unsigned long active_slabs = 0;
4165	unsigned long num_slabs, free_objects = 0, shared_avail = 0;
4166	const char *name;
4167	char *error = NULL;
4168	int node;
4169	struct kmem_cache_node *n;
4170
4171	active_objs = 0;
4172	num_slabs = 0;
4173	for_each_online_node(node) {
4174	n = cachep->node[node];
4175	if (!n)
4176	continue;
4177
4178	check_irq_on();
4179	spin_lock_irq(&n->list_lock);
4180
4181	list_for_each_entry(slabp, &n->slabs_full, list) {
4182	if (slabp->inuse != cachep->num && !error)
4183	error = "slabs_full accounting error";
4184	active_objs += cachep->num;
4185	active_slabs++;
4186	}
4187	list_for_each_entry(slabp, &n->slabs_partial, list) {
4188	if (slabp->inuse == cachep->num && !error)
4189	error = "slabs_partial inuse accounting error";
4190	if (!slabp->inuse && !error)
4191	error = "slabs_partial/inuse accounting error";
4192	active_objs += slabp->inuse;
4193	active_slabs++;
4194	}
4195	list_for_each_entry(slabp, &n->slabs_free, list) {
4196	if (slabp->inuse && !error)
4197	error = "slabs_free/inuse accounting error";
4198	num_slabs++;
4199	}
4200	free_objects += n->free_objects;
4201	if (n->shared)
4202	shared_avail += n->shared->avail;
4203
4204	spin_unlock_irq(&n->list_lock);
4205	}
4206	num_slabs += active_slabs;
4207	num_objs = num_slabs * cachep->num;
4208	if (num_objs - active_objs != free_objects && !error)
4209	error = "free_objects accounting error";
4210
4211	name = cachep->name;
4212	if (error)
4213	printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
4214
4215	sinfo->active_objs = active_objs;
4216	sinfo->num_objs = num_objs;
4217	sinfo->active_slabs = active_slabs;
4218	sinfo->num_slabs = num_slabs;
4219	sinfo->shared_avail = shared_avail;
4220	sinfo->limit = cachep->limit;
4221	sinfo->batchcount = cachep->batchcount;
4222	sinfo->shared = cachep->shared;
4223	sinfo->objects_per_slab = cachep->num;
4224	sinfo->cache_order = cachep->gfporder;
4225	}
4226
4227	void slabinfo_show_stats(struct seq_file m, struct kmem_cache cachep)
4228	{
4229	#if STATS
4230	{ /* node stats */
4231	unsigned long high = cachep->high_mark;
4232	unsigned long allocs = cachep->num_allocations;
4233	unsigned long grown = cachep->grown;
4234	unsigned long reaped = cachep->reaped;
4235	unsigned long errors = cachep->errors;
4236	unsigned long max_freeable = cachep->max_freeable;
4237	unsigned long node_allocs = cachep->node_allocs;
4238	unsigned long node_frees = cachep->node_frees;
4239	unsigned long overflows = cachep->node_overflow;
4240
4241	seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4242	"%4lu %4lu %4lu %4lu %4lu",
4243	allocs, high, grown,
4244	reaped, errors, max_freeable, node_allocs,
4245	node_frees, overflows);
4246	}
4247	/* cpu stats */
4248	{
4249	unsigned long allochit = atomic_read(&cachep->allochit);
4250	unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4251	unsigned long freehit = atomic_read(&cachep->freehit);
4252	unsigned long freemiss = atomic_read(&cachep->freemiss);
4253
4254	seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
4255	allochit, allocmiss, freehit, freemiss);
4256	}
4257	#endif
4258	}
4259
4260	#define MAX_SLABINFO_WRITE 128
4261	/**
4262	* slabinfo_write - Tuning for the slab allocator
4263	* @file: unused
4264	* @buffer: user buffer
4265	* @count: data length
4266	* @ppos: unused
4267	*/
4268	ssize_t slabinfo_write(struct file file, const char __user buffer,
4269	size_t count, loff_t *ppos)
4270	{
4271	char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
4272	int limit, batchcount, shared, res;
4273	struct kmem_cache *cachep;
4274
4275	if (count > MAX_SLABINFO_WRITE)
4276	return -EINVAL;
4277	if (copy_from_user(&kbuf, buffer, count))
4278	return -EFAULT;
4279	kbuf[MAX_SLABINFO_WRITE] = '\0';
4280
4281	tmp = strchr(kbuf, ' ');
4282	if (!tmp)
4283	return -EINVAL;
4284	*tmp = '\0';
4285	tmp++;
4286	if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4287	return -EINVAL;
4288
4289	/* Find the cache in the chain of caches. */
4290	mutex_lock(&slab_mutex);
4291	res = -EINVAL;
4292	list_for_each_entry(cachep, &slab_caches, list) {
4293	if (!strcmp(cachep->name, kbuf)) {
4294	if (limit < 1 \|\| batchcount < 1 \|\|
4295	batchcount > limit \|\| shared < 0) {
4296	res = 0;
4297	} else {
4298	res = do_tune_cpucache(cachep, limit,
4299	batchcount, shared,
4300	GFP_KERNEL);
4301	}
4302	break;
4303	}
4304	}
4305	mutex_unlock(&slab_mutex);
4306	if (res >= 0)
4307	res = count;
4308	return res;
4309	}
4310
4311	#ifdef CONFIG_DEBUG_SLAB_LEAK
4312
4313	static void leaks_start(struct seq_file m, loff_t *pos)
4314	{
4315	mutex_lock(&slab_mutex);
4316	return seq_list_start(&slab_caches, *pos);
4317	}
4318
4319	static inline int add_caller(unsigned long *n, unsigned long v)
4320	{
4321	unsigned long *p;
4322	int l;
4323	if (!v)
4324	return 1;
4325	l = n[1];
4326	p = n + 2;
4327	while (l) {
4328	int i = l/2;
4329	unsigned long q = p + 2 i;
4330	if (*q == v) {
4331	q[1]++;
4332	return 1;
4333	}
4334	if (*q > v) {
4335	l = i;
4336	} else {
4337	p = q + 2;
4338	l -= i + 1;
4339	}
4340	}
4341	if (++n[1] == n[0])
4342	return 0;
4343	memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void )p - (void )n));
4344	p[0] = v;
4345	p[1] = 1;
4346	return 1;
4347	}
4348
4349	static void handle_slab(unsigned long n, struct kmem_cache c, struct slab *s)
4350	{
4351	void *p;
4352	int i;
4353	if (n[0] == n[1])
4354	return;
4355	for (i = 0, p = s->s_mem; i < c->num; i++, p += c->size) {
4356	if (slab_bufctl(s)[i] != BUFCTL_ACTIVE)
4357	continue;
4358	if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4359	return;
4360	}
4361	}
4362
4363	static void show_symbol(struct seq_file *m, unsigned long address)
4364	{
4365	#ifdef CONFIG_KALLSYMS
4366	unsigned long offset, size;
4367	char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4368
4369	if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4370	seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4371	if (modname[0])
4372	seq_printf(m, " [%s]", modname);
4373	return;
4374	}
4375	#endif
4376	seq_printf(m, "%p", (void *)address);
4377	}
4378
4379	static int leaks_show(struct seq_file m, void p)
4380	{
4381	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4382	struct slab *slabp;
4383	struct kmem_cache_node *n;
4384	const char *name;
4385	unsigned long *x = m->private;
4386	int node;
4387	int i;
4388
4389	if (!(cachep->flags & SLAB_STORE_USER))
4390	return 0;
4391	if (!(cachep->flags & SLAB_RED_ZONE))
4392	return 0;
4393
4394	/* OK, we can do it */
4395
4396	x[1] = 0;
4397
4398	for_each_online_node(node) {
4399	n = cachep->node[node];
4400	if (!n)
4401	continue;
4402
4403	check_irq_on();
4404	spin_lock_irq(&n->list_lock);
4405
4406	list_for_each_entry(slabp, &n->slabs_full, list)
4407	handle_slab(x, cachep, slabp);
4408	list_for_each_entry(slabp, &n->slabs_partial, list)
4409	handle_slab(x, cachep, slabp);
4410	spin_unlock_irq(&n->list_lock);
4411	}
4412	name = cachep->name;
4413	if (x[0] == x[1]) {
4414	/* Increase the buffer size */
4415	mutex_unlock(&slab_mutex);
4416	m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4417	if (!m->private) {
4418	/* Too bad, we are really out */
4419	m->private = x;
4420	mutex_lock(&slab_mutex);
4421	return -ENOMEM;
4422	}
4423	(unsigned long )m->private = x[0] * 2;
4424	kfree(x);
4425	mutex_lock(&slab_mutex);
4426	/* Now make sure this entry will be retried */
4427	m->count = m->size;
4428	return 0;
4429	}
4430	for (i = 0; i < x[1]; i++) {
4431	seq_printf(m, "%s: %lu ", name, x[2*i+3]);
4432	show_symbol(m, x[2*i+2]);
4433	seq_putc(m, '\n');
4434	}
4435
4436	return 0;
4437	}
4438
4439	static const struct seq_operations slabstats_op = {
4440	.start = leaks_start,
4441	.next = slab_next,
4442	.stop = slab_stop,
4443	.show = leaks_show,
4444	};
4445
4446	static int slabstats_open(struct inode inode, struct file file)
4447	{
4448	unsigned long *n = kzalloc(PAGE_SIZE, GFP_KERNEL);
4449	int ret = -ENOMEM;
4450	if (n) {
4451	ret = seq_open(file, &slabstats_op);
4452	if (!ret) {
4453	struct seq_file *m = file->private_data;
4454	n = PAGE_SIZE / (2 sizeof(unsigned long));
4455	m->private = n;
4456	n = NULL;
4457	}
4458	kfree(n);
4459	}
4460	return ret;
4461	}
4462
4463	static const struct file_operations proc_slabstats_operations = {
4464	.open = slabstats_open,
4465	.read = seq_read,
4466	.llseek = seq_lseek,
4467	.release = seq_release_private,
4468	};
4469	#endif
4470
4471	static int __init slab_proc_init(void)
4472	{
4473	#ifdef CONFIG_DEBUG_SLAB_LEAK
4474	proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4475	#endif
4476	return 0;
4477	}
4478	module_init(slab_proc_init);
4479	#endif
4480
4481	/**
4482	* ksize - get the actual amount of memory allocated for a given object
4483	* @objp: Pointer to the object
4484	*
4485	* kmalloc may internally round up allocations and return more memory
4486	* than requested. ksize() can be used to determine the actual amount of
4487	* memory allocated. The caller may use this additional memory, even though
4488	* a smaller amount of memory was initially specified with the kmalloc call.
4489	* The caller must guarantee that objp points to a valid object previously
4490	* allocated with either kmalloc() or kmem_cache_alloc(). The object
4491	* must not be freed during the duration of the call.
4492	*/
4493	size_t ksize(const void *objp)
4494	{
4495	BUG_ON(!objp);
4496	if (unlikely(objp == ZERO_SIZE_PTR))
4497	return 0;
4498
4499	return virt_to_cache(objp)->object_size;
4500	}
4501	EXPORT_SYMBOL(ksize);
4502

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/slab.c

interactive