Kernel

Kernel Git Source Tree

Root/mm/memcontrol.c

1	/* memcontrol.c - Memory Controller
2	*
3	* Copyright IBM Corporation, 2007
4	* Author Balbir Singh <balbir@linux.vnet.ibm.com>
5	*
6	* Copyright 2007 OpenVZ SWsoft Inc
7	* Author: Pavel Emelianov <xemul@openvz.org>
8	*
9	* Memory thresholds
10	* Copyright (C) 2009 Nokia Corporation
11	* Author: Kirill A. Shutemov
12	*
13	* Kernel Memory Controller
14	* Copyright (C) 2012 Parallels Inc. and Google Inc.
15	* Authors: Glauber Costa and Suleiman Souhlal
16	*
17	* This program is free software; you can redistribute it and/or modify
18	* it under the terms of the GNU General Public License as published by
19	* the Free Software Foundation; either version 2 of the License, or
20	* (at your option) any later version.
21	*
22	* This program is distributed in the hope that it will be useful,
23	* but WITHOUT ANY WARRANTY; without even the implied warranty of
24	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25	* GNU General Public License for more details.
26	*/
27
28	#include <linux/res_counter.h>
29	#include <linux/memcontrol.h>
30	#include <linux/cgroup.h>
31	#include <linux/mm.h>
32	#include <linux/hugetlb.h>
33	#include <linux/pagemap.h>
34	#include <linux/smp.h>
35	#include <linux/page-flags.h>
36	#include <linux/backing-dev.h>
37	#include <linux/bit_spinlock.h>
38	#include <linux/rcupdate.h>
39	#include <linux/limits.h>
40	#include <linux/export.h>
41	#include <linux/mutex.h>
42	#include <linux/rbtree.h>
43	#include <linux/slab.h>
44	#include <linux/swap.h>
45	#include <linux/swapops.h>
46	#include <linux/spinlock.h>
47	#include <linux/eventfd.h>
48	#include <linux/sort.h>
49	#include <linux/fs.h>
50	#include <linux/seq_file.h>
51	#include <linux/vmalloc.h>
52	#include <linux/vmpressure.h>
53	#include <linux/mm_inline.h>
54	#include <linux/page_cgroup.h>
55	#include <linux/cpu.h>
56	#include <linux/oom.h>
57	#include <linux/lockdep.h>
58	#include "internal.h"
59	#include <net/sock.h>
60	#include <net/ip.h>
61	#include <net/tcp_memcontrol.h>
62
63	#include <asm/uaccess.h>
64
65	#include <trace/events/vmscan.h>
66
67	struct cgroup_subsys mem_cgroup_subsys __read_mostly;
68	EXPORT_SYMBOL(mem_cgroup_subsys);
69
70	#define MEM_CGROUP_RECLAIM_RETRIES 5
71	static struct mem_cgroup *root_mem_cgroup __read_mostly;
72
73	#ifdef CONFIG_MEMCG_SWAP
74	/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
75	int do_swap_account __read_mostly;
76
77	/* for remember boot option*/
78	#ifdef CONFIG_MEMCG_SWAP_ENABLED
79	static int really_do_swap_account __initdata = 1;
80	#else
81	static int really_do_swap_account __initdata = 0;
82	#endif
83
84	#else
85	#define do_swap_account 0
86	#endif
87
88
89	static const char * const mem_cgroup_stat_names[] = {
90	"cache",
91	"rss",
92	"rss_huge",
93	"mapped_file",
94	"writeback",
95	"swap",
96	};
97
98	enum mem_cgroup_events_index {
99	MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
100	MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
101	MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
102	MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
103	MEM_CGROUP_EVENTS_NSTATS,
104	};
105
106	static const char * const mem_cgroup_events_names[] = {
107	"pgpgin",
108	"pgpgout",
109	"pgfault",
110	"pgmajfault",
111	};
112
113	static const char * const mem_cgroup_lru_names[] = {
114	"inactive_anon",
115	"active_anon",
116	"inactive_file",
117	"active_file",
118	"unevictable",
119	};
120
121	/*
122	* Per memcg event counter is incremented at every pagein/pageout. With THP,
123	* it will be incremated by the number of pages. This counter is used for
124	* for trigger some periodic events. This is straightforward and better
125	* than using jiffies etc. to handle periodic memcg event.
126	*/
127	enum mem_cgroup_events_target {
128	MEM_CGROUP_TARGET_THRESH,
129	MEM_CGROUP_TARGET_SOFTLIMIT,
130	MEM_CGROUP_TARGET_NUMAINFO,
131	MEM_CGROUP_NTARGETS,
132	};
133	#define THRESHOLDS_EVENTS_TARGET 128
134	#define SOFTLIMIT_EVENTS_TARGET 1024
135	#define NUMAINFO_EVENTS_TARGET 1024
136
137	struct mem_cgroup_stat_cpu {
138	long count[MEM_CGROUP_STAT_NSTATS];
139	unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
140	unsigned long nr_page_events;
141	unsigned long targets[MEM_CGROUP_NTARGETS];
142	};
143
144	struct mem_cgroup_reclaim_iter {
145	/*
146	* last scanned hierarchy member. Valid only if last_dead_count
147	* matches memcg->dead_count of the hierarchy root group.
148	*/
149	struct mem_cgroup *last_visited;
150	unsigned long last_dead_count;
151
152	/* scan generation, increased every round-trip */
153	unsigned int generation;
154	};
155
156	/*
157	* per-zone information in memory controller.
158	*/
159	struct mem_cgroup_per_zone {
160	struct lruvec lruvec;
161	unsigned long lru_size[NR_LRU_LISTS];
162
163	struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
164
165	struct rb_node tree_node; /* RB tree node */
166	unsigned long long usage_in_excess;/* Set to the value by which */
167	/* the soft limit is exceeded*/
168	bool on_tree;
169	struct mem_cgroup memcg; / Back pointer, we cannot */
170	/* use container_of */
171	};
172
173	struct mem_cgroup_per_node {
174	struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
175	};
176
177	/*
178	* Cgroups above their limits are maintained in a RB-Tree, independent of
179	* their hierarchy representation
180	*/
181
182	struct mem_cgroup_tree_per_zone {
183	struct rb_root rb_root;
184	spinlock_t lock;
185	};
186
187	struct mem_cgroup_tree_per_node {
188	struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
189	};
190
191	struct mem_cgroup_tree {
192	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
193	};
194
195	static struct mem_cgroup_tree soft_limit_tree __read_mostly;
196
197	struct mem_cgroup_threshold {
198	struct eventfd_ctx *eventfd;
199	u64 threshold;
200	};
201
202	/* For threshold */
203	struct mem_cgroup_threshold_ary {
204	/* An array index points to threshold just below or equal to usage. */
205	int current_threshold;
206	/* Size of entries[] */
207	unsigned int size;
208	/* Array of thresholds */
209	struct mem_cgroup_threshold entries[0];
210	};
211
212	struct mem_cgroup_thresholds {
213	/* Primary thresholds array */
214	struct mem_cgroup_threshold_ary *primary;
215	/*
216	* Spare threshold array.
217	* This is needed to make mem_cgroup_unregister_event() "never fail".
218	* It must be able to store at least primary->size - 1 entries.
219	*/
220	struct mem_cgroup_threshold_ary *spare;
221	};
222
223	/* for OOM */
224	struct mem_cgroup_eventfd_list {
225	struct list_head list;
226	struct eventfd_ctx *eventfd;
227	};
228
229	static void mem_cgroup_threshold(struct mem_cgroup *memcg);
230	static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
231
232	/*
233	* The memory controller data structure. The memory controller controls both
234	* page cache and RSS per cgroup. We would eventually like to provide
235	* statistics based on the statistics developed by Rik Van Riel for clock-pro,
236	* to help the administrator determine what knobs to tune.
237	*
238	* TODO: Add a water mark for the memory controller. Reclaim will begin when
239	* we hit the water mark. May be even add a low water mark, such that
240	* no reclaim occurs from a cgroup at it's low water mark, this is
241	* a feature that will be implemented much later in the future.
242	*/
243	struct mem_cgroup {
244	struct cgroup_subsys_state css;
245	/*
246	* the counter to account for memory usage
247	*/
248	struct res_counter res;
249
250	/* vmpressure notifications */
251	struct vmpressure vmpressure;
252
253	/*
254	* the counter to account for mem+swap usage.
255	*/
256	struct res_counter memsw;
257
258	/*
259	* the counter to account for kernel memory usage.
260	*/
261	struct res_counter kmem;
262	/*
263	* Should the accounting and control be hierarchical, per subtree?
264	*/
265	bool use_hierarchy;
266	unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_, below /
267
268	bool oom_lock;
269	atomic_t under_oom;
270	atomic_t oom_wakeups;
271
272	int swappiness;
273	/* OOM-Killer disable */
274	int oom_kill_disable;
275
276	/* set when res.limit == memsw.limit */
277	bool memsw_is_minimum;
278
279	/* protect arrays of thresholds */
280	struct mutex thresholds_lock;
281
282	/* thresholds for memory usage. RCU-protected */
283	struct mem_cgroup_thresholds thresholds;
284
285	/* thresholds for mem+swap usage. RCU-protected */
286	struct mem_cgroup_thresholds memsw_thresholds;
287
288	/* For oom notifier event fd */
289	struct list_head oom_notify;
290
291	/*
292	* Should we move charges of a task when a task is moved into this
293	* mem_cgroup ? And what type of charges should we move ?
294	*/
295	unsigned long move_charge_at_immigrate;
296	/*
297	* set > 0 if pages under this cgroup are moving to other cgroup.
298	*/
299	atomic_t moving_account;
300	/* taken only while moving_account > 0 */
301	spinlock_t move_lock;
302	/*
303	* percpu counter.
304	*/
305	struct mem_cgroup_stat_cpu __percpu *stat;
306	/*
307	* used when a cpu is offlined or other synchronizations
308	* See mem_cgroup_read_stat().
309	*/
310	struct mem_cgroup_stat_cpu nocpu_base;
311	spinlock_t pcp_counter_lock;
312
313	atomic_t dead_count;
314	#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
315	struct tcp_memcontrol tcp_mem;
316	#endif
317	#if defined(CONFIG_MEMCG_KMEM)
318	/* analogous to slab_common's slab_caches list. per-memcg */
319	struct list_head memcg_slab_caches;
320	/* Not a spinlock, we can take a lot of time walking the list */
321	struct mutex slab_caches_mutex;
322	/* Index in the kmem_cache->memcg_params->memcg_caches array */
323	int kmemcg_id;
324	#endif
325
326	int last_scanned_node;
327	#if MAX_NUMNODES > 1
328	nodemask_t scan_nodes;
329	atomic_t numainfo_events;
330	atomic_t numainfo_updating;
331	#endif
332
333	struct mem_cgroup_per_node *nodeinfo[0];
334	/* WARNING: nodeinfo must be the last member here */
335	};
336
337	static size_t memcg_size(void)
338	{
339	return sizeof(struct mem_cgroup) +
340	nr_node_ids * sizeof(struct mem_cgroup_per_node);
341	}
342
343	/* internal only representation about the status of kmem accounting. */
344	enum {
345	KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
346	KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
347	KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
348	};
349
350	/* We account when limit is on, but only after call sites are patched */
351	#define KMEM_ACCOUNTED_MASK \
352	((1 << KMEM_ACCOUNTED_ACTIVE) \| (1 << KMEM_ACCOUNTED_ACTIVATED))
353
354	#ifdef CONFIG_MEMCG_KMEM
355	static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
356	{
357	set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
358	}
359
360	static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
361	{
362	return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
363	}
364
365	static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
366	{
367	set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
368	}
369
370	static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
371	{
372	clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
373	}
374
375	static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
376	{
377	/*
378	* Our caller must use css_get() first, because memcg_uncharge_kmem()
379	* will call css_put() if it sees the memcg is dead.
380	*/
381	smp_wmb();
382	if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
383	set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
384	}
385
386	static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
387	{
388	return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
389	&memcg->kmem_account_flags);
390	}
391	#endif
392
393	/* Stuffs for move charges at task migration. */
394	/*
395	* Types of charges to be moved. "move_charge_at_immitgrate" and
396	* "immigrate_flags" are treated as a left-shifted bitmap of these types.
397	*/
398	enum move_type {
399	MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
400	MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
401	NR_MOVE_TYPE,
402	};
403
404	/* "mc" and its members are protected by cgroup_mutex */
405	static struct move_charge_struct {
406	spinlock_t lock; /* for from, to */
407	struct mem_cgroup *from;
408	struct mem_cgroup *to;
409	unsigned long immigrate_flags;
410	unsigned long precharge;
411	unsigned long moved_charge;
412	unsigned long moved_swap;
413	struct task_struct moving_task; / a task moving charges */
414	wait_queue_head_t waitq; /* a waitq for other context */
415	} mc = {
416	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
417	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
418	};
419
420	static bool move_anon(void)
421	{
422	return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
423	}
424
425	static bool move_file(void)
426	{
427	return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
428	}
429
430	/*
431	* Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
432	* limit reclaim to prevent infinite loops, if they ever occur.
433	*/
434	#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
435	#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
436
437	enum charge_type {
438	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
439	MEM_CGROUP_CHARGE_TYPE_ANON,
440	MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
441	MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
442	NR_CHARGE_TYPE,
443	};
444
445	/* for encoding cft->private value on file */
446	enum res_type {
447	_MEM,
448	_MEMSWAP,
449	_OOM_TYPE,
450	_KMEM,
451	};
452
453	#define MEMFILE_PRIVATE(x, val) ((x) << 16 \| (val))
454	#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
455	#define MEMFILE_ATTR(val) ((val) & 0xffff)
456	/* Used for OOM nofiier */
457	#define OOM_CONTROL (0)
458
459	/*
460	* Reclaim flags for mem_cgroup_hierarchical_reclaim
461	*/
462	#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
463	#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
464	#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
465	#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
466
467	/*
468	* The memcg_create_mutex will be held whenever a new cgroup is created.
469	* As a consequence, any change that needs to protect against new child cgroups
470	* appearing has to hold it as well.
471	*/
472	static DEFINE_MUTEX(memcg_create_mutex);
473
474	struct mem_cgroup mem_cgroup_from_css(struct cgroup_subsys_state s)
475	{
476	return s ? container_of(s, struct mem_cgroup, css) : NULL;
477	}
478
479	/* Some nice accessors for the vmpressure. */
480	struct vmpressure memcg_to_vmpressure(struct mem_cgroup memcg)
481	{
482	if (!memcg)
483	memcg = root_mem_cgroup;
484	return &memcg->vmpressure;
485	}
486
487	struct cgroup_subsys_state vmpressure_to_css(struct vmpressure vmpr)
488	{
489	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
490	}
491
492	struct vmpressure css_to_vmpressure(struct cgroup_subsys_state css)
493	{
494	return &mem_cgroup_from_css(css)->vmpressure;
495	}
496
497	static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
498	{
499	return (memcg == root_mem_cgroup);
500	}
501
502	/* Writing them here to avoid exposing memcg's inner layout */
503	#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
504
505	void sock_update_memcg(struct sock *sk)
506	{
507	if (mem_cgroup_sockets_enabled) {
508	struct mem_cgroup *memcg;
509	struct cg_proto *cg_proto;
510
511	BUG_ON(!sk->sk_prot->proto_cgroup);
512
513	/* Socket cloning can throw us here with sk_cgrp already
514	* filled. It won't however, necessarily happen from
515	* process context. So the test for root memcg given
516	* the current task's memcg won't help us in this case.
517	*
518	* Respecting the original socket's memcg is a better
519	* decision in this case.
520	*/
521	if (sk->sk_cgrp) {
522	BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
523	css_get(&sk->sk_cgrp->memcg->css);
524	return;
525	}
526
527	rcu_read_lock();
528	memcg = mem_cgroup_from_task(current);
529	cg_proto = sk->sk_prot->proto_cgroup(memcg);
530	if (!mem_cgroup_is_root(memcg) &&
531	memcg_proto_active(cg_proto) && css_tryget(&memcg->css)) {
532	sk->sk_cgrp = cg_proto;
533	}
534	rcu_read_unlock();
535	}
536	}
537	EXPORT_SYMBOL(sock_update_memcg);
538
539	void sock_release_memcg(struct sock *sk)
540	{
541	if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
542	struct mem_cgroup *memcg;
543	WARN_ON(!sk->sk_cgrp->memcg);
544	memcg = sk->sk_cgrp->memcg;
545	css_put(&sk->sk_cgrp->memcg->css);
546	}
547	}
548
549	struct cg_proto tcp_proto_cgroup(struct mem_cgroup memcg)
550	{
551	if (!memcg \|\| mem_cgroup_is_root(memcg))
552	return NULL;
553
554	return &memcg->tcp_mem.cg_proto;
555	}
556	EXPORT_SYMBOL(tcp_proto_cgroup);
557
558	static void disarm_sock_keys(struct mem_cgroup *memcg)
559	{
560	if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
561	return;
562	static_key_slow_dec(&memcg_socket_limit_enabled);
563	}
564	#else
565	static void disarm_sock_keys(struct mem_cgroup *memcg)
566	{
567	}
568	#endif
569
570	#ifdef CONFIG_MEMCG_KMEM
571	/*
572	* This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
573	* There are two main reasons for not using the css_id for this:
574	* 1) this works better in sparse environments, where we have a lot of memcgs,
575	* but only a few kmem-limited. Or also, if we have, for instance, 200
576	* memcgs, and none but the 200th is kmem-limited, we'd have to have a
577	* 200 entry array for that.
578	*
579	* 2) In order not to violate the cgroup API, we would like to do all memory
580	* allocation in ->create(). At that point, we haven't yet allocated the
581	* css_id. Having a separate index prevents us from messing with the cgroup
582	* core for this
583	*
584	* The current size of the caches array is stored in
585	* memcg_limited_groups_array_size. It will double each time we have to
586	* increase it.
587	*/
588	static DEFINE_IDA(kmem_limited_groups);
589	int memcg_limited_groups_array_size;
590
591	/*
592	* MIN_SIZE is different than 1, because we would like to avoid going through
593	* the alloc/free process all the time. In a small machine, 4 kmem-limited
594	* cgroups is a reasonable guess. In the future, it could be a parameter or
595	* tunable, but that is strictly not necessary.
596	*
597	* MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
598	* this constant directly from cgroup, but it is understandable that this is
599	* better kept as an internal representation in cgroup.c. In any case, the
600	* css_id space is not getting any smaller, and we don't have to necessarily
601	* increase ours as well if it increases.
602	*/
603	#define MEMCG_CACHES_MIN_SIZE 4
604	#define MEMCG_CACHES_MAX_SIZE 65535
605
606	/*
607	* A lot of the calls to the cache allocation functions are expected to be
608	* inlined by the compiler. Since the calls to memcg_kmem_get_cache are
609	* conditional to this static branch, we'll have to allow modules that does
610	* kmem_cache_alloc and the such to see this symbol as well
611	*/
612	struct static_key memcg_kmem_enabled_key;
613	EXPORT_SYMBOL(memcg_kmem_enabled_key);
614
615	static void disarm_kmem_keys(struct mem_cgroup *memcg)
616	{
617	if (memcg_kmem_is_active(memcg)) {
618	static_key_slow_dec(&memcg_kmem_enabled_key);
619	ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
620	}
621	/*
622	* This check can't live in kmem destruction function,
623	* since the charges will outlive the cgroup
624	*/
625	WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
626	}
627	#else
628	static void disarm_kmem_keys(struct mem_cgroup *memcg)
629	{
630	}
631	#endif /* CONFIG_MEMCG_KMEM */
632
633	static void disarm_static_keys(struct mem_cgroup *memcg)
634	{
635	disarm_sock_keys(memcg);
636	disarm_kmem_keys(memcg);
637	}
638
639	static void drain_all_stock_async(struct mem_cgroup *memcg);
640
641	static struct mem_cgroup_per_zone *
642	mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
643	{
644	VM_BUG_ON((unsigned)nid >= nr_node_ids);
645	return &memcg->nodeinfo[nid]->zoneinfo[zid];
646	}
647
648	struct cgroup_subsys_state mem_cgroup_css(struct mem_cgroup memcg)
649	{
650	return &memcg->css;
651	}
652
653	static struct mem_cgroup_per_zone *
654	page_cgroup_zoneinfo(struct mem_cgroup memcg, struct page page)
655	{
656	int nid = page_to_nid(page);
657	int zid = page_zonenum(page);
658
659	return mem_cgroup_zoneinfo(memcg, nid, zid);
660	}
661
662	static struct mem_cgroup_tree_per_zone *
663	soft_limit_tree_node_zone(int nid, int zid)
664	{
665	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
666	}
667
668	static struct mem_cgroup_tree_per_zone *
669	soft_limit_tree_from_page(struct page *page)
670	{
671	int nid = page_to_nid(page);
672	int zid = page_zonenum(page);
673
674	return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
675	}
676
677	static void
678	__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
679	struct mem_cgroup_per_zone *mz,
680	struct mem_cgroup_tree_per_zone *mctz,
681	unsigned long long new_usage_in_excess)
682	{
683	struct rb_node **p = &mctz->rb_root.rb_node;
684	struct rb_node *parent = NULL;
685	struct mem_cgroup_per_zone *mz_node;
686
687	if (mz->on_tree)
688	return;
689
690	mz->usage_in_excess = new_usage_in_excess;
691	if (!mz->usage_in_excess)
692	return;
693	while (*p) {
694	parent = *p;
695	mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
696	tree_node);
697	if (mz->usage_in_excess < mz_node->usage_in_excess)
698	p = &(*p)->rb_left;
699	/*
700	* We can't avoid mem cgroups that are over their soft
701	* limit by the same amount
702	*/
703	else if (mz->usage_in_excess >= mz_node->usage_in_excess)
704	p = &(*p)->rb_right;
705	}
706	rb_link_node(&mz->tree_node, parent, p);
707	rb_insert_color(&mz->tree_node, &mctz->rb_root);
708	mz->on_tree = true;
709	}
710
711	static void
712	__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
713	struct mem_cgroup_per_zone *mz,
714	struct mem_cgroup_tree_per_zone *mctz)
715	{
716	if (!mz->on_tree)
717	return;
718	rb_erase(&mz->tree_node, &mctz->rb_root);
719	mz->on_tree = false;
720	}
721
722	static void
723	mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
724	struct mem_cgroup_per_zone *mz,
725	struct mem_cgroup_tree_per_zone *mctz)
726	{
727	spin_lock(&mctz->lock);
728	__mem_cgroup_remove_exceeded(memcg, mz, mctz);
729	spin_unlock(&mctz->lock);
730	}
731
732
733	static void mem_cgroup_update_tree(struct mem_cgroup memcg, struct page page)
734	{
735	unsigned long long excess;
736	struct mem_cgroup_per_zone *mz;
737	struct mem_cgroup_tree_per_zone *mctz;
738	int nid = page_to_nid(page);
739	int zid = page_zonenum(page);
740	mctz = soft_limit_tree_from_page(page);
741
742	/*
743	* Necessary to update all ancestors when hierarchy is used.
744	* because their event counter is not touched.
745	*/
746	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
747	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
748	excess = res_counter_soft_limit_excess(&memcg->res);
749	/*
750	* We have to update the tree if mz is on RB-tree or
751	* mem is over its softlimit.
752	*/
753	if (excess \|\| mz->on_tree) {
754	spin_lock(&mctz->lock);
755	/* if on-tree, remove it */
756	if (mz->on_tree)
757	__mem_cgroup_remove_exceeded(memcg, mz, mctz);
758	/*
759	* Insert again. mz->usage_in_excess will be updated.
760	* If excess is 0, no tree ops.
761	*/
762	__mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
763	spin_unlock(&mctz->lock);
764	}
765	}
766	}
767
768	static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
769	{
770	int node, zone;
771	struct mem_cgroup_per_zone *mz;
772	struct mem_cgroup_tree_per_zone *mctz;
773
774	for_each_node(node) {
775	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
776	mz = mem_cgroup_zoneinfo(memcg, node, zone);
777	mctz = soft_limit_tree_node_zone(node, zone);
778	mem_cgroup_remove_exceeded(memcg, mz, mctz);
779	}
780	}
781	}
782
783	static struct mem_cgroup_per_zone *
784	__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
785	{
786	struct rb_node *rightmost = NULL;
787	struct mem_cgroup_per_zone *mz;
788
789	retry:
790	mz = NULL;
791	rightmost = rb_last(&mctz->rb_root);
792	if (!rightmost)
793	goto done; /* Nothing to reclaim from */
794
795	mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
796	/*
797	* Remove the node now but someone else can add it back,
798	* we will to add it back at the end of reclaim to its correct
799	* position in the tree.
800	*/
801	__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
802	if (!res_counter_soft_limit_excess(&mz->memcg->res) \|\|
803	!css_tryget(&mz->memcg->css))
804	goto retry;
805	done:
806	return mz;
807	}
808
809	static struct mem_cgroup_per_zone *
810	mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
811	{
812	struct mem_cgroup_per_zone *mz;
813
814	spin_lock(&mctz->lock);
815	mz = __mem_cgroup_largest_soft_limit_node(mctz);
816	spin_unlock(&mctz->lock);
817	return mz;
818	}
819
820	/*
821	* Implementation Note: reading percpu statistics for memcg.
822	*
823	* Both of vmstat[] and percpu_counter has threshold and do periodic
824	* synchronization to implement "quick" read. There are trade-off between
825	* reading cost and precision of value. Then, we may have a chance to implement
826	* a periodic synchronizion of counter in memcg's counter.
827	*
828	* But this _read() function is used for user interface now. The user accounts
829	* memory usage by memory cgroup and he _always_ requires exact value because
830	* he accounts memory. Even if we provide quick-and-fuzzy read, we always
831	* have to visit all online cpus and make sum. So, for now, unnecessary
832	* synchronization is not implemented. (just implemented for cpu hotplug)
833	*
834	* If there are kernel internal actions which can make use of some not-exact
835	* value, and reading all cpu value can be performance bottleneck in some
836	* common workload, threashold and synchonization as vmstat[] should be
837	* implemented.
838	*/
839	static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
840	enum mem_cgroup_stat_index idx)
841	{
842	long val = 0;
843	int cpu;
844
845	get_online_cpus();
846	for_each_online_cpu(cpu)
847	val += per_cpu(memcg->stat->count[idx], cpu);
848	#ifdef CONFIG_HOTPLUG_CPU
849	spin_lock(&memcg->pcp_counter_lock);
850	val += memcg->nocpu_base.count[idx];
851	spin_unlock(&memcg->pcp_counter_lock);
852	#endif
853	put_online_cpus();
854	return val;
855	}
856
857	static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
858	bool charge)
859	{
860	int val = (charge) ? 1 : -1;
861	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
862	}
863
864	static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
865	enum mem_cgroup_events_index idx)
866	{
867	unsigned long val = 0;
868	int cpu;
869
870	get_online_cpus();
871	for_each_online_cpu(cpu)
872	val += per_cpu(memcg->stat->events[idx], cpu);
873	#ifdef CONFIG_HOTPLUG_CPU
874	spin_lock(&memcg->pcp_counter_lock);
875	val += memcg->nocpu_base.events[idx];
876	spin_unlock(&memcg->pcp_counter_lock);
877	#endif
878	put_online_cpus();
879	return val;
880	}
881
882	static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
883	struct page *page,
884	bool anon, int nr_pages)
885	{
886	preempt_disable();
887
888	/*
889	* Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
890	* counted as CACHE even if it's on ANON LRU.
891	*/
892	if (anon)
893	__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
894	nr_pages);
895	else
896	__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
897	nr_pages);
898
899	if (PageTransHuge(page))
900	__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
901	nr_pages);
902
903	/* pagein of a big page is an event. So, ignore page size */
904	if (nr_pages > 0)
905	__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
906	else {
907	__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
908	nr_pages = -nr_pages; /* for event */
909	}
910
911	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
912
913	preempt_enable();
914	}
915
916	unsigned long
917	mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
918	{
919	struct mem_cgroup_per_zone *mz;
920
921	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
922	return mz->lru_size[lru];
923	}
924
925	static unsigned long
926	mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
927	unsigned int lru_mask)
928	{
929	struct mem_cgroup_per_zone *mz;
930	enum lru_list lru;
931	unsigned long ret = 0;
932
933	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
934
935	for_each_lru(lru) {
936	if (BIT(lru) & lru_mask)
937	ret += mz->lru_size[lru];
938	}
939	return ret;
940	}
941
942	static unsigned long
943	mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
944	int nid, unsigned int lru_mask)
945	{
946	u64 total = 0;
947	int zid;
948
949	for (zid = 0; zid < MAX_NR_ZONES; zid++)
950	total += mem_cgroup_zone_nr_lru_pages(memcg,
951	nid, zid, lru_mask);
952
953	return total;
954	}
955
956	static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
957	unsigned int lru_mask)
958	{
959	int nid;
960	u64 total = 0;
961
962	for_each_node_state(nid, N_MEMORY)
963	total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
964	return total;
965	}
966
967	static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
968	enum mem_cgroup_events_target target)
969	{
970	unsigned long val, next;
971
972	val = __this_cpu_read(memcg->stat->nr_page_events);
973	next = __this_cpu_read(memcg->stat->targets[target]);
974	/* from time_after() in jiffies.h */
975	if ((long)next - (long)val < 0) {
976	switch (target) {
977	case MEM_CGROUP_TARGET_THRESH:
978	next = val + THRESHOLDS_EVENTS_TARGET;
979	break;
980	case MEM_CGROUP_TARGET_SOFTLIMIT:
981	next = val + SOFTLIMIT_EVENTS_TARGET;
982	break;
983	case MEM_CGROUP_TARGET_NUMAINFO:
984	next = val + NUMAINFO_EVENTS_TARGET;
985	break;
986	default:
987	break;
988	}
989	__this_cpu_write(memcg->stat->targets[target], next);
990	return true;
991	}
992	return false;
993	}
994
995	/*
996	* Check events in order.
997	*
998	*/
999	static void memcg_check_events(struct mem_cgroup memcg, struct page page)
1000	{
1001	preempt_disable();
1002	/* threshold event is triggered in finer grain than soft limit */
1003	if (unlikely(mem_cgroup_event_ratelimit(memcg,
1004	MEM_CGROUP_TARGET_THRESH))) {
1005	bool do_softlimit;
1006	bool do_numainfo __maybe_unused;
1007
1008	do_softlimit = mem_cgroup_event_ratelimit(memcg,
1009	MEM_CGROUP_TARGET_SOFTLIMIT);
1010	#if MAX_NUMNODES > 1
1011	do_numainfo = mem_cgroup_event_ratelimit(memcg,
1012	MEM_CGROUP_TARGET_NUMAINFO);
1013	#endif
1014	preempt_enable();
1015
1016	mem_cgroup_threshold(memcg);
1017	if (unlikely(do_softlimit))
1018	mem_cgroup_update_tree(memcg, page);
1019	#if MAX_NUMNODES > 1
1020	if (unlikely(do_numainfo))
1021	atomic_inc(&memcg->numainfo_events);
1022	#endif
1023	} else
1024	preempt_enable();
1025	}
1026
1027	struct mem_cgroup mem_cgroup_from_task(struct task_struct p)
1028	{
1029	/*
1030	* mm_update_next_owner() may clear mm->owner to NULL
1031	* if it races with swapoff, page migration, etc.
1032	* So this can be called with p == NULL.
1033	*/
1034	if (unlikely(!p))
1035	return NULL;
1036
1037	return mem_cgroup_from_css(task_css(p, mem_cgroup_subsys_id));
1038	}
1039
1040	struct mem_cgroup try_get_mem_cgroup_from_mm(struct mm_struct mm)
1041	{
1042	struct mem_cgroup *memcg = NULL;
1043
1044	if (!mm)
1045	return NULL;
1046	/*
1047	* Because we have no locks, mm->owner's may be being moved to other
1048	* cgroup. We use css_tryget() here even if this looks
1049	* pessimistic (rather than adding locks here).
1050	*/
1051	rcu_read_lock();
1052	do {
1053	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1054	if (unlikely(!memcg))
1055	break;
1056	} while (!css_tryget(&memcg->css));
1057	rcu_read_unlock();
1058	return memcg;
1059	}
1060
1061	/*
1062	* Returns a next (in a pre-order walk) alive memcg (with elevated css
1063	* ref. count) or NULL if the whole root's subtree has been visited.
1064	*
1065	* helper function to be used by mem_cgroup_iter
1066	*/
1067	static struct mem_cgroup __mem_cgroup_iter_next(struct mem_cgroup root,
1068	struct mem_cgroup *last_visited)
1069	{
1070	struct cgroup_subsys_state prev_css, next_css;
1071
1072	prev_css = last_visited ? &last_visited->css : NULL;
1073	skip_node:
1074	next_css = css_next_descendant_pre(prev_css, &root->css);
1075
1076	/*
1077	* Even if we found a group we have to make sure it is
1078	* alive. css && !memcg means that the groups should be
1079	* skipped and we should continue the tree walk.
1080	* last_visited css is safe to use because it is
1081	* protected by css_get and the tree walk is rcu safe.
1082	*/
1083	if (next_css) {
1084	struct mem_cgroup *mem = mem_cgroup_from_css(next_css);
1085
1086	if (css_tryget(&mem->css))
1087	return mem;
1088	else {
1089	prev_css = next_css;
1090	goto skip_node;
1091	}
1092	}
1093
1094	return NULL;
1095	}
1096
1097	static void mem_cgroup_iter_invalidate(struct mem_cgroup *root)
1098	{
1099	/*
1100	* When a group in the hierarchy below root is destroyed, the
1101	* hierarchy iterator can no longer be trusted since it might
1102	* have pointed to the destroyed group. Invalidate it.
1103	*/
1104	atomic_inc(&root->dead_count);
1105	}
1106
1107	static struct mem_cgroup *
1108	mem_cgroup_iter_load(struct mem_cgroup_reclaim_iter *iter,
1109	struct mem_cgroup *root,
1110	int *sequence)
1111	{
1112	struct mem_cgroup *position = NULL;
1113	/*
1114	* A cgroup destruction happens in two stages: offlining and
1115	* release. They are separated by a RCU grace period.
1116	*
1117	* If the iterator is valid, we may still race with an
1118	* offlining. The RCU lock ensures the object won't be
1119	* released, tryget will fail if we lost the race.
1120	*/
1121	*sequence = atomic_read(&root->dead_count);
1122	if (iter->last_dead_count == *sequence) {
1123	smp_rmb();
1124	position = iter->last_visited;
1125	if (position && !css_tryget(&position->css))
1126	position = NULL;
1127	}
1128	return position;
1129	}
1130
1131	static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
1132	struct mem_cgroup *last_visited,
1133	struct mem_cgroup *new_position,
1134	int sequence)
1135	{
1136	if (last_visited)
1137	css_put(&last_visited->css);
1138	/*
1139	* We store the sequence count from the time @last_visited was
1140	* loaded successfully instead of rereading it here so that we
1141	* don't lose destruction events in between. We could have
1142	* raced with the destruction of @new_position after all.
1143	*/
1144	iter->last_visited = new_position;
1145	smp_wmb();
1146	iter->last_dead_count = sequence;
1147	}
1148
1149	/**
1150	* mem_cgroup_iter - iterate over memory cgroup hierarchy
1151	* @root: hierarchy root
1152	* @prev: previously returned memcg, NULL on first invocation
1153	* @reclaim: cookie for shared reclaim walks, NULL for full walks
1154	*
1155	* Returns references to children of the hierarchy below @root, or
1156	* @root itself, or %NULL after a full round-trip.
1157	*
1158	* Caller must pass the return value in @prev on subsequent
1159	* invocations for reference counting, or use mem_cgroup_iter_break()
1160	* to cancel a hierarchy walk before the round-trip is complete.
1161	*
1162	* Reclaimers can specify a zone and a priority level in @reclaim to
1163	* divide up the memcgs in the hierarchy among all concurrent
1164	* reclaimers operating on the same zone and priority.
1165	*/
1166	struct mem_cgroup mem_cgroup_iter(struct mem_cgroup root,
1167	struct mem_cgroup *prev,
1168	struct mem_cgroup_reclaim_cookie *reclaim)
1169	{
1170	struct mem_cgroup *memcg = NULL;
1171	struct mem_cgroup *last_visited = NULL;
1172
1173	if (mem_cgroup_disabled())
1174	return NULL;
1175
1176	if (!root)
1177	root = root_mem_cgroup;
1178
1179	if (prev && !reclaim)
1180	last_visited = prev;
1181
1182	if (!root->use_hierarchy && root != root_mem_cgroup) {
1183	if (prev)
1184	goto out_css_put;
1185	return root;
1186	}
1187
1188	rcu_read_lock();
1189	while (!memcg) {
1190	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1191	int uninitialized_var(seq);
1192
1193	if (reclaim) {
1194	int nid = zone_to_nid(reclaim->zone);
1195	int zid = zone_idx(reclaim->zone);
1196	struct mem_cgroup_per_zone *mz;
1197
1198	mz = mem_cgroup_zoneinfo(root, nid, zid);
1199	iter = &mz->reclaim_iter[reclaim->priority];
1200	if (prev && reclaim->generation != iter->generation) {
1201	iter->last_visited = NULL;
1202	goto out_unlock;
1203	}
1204
1205	last_visited = mem_cgroup_iter_load(iter, root, &seq);
1206	}
1207
1208	memcg = __mem_cgroup_iter_next(root, last_visited);
1209
1210	if (reclaim) {
1211	mem_cgroup_iter_update(iter, last_visited, memcg, seq);
1212
1213	if (!memcg)
1214	iter->generation++;
1215	else if (!prev && memcg)
1216	reclaim->generation = iter->generation;
1217	}
1218
1219	if (prev && !memcg)
1220	goto out_unlock;
1221	}
1222	out_unlock:
1223	rcu_read_unlock();
1224	out_css_put:
1225	if (prev && prev != root)
1226	css_put(&prev->css);
1227
1228	return memcg;
1229	}
1230
1231	/**
1232	* mem_cgroup_iter_break - abort a hierarchy walk prematurely
1233	* @root: hierarchy root
1234	* @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1235	*/
1236	void mem_cgroup_iter_break(struct mem_cgroup *root,
1237	struct mem_cgroup *prev)
1238	{
1239	if (!root)
1240	root = root_mem_cgroup;
1241	if (prev && prev != root)
1242	css_put(&prev->css);
1243	}
1244
1245	/*
1246	* Iteration constructs for visiting all cgroups (under a tree). If
1247	* loops are exited prematurely (break), mem_cgroup_iter_break() must
1248	* be used for reference counting.
1249	*/
1250	#define for_each_mem_cgroup_tree(iter, root) \
1251	for (iter = mem_cgroup_iter(root, NULL, NULL); \
1252	iter != NULL; \
1253	iter = mem_cgroup_iter(root, iter, NULL))
1254
1255	#define for_each_mem_cgroup(iter) \
1256	for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
1257	iter != NULL; \
1258	iter = mem_cgroup_iter(NULL, iter, NULL))
1259
1260	void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1261	{
1262	struct mem_cgroup *memcg;
1263
1264	rcu_read_lock();
1265	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1266	if (unlikely(!memcg))
1267	goto out;
1268
1269	switch (idx) {
1270	case PGFAULT:
1271	this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1272	break;
1273	case PGMAJFAULT:
1274	this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1275	break;
1276	default:
1277	BUG();
1278	}
1279	out:
1280	rcu_read_unlock();
1281	}
1282	EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1283
1284	/**
1285	* mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1286	* @zone: zone of the wanted lruvec
1287	* @memcg: memcg of the wanted lruvec
1288	*
1289	* Returns the lru list vector holding pages for the given @zone and
1290	* @mem. This can be the global zone lruvec, if the memory controller
1291	* is disabled.
1292	*/
1293	struct lruvec mem_cgroup_zone_lruvec(struct zone zone,
1294	struct mem_cgroup *memcg)
1295	{
1296	struct mem_cgroup_per_zone *mz;
1297	struct lruvec *lruvec;
1298
1299	if (mem_cgroup_disabled()) {
1300	lruvec = &zone->lruvec;
1301	goto out;
1302	}
1303
1304	mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1305	lruvec = &mz->lruvec;
1306	out:
1307	/*
1308	* Since a node can be onlined after the mem_cgroup was created,
1309	* we have to be prepared to initialize lruvec->zone here;
1310	* and if offlined then reonlined, we need to reinitialize it.
1311	*/
1312	if (unlikely(lruvec->zone != zone))
1313	lruvec->zone = zone;
1314	return lruvec;
1315	}
1316
1317	/*
1318	* Following LRU functions are allowed to be used without PCG_LOCK.
1319	* Operations are called by routine of global LRU independently from memcg.
1320	* What we have to take care of here is validness of pc->mem_cgroup.
1321	*
1322	* Changes to pc->mem_cgroup happens when
1323	* 1. charge
1324	* 2. moving account
1325	* In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
1326	* It is added to LRU before charge.
1327	* If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
1328	* When moving account, the page is not on LRU. It's isolated.
1329	*/
1330
1331	/**
1332	* mem_cgroup_page_lruvec - return lruvec for adding an lru page
1333	* @page: the page
1334	* @zone: zone of the page
1335	*/
1336	struct lruvec mem_cgroup_page_lruvec(struct page page, struct zone *zone)
1337	{
1338	struct mem_cgroup_per_zone *mz;
1339	struct mem_cgroup *memcg;
1340	struct page_cgroup *pc;
1341	struct lruvec *lruvec;
1342
1343	if (mem_cgroup_disabled()) {
1344	lruvec = &zone->lruvec;
1345	goto out;
1346	}
1347
1348	pc = lookup_page_cgroup(page);
1349	memcg = pc->mem_cgroup;
1350
1351	/*
1352	* Surreptitiously switch any uncharged offlist page to root:
1353	* an uncharged page off lru does nothing to secure
1354	* its former mem_cgroup from sudden removal.
1355	*
1356	* Our caller holds lru_lock, and PageCgroupUsed is updated
1357	* under page_cgroup lock: between them, they make all uses
1358	* of pc->mem_cgroup safe.
1359	*/
1360	if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1361	pc->mem_cgroup = memcg = root_mem_cgroup;
1362
1363	mz = page_cgroup_zoneinfo(memcg, page);
1364	lruvec = &mz->lruvec;
1365	out:
1366	/*
1367	* Since a node can be onlined after the mem_cgroup was created,
1368	* we have to be prepared to initialize lruvec->zone here;
1369	* and if offlined then reonlined, we need to reinitialize it.
1370	*/
1371	if (unlikely(lruvec->zone != zone))
1372	lruvec->zone = zone;
1373	return lruvec;
1374	}
1375
1376	/**
1377	* mem_cgroup_update_lru_size - account for adding or removing an lru page
1378	* @lruvec: mem_cgroup per zone lru vector
1379	* @lru: index of lru list the page is sitting on
1380	* @nr_pages: positive when adding or negative when removing
1381	*
1382	* This function must be called when a page is added to or removed from an
1383	* lru list.
1384	*/
1385	void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1386	int nr_pages)
1387	{
1388	struct mem_cgroup_per_zone *mz;
1389	unsigned long *lru_size;
1390
1391	if (mem_cgroup_disabled())
1392	return;
1393
1394	mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1395	lru_size = mz->lru_size + lru;
1396	*lru_size += nr_pages;
1397	VM_BUG_ON((long)(*lru_size) < 0);
1398	}
1399
1400	/*
1401	* Checks whether given mem is same or in the root_mem_cgroup's
1402	* hierarchy subtree
1403	*/
1404	bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1405	struct mem_cgroup *memcg)
1406	{
1407	if (root_memcg == memcg)
1408	return true;
1409	if (!root_memcg->use_hierarchy \|\| !memcg)
1410	return false;
1411	return css_is_ancestor(&memcg->css, &root_memcg->css);
1412	}
1413
1414	static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1415	struct mem_cgroup *memcg)
1416	{
1417	bool ret;
1418
1419	rcu_read_lock();
1420	ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1421	rcu_read_unlock();
1422	return ret;
1423	}
1424
1425	bool task_in_mem_cgroup(struct task_struct *task,
1426	const struct mem_cgroup *memcg)
1427	{
1428	struct mem_cgroup *curr = NULL;
1429	struct task_struct *p;
1430	bool ret;
1431
1432	p = find_lock_task_mm(task);
1433	if (p) {
1434	curr = try_get_mem_cgroup_from_mm(p->mm);
1435	task_unlock(p);
1436	} else {
1437	/*
1438	* All threads may have already detached their mm's, but the oom
1439	* killer still needs to detect if they have already been oom
1440	* killed to prevent needlessly killing additional tasks.
1441	*/
1442	rcu_read_lock();
1443	curr = mem_cgroup_from_task(task);
1444	if (curr)
1445	css_get(&curr->css);
1446	rcu_read_unlock();
1447	}
1448	if (!curr)
1449	return false;
1450	/*
1451	* We should check use_hierarchy of "memcg" not "curr". Because checking
1452	* use_hierarchy of "curr" here make this function true if hierarchy is
1453	* enabled in "curr" and "curr" is a child of "memcg" in cgroup
1454	* hierarchy(even if use_hierarchy is disabled in "memcg").
1455	*/
1456	ret = mem_cgroup_same_or_subtree(memcg, curr);
1457	css_put(&curr->css);
1458	return ret;
1459	}
1460
1461	int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1462	{
1463	unsigned long inactive_ratio;
1464	unsigned long inactive;
1465	unsigned long active;
1466	unsigned long gb;
1467
1468	inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1469	active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1470
1471	gb = (inactive + active) >> (30 - PAGE_SHIFT);
1472	if (gb)
1473	inactive_ratio = int_sqrt(10 * gb);
1474	else
1475	inactive_ratio = 1;
1476
1477	return inactive * inactive_ratio < active;
1478	}
1479
1480	#define mem_cgroup_from_res_counter(counter, member) \
1481	container_of(counter, struct mem_cgroup, member)
1482
1483	/**
1484	* mem_cgroup_margin - calculate chargeable space of a memory cgroup
1485	* @memcg: the memory cgroup
1486	*
1487	* Returns the maximum amount of memory @mem can be charged with, in
1488	* pages.
1489	*/
1490	static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1491	{
1492	unsigned long long margin;
1493
1494	margin = res_counter_margin(&memcg->res);
1495	if (do_swap_account)
1496	margin = min(margin, res_counter_margin(&memcg->memsw));
1497	return margin >> PAGE_SHIFT;
1498	}
1499
1500	int mem_cgroup_swappiness(struct mem_cgroup *memcg)
1501	{
1502	/* root ? */
1503	if (!css_parent(&memcg->css))
1504	return vm_swappiness;
1505
1506	return memcg->swappiness;
1507	}
1508
1509	/*
1510	* memcg->moving_account is used for checking possibility that some thread is
1511	* calling move_account(). When a thread on CPU-A starts moving pages under
1512	* a memcg, other threads should check memcg->moving_account under
1513	* rcu_read_lock(), like this:
1514	*
1515	* CPU-A CPU-B
1516	* rcu_read_lock()
1517	* memcg->moving_account+1 if (memcg->mocing_account)
1518	* take heavy locks.
1519	* synchronize_rcu() update something.
1520	* rcu_read_unlock()
1521	* start move here.
1522	*/
1523
1524	/* for quick checking without looking up memcg */
1525	atomic_t memcg_moving __read_mostly;
1526
1527	static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1528	{
1529	atomic_inc(&memcg_moving);
1530	atomic_inc(&memcg->moving_account);
1531	synchronize_rcu();
1532	}
1533
1534	static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1535	{
1536	/*
1537	* Now, mem_cgroup_clear_mc() may call this function with NULL.
1538	* We check NULL in callee rather than caller.
1539	*/
1540	if (memcg) {
1541	atomic_dec(&memcg_moving);
1542	atomic_dec(&memcg->moving_account);
1543	}
1544	}
1545
1546	/*
1547	* 2 routines for checking "mem" is under move_account() or not.
1548	*
1549	* mem_cgroup_stolen() - checking whether a cgroup is mc.from or not. This
1550	* is used for avoiding races in accounting. If true,
1551	* pc->mem_cgroup may be overwritten.
1552	*
1553	* mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1554	* under hierarchy of moving cgroups. This is for
1555	* waiting at hith-memory prressure caused by "move".
1556	*/
1557
1558	static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1559	{
1560	VM_BUG_ON(!rcu_read_lock_held());
1561	return atomic_read(&memcg->moving_account) > 0;
1562	}
1563
1564	static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1565	{
1566	struct mem_cgroup *from;
1567	struct mem_cgroup *to;
1568	bool ret = false;
1569	/*
1570	* Unlike task_move routines, we access mc.to, mc.from not under
1571	* mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1572	*/
1573	spin_lock(&mc.lock);
1574	from = mc.from;
1575	to = mc.to;
1576	if (!from)
1577	goto unlock;
1578
1579	ret = mem_cgroup_same_or_subtree(memcg, from)
1580	\|\| mem_cgroup_same_or_subtree(memcg, to);
1581	unlock:
1582	spin_unlock(&mc.lock);
1583	return ret;
1584	}
1585
1586	static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1587	{
1588	if (mc.moving_task && current != mc.moving_task) {
1589	if (mem_cgroup_under_move(memcg)) {
1590	DEFINE_WAIT(wait);
1591	prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1592	/* moving charge context might have finished. */
1593	if (mc.moving_task)
1594	schedule();
1595	finish_wait(&mc.waitq, &wait);
1596	return true;
1597	}
1598	}
1599	return false;
1600	}
1601
1602	/*
1603	* Take this lock when
1604	* - a code tries to modify page's memcg while it's USED.
1605	* - a code tries to modify page state accounting in a memcg.
1606	* see mem_cgroup_stolen(), too.
1607	*/
1608	static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
1609	unsigned long *flags)
1610	{
1611	spin_lock_irqsave(&memcg->move_lock, *flags);
1612	}
1613
1614	static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
1615	unsigned long *flags)
1616	{
1617	spin_unlock_irqrestore(&memcg->move_lock, *flags);
1618	}
1619
1620	#define K(x) ((x) << (PAGE_SHIFT-10))
1621	/**
1622	* mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1623	* @memcg: The memory cgroup that went over limit
1624	* @p: Task that is going to be killed
1625	*
1626	* NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1627	* enabled
1628	*/
1629	void mem_cgroup_print_oom_info(struct mem_cgroup memcg, struct task_struct p)
1630	{
1631	struct cgroup *task_cgrp;
1632	struct cgroup *mem_cgrp;
1633	/*
1634	* Need a buffer in BSS, can't rely on allocations. The code relies
1635	* on the assumption that OOM is serialized for memory controller.
1636	* If this assumption is broken, revisit this code.
1637	*/
1638	static char memcg_name[PATH_MAX];
1639	int ret;
1640	struct mem_cgroup *iter;
1641	unsigned int i;
1642
1643	if (!p)
1644	return;
1645
1646	rcu_read_lock();
1647
1648	mem_cgrp = memcg->css.cgroup;
1649	task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1650
1651	ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1652	if (ret < 0) {
1653	/*
1654	* Unfortunately, we are unable to convert to a useful name
1655	* But we'll still print out the usage information
1656	*/
1657	rcu_read_unlock();
1658	goto done;
1659	}
1660	rcu_read_unlock();
1661
1662	pr_info("Task in %s killed", memcg_name);
1663
1664	rcu_read_lock();
1665	ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1666	if (ret < 0) {
1667	rcu_read_unlock();
1668	goto done;
1669	}
1670	rcu_read_unlock();
1671
1672	/*
1673	* Continues from above, so we don't need an KERN_ level
1674	*/
1675	pr_cont(" as a result of limit of %s\n", memcg_name);
1676	done:
1677
1678	pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
1679	res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1680	res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1681	res_counter_read_u64(&memcg->res, RES_FAILCNT));
1682	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
1683	res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1684	res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1685	res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1686	pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1687	res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
1688	res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
1689	res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
1690
1691	for_each_mem_cgroup_tree(iter, memcg) {
1692	pr_info("Memory cgroup stats");
1693
1694	rcu_read_lock();
1695	ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
1696	if (!ret)
1697	pr_cont(" for %s", memcg_name);
1698	rcu_read_unlock();
1699	pr_cont(":");
1700
1701	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1702	if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1703	continue;
1704	pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
1705	K(mem_cgroup_read_stat(iter, i)));
1706	}
1707
1708	for (i = 0; i < NR_LRU_LISTS; i++)
1709	pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1710	K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1711
1712	pr_cont("\n");
1713	}
1714	}
1715
1716	/*
1717	* This function returns the number of memcg under hierarchy tree. Returns
1718	* 1(self count) if no children.
1719	*/
1720	static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1721	{
1722	int num = 0;
1723	struct mem_cgroup *iter;
1724
1725	for_each_mem_cgroup_tree(iter, memcg)
1726	num++;
1727	return num;
1728	}
1729
1730	/*
1731	* Return the memory (and swap, if configured) limit for a memcg.
1732	*/
1733	static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1734	{
1735	u64 limit;
1736
1737	limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1738
1739	/*
1740	* Do not consider swap space if we cannot swap due to swappiness
1741	*/
1742	if (mem_cgroup_swappiness(memcg)) {
1743	u64 memsw;
1744
1745	limit += total_swap_pages << PAGE_SHIFT;
1746	memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1747
1748	/*
1749	* If memsw is finite and limits the amount of swap space
1750	* available to this memcg, return that limit.
1751	*/
1752	limit = min(limit, memsw);
1753	}
1754
1755	return limit;
1756	}
1757
1758	static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1759	int order)
1760	{
1761	struct mem_cgroup *iter;
1762	unsigned long chosen_points = 0;
1763	unsigned long totalpages;
1764	unsigned int points = 0;
1765	struct task_struct *chosen = NULL;
1766
1767	/*
1768	* If current has a pending SIGKILL or is exiting, then automatically
1769	* select it. The goal is to allow it to allocate so that it may
1770	* quickly exit and free its memory.
1771	*/
1772	if (fatal_signal_pending(current) \|\| current->flags & PF_EXITING) {
1773	set_thread_flag(TIF_MEMDIE);
1774	return;
1775	}
1776
1777	check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1778	totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
1779	for_each_mem_cgroup_tree(iter, memcg) {
1780	struct css_task_iter it;
1781	struct task_struct *task;
1782
1783	css_task_iter_start(&iter->css, &it);
1784	while ((task = css_task_iter_next(&it))) {
1785	switch (oom_scan_process_thread(task, totalpages, NULL,
1786	false)) {
1787	case OOM_SCAN_SELECT:
1788	if (chosen)
1789	put_task_struct(chosen);
1790	chosen = task;
1791	chosen_points = ULONG_MAX;
1792	get_task_struct(chosen);
1793	/* fall through */
1794	case OOM_SCAN_CONTINUE:
1795	continue;
1796	case OOM_SCAN_ABORT:
1797	css_task_iter_end(&it);
1798	mem_cgroup_iter_break(memcg, iter);
1799	if (chosen)
1800	put_task_struct(chosen);
1801	return;
1802	case OOM_SCAN_OK:
1803	break;
1804	};
1805	points = oom_badness(task, memcg, NULL, totalpages);
1806	if (points > chosen_points) {
1807	if (chosen)
1808	put_task_struct(chosen);
1809	chosen = task;
1810	chosen_points = points;
1811	get_task_struct(chosen);
1812	}
1813	}
1814	css_task_iter_end(&it);
1815	}
1816
1817	if (!chosen)
1818	return;
1819	points = chosen_points * 1000 / totalpages;
1820	oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1821	NULL, "Memory cgroup out of memory");
1822	}
1823
1824	static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
1825	gfp_t gfp_mask,
1826	unsigned long flags)
1827	{
1828	unsigned long total = 0;
1829	bool noswap = false;
1830	int loop;
1831
1832	if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
1833	noswap = true;
1834	if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
1835	noswap = true;
1836
1837	for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
1838	if (loop)
1839	drain_all_stock_async(memcg);
1840	total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
1841	/*
1842	* Allow limit shrinkers, which are triggered directly
1843	* by userspace, to catch signals and stop reclaim
1844	* after minimal progress, regardless of the margin.
1845	*/
1846	if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
1847	break;
1848	if (mem_cgroup_margin(memcg))
1849	break;
1850	/*
1851	* If nothing was reclaimed after two attempts, there
1852	* may be no reclaimable pages in this hierarchy.
1853	*/
1854	if (loop && !total)
1855	break;
1856	}
1857	return total;
1858	}
1859
1860	/**
1861	* test_mem_cgroup_node_reclaimable
1862	* @memcg: the target memcg
1863	* @nid: the node ID to be checked.
1864	* @noswap : specify true here if the user wants flle only information.
1865	*
1866	* This function returns whether the specified memcg contains any
1867	* reclaimable pages on a node. Returns true if there are any reclaimable
1868	* pages in the node.
1869	*/
1870	static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1871	int nid, bool noswap)
1872	{
1873	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1874	return true;
1875	if (noswap \|\| !total_swap_pages)
1876	return false;
1877	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1878	return true;
1879	return false;
1880
1881	}
1882	#if MAX_NUMNODES > 1
1883
1884	/*
1885	* Always updating the nodemask is not very good - even if we have an empty
1886	* list or the wrong list here, we can start from some node and traverse all
1887	* nodes based on the zonelist. So update the list loosely once per 10 secs.
1888	*
1889	*/
1890	static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1891	{
1892	int nid;
1893	/*
1894	* numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1895	* pagein/pageout changes since the last update.
1896	*/
1897	if (!atomic_read(&memcg->numainfo_events))
1898	return;
1899	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1900	return;
1901
1902	/* make a nodemask where this memcg uses memory from */
1903	memcg->scan_nodes = node_states[N_MEMORY];
1904
1905	for_each_node_mask(nid, node_states[N_MEMORY]) {
1906
1907	if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1908	node_clear(nid, memcg->scan_nodes);
1909	}
1910
1911	atomic_set(&memcg->numainfo_events, 0);
1912	atomic_set(&memcg->numainfo_updating, 0);
1913	}
1914
1915	/*
1916	* Selecting a node where we start reclaim from. Because what we need is just
1917	* reducing usage counter, start from anywhere is O,K. Considering
1918	* memory reclaim from current node, there are pros. and cons.
1919	*
1920	* Freeing memory from current node means freeing memory from a node which
1921	* we'll use or we've used. So, it may make LRU bad. And if several threads
1922	* hit limits, it will see a contention on a node. But freeing from remote
1923	* node means more costs for memory reclaim because of memory latency.
1924	*
1925	* Now, we use round-robin. Better algorithm is welcomed.
1926	*/
1927	int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1928	{
1929	int node;
1930
1931	mem_cgroup_may_update_nodemask(memcg);
1932	node = memcg->last_scanned_node;
1933
1934	node = next_node(node, memcg->scan_nodes);
1935	if (node == MAX_NUMNODES)
1936	node = first_node(memcg->scan_nodes);
1937	/*
1938	* We call this when we hit limit, not when pages are added to LRU.
1939	* No LRU may hold pages because all pages are UNEVICTABLE or
1940	* memcg is too small and all pages are not on LRU. In that case,
1941	* we use curret node.
1942	*/
1943	if (unlikely(node == MAX_NUMNODES))
1944	node = numa_node_id();
1945
1946	memcg->last_scanned_node = node;
1947	return node;
1948	}
1949
1950	/*
1951	* Check all nodes whether it contains reclaimable pages or not.
1952	* For quick scan, we make use of scan_nodes. This will allow us to skip
1953	* unused nodes. But scan_nodes is lazily updated and may not cotain
1954	* enough new information. We need to do double check.
1955	*/
1956	static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1957	{
1958	int nid;
1959
1960	/*
1961	* quick check...making use of scan_node.
1962	* We can skip unused nodes.
1963	*/
1964	if (!nodes_empty(memcg->scan_nodes)) {
1965	for (nid = first_node(memcg->scan_nodes);
1966	nid < MAX_NUMNODES;
1967	nid = next_node(nid, memcg->scan_nodes)) {
1968
1969	if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1970	return true;
1971	}
1972	}
1973	/*
1974	* Check rest of nodes.
1975	*/
1976	for_each_node_state(nid, N_MEMORY) {
1977	if (node_isset(nid, memcg->scan_nodes))
1978	continue;
1979	if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1980	return true;
1981	}
1982	return false;
1983	}
1984
1985	#else
1986	int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1987	{
1988	return 0;
1989	}
1990
1991	static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1992	{
1993	return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
1994	}
1995	#endif
1996
1997	static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1998	struct zone *zone,
1999	gfp_t gfp_mask,
2000	unsigned long *total_scanned)
2001	{
2002	struct mem_cgroup *victim = NULL;
2003	int total = 0;
2004	int loop = 0;
2005	unsigned long excess;
2006	unsigned long nr_scanned;
2007	struct mem_cgroup_reclaim_cookie reclaim = {
2008	.zone = zone,
2009	.priority = 0,
2010	};
2011
2012	excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
2013
2014	while (1) {
2015	victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
2016	if (!victim) {
2017	loop++;
2018	if (loop >= 2) {
2019	/*
2020	* If we have not been able to reclaim
2021	* anything, it might because there are
2022	* no reclaimable pages under this hierarchy
2023	*/
2024	if (!total)
2025	break;
2026	/*
2027	* We want to do more targeted reclaim.
2028	* excess >> 2 is not to excessive so as to
2029	* reclaim too much, nor too less that we keep
2030	* coming back to reclaim from this cgroup
2031	*/
2032	if (total >= (excess >> 2) \|\|
2033	(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
2034	break;
2035	}
2036	continue;
2037	}
2038	if (!mem_cgroup_reclaimable(victim, false))
2039	continue;
2040	total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
2041	zone, &nr_scanned);
2042	*total_scanned += nr_scanned;
2043	if (!res_counter_soft_limit_excess(&root_memcg->res))
2044	break;
2045	}
2046	mem_cgroup_iter_break(root_memcg, victim);
2047	return total;
2048	}
2049
2050	#ifdef CONFIG_LOCKDEP
2051	static struct lockdep_map memcg_oom_lock_dep_map = {
2052	.name = "memcg_oom_lock",
2053	};
2054	#endif
2055
2056	static DEFINE_SPINLOCK(memcg_oom_lock);
2057
2058	/*
2059	* Check OOM-Killer is already running under our hierarchy.
2060	* If someone is running, return false.
2061	*/
2062	static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
2063	{
2064	struct mem_cgroup iter, failed = NULL;
2065
2066	spin_lock(&memcg_oom_lock);
2067
2068	for_each_mem_cgroup_tree(iter, memcg) {
2069	if (iter->oom_lock) {
2070	/*
2071	* this subtree of our hierarchy is already locked
2072	* so we cannot give a lock.
2073	*/
2074	failed = iter;
2075	mem_cgroup_iter_break(memcg, iter);
2076	break;
2077	} else
2078	iter->oom_lock = true;
2079	}
2080
2081	if (failed) {
2082	/*
2083	* OK, we failed to lock the whole subtree so we have
2084	* to clean up what we set up to the failing subtree
2085	*/
2086	for_each_mem_cgroup_tree(iter, memcg) {
2087	if (iter == failed) {
2088	mem_cgroup_iter_break(memcg, iter);
2089	break;
2090	}
2091	iter->oom_lock = false;
2092	}
2093	} else
2094	mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
2095
2096	spin_unlock(&memcg_oom_lock);
2097
2098	return !failed;
2099	}
2100
2101	static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2102	{
2103	struct mem_cgroup *iter;
2104
2105	spin_lock(&memcg_oom_lock);
2106	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
2107	for_each_mem_cgroup_tree(iter, memcg)
2108	iter->oom_lock = false;
2109	spin_unlock(&memcg_oom_lock);
2110	}
2111
2112	static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2113	{
2114	struct mem_cgroup *iter;
2115
2116	for_each_mem_cgroup_tree(iter, memcg)
2117	atomic_inc(&iter->under_oom);
2118	}
2119
2120	static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2121	{
2122	struct mem_cgroup *iter;
2123
2124	/*
2125	* When a new child is created while the hierarchy is under oom,
2126	* mem_cgroup_oom_lock() may not be called. We have to use
2127	* atomic_add_unless() here.
2128	*/
2129	for_each_mem_cgroup_tree(iter, memcg)
2130	atomic_add_unless(&iter->under_oom, -1, 0);
2131	}
2132
2133	static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2134
2135	struct oom_wait_info {
2136	struct mem_cgroup *memcg;
2137	wait_queue_t wait;
2138	};
2139
2140	static int memcg_oom_wake_function(wait_queue_t *wait,
2141	unsigned mode, int sync, void *arg)
2142	{
2143	struct mem_cgroup wake_memcg = (struct mem_cgroup )arg;
2144	struct mem_cgroup *oom_wait_memcg;
2145	struct oom_wait_info *oom_wait_info;
2146
2147	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2148	oom_wait_memcg = oom_wait_info->memcg;
2149
2150	/*
2151	* Both of oom_wait_info->memcg and wake_memcg are stable under us.
2152	* Then we can use css_is_ancestor without taking care of RCU.
2153	*/
2154	if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
2155	&& !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
2156	return 0;
2157	return autoremove_wake_function(wait, mode, sync, arg);
2158	}
2159
2160	static void memcg_wakeup_oom(struct mem_cgroup *memcg)
2161	{
2162	atomic_inc(&memcg->oom_wakeups);
2163	/* for filtering, pass "memcg" as argument. */
2164	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
2165	}
2166
2167	static void memcg_oom_recover(struct mem_cgroup *memcg)
2168	{
2169	if (memcg && atomic_read(&memcg->under_oom))
2170	memcg_wakeup_oom(memcg);
2171	}
2172
2173	static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
2174	{
2175	if (!current->memcg_oom.may_oom)
2176	return;
2177	/*
2178	* We are in the middle of the charge context here, so we
2179	* don't want to block when potentially sitting on a callstack
2180	* that holds all kinds of filesystem and mm locks.
2181	*
2182	* Also, the caller may handle a failed allocation gracefully
2183	* (like optional page cache readahead) and so an OOM killer
2184	* invocation might not even be necessary.
2185	*
2186	* That's why we don't do anything here except remember the
2187	* OOM context and then deal with it at the end of the page
2188	* fault when the stack is unwound, the locks are released,
2189	* and when we know whether the fault was overall successful.
2190	*/
2191	css_get(&memcg->css);
2192	current->memcg_oom.memcg = memcg;
2193	current->memcg_oom.gfp_mask = mask;
2194	current->memcg_oom.order = order;
2195	}
2196
2197	/**
2198	* mem_cgroup_oom_synchronize - complete memcg OOM handling
2199	* @handle: actually kill/wait or just clean up the OOM state
2200	*
2201	* This has to be called at the end of a page fault if the memcg OOM
2202	* handler was enabled.
2203	*
2204	* Memcg supports userspace OOM handling where failed allocations must
2205	* sleep on a waitqueue until the userspace task resolves the
2206	* situation. Sleeping directly in the charge context with all kinds
2207	* of locks held is not a good idea, instead we remember an OOM state
2208	* in the task and mem_cgroup_oom_synchronize() has to be called at
2209	* the end of the page fault to complete the OOM handling.
2210	*
2211	* Returns %true if an ongoing memcg OOM situation was detected and
2212	* completed, %false otherwise.
2213	*/
2214	bool mem_cgroup_oom_synchronize(bool handle)
2215	{
2216	struct mem_cgroup *memcg = current->memcg_oom.memcg;
2217	struct oom_wait_info owait;
2218	bool locked;
2219
2220	/* OOM is global, do not handle */
2221	if (!memcg)
2222	return false;
2223
2224	if (!handle)
2225	goto cleanup;
2226
2227	owait.memcg = memcg;
2228	owait.wait.flags = 0;
2229	owait.wait.func = memcg_oom_wake_function;
2230	owait.wait.private = current;
2231	INIT_LIST_HEAD(&owait.wait.task_list);
2232
2233	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2234	mem_cgroup_mark_under_oom(memcg);
2235
2236	locked = mem_cgroup_oom_trylock(memcg);
2237
2238	if (locked)
2239	mem_cgroup_oom_notify(memcg);
2240
2241	if (locked && !memcg->oom_kill_disable) {
2242	mem_cgroup_unmark_under_oom(memcg);
2243	finish_wait(&memcg_oom_waitq, &owait.wait);
2244	mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
2245	current->memcg_oom.order);
2246	} else {
2247	schedule();
2248	mem_cgroup_unmark_under_oom(memcg);
2249	finish_wait(&memcg_oom_waitq, &owait.wait);
2250	}
2251
2252	if (locked) {
2253	mem_cgroup_oom_unlock(memcg);
2254	/*
2255	* There is no guarantee that an OOM-lock contender
2256	* sees the wakeups triggered by the OOM kill
2257	* uncharges. Wake any sleepers explicitely.
2258	*/
2259	memcg_oom_recover(memcg);
2260	}
2261	cleanup:
2262	current->memcg_oom.memcg = NULL;
2263	css_put(&memcg->css);
2264	return true;
2265	}
2266
2267	/*
2268	* Currently used to update mapped file statistics, but the routine can be
2269	* generalized to update other statistics as well.
2270	*
2271	* Notes: Race condition
2272	*
2273	* We usually use page_cgroup_lock() for accessing page_cgroup member but
2274	* it tends to be costly. But considering some conditions, we doesn't need
2275	* to do so _always_.
2276	*
2277	* Considering "charge", lock_page_cgroup() is not required because all
2278	* file-stat operations happen after a page is attached to radix-tree. There
2279	* are no race with "charge".
2280	*
2281	* Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2282	* at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2283	* if there are race with "uncharge". Statistics itself is properly handled
2284	* by flags.
2285	*
2286	* Considering "move", this is an only case we see a race. To make the race
2287	* small, we check mm->moving_account and detect there are possibility of race
2288	* If there is, we take a lock.
2289	*/
2290
2291	void __mem_cgroup_begin_update_page_stat(struct page *page,
2292	bool locked, unsigned long flags)
2293	{
2294	struct mem_cgroup *memcg;
2295	struct page_cgroup *pc;
2296
2297	pc = lookup_page_cgroup(page);
2298	again:
2299	memcg = pc->mem_cgroup;
2300	if (unlikely(!memcg \|\| !PageCgroupUsed(pc)))
2301	return;
2302	/*
2303	* If this memory cgroup is not under account moving, we don't
2304	* need to take move_lock_mem_cgroup(). Because we already hold
2305	* rcu_read_lock(), any calls to move_account will be delayed until
2306	* rcu_read_unlock() if mem_cgroup_stolen() == true.
2307	*/
2308	if (!mem_cgroup_stolen(memcg))
2309	return;
2310
2311	move_lock_mem_cgroup(memcg, flags);
2312	if (memcg != pc->mem_cgroup \|\| !PageCgroupUsed(pc)) {
2313	move_unlock_mem_cgroup(memcg, flags);
2314	goto again;
2315	}
2316	*locked = true;
2317	}
2318
2319	void __mem_cgroup_end_update_page_stat(struct page page, unsigned long flags)
2320	{
2321	struct page_cgroup *pc = lookup_page_cgroup(page);
2322
2323	/*
2324	* It's guaranteed that pc->mem_cgroup never changes while
2325	* lock is held because a routine modifies pc->mem_cgroup
2326	* should take move_lock_mem_cgroup().
2327	*/
2328	move_unlock_mem_cgroup(pc->mem_cgroup, flags);
2329	}
2330
2331	void mem_cgroup_update_page_stat(struct page *page,
2332	enum mem_cgroup_stat_index idx, int val)
2333	{
2334	struct mem_cgroup *memcg;
2335	struct page_cgroup *pc = lookup_page_cgroup(page);
2336	unsigned long uninitialized_var(flags);
2337
2338	if (mem_cgroup_disabled())
2339	return;
2340
2341	VM_BUG_ON(!rcu_read_lock_held());
2342	memcg = pc->mem_cgroup;
2343	if (unlikely(!memcg \|\| !PageCgroupUsed(pc)))
2344	return;
2345
2346	this_cpu_add(memcg->stat->count[idx], val);
2347	}
2348
2349	/*
2350	* size of first charge trial. "32" comes from vmscan.c's magic value.
2351	* TODO: maybe necessary to use big numbers in big irons.
2352	*/
2353	#define CHARGE_BATCH 32U
2354	struct memcg_stock_pcp {
2355	struct mem_cgroup cached; / this never be root cgroup */
2356	unsigned int nr_pages;
2357	struct work_struct work;
2358	unsigned long flags;
2359	#define FLUSHING_CACHED_CHARGE 0
2360	};
2361	static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2362	static DEFINE_MUTEX(percpu_charge_mutex);
2363
2364	/**
2365	* consume_stock: Try to consume stocked charge on this cpu.
2366	* @memcg: memcg to consume from.
2367	* @nr_pages: how many pages to charge.
2368	*
2369	* The charges will only happen if @memcg matches the current cpu's memcg
2370	* stock, and at least @nr_pages are available in that stock. Failure to
2371	* service an allocation will refill the stock.
2372	*
2373	* returns true if successful, false otherwise.
2374	*/
2375	static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2376	{
2377	struct memcg_stock_pcp *stock;
2378	bool ret = true;
2379
2380	if (nr_pages > CHARGE_BATCH)
2381	return false;
2382
2383	stock = &get_cpu_var(memcg_stock);
2384	if (memcg == stock->cached && stock->nr_pages >= nr_pages)
2385	stock->nr_pages -= nr_pages;
2386	else /* need to call res_counter_charge */
2387	ret = false;
2388	put_cpu_var(memcg_stock);
2389	return ret;
2390	}
2391
2392	/*
2393	* Returns stocks cached in percpu to res_counter and reset cached information.
2394	*/
2395	static void drain_stock(struct memcg_stock_pcp *stock)
2396	{
2397	struct mem_cgroup *old = stock->cached;
2398
2399	if (stock->nr_pages) {
2400	unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2401
2402	res_counter_uncharge(&old->res, bytes);
2403	if (do_swap_account)
2404	res_counter_uncharge(&old->memsw, bytes);
2405	stock->nr_pages = 0;
2406	}
2407	stock->cached = NULL;
2408	}
2409
2410	/*
2411	* This must be called under preempt disabled or must be called by
2412	* a thread which is pinned to local cpu.
2413	*/
2414	static void drain_local_stock(struct work_struct *dummy)
2415	{
2416	struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2417	drain_stock(stock);
2418	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2419	}
2420
2421	static void __init memcg_stock_init(void)
2422	{
2423	int cpu;
2424
2425	for_each_possible_cpu(cpu) {
2426	struct memcg_stock_pcp *stock =
2427	&per_cpu(memcg_stock, cpu);
2428	INIT_WORK(&stock->work, drain_local_stock);
2429	}
2430	}
2431
2432	/*
2433	* Cache charges(val) which is from res_counter, to local per_cpu area.
2434	* This will be consumed by consume_stock() function, later.
2435	*/
2436	static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2437	{
2438	struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2439
2440	if (stock->cached != memcg) { /* reset if necessary */
2441	drain_stock(stock);
2442	stock->cached = memcg;
2443	}
2444	stock->nr_pages += nr_pages;
2445	put_cpu_var(memcg_stock);
2446	}
2447
2448	/*
2449	* Drains all per-CPU charge caches for given root_memcg resp. subtree
2450	* of the hierarchy under it. sync flag says whether we should block
2451	* until the work is done.
2452	*/
2453	static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2454	{
2455	int cpu, curcpu;
2456
2457	/* Notify other cpus that system-wide "drain" is running */
2458	get_online_cpus();
2459	curcpu = get_cpu();
2460	for_each_online_cpu(cpu) {
2461	struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2462	struct mem_cgroup *memcg;
2463
2464	memcg = stock->cached;
2465	if (!memcg \|\| !stock->nr_pages)
2466	continue;
2467	if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2468	continue;
2469	if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2470	if (cpu == curcpu)
2471	drain_local_stock(&stock->work);
2472	else
2473	schedule_work_on(cpu, &stock->work);
2474	}
2475	}
2476	put_cpu();
2477
2478	if (!sync)
2479	goto out;
2480
2481	for_each_online_cpu(cpu) {
2482	struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2483	if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2484	flush_work(&stock->work);
2485	}
2486	out:
2487	put_online_cpus();
2488	}
2489
2490	/*
2491	* Tries to drain stocked charges in other cpus. This function is asynchronous
2492	* and just put a work per cpu for draining localy on each cpu. Caller can
2493	* expects some charges will be back to res_counter later but cannot wait for
2494	* it.
2495	*/
2496	static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2497	{
2498	/*
2499	* If someone calls draining, avoid adding more kworker runs.
2500	*/
2501	if (!mutex_trylock(&percpu_charge_mutex))
2502	return;
2503	drain_all_stock(root_memcg, false);
2504	mutex_unlock(&percpu_charge_mutex);
2505	}
2506
2507	/* This is a synchronous drain interface. */
2508	static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2509	{
2510	/* called when force_empty is called */
2511	mutex_lock(&percpu_charge_mutex);
2512	drain_all_stock(root_memcg, true);
2513	mutex_unlock(&percpu_charge_mutex);
2514	}
2515
2516	/*
2517	* This function drains percpu counter value from DEAD cpu and
2518	* move it to local cpu. Note that this function can be preempted.
2519	*/
2520	static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2521	{
2522	int i;
2523
2524	spin_lock(&memcg->pcp_counter_lock);
2525	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2526	long x = per_cpu(memcg->stat->count[i], cpu);
2527
2528	per_cpu(memcg->stat->count[i], cpu) = 0;
2529	memcg->nocpu_base.count[i] += x;
2530	}
2531	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2532	unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2533
2534	per_cpu(memcg->stat->events[i], cpu) = 0;
2535	memcg->nocpu_base.events[i] += x;
2536	}
2537	spin_unlock(&memcg->pcp_counter_lock);
2538	}
2539
2540	static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
2541	unsigned long action,
2542	void *hcpu)
2543	{
2544	int cpu = (unsigned long)hcpu;
2545	struct memcg_stock_pcp *stock;
2546	struct mem_cgroup *iter;
2547
2548	if (action == CPU_ONLINE)
2549	return NOTIFY_OK;
2550
2551	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2552	return NOTIFY_OK;
2553
2554	for_each_mem_cgroup(iter)
2555	mem_cgroup_drain_pcp_counter(iter, cpu);
2556
2557	stock = &per_cpu(memcg_stock, cpu);
2558	drain_stock(stock);
2559	return NOTIFY_OK;
2560	}
2561
2562
2563	/* See __mem_cgroup_try_charge() for details */
2564	enum {
2565	CHARGE_OK, /* success */
2566	CHARGE_RETRY, /* need to retry but retry is not bad */
2567	CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
2568	CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
2569	};
2570
2571	static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2572	unsigned int nr_pages, unsigned int min_pages,
2573	bool invoke_oom)
2574	{
2575	unsigned long csize = nr_pages * PAGE_SIZE;
2576	struct mem_cgroup *mem_over_limit;
2577	struct res_counter *fail_res;
2578	unsigned long flags = 0;
2579	int ret;
2580
2581	ret = res_counter_charge(&memcg->res, csize, &fail_res);
2582
2583	if (likely(!ret)) {
2584	if (!do_swap_account)
2585	return CHARGE_OK;
2586	ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2587	if (likely(!ret))
2588	return CHARGE_OK;
2589
2590	res_counter_uncharge(&memcg->res, csize);
2591	mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2592	flags \|= MEM_CGROUP_RECLAIM_NOSWAP;
2593	} else
2594	mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2595	/*
2596	* Never reclaim on behalf of optional batching, retry with a
2597	* single page instead.
2598	*/
2599	if (nr_pages > min_pages)
2600	return CHARGE_RETRY;
2601
2602	if (!(gfp_mask & __GFP_WAIT))
2603	return CHARGE_WOULDBLOCK;
2604
2605	if (gfp_mask & __GFP_NORETRY)
2606	return CHARGE_NOMEM;
2607
2608	ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2609	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2610	return CHARGE_RETRY;
2611	/*
2612	* Even though the limit is exceeded at this point, reclaim
2613	* may have been able to free some pages. Retry the charge
2614	* before killing the task.
2615	*
2616	* Only for regular pages, though: huge pages are rather
2617	* unlikely to succeed so close to the limit, and we fall back
2618	* to regular pages anyway in case of failure.
2619	*/
2620	if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
2621	return CHARGE_RETRY;
2622
2623	/*
2624	* At task move, charge accounts can be doubly counted. So, it's
2625	* better to wait until the end of task_move if something is going on.
2626	*/
2627	if (mem_cgroup_wait_acct_move(mem_over_limit))
2628	return CHARGE_RETRY;
2629
2630	if (invoke_oom)
2631	mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize));
2632
2633	return CHARGE_NOMEM;
2634	}
2635
2636	/*
2637	* __mem_cgroup_try_charge() does
2638	* 1. detect memcg to be charged against from passed mm and ptr,
2639	* 2. update res_counter
2640	* 3. call memory reclaim if necessary.
2641	*
2642	* In some special case, if the task is fatal, fatal_signal_pending() or
2643	* has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
2644	* to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
2645	* as possible without any hazards. 2: all pages should have a valid
2646	* pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
2647	* pointer, that is treated as a charge to root_mem_cgroup.
2648	*
2649	* So __mem_cgroup_try_charge() will return
2650	* 0 ... on success, filling *ptr with a valid memcg pointer.
2651	* -ENOMEM ... charge failure because of resource limits.
2652	* -EINTR ... if thread is fatal. *ptr is filled with root_mem_cgroup.
2653	*
2654	* Unlike the exported interface, an "oom" parameter is added. if oom==true,
2655	* the oom-killer can be invoked.
2656	*/
2657	static int __mem_cgroup_try_charge(struct mm_struct *mm,
2658	gfp_t gfp_mask,
2659	unsigned int nr_pages,
2660	struct mem_cgroup **ptr,
2661	bool oom)
2662	{
2663	unsigned int batch = max(CHARGE_BATCH, nr_pages);
2664	int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2665	struct mem_cgroup *memcg = NULL;
2666	int ret;
2667
2668	/*
2669	* Unlike gloval-vm's OOM-kill, we're not in memory shortage
2670	* in system level. So, allow to go ahead dying process in addition to
2671	* MEMDIE process.
2672	*/
2673	if (unlikely(test_thread_flag(TIF_MEMDIE)
2674	\|\| fatal_signal_pending(current)))
2675	goto bypass;
2676
2677	if (unlikely(task_in_memcg_oom(current)))
2678	goto bypass;
2679
2680	/*
2681	* We always charge the cgroup the mm_struct belongs to.
2682	* The mm_struct's mem_cgroup changes on task migration if the
2683	* thread group leader migrates. It's possible that mm is not
2684	* set, if so charge the root memcg (happens for pagecache usage).
2685	*/
2686	if (!*ptr && !mm)
2687	*ptr = root_mem_cgroup;
2688	again:
2689	if (ptr) { / css should be a valid one */
2690	memcg = *ptr;
2691	if (mem_cgroup_is_root(memcg))
2692	goto done;
2693	if (consume_stock(memcg, nr_pages))
2694	goto done;
2695	css_get(&memcg->css);
2696	} else {
2697	struct task_struct *p;
2698
2699	rcu_read_lock();
2700	p = rcu_dereference(mm->owner);
2701	/*
2702	* Because we don't have task_lock(), "p" can exit.
2703	* In that case, "memcg" can point to root or p can be NULL with
2704	* race with swapoff. Then, we have small risk of mis-accouning.
2705	* But such kind of mis-account by race always happens because
2706	* we don't have cgroup_mutex(). It's overkill and we allo that
2707	* small race, here.
2708	* (*) swapoff at el will charge against mm-struct not against
2709	* task-struct. So, mm->owner can be NULL.
2710	*/
2711	memcg = mem_cgroup_from_task(p);
2712	if (!memcg)
2713	memcg = root_mem_cgroup;
2714	if (mem_cgroup_is_root(memcg)) {
2715	rcu_read_unlock();
2716	goto done;
2717	}
2718	if (consume_stock(memcg, nr_pages)) {
2719	/*
2720	* It seems dagerous to access memcg without css_get().
2721	* But considering how consume_stok works, it's not
2722	* necessary. If consume_stock success, some charges
2723	* from this memcg are cached on this cpu. So, we
2724	* don't need to call css_get()/css_tryget() before
2725	* calling consume_stock().
2726	*/
2727	rcu_read_unlock();
2728	goto done;
2729	}
2730	/* after here, we may be blocked. we need to get refcnt */
2731	if (!css_tryget(&memcg->css)) {
2732	rcu_read_unlock();
2733	goto again;
2734	}
2735	rcu_read_unlock();
2736	}
2737
2738	do {
2739	bool invoke_oom = oom && !nr_oom_retries;
2740
2741	/* If killed, bypass charge */
2742	if (fatal_signal_pending(current)) {
2743	css_put(&memcg->css);
2744	goto bypass;
2745	}
2746
2747	ret = mem_cgroup_do_charge(memcg, gfp_mask, batch,
2748	nr_pages, invoke_oom);
2749	switch (ret) {
2750	case CHARGE_OK:
2751	break;
2752	case CHARGE_RETRY: /* not in OOM situation but retry */
2753	batch = nr_pages;
2754	css_put(&memcg->css);
2755	memcg = NULL;
2756	goto again;
2757	case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2758	css_put(&memcg->css);
2759	goto nomem;
2760	case CHARGE_NOMEM: /* OOM routine works */
2761	if (!oom \|\| invoke_oom) {
2762	css_put(&memcg->css);
2763	goto nomem;
2764	}
2765	nr_oom_retries--;
2766	break;
2767	}
2768	} while (ret != CHARGE_OK);
2769
2770	if (batch > nr_pages)
2771	refill_stock(memcg, batch - nr_pages);
2772	css_put(&memcg->css);
2773	done:
2774	*ptr = memcg;
2775	return 0;
2776	nomem:
2777	if (!(gfp_mask & __GFP_NOFAIL)) {
2778	*ptr = NULL;
2779	return -ENOMEM;
2780	}
2781	bypass:
2782	*ptr = root_mem_cgroup;
2783	return -EINTR;
2784	}
2785
2786	/*
2787	* Somemtimes we have to undo a charge we got by try_charge().
2788	* This function is for that and do uncharge, put css's refcnt.
2789	* gotten by try_charge().
2790	*/
2791	static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2792	unsigned int nr_pages)
2793	{
2794	if (!mem_cgroup_is_root(memcg)) {
2795	unsigned long bytes = nr_pages * PAGE_SIZE;
2796
2797	res_counter_uncharge(&memcg->res, bytes);
2798	if (do_swap_account)
2799	res_counter_uncharge(&memcg->memsw, bytes);
2800	}
2801	}
2802
2803	/*
2804	* Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
2805	* This is useful when moving usage to parent cgroup.
2806	*/
2807	static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
2808	unsigned int nr_pages)
2809	{
2810	unsigned long bytes = nr_pages * PAGE_SIZE;
2811
2812	if (mem_cgroup_is_root(memcg))
2813	return;
2814
2815	res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
2816	if (do_swap_account)
2817	res_counter_uncharge_until(&memcg->memsw,
2818	memcg->memsw.parent, bytes);
2819	}
2820
2821	/*
2822	* A helper function to get mem_cgroup from ID. must be called under
2823	* rcu_read_lock(). The caller is responsible for calling css_tryget if
2824	* the mem_cgroup is used for charging. (dropping refcnt from swap can be
2825	* called against removed memcg.)
2826	*/
2827	static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2828	{
2829	struct cgroup_subsys_state *css;
2830
2831	/* ID 0 is unused ID */
2832	if (!id)
2833	return NULL;
2834	css = css_lookup(&mem_cgroup_subsys, id);
2835	if (!css)
2836	return NULL;
2837	return mem_cgroup_from_css(css);
2838	}
2839
2840	struct mem_cgroup try_get_mem_cgroup_from_page(struct page page)
2841	{
2842	struct mem_cgroup *memcg = NULL;
2843	struct page_cgroup *pc;
2844	unsigned short id;
2845	swp_entry_t ent;
2846
2847	VM_BUG_ON(!PageLocked(page));
2848
2849	pc = lookup_page_cgroup(page);
2850	lock_page_cgroup(pc);
2851	if (PageCgroupUsed(pc)) {
2852	memcg = pc->mem_cgroup;
2853	if (memcg && !css_tryget(&memcg->css))
2854	memcg = NULL;
2855	} else if (PageSwapCache(page)) {
2856	ent.val = page_private(page);
2857	id = lookup_swap_cgroup_id(ent);
2858	rcu_read_lock();
2859	memcg = mem_cgroup_lookup(id);
2860	if (memcg && !css_tryget(&memcg->css))
2861	memcg = NULL;
2862	rcu_read_unlock();
2863	}
2864	unlock_page_cgroup(pc);
2865	return memcg;
2866	}
2867
2868	static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2869	struct page *page,
2870	unsigned int nr_pages,
2871	enum charge_type ctype,
2872	bool lrucare)
2873	{
2874	struct page_cgroup *pc = lookup_page_cgroup(page);
2875	struct zone *uninitialized_var(zone);
2876	struct lruvec *lruvec;
2877	bool was_on_lru = false;
2878	bool anon;
2879
2880	lock_page_cgroup(pc);
2881	VM_BUG_ON(PageCgroupUsed(pc));
2882	/*
2883	* we don't need page_cgroup_lock about tail pages, becase they are not
2884	* accessed by any other context at this point.
2885	*/
2886
2887	/*
2888	* In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2889	* may already be on some other mem_cgroup's LRU. Take care of it.
2890	*/
2891	if (lrucare) {
2892	zone = page_zone(page);
2893	spin_lock_irq(&zone->lru_lock);
2894	if (PageLRU(page)) {
2895	lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2896	ClearPageLRU(page);
2897	del_page_from_lru_list(page, lruvec, page_lru(page));
2898	was_on_lru = true;
2899	}
2900	}
2901
2902	pc->mem_cgroup = memcg;
2903	/*
2904	* We access a page_cgroup asynchronously without lock_page_cgroup().
2905	* Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2906	* is accessed after testing USED bit. To make pc->mem_cgroup visible
2907	* before USED bit, we need memory barrier here.
2908	* See mem_cgroup_add_lru_list(), etc.
2909	*/
2910	smp_wmb();
2911	SetPageCgroupUsed(pc);
2912
2913	if (lrucare) {
2914	if (was_on_lru) {
2915	lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2916	VM_BUG_ON(PageLRU(page));
2917	SetPageLRU(page);
2918	add_page_to_lru_list(page, lruvec, page_lru(page));
2919	}
2920	spin_unlock_irq(&zone->lru_lock);
2921	}
2922
2923	if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2924	anon = true;
2925	else
2926	anon = false;
2927
2928	mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
2929	unlock_page_cgroup(pc);
2930
2931	/*
2932	* "charge_statistics" updated event counter. Then, check it.
2933	* Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2934	* if they exceeds softlimit.
2935	*/
2936	memcg_check_events(memcg, page);
2937	}
2938
2939	static DEFINE_MUTEX(set_limit_mutex);
2940
2941	#ifdef CONFIG_MEMCG_KMEM
2942	static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
2943	{
2944	return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
2945	(memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
2946	}
2947
2948	/*
2949	* This is a bit cumbersome, but it is rarely used and avoids a backpointer
2950	* in the memcg_cache_params struct.
2951	*/
2952	static struct kmem_cache memcg_params_to_cache(struct memcg_cache_params p)
2953	{
2954	struct kmem_cache *cachep;
2955
2956	VM_BUG_ON(p->is_root_cache);
2957	cachep = p->root_cache;
2958	return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
2959	}
2960
2961	#ifdef CONFIG_SLABINFO
2962	static int mem_cgroup_slabinfo_read(struct cgroup_subsys_state *css,
2963	struct cftype cft, struct seq_file m)
2964	{
2965	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2966	struct memcg_cache_params *params;
2967
2968	if (!memcg_can_account_kmem(memcg))
2969	return -EIO;
2970
2971	print_slabinfo_header(m);
2972
2973	mutex_lock(&memcg->slab_caches_mutex);
2974	list_for_each_entry(params, &memcg->memcg_slab_caches, list)
2975	cache_show(memcg_params_to_cache(params), m);
2976	mutex_unlock(&memcg->slab_caches_mutex);
2977
2978	return 0;
2979	}
2980	#endif
2981
2982	static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
2983	{
2984	struct res_counter *fail_res;
2985	struct mem_cgroup *_memcg;
2986	int ret = 0;
2987	bool may_oom;
2988
2989	ret = res_counter_charge(&memcg->kmem, size, &fail_res);
2990	if (ret)
2991	return ret;
2992
2993	/*
2994	* Conditions under which we can wait for the oom_killer. Those are
2995	* the same conditions tested by the core page allocator
2996	*/
2997	may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);
2998
2999	_memcg = memcg;
3000	ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
3001	&_memcg, may_oom);
3002
3003	if (ret == -EINTR) {
3004	/*
3005	* __mem_cgroup_try_charge() chosed to bypass to root due to
3006	* OOM kill or fatal signal. Since our only options are to
3007	* either fail the allocation or charge it to this cgroup, do
3008	* it as a temporary condition. But we can't fail. From a
3009	* kmem/slab perspective, the cache has already been selected,
3010	* by mem_cgroup_kmem_get_cache(), so it is too late to change
3011	* our minds.
3012	*
3013	* This condition will only trigger if the task entered
3014	* memcg_charge_kmem in a sane state, but was OOM-killed during
3015	* __mem_cgroup_try_charge() above. Tasks that were already
3016	* dying when the allocation triggers should have been already
3017	* directed to the root cgroup in memcontrol.h
3018	*/
3019	res_counter_charge_nofail(&memcg->res, size, &fail_res);
3020	if (do_swap_account)
3021	res_counter_charge_nofail(&memcg->memsw, size,
3022	&fail_res);
3023	ret = 0;
3024	} else if (ret)
3025	res_counter_uncharge(&memcg->kmem, size);
3026
3027	return ret;
3028	}
3029
3030	static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
3031	{
3032	res_counter_uncharge(&memcg->res, size);
3033	if (do_swap_account)
3034	res_counter_uncharge(&memcg->memsw, size);
3035
3036	/* Not down to 0 */
3037	if (res_counter_uncharge(&memcg->kmem, size))
3038	return;
3039
3040	/*
3041	* Releases a reference taken in kmem_cgroup_css_offline in case
3042	* this last uncharge is racing with the offlining code or it is
3043	* outliving the memcg existence.
3044	*
3045	* The memory barrier imposed by test&clear is paired with the
3046	* explicit one in memcg_kmem_mark_dead().
3047	*/
3048	if (memcg_kmem_test_and_clear_dead(memcg))
3049	css_put(&memcg->css);
3050	}
3051
3052	void memcg_cache_list_add(struct mem_cgroup memcg, struct kmem_cache cachep)
3053	{
3054	if (!memcg)
3055	return;
3056
3057	mutex_lock(&memcg->slab_caches_mutex);
3058	list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
3059	mutex_unlock(&memcg->slab_caches_mutex);
3060	}
3061
3062	/*
3063	* helper for acessing a memcg's index. It will be used as an index in the
3064	* child cache array in kmem_cache, and also to derive its name. This function
3065	* will return -1 when this is not a kmem-limited memcg.
3066	*/
3067	int memcg_cache_id(struct mem_cgroup *memcg)
3068	{
3069	return memcg ? memcg->kmemcg_id : -1;
3070	}
3071
3072	/*
3073	* This ends up being protected by the set_limit mutex, during normal
3074	* operation, because that is its main call site.
3075	*
3076	* But when we create a new cache, we can call this as well if its parent
3077	* is kmem-limited. That will have to hold set_limit_mutex as well.
3078	*/
3079	int memcg_update_cache_sizes(struct mem_cgroup *memcg)
3080	{
3081	int num, ret;
3082
3083	num = ida_simple_get(&kmem_limited_groups,
3084	0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
3085	if (num < 0)
3086	return num;
3087	/*
3088	* After this point, kmem_accounted (that we test atomically in
3089	* the beginning of this conditional), is no longer 0. This
3090	* guarantees only one process will set the following boolean
3091	* to true. We don't need test_and_set because we're protected
3092	* by the set_limit_mutex anyway.
3093	*/
3094	memcg_kmem_set_activated(memcg);
3095
3096	ret = memcg_update_all_caches(num+1);
3097	if (ret) {
3098	ida_simple_remove(&kmem_limited_groups, num);
3099	memcg_kmem_clear_activated(memcg);
3100	return ret;
3101	}
3102
3103	memcg->kmemcg_id = num;
3104	INIT_LIST_HEAD(&memcg->memcg_slab_caches);
3105	mutex_init(&memcg->slab_caches_mutex);
3106	return 0;
3107	}
3108
3109	static size_t memcg_caches_array_size(int num_groups)
3110	{
3111	ssize_t size;
3112	if (num_groups <= 0)
3113	return 0;
3114
3115	size = 2 * num_groups;
3116	if (size < MEMCG_CACHES_MIN_SIZE)
3117	size = MEMCG_CACHES_MIN_SIZE;
3118	else if (size > MEMCG_CACHES_MAX_SIZE)
3119	size = MEMCG_CACHES_MAX_SIZE;
3120
3121	return size;
3122	}
3123
3124	/*
3125	* We should update the current array size iff all caches updates succeed. This
3126	* can only be done from the slab side. The slab mutex needs to be held when
3127	* calling this.
3128	*/
3129	void memcg_update_array_size(int num)
3130	{
3131	if (num > memcg_limited_groups_array_size)
3132	memcg_limited_groups_array_size = memcg_caches_array_size(num);
3133	}
3134
3135	static void kmem_cache_destroy_work_func(struct work_struct *w);
3136
3137	int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
3138	{
3139	struct memcg_cache_params *cur_params = s->memcg_params;
3140
3141	VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
3142
3143	if (num_groups > memcg_limited_groups_array_size) {
3144	int i;
3145	ssize_t size = memcg_caches_array_size(num_groups);
3146
3147	size = sizeof(void );
3148	size += offsetof(struct memcg_cache_params, memcg_caches);
3149
3150	s->memcg_params = kzalloc(size, GFP_KERNEL);
3151	if (!s->memcg_params) {
3152	s->memcg_params = cur_params;
3153	return -ENOMEM;
3154	}
3155
3156	s->memcg_params->is_root_cache = true;
3157
3158	/*
3159	* There is the chance it will be bigger than
3160	* memcg_limited_groups_array_size, if we failed an allocation
3161	* in a cache, in which case all caches updated before it, will
3162	* have a bigger array.
3163	*
3164	* But if that is the case, the data after
3165	* memcg_limited_groups_array_size is certainly unused
3166	*/
3167	for (i = 0; i < memcg_limited_groups_array_size; i++) {
3168	if (!cur_params->memcg_caches[i])
3169	continue;
3170	s->memcg_params->memcg_caches[i] =
3171	cur_params->memcg_caches[i];
3172	}
3173
3174	/*
3175	* Ideally, we would wait until all caches succeed, and only
3176	* then free the old one. But this is not worth the extra
3177	* pointer per-cache we'd have to have for this.
3178	*
3179	* It is not a big deal if some caches are left with a size
3180	* bigger than the others. And all updates will reset this
3181	* anyway.
3182	*/
3183	kfree(cur_params);
3184	}
3185	return 0;
3186	}
3187
3188	int memcg_register_cache(struct mem_cgroup memcg, struct kmem_cache s,
3189	struct kmem_cache *root_cache)
3190	{
3191	size_t size;
3192
3193	if (!memcg_kmem_enabled())
3194	return 0;
3195
3196	if (!memcg) {
3197	size = offsetof(struct memcg_cache_params, memcg_caches);
3198	size += memcg_limited_groups_array_size * sizeof(void *);
3199	} else
3200	size = sizeof(struct memcg_cache_params);
3201
3202	s->memcg_params = kzalloc(size, GFP_KERNEL);
3203	if (!s->memcg_params)
3204	return -ENOMEM;
3205
3206	if (memcg) {
3207	s->memcg_params->memcg = memcg;
3208	s->memcg_params->root_cache = root_cache;
3209	INIT_WORK(&s->memcg_params->destroy,
3210	kmem_cache_destroy_work_func);
3211	} else
3212	s->memcg_params->is_root_cache = true;
3213
3214	return 0;
3215	}
3216
3217	void memcg_release_cache(struct kmem_cache *s)
3218	{
3219	struct kmem_cache *root;
3220	struct mem_cgroup *memcg;
3221	int id;
3222
3223	/*
3224	* This happens, for instance, when a root cache goes away before we
3225	* add any memcg.
3226	*/
3227	if (!s->memcg_params)
3228	return;
3229
3230	if (s->memcg_params->is_root_cache)
3231	goto out;
3232
3233	memcg = s->memcg_params->memcg;
3234	id = memcg_cache_id(memcg);
3235
3236	root = s->memcg_params->root_cache;
3237	root->memcg_params->memcg_caches[id] = NULL;
3238
3239	mutex_lock(&memcg->slab_caches_mutex);
3240	list_del(&s->memcg_params->list);
3241	mutex_unlock(&memcg->slab_caches_mutex);
3242
3243	css_put(&memcg->css);
3244	out:
3245	kfree(s->memcg_params);
3246	}
3247
3248	/*
3249	* During the creation a new cache, we need to disable our accounting mechanism
3250	* altogether. This is true even if we are not creating, but rather just
3251	* enqueing new caches to be created.
3252	*
3253	* This is because that process will trigger allocations; some visible, like
3254	* explicit kmallocs to auxiliary data structures, name strings and internal
3255	* cache structures; some well concealed, like INIT_WORK() that can allocate
3256	* objects during debug.
3257	*
3258	* If any allocation happens during memcg_kmem_get_cache, we will recurse back
3259	* to it. This may not be a bounded recursion: since the first cache creation
3260	* failed to complete (waiting on the allocation), we'll just try to create the
3261	* cache again, failing at the same point.
3262	*
3263	* memcg_kmem_get_cache is prepared to abort after seeing a positive count of
3264	* memcg_kmem_skip_account. So we enclose anything that might allocate memory
3265	* inside the following two functions.
3266	*/
3267	static inline void memcg_stop_kmem_account(void)
3268	{
3269	VM_BUG_ON(!current->mm);
3270	current->memcg_kmem_skip_account++;
3271	}
3272
3273	static inline void memcg_resume_kmem_account(void)
3274	{
3275	VM_BUG_ON(!current->mm);
3276	current->memcg_kmem_skip_account--;
3277	}
3278
3279	static void kmem_cache_destroy_work_func(struct work_struct *w)
3280	{
3281	struct kmem_cache *cachep;
3282	struct memcg_cache_params *p;
3283
3284	p = container_of(w, struct memcg_cache_params, destroy);
3285
3286	cachep = memcg_params_to_cache(p);
3287
3288	/*
3289	* If we get down to 0 after shrink, we could delete right away.
3290	* However, memcg_release_pages() already puts us back in the workqueue
3291	* in that case. If we proceed deleting, we'll get a dangling
3292	* reference, and removing the object from the workqueue in that case
3293	* is unnecessary complication. We are not a fast path.
3294	*
3295	* Note that this case is fundamentally different from racing with
3296	* shrink_slab(): if memcg_cgroup_destroy_cache() is called in
3297	* kmem_cache_shrink, not only we would be reinserting a dead cache
3298	* into the queue, but doing so from inside the worker racing to
3299	* destroy it.
3300	*
3301	* So if we aren't down to zero, we'll just schedule a worker and try
3302	* again
3303	*/
3304	if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
3305	kmem_cache_shrink(cachep);
3306	if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
3307	return;
3308	} else
3309	kmem_cache_destroy(cachep);
3310	}
3311
3312	void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
3313	{
3314	if (!cachep->memcg_params->dead)
3315	return;
3316
3317	/*
3318	* There are many ways in which we can get here.
3319	*
3320	* We can get to a memory-pressure situation while the delayed work is
3321	* still pending to run. The vmscan shrinkers can then release all
3322	* cache memory and get us to destruction. If this is the case, we'll
3323	* be executed twice, which is a bug (the second time will execute over
3324	* bogus data). In this case, cancelling the work should be fine.
3325	*
3326	* But we can also get here from the worker itself, if
3327	* kmem_cache_shrink is enough to shake all the remaining objects and
3328	* get the page count to 0. In this case, we'll deadlock if we try to
3329	* cancel the work (the worker runs with an internal lock held, which
3330	* is the same lock we would hold for cancel_work_sync().)
3331	*
3332	* Since we can't possibly know who got us here, just refrain from
3333	* running if there is already work pending
3334	*/
3335	if (work_pending(&cachep->memcg_params->destroy))
3336	return;
3337	/*
3338	* We have to defer the actual destroying to a workqueue, because
3339	* we might currently be in a context that cannot sleep.
3340	*/
3341	schedule_work(&cachep->memcg_params->destroy);
3342	}
3343
3344	/*
3345	* This lock protects updaters, not readers. We want readers to be as fast as
3346	* they can, and they will either see NULL or a valid cache value. Our model
3347	* allow them to see NULL, in which case the root memcg will be selected.
3348	*
3349	* We need this lock because multiple allocations to the same cache from a non
3350	* will span more than one worker. Only one of them can create the cache.
3351	*/
3352	static DEFINE_MUTEX(memcg_cache_mutex);
3353
3354	/*
3355	* Called with memcg_cache_mutex held
3356	*/
3357	static struct kmem_cache kmem_cache_dup(struct mem_cgroup memcg,
3358	struct kmem_cache *s)
3359	{
3360	struct kmem_cache *new;
3361	static char *tmp_name = NULL;
3362
3363	lockdep_assert_held(&memcg_cache_mutex);
3364
3365	/*
3366	* kmem_cache_create_memcg duplicates the given name and
3367	* cgroup_name for this name requires RCU context.
3368	* This static temporary buffer is used to prevent from
3369	* pointless shortliving allocation.
3370	*/
3371	if (!tmp_name) {
3372	tmp_name = kmalloc(PATH_MAX, GFP_KERNEL);
3373	if (!tmp_name)
3374	return NULL;
3375	}
3376
3377	rcu_read_lock();
3378	snprintf(tmp_name, PATH_MAX, "%s(%d:%s)", s->name,
3379	memcg_cache_id(memcg), cgroup_name(memcg->css.cgroup));
3380	rcu_read_unlock();
3381
3382	new = kmem_cache_create_memcg(memcg, tmp_name, s->object_size, s->align,
3383	(s->flags & ~SLAB_PANIC), s->ctor, s);
3384
3385	if (new)
3386	new->allocflags \|= __GFP_KMEMCG;
3387
3388	return new;
3389	}
3390
3391	static struct kmem_cache memcg_create_kmem_cache(struct mem_cgroup memcg,
3392	struct kmem_cache *cachep)
3393	{
3394	struct kmem_cache *new_cachep;
3395	int idx;
3396
3397	BUG_ON(!memcg_can_account_kmem(memcg));
3398
3399	idx = memcg_cache_id(memcg);
3400
3401	mutex_lock(&memcg_cache_mutex);
3402	new_cachep = cachep->memcg_params->memcg_caches[idx];
3403	if (new_cachep) {
3404	css_put(&memcg->css);
3405	goto out;
3406	}
3407
3408	new_cachep = kmem_cache_dup(memcg, cachep);
3409	if (new_cachep == NULL) {
3410	new_cachep = cachep;
3411	css_put(&memcg->css);
3412	goto out;
3413	}
3414
3415	atomic_set(&new_cachep->memcg_params->nr_pages , 0);
3416
3417	cachep->memcg_params->memcg_caches[idx] = new_cachep;
3418	/*
3419	* the readers won't lock, make sure everybody sees the updated value,
3420	* so they won't put stuff in the queue again for no reason
3421	*/
3422	wmb();
3423	out:
3424	mutex_unlock(&memcg_cache_mutex);
3425	return new_cachep;
3426	}
3427
3428	void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
3429	{
3430	struct kmem_cache *c;
3431	int i;
3432
3433	if (!s->memcg_params)
3434	return;
3435	if (!s->memcg_params->is_root_cache)
3436	return;
3437
3438	/*
3439	* If the cache is being destroyed, we trust that there is no one else
3440	* requesting objects from it. Even if there are, the sanity checks in
3441	* kmem_cache_destroy should caught this ill-case.
3442	*
3443	* Still, we don't want anyone else freeing memcg_caches under our
3444	* noses, which can happen if a new memcg comes to life. As usual,
3445	* we'll take the set_limit_mutex to protect ourselves against this.
3446	*/
3447	mutex_lock(&set_limit_mutex);
3448	for (i = 0; i < memcg_limited_groups_array_size; i++) {
3449	c = s->memcg_params->memcg_caches[i];
3450	if (!c)
3451	continue;
3452
3453	/*
3454	* We will now manually delete the caches, so to avoid races
3455	* we need to cancel all pending destruction workers and
3456	* proceed with destruction ourselves.
3457	*
3458	* kmem_cache_destroy() will call kmem_cache_shrink internally,
3459	* and that could spawn the workers again: it is likely that
3460	* the cache still have active pages until this very moment.
3461	* This would lead us back to mem_cgroup_destroy_cache.
3462	*
3463	* But that will not execute at all if the "dead" flag is not
3464	* set, so flip it down to guarantee we are in control.
3465	*/
3466	c->memcg_params->dead = false;
3467	cancel_work_sync(&c->memcg_params->destroy);
3468	kmem_cache_destroy(c);
3469	}
3470	mutex_unlock(&set_limit_mutex);
3471	}
3472
3473	struct create_work {
3474	struct mem_cgroup *memcg;
3475	struct kmem_cache *cachep;
3476	struct work_struct work;
3477	};
3478
3479	static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3480	{
3481	struct kmem_cache *cachep;
3482	struct memcg_cache_params *params;
3483
3484	if (!memcg_kmem_is_active(memcg))
3485	return;
3486
3487	mutex_lock(&memcg->slab_caches_mutex);
3488	list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
3489	cachep = memcg_params_to_cache(params);
3490	cachep->memcg_params->dead = true;
3491	schedule_work(&cachep->memcg_params->destroy);
3492	}
3493	mutex_unlock(&memcg->slab_caches_mutex);
3494	}
3495
3496	static void memcg_create_cache_work_func(struct work_struct *w)
3497	{
3498	struct create_work *cw;
3499
3500	cw = container_of(w, struct create_work, work);
3501	memcg_create_kmem_cache(cw->memcg, cw->cachep);
3502	kfree(cw);
3503	}
3504
3505	/*
3506	* Enqueue the creation of a per-memcg kmem_cache.
3507	*/
3508	static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3509	struct kmem_cache *cachep)
3510	{
3511	struct create_work *cw;
3512
3513	cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
3514	if (cw == NULL) {
3515	css_put(&memcg->css);
3516	return;
3517	}
3518
3519	cw->memcg = memcg;
3520	cw->cachep = cachep;
3521
3522	INIT_WORK(&cw->work, memcg_create_cache_work_func);
3523	schedule_work(&cw->work);
3524	}
3525
3526	static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3527	struct kmem_cache *cachep)
3528	{
3529	/*
3530	* We need to stop accounting when we kmalloc, because if the
3531	* corresponding kmalloc cache is not yet created, the first allocation
3532	* in __memcg_create_cache_enqueue will recurse.
3533	*
3534	* However, it is better to enclose the whole function. Depending on
3535	* the debugging options enabled, INIT_WORK(), for instance, can
3536	* trigger an allocation. This too, will make us recurse. Because at
3537	* this point we can't allow ourselves back into memcg_kmem_get_cache,
3538	* the safest choice is to do it like this, wrapping the whole function.
3539	*/
3540	memcg_stop_kmem_account();
3541	__memcg_create_cache_enqueue(memcg, cachep);
3542	memcg_resume_kmem_account();
3543	}
3544	/*
3545	* Return the kmem_cache we're supposed to use for a slab allocation.
3546	* We try to use the current memcg's version of the cache.
3547	*
3548	* If the cache does not exist yet, if we are the first user of it,
3549	* we either create it immediately, if possible, or create it asynchronously
3550	* in a workqueue.
3551	* In the latter case, we will let the current allocation go through with
3552	* the original cache.
3553	*
3554	* Can't be called in interrupt context or from kernel threads.
3555	* This function needs to be called with rcu_read_lock() held.
3556	*/
3557	struct kmem_cache __memcg_kmem_get_cache(struct kmem_cache cachep,
3558	gfp_t gfp)
3559	{
3560	struct mem_cgroup *memcg;
3561	int idx;
3562
3563	VM_BUG_ON(!cachep->memcg_params);
3564	VM_BUG_ON(!cachep->memcg_params->is_root_cache);
3565
3566	if (!current->mm \|\| current->memcg_kmem_skip_account)
3567	return cachep;
3568
3569	rcu_read_lock();
3570	memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
3571
3572	if (!memcg_can_account_kmem(memcg))
3573	goto out;
3574
3575	idx = memcg_cache_id(memcg);
3576
3577	/*
3578	* barrier to mare sure we're always seeing the up to date value. The
3579	* code updating memcg_caches will issue a write barrier to match this.
3580	*/
3581	read_barrier_depends();
3582	if (likely(cachep->memcg_params->memcg_caches[idx])) {
3583	cachep = cachep->memcg_params->memcg_caches[idx];
3584	goto out;
3585	}
3586
3587	/* The corresponding put will be done in the workqueue. */
3588	if (!css_tryget(&memcg->css))
3589	goto out;
3590	rcu_read_unlock();
3591
3592	/*
3593	* If we are in a safe context (can wait, and not in interrupt
3594	* context), we could be be predictable and return right away.
3595	* This would guarantee that the allocation being performed
3596	* already belongs in the new cache.
3597	*
3598	* However, there are some clashes that can arrive from locking.
3599	* For instance, because we acquire the slab_mutex while doing
3600	* kmem_cache_dup, this means no further allocation could happen
3601	* with the slab_mutex held.
3602	*
3603	* Also, because cache creation issue get_online_cpus(), this
3604	* creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
3605	* that ends up reversed during cpu hotplug. (cpuset allocates
3606	* a bunch of GFP_KERNEL memory during cpuup). Due to all that,
3607	* better to defer everything.
3608	*/
3609	memcg_create_cache_enqueue(memcg, cachep);
3610	return cachep;
3611	out:
3612	rcu_read_unlock();
3613	return cachep;
3614	}
3615	EXPORT_SYMBOL(__memcg_kmem_get_cache);
3616
3617	/*
3618	* We need to verify if the allocation against current->mm->owner's memcg is
3619	* possible for the given order. But the page is not allocated yet, so we'll
3620	* need a further commit step to do the final arrangements.
3621	*
3622	* It is possible for the task to switch cgroups in this mean time, so at
3623	* commit time, we can't rely on task conversion any longer. We'll then use
3624	* the handle argument to return to the caller which cgroup we should commit
3625	* against. We could also return the memcg directly and avoid the pointer
3626	* passing, but a boolean return value gives better semantics considering
3627	* the compiled-out case as well.
3628	*
3629	* Returning true means the allocation is possible.
3630	*/
3631	bool
3632	__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
3633	{
3634	struct mem_cgroup *memcg;
3635	int ret;
3636
3637	*_memcg = NULL;
3638
3639	/*
3640	* Disabling accounting is only relevant for some specific memcg
3641	* internal allocations. Therefore we would initially not have such
3642	* check here, since direct calls to the page allocator that are marked
3643	* with GFP_KMEMCG only happen outside memcg core. We are mostly
3644	* concerned with cache allocations, and by having this test at
3645	* memcg_kmem_get_cache, we are already able to relay the allocation to
3646	* the root cache and bypass the memcg cache altogether.
3647	*
3648	* There is one exception, though: the SLUB allocator does not create
3649	* large order caches, but rather service large kmallocs directly from
3650	* the page allocator. Therefore, the following sequence when backed by
3651	* the SLUB allocator:
3652	*
3653	* memcg_stop_kmem_account();
3654	* kmalloc(<large_number>)
3655	* memcg_resume_kmem_account();
3656	*
3657	* would effectively ignore the fact that we should skip accounting,
3658	* since it will drive us directly to this function without passing
3659	* through the cache selector memcg_kmem_get_cache. Such large
3660	* allocations are extremely rare but can happen, for instance, for the
3661	* cache arrays. We bring this test here.
3662	*/
3663	if (!current->mm \|\| current->memcg_kmem_skip_account)
3664	return true;
3665
3666	memcg = try_get_mem_cgroup_from_mm(current->mm);
3667
3668	/*
3669	* very rare case described in mem_cgroup_from_task. Unfortunately there
3670	* isn't much we can do without complicating this too much, and it would
3671	* be gfp-dependent anyway. Just let it go
3672	*/
3673	if (unlikely(!memcg))
3674	return true;
3675
3676	if (!memcg_can_account_kmem(memcg)) {
3677	css_put(&memcg->css);
3678	return true;
3679	}
3680
3681	ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
3682	if (!ret)
3683	*_memcg = memcg;
3684
3685	css_put(&memcg->css);
3686	return (ret == 0);
3687	}
3688
3689	void __memcg_kmem_commit_charge(struct page page, struct mem_cgroup memcg,
3690	int order)
3691	{
3692	struct page_cgroup *pc;
3693
3694	VM_BUG_ON(mem_cgroup_is_root(memcg));
3695
3696	/* The page allocation failed. Revert */
3697	if (!page) {
3698	memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3699	return;
3700	}
3701
3702	pc = lookup_page_cgroup(page);
3703	lock_page_cgroup(pc);
3704	pc->mem_cgroup = memcg;
3705	SetPageCgroupUsed(pc);
3706	unlock_page_cgroup(pc);
3707	}
3708
3709	void __memcg_kmem_uncharge_pages(struct page *page, int order)
3710	{
3711	struct mem_cgroup *memcg = NULL;
3712	struct page_cgroup *pc;
3713
3714
3715	pc = lookup_page_cgroup(page);
3716	/*
3717	* Fast unlocked return. Theoretically might have changed, have to
3718	* check again after locking.
3719	*/
3720	if (!PageCgroupUsed(pc))
3721	return;
3722
3723	lock_page_cgroup(pc);
3724	if (PageCgroupUsed(pc)) {
3725	memcg = pc->mem_cgroup;
3726	ClearPageCgroupUsed(pc);
3727	}
3728	unlock_page_cgroup(pc);
3729
3730	/*
3731	* We trust that only if there is a memcg associated with the page, it
3732	* is a valid allocation
3733	*/
3734	if (!memcg)
3735	return;
3736
3737	VM_BUG_ON(mem_cgroup_is_root(memcg));
3738	memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3739	}
3740	#else
3741	static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3742	{
3743	}
3744	#endif /* CONFIG_MEMCG_KMEM */
3745
3746	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3747
3748	#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK \| 1 << PCG_MIGRATION)
3749	/*
3750	* Because tail pages are not marked as "used", set it. We're under
3751	* zone->lru_lock, 'splitting on pmd' and compound_lock.
3752	* charge/uncharge will be never happen and move_account() is done under
3753	* compound_lock(), so we don't have to take care of races.
3754	*/
3755	void mem_cgroup_split_huge_fixup(struct page *head)
3756	{
3757	struct page_cgroup *head_pc = lookup_page_cgroup(head);
3758	struct page_cgroup *pc;
3759	struct mem_cgroup *memcg;
3760	int i;
3761
3762	if (mem_cgroup_disabled())
3763	return;
3764
3765	memcg = head_pc->mem_cgroup;
3766	for (i = 1; i < HPAGE_PMD_NR; i++) {
3767	pc = head_pc + i;
3768	pc->mem_cgroup = memcg;
3769	smp_wmb();/* see __commit_charge() */
3770	pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
3771	}
3772	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
3773	HPAGE_PMD_NR);
3774	}
3775	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3776
3777	static inline
3778	void mem_cgroup_move_account_page_stat(struct mem_cgroup *from,
3779	struct mem_cgroup *to,
3780	unsigned int nr_pages,
3781	enum mem_cgroup_stat_index idx)
3782	{
3783	/* Update stat data for mem_cgroup */
3784	preempt_disable();
3785	__this_cpu_sub(from->stat->count[idx], nr_pages);
3786	__this_cpu_add(to->stat->count[idx], nr_pages);
3787	preempt_enable();
3788	}
3789
3790	/**
3791	* mem_cgroup_move_account - move account of the page
3792	* @page: the page
3793	* @nr_pages: number of regular pages (>1 for huge pages)
3794	* @pc: page_cgroup of the page.
3795	* @from: mem_cgroup which the page is moved from.
3796	* @to: mem_cgroup which the page is moved to. @from != @to.
3797	*
3798	* The caller must confirm following.
3799	* - page is not on LRU (isolate_page() is useful.)
3800	* - compound_lock is held when nr_pages > 1
3801	*
3802	* This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
3803	* from old cgroup.
3804	*/
3805	static int mem_cgroup_move_account(struct page *page,
3806	unsigned int nr_pages,
3807	struct page_cgroup *pc,
3808	struct mem_cgroup *from,
3809	struct mem_cgroup *to)
3810	{
3811	unsigned long flags;
3812	int ret;
3813	bool anon = PageAnon(page);
3814
3815	VM_BUG_ON(from == to);
3816	VM_BUG_ON(PageLRU(page));
3817	/*
3818	* The page is isolated from LRU. So, collapse function
3819	* will not handle this page. But page splitting can happen.
3820	* Do this check under compound_page_lock(). The caller should
3821	* hold it.
3822	*/
3823	ret = -EBUSY;
3824	if (nr_pages > 1 && !PageTransHuge(page))
3825	goto out;
3826
3827	lock_page_cgroup(pc);
3828
3829	ret = -EINVAL;
3830	if (!PageCgroupUsed(pc) \|\| pc->mem_cgroup != from)
3831	goto unlock;
3832
3833	move_lock_mem_cgroup(from, &flags);
3834
3835	if (!anon && page_mapped(page))
3836	mem_cgroup_move_account_page_stat(from, to, nr_pages,
3837	MEM_CGROUP_STAT_FILE_MAPPED);
3838
3839	if (PageWriteback(page))
3840	mem_cgroup_move_account_page_stat(from, to, nr_pages,
3841	MEM_CGROUP_STAT_WRITEBACK);
3842
3843	mem_cgroup_charge_statistics(from, page, anon, -nr_pages);
3844
3845	/* caller should have done css_get */
3846	pc->mem_cgroup = to;
3847	mem_cgroup_charge_statistics(to, page, anon, nr_pages);
3848	move_unlock_mem_cgroup(from, &flags);
3849	ret = 0;
3850	unlock:
3851	unlock_page_cgroup(pc);
3852	/*
3853	* check events
3854	*/
3855	memcg_check_events(to, page);
3856	memcg_check_events(from, page);
3857	out:
3858	return ret;
3859	}
3860
3861	/**
3862	* mem_cgroup_move_parent - moves page to the parent group
3863	* @page: the page to move
3864	* @pc: page_cgroup of the page
3865	* @child: page's cgroup
3866	*
3867	* move charges to its parent or the root cgroup if the group has no
3868	* parent (aka use_hierarchy==0).
3869	* Although this might fail (get_page_unless_zero, isolate_lru_page or
3870	* mem_cgroup_move_account fails) the failure is always temporary and
3871	* it signals a race with a page removal/uncharge or migration. In the
3872	* first case the page is on the way out and it will vanish from the LRU
3873	* on the next attempt and the call should be retried later.
3874	* Isolation from the LRU fails only if page has been isolated from
3875	* the LRU since we looked at it and that usually means either global
3876	* reclaim or migration going on. The page will either get back to the
3877	* LRU or vanish.
3878	* Finaly mem_cgroup_move_account fails only if the page got uncharged
3879	* (!PageCgroupUsed) or moved to a different group. The page will
3880	* disappear in the next attempt.
3881	*/
3882	static int mem_cgroup_move_parent(struct page *page,
3883	struct page_cgroup *pc,
3884	struct mem_cgroup *child)
3885	{
3886	struct mem_cgroup *parent;
3887	unsigned int nr_pages;
3888	unsigned long uninitialized_var(flags);
3889	int ret;
3890
3891	VM_BUG_ON(mem_cgroup_is_root(child));
3892
3893	ret = -EBUSY;
3894	if (!get_page_unless_zero(page))
3895	goto out;
3896	if (isolate_lru_page(page))
3897	goto put;
3898
3899	nr_pages = hpage_nr_pages(page);
3900
3901	parent = parent_mem_cgroup(child);
3902	/*
3903	* If no parent, move charges to root cgroup.
3904	*/
3905	if (!parent)
3906	parent = root_mem_cgroup;
3907
3908	if (nr_pages > 1) {
3909	VM_BUG_ON(!PageTransHuge(page));
3910	flags = compound_lock_irqsave(page);
3911	}
3912
3913	ret = mem_cgroup_move_account(page, nr_pages,
3914	pc, child, parent);
3915	if (!ret)
3916	__mem_cgroup_cancel_local_charge(child, nr_pages);
3917
3918	if (nr_pages > 1)
3919	compound_unlock_irqrestore(page, flags);
3920	putback_lru_page(page);
3921	put:
3922	put_page(page);
3923	out:
3924	return ret;
3925	}
3926
3927	/*
3928	* Charge the memory controller for page usage.
3929	* Return
3930	* 0 if the charge was successful
3931	* < 0 if the cgroup is over its limit
3932	*/
3933	static int mem_cgroup_charge_common(struct page page, struct mm_struct mm,
3934	gfp_t gfp_mask, enum charge_type ctype)
3935	{
3936	struct mem_cgroup *memcg = NULL;
3937	unsigned int nr_pages = 1;
3938	bool oom = true;
3939	int ret;
3940
3941	if (PageTransHuge(page)) {
3942	nr_pages <<= compound_order(page);
3943	VM_BUG_ON(!PageTransHuge(page));
3944	/*
3945	* Never OOM-kill a process for a huge page. The
3946	* fault handler will fall back to regular pages.
3947	*/
3948	oom = false;
3949	}
3950
3951	ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
3952	if (ret == -ENOMEM)
3953	return ret;
3954	__mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
3955	return 0;
3956	}
3957
3958	int mem_cgroup_newpage_charge(struct page *page,
3959	struct mm_struct *mm, gfp_t gfp_mask)
3960	{
3961	if (mem_cgroup_disabled())
3962	return 0;
3963	VM_BUG_ON(page_mapped(page));
3964	VM_BUG_ON(page->mapping && !PageAnon(page));
3965	VM_BUG_ON(!mm);
3966	return mem_cgroup_charge_common(page, mm, gfp_mask,
3967	MEM_CGROUP_CHARGE_TYPE_ANON);
3968	}
3969
3970	/*
3971	* While swap-in, try_charge -> commit or cancel, the page is locked.
3972	* And when try_charge() successfully returns, one refcnt to memcg without
3973	* struct page_cgroup is acquired. This refcnt will be consumed by
3974	* "commit()" or removed by "cancel()"
3975	*/
3976	static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
3977	struct page *page,
3978	gfp_t mask,
3979	struct mem_cgroup **memcgp)
3980	{
3981	struct mem_cgroup *memcg;
3982	struct page_cgroup *pc;
3983	int ret;
3984
3985	pc = lookup_page_cgroup(page);
3986	/*
3987	* Every swap fault against a single page tries to charge the
3988	* page, bail as early as possible. shmem_unuse() encounters
3989	* already charged pages, too. The USED bit is protected by
3990	* the page lock, which serializes swap cache removal, which
3991	* in turn serializes uncharging.
3992	*/
3993	if (PageCgroupUsed(pc))
3994	return 0;
3995	if (!do_swap_account)
3996	goto charge_cur_mm;
3997	memcg = try_get_mem_cgroup_from_page(page);
3998	if (!memcg)
3999	goto charge_cur_mm;
4000	*memcgp = memcg;
4001	ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
4002	css_put(&memcg->css);
4003	if (ret == -EINTR)
4004	ret = 0;
4005	return ret;
4006	charge_cur_mm:
4007	ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
4008	if (ret == -EINTR)
4009	ret = 0;
4010	return ret;
4011	}
4012
4013	int mem_cgroup_try_charge_swapin(struct mm_struct mm, struct page page,
4014	gfp_t gfp_mask, struct mem_cgroup **memcgp)
4015	{
4016	*memcgp = NULL;
4017	if (mem_cgroup_disabled())
4018	return 0;
4019	/*
4020	* A racing thread's fault, or swapoff, may have already
4021	* updated the pte, and even removed page from swap cache: in
4022	* those cases unuse_pte()'s pte_same() test will fail; but
4023	* there's also a KSM case which does need to charge the page.
4024	*/
4025	if (!PageSwapCache(page)) {
4026	int ret;
4027
4028	ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
4029	if (ret == -EINTR)
4030	ret = 0;
4031	return ret;
4032	}
4033	return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
4034	}
4035
4036	void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
4037	{
4038	if (mem_cgroup_disabled())
4039	return;
4040	if (!memcg)
4041	return;
4042	__mem_cgroup_cancel_charge(memcg, 1);
4043	}
4044
4045	static void
4046	__mem_cgroup_commit_charge_swapin(struct page page, struct mem_cgroup memcg,
4047	enum charge_type ctype)
4048	{
4049	if (mem_cgroup_disabled())
4050	return;
4051	if (!memcg)
4052	return;
4053
4054	__mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
4055	/*
4056	* Now swap is on-memory. This means this page may be
4057	* counted both as mem and swap....double count.
4058	* Fix it by uncharging from memsw. Basically, this SwapCache is stable
4059	* under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
4060	* may call delete_from_swap_cache() before reach here.
4061	*/
4062	if (do_swap_account && PageSwapCache(page)) {
4063	swp_entry_t ent = {.val = page_private(page)};
4064	mem_cgroup_uncharge_swap(ent);
4065	}
4066	}
4067
4068	void mem_cgroup_commit_charge_swapin(struct page *page,
4069	struct mem_cgroup *memcg)
4070	{
4071	__mem_cgroup_commit_charge_swapin(page, memcg,
4072	MEM_CGROUP_CHARGE_TYPE_ANON);
4073	}
4074
4075	int mem_cgroup_cache_charge(struct page page, struct mm_struct mm,
4076	gfp_t gfp_mask)
4077	{
4078	struct mem_cgroup *memcg = NULL;
4079	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4080	int ret;
4081
4082	if (mem_cgroup_disabled())
4083	return 0;
4084	if (PageCompound(page))
4085	return 0;
4086
4087	if (!PageSwapCache(page))
4088	ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
4089	else { /* page is swapcache/shmem */
4090	ret = __mem_cgroup_try_charge_swapin(mm, page,
4091	gfp_mask, &memcg);
4092	if (!ret)
4093	__mem_cgroup_commit_charge_swapin(page, memcg, type);
4094	}
4095	return ret;
4096	}
4097
4098	static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
4099	unsigned int nr_pages,
4100	const enum charge_type ctype)
4101	{
4102	struct memcg_batch_info *batch = NULL;
4103	bool uncharge_memsw = true;
4104
4105	/* If swapout, usage of swap doesn't decrease */
4106	if (!do_swap_account \|\| ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
4107	uncharge_memsw = false;
4108
4109	batch = &current->memcg_batch;
4110	/*
4111	* In usual, we do css_get() when we remember memcg pointer.
4112	* But in this case, we keep res->usage until end of a series of
4113	* uncharges. Then, it's ok to ignore memcg's refcnt.
4114	*/
4115	if (!batch->memcg)
4116	batch->memcg = memcg;
4117	/*
4118	* do_batch > 0 when unmapping pages or inode invalidate/truncate.
4119	* In those cases, all pages freed continuously can be expected to be in
4120	* the same cgroup and we have chance to coalesce uncharges.
4121	* But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
4122	* because we want to do uncharge as soon as possible.
4123	*/
4124
4125	if (!batch->do_batch \|\| test_thread_flag(TIF_MEMDIE))
4126	goto direct_uncharge;
4127
4128	if (nr_pages > 1)
4129	goto direct_uncharge;
4130
4131	/*
4132	* In typical case, batch->memcg == mem. This means we can
4133	* merge a series of uncharges to an uncharge of res_counter.
4134	* If not, we uncharge res_counter ony by one.
4135	*/
4136	if (batch->memcg != memcg)
4137	goto direct_uncharge;
4138	/* remember freed charge and uncharge it later */
4139	batch->nr_pages++;
4140	if (uncharge_memsw)
4141	batch->memsw_nr_pages++;
4142	return;
4143	direct_uncharge:
4144	res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
4145	if (uncharge_memsw)
4146	res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
4147	if (unlikely(batch->memcg != memcg))
4148	memcg_oom_recover(memcg);
4149	}
4150
4151	/*
4152	* uncharge if !page_mapped(page)
4153	*/
4154	static struct mem_cgroup *
4155	__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
4156	bool end_migration)
4157	{
4158	struct mem_cgroup *memcg = NULL;
4159	unsigned int nr_pages = 1;
4160	struct page_cgroup *pc;
4161	bool anon;
4162
4163	if (mem_cgroup_disabled())
4164	return NULL;
4165
4166	if (PageTransHuge(page)) {
4167	nr_pages <<= compound_order(page);
4168	VM_BUG_ON(!PageTransHuge(page));
4169	}
4170	/*
4171	* Check if our page_cgroup is valid
4172	*/
4173	pc = lookup_page_cgroup(page);
4174	if (unlikely(!PageCgroupUsed(pc)))
4175	return NULL;
4176
4177	lock_page_cgroup(pc);
4178
4179	memcg = pc->mem_cgroup;
4180
4181	if (!PageCgroupUsed(pc))
4182	goto unlock_out;
4183
4184	anon = PageAnon(page);
4185
4186	switch (ctype) {
4187	case MEM_CGROUP_CHARGE_TYPE_ANON:
4188	/*
4189	* Generally PageAnon tells if it's the anon statistics to be
4190	* updated; but sometimes e.g. mem_cgroup_uncharge_page() is
4191	* used before page reached the stage of being marked PageAnon.
4192	*/
4193	anon = true;
4194	/* fallthrough */
4195	case MEM_CGROUP_CHARGE_TYPE_DROP:
4196	/* See mem_cgroup_prepare_migration() */
4197	if (page_mapped(page))
4198	goto unlock_out;
4199	/*
4200	* Pages under migration may not be uncharged. But
4201	* end_migration() /must/ be the one uncharging the
4202	* unused post-migration page and so it has to call
4203	* here with the migration bit still set. See the
4204	* res_counter handling below.
4205	*/
4206	if (!end_migration && PageCgroupMigration(pc))
4207	goto unlock_out;
4208	break;
4209	case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
4210	if (!PageAnon(page)) { /* Shared memory */
4211	if (page->mapping && !page_is_file_cache(page))
4212	goto unlock_out;
4213	} else if (page_mapped(page)) /* Anon */
4214	goto unlock_out;
4215	break;
4216	default:
4217	break;
4218	}
4219
4220	mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages);
4221
4222	ClearPageCgroupUsed(pc);
4223	/*
4224	* pc->mem_cgroup is not cleared here. It will be accessed when it's
4225	* freed from LRU. This is safe because uncharged page is expected not
4226	* to be reused (freed soon). Exception is SwapCache, it's handled by
4227	* special functions.
4228	*/
4229
4230	unlock_page_cgroup(pc);
4231	/*
4232	* even after unlock, we have memcg->res.usage here and this memcg
4233	* will never be freed, so it's safe to call css_get().
4234	*/
4235	memcg_check_events(memcg, page);
4236	if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
4237	mem_cgroup_swap_statistics(memcg, true);
4238	css_get(&memcg->css);
4239	}
4240	/*
4241	* Migration does not charge the res_counter for the
4242	* replacement page, so leave it alone when phasing out the
4243	* page that is unused after the migration.
4244	*/
4245	if (!end_migration && !mem_cgroup_is_root(memcg))
4246	mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
4247
4248	return memcg;
4249
4250	unlock_out:
4251	unlock_page_cgroup(pc);
4252	return NULL;
4253	}
4254
4255	void mem_cgroup_uncharge_page(struct page *page)
4256	{
4257	/* early check. */
4258	if (page_mapped(page))
4259	return;
4260	VM_BUG_ON(page->mapping && !PageAnon(page));
4261	/*
4262	* If the page is in swap cache, uncharge should be deferred
4263	* to the swap path, which also properly accounts swap usage
4264	* and handles memcg lifetime.
4265	*
4266	* Note that this check is not stable and reclaim may add the
4267	* page to swap cache at any time after this. However, if the
4268	* page is not in swap cache by the time page->mapcount hits
4269	* 0, there won't be any page table references to the swap
4270	* slot, and reclaim will free it and not actually write the
4271	* page to disk.
4272	*/
4273	if (PageSwapCache(page))
4274	return;
4275	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
4276	}
4277
4278	void mem_cgroup_uncharge_cache_page(struct page *page)
4279	{
4280	VM_BUG_ON(page_mapped(page));
4281	VM_BUG_ON(page->mapping);
4282	__mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
4283	}
4284
4285	/*
4286	* Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
4287	* In that cases, pages are freed continuously and we can expect pages
4288	* are in the same memcg. All these calls itself limits the number of
4289	* pages freed at once, then uncharge_start/end() is called properly.
4290	* This may be called prural(2) times in a context,
4291	*/
4292
4293	void mem_cgroup_uncharge_start(void)
4294	{
4295	current->memcg_batch.do_batch++;
4296	/* We can do nest. */
4297	if (current->memcg_batch.do_batch == 1) {
4298	current->memcg_batch.memcg = NULL;
4299	current->memcg_batch.nr_pages = 0;
4300	current->memcg_batch.memsw_nr_pages = 0;
4301	}
4302	}
4303
4304	void mem_cgroup_uncharge_end(void)
4305	{
4306	struct memcg_batch_info *batch = &current->memcg_batch;
4307
4308	if (!batch->do_batch)
4309	return;
4310
4311	batch->do_batch--;
4312	if (batch->do_batch) /* If stacked, do nothing. */
4313	return;
4314
4315	if (!batch->memcg)
4316	return;
4317	/*
4318	* This "batch->memcg" is valid without any css_get/put etc...
4319	* bacause we hide charges behind us.
4320	*/
4321	if (batch->nr_pages)
4322	res_counter_uncharge(&batch->memcg->res,
4323	batch->nr_pages * PAGE_SIZE);
4324	if (batch->memsw_nr_pages)
4325	res_counter_uncharge(&batch->memcg->memsw,
4326	batch->memsw_nr_pages * PAGE_SIZE);
4327	memcg_oom_recover(batch->memcg);
4328	/* forget this pointer (for sanity check) */
4329	batch->memcg = NULL;
4330	}
4331
4332	#ifdef CONFIG_SWAP
4333	/*
4334	* called after __delete_from_swap_cache() and drop "page" account.
4335	* memcg information is recorded to swap_cgroup of "ent"
4336	*/
4337	void
4338	mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
4339	{
4340	struct mem_cgroup *memcg;
4341	int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
4342
4343	if (!swapout) /* this was a swap cache but the swap is unused ! */
4344	ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
4345
4346	memcg = __mem_cgroup_uncharge_common(page, ctype, false);
4347
4348	/*
4349	* record memcg information, if swapout && memcg != NULL,
4350	* css_get() was called in uncharge().
4351	*/
4352	if (do_swap_account && swapout && memcg)
4353	swap_cgroup_record(ent, css_id(&memcg->css));
4354	}
4355	#endif
4356
4357	#ifdef CONFIG_MEMCG_SWAP
4358	/*
4359	* called from swap_entry_free(). remove record in swap_cgroup and
4360	* uncharge "memsw" account.
4361	*/
4362	void mem_cgroup_uncharge_swap(swp_entry_t ent)
4363	{
4364	struct mem_cgroup *memcg;
4365	unsigned short id;
4366
4367	if (!do_swap_account)
4368	return;
4369
4370	id = swap_cgroup_record(ent, 0);
4371	rcu_read_lock();
4372	memcg = mem_cgroup_lookup(id);
4373	if (memcg) {
4374	/*
4375	* We uncharge this because swap is freed.
4376	* This memcg can be obsolete one. We avoid calling css_tryget
4377	*/
4378	if (!mem_cgroup_is_root(memcg))
4379	res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
4380	mem_cgroup_swap_statistics(memcg, false);
4381	css_put(&memcg->css);
4382	}
4383	rcu_read_unlock();
4384	}
4385
4386	/**
4387	* mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
4388	* @entry: swap entry to be moved
4389	* @from: mem_cgroup which the entry is moved from
4390	* @to: mem_cgroup which the entry is moved to
4391	*
4392	* It succeeds only when the swap_cgroup's record for this entry is the same
4393	* as the mem_cgroup's id of @from.
4394	*
4395	* Returns 0 on success, -EINVAL on failure.
4396	*
4397	* The caller must have charged to @to, IOW, called res_counter_charge() about
4398	* both res and memsw, and called css_get().
4399	*/
4400	static int mem_cgroup_move_swap_account(swp_entry_t entry,
4401	struct mem_cgroup from, struct mem_cgroup to)
4402	{
4403	unsigned short old_id, new_id;
4404
4405	old_id = css_id(&from->css);
4406	new_id = css_id(&to->css);
4407
4408	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
4409	mem_cgroup_swap_statistics(from, false);
4410	mem_cgroup_swap_statistics(to, true);
4411	/*
4412	* This function is only called from task migration context now.
4413	* It postpones res_counter and refcount handling till the end
4414	* of task migration(mem_cgroup_clear_mc()) for performance
4415	* improvement. But we cannot postpone css_get(to) because if
4416	* the process that has been moved to @to does swap-in, the
4417	* refcount of @to might be decreased to 0.
4418	*
4419	* We are in attach() phase, so the cgroup is guaranteed to be
4420	* alive, so we can just call css_get().
4421	*/
4422	css_get(&to->css);
4423	return 0;
4424	}
4425	return -EINVAL;
4426	}
4427	#else
4428	static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
4429	struct mem_cgroup from, struct mem_cgroup to)
4430	{
4431	return -EINVAL;
4432	}
4433	#endif
4434
4435	/*
4436	* Before starting migration, account PAGE_SIZE to mem_cgroup that the old
4437	* page belongs to.
4438	*/
4439	void mem_cgroup_prepare_migration(struct page page, struct page newpage,
4440	struct mem_cgroup **memcgp)
4441	{
4442	struct mem_cgroup *memcg = NULL;
4443	unsigned int nr_pages = 1;
4444	struct page_cgroup *pc;
4445	enum charge_type ctype;
4446
4447	*memcgp = NULL;
4448
4449	if (mem_cgroup_disabled())
4450	return;
4451
4452	if (PageTransHuge(page))
4453	nr_pages <<= compound_order(page);
4454
4455	pc = lookup_page_cgroup(page);
4456	lock_page_cgroup(pc);
4457	if (PageCgroupUsed(pc)) {
4458	memcg = pc->mem_cgroup;
4459	css_get(&memcg->css);
4460	/*
4461	* At migrating an anonymous page, its mapcount goes down
4462	* to 0 and uncharge() will be called. But, even if it's fully
4463	* unmapped, migration may fail and this page has to be
4464	* charged again. We set MIGRATION flag here and delay uncharge
4465	* until end_migration() is called
4466	*
4467	* Corner Case Thinking
4468	* A)
4469	* When the old page was mapped as Anon and it's unmap-and-freed
4470	* while migration was ongoing.
4471	* If unmap finds the old page, uncharge() of it will be delayed
4472	* until end_migration(). If unmap finds a new page, it's
4473	* uncharged when it make mapcount to be 1->0. If unmap code
4474	* finds swap_migration_entry, the new page will not be mapped
4475	* and end_migration() will find it(mapcount==0).
4476	*
4477	* B)
4478	* When the old page was mapped but migraion fails, the kernel
4479	* remaps it. A charge for it is kept by MIGRATION flag even
4480	* if mapcount goes down to 0. We can do remap successfully
4481	* without charging it again.
4482	*
4483	* C)
4484	* The "old" page is under lock_page() until the end of
4485	* migration, so, the old page itself will not be swapped-out.
4486	* If the new page is swapped out before end_migraton, our
4487	* hook to usual swap-out path will catch the event.
4488	*/
4489	if (PageAnon(page))
4490	SetPageCgroupMigration(pc);
4491	}
4492	unlock_page_cgroup(pc);
4493	/*
4494	* If the page is not charged at this point,
4495	* we return here.
4496	*/
4497	if (!memcg)
4498	return;
4499
4500	*memcgp = memcg;
4501	/*
4502	* We charge new page before it's used/mapped. So, even if unlock_page()
4503	* is called before end_migration, we can catch all events on this new
4504	* page. In the case new page is migrated but not remapped, new page's
4505	* mapcount will be finally 0 and we call uncharge in end_migration().
4506	*/
4507	if (PageAnon(page))
4508	ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
4509	else
4510	ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
4511	/*
4512	* The page is committed to the memcg, but it's not actually
4513	* charged to the res_counter since we plan on replacing the
4514	* old one and only one page is going to be left afterwards.
4515	*/
4516	__mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
4517	}
4518
4519	/* remove redundant charge if migration failed*/
4520	void mem_cgroup_end_migration(struct mem_cgroup *memcg,
4521	struct page oldpage, struct page newpage, bool migration_ok)
4522	{
4523	struct page used, unused;
4524	struct page_cgroup *pc;
4525	bool anon;
4526
4527	if (!memcg)
4528	return;
4529
4530	if (!migration_ok) {
4531	used = oldpage;
4532	unused = newpage;
4533	} else {
4534	used = newpage;
4535	unused = oldpage;
4536	}
4537	anon = PageAnon(used);
4538	__mem_cgroup_uncharge_common(unused,
4539	anon ? MEM_CGROUP_CHARGE_TYPE_ANON
4540	: MEM_CGROUP_CHARGE_TYPE_CACHE,
4541	true);
4542	css_put(&memcg->css);
4543	/*
4544	* We disallowed uncharge of pages under migration because mapcount
4545	* of the page goes down to zero, temporarly.
4546	* Clear the flag and check the page should be charged.
4547	*/
4548	pc = lookup_page_cgroup(oldpage);
4549	lock_page_cgroup(pc);
4550	ClearPageCgroupMigration(pc);
4551	unlock_page_cgroup(pc);
4552
4553	/*
4554	* If a page is a file cache, radix-tree replacement is very atomic
4555	* and we can skip this check. When it was an Anon page, its mapcount
4556	* goes down to 0. But because we added MIGRATION flage, it's not
4557	* uncharged yet. There are several case but page->mapcount check
4558	* and USED bit check in mem_cgroup_uncharge_page() will do enough
4559	* check. (see prepare_charge() also)
4560	*/
4561	if (anon)
4562	mem_cgroup_uncharge_page(used);
4563	}
4564
4565	/*
4566	* At replace page cache, newpage is not under any memcg but it's on
4567	* LRU. So, this function doesn't touch res_counter but handles LRU
4568	* in correct way. Both pages are locked so we cannot race with uncharge.
4569	*/
4570	void mem_cgroup_replace_page_cache(struct page *oldpage,
4571	struct page *newpage)
4572	{
4573	struct mem_cgroup *memcg = NULL;
4574	struct page_cgroup *pc;
4575	enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4576
4577	if (mem_cgroup_disabled())
4578	return;
4579
4580	pc = lookup_page_cgroup(oldpage);
4581	/* fix accounting on old pages */
4582	lock_page_cgroup(pc);
4583	if (PageCgroupUsed(pc)) {
4584	memcg = pc->mem_cgroup;
4585	mem_cgroup_charge_statistics(memcg, oldpage, false, -1);
4586	ClearPageCgroupUsed(pc);
4587	}
4588	unlock_page_cgroup(pc);
4589
4590	/*
4591	* When called from shmem_replace_page(), in some cases the
4592	* oldpage has already been charged, and in some cases not.
4593	*/
4594	if (!memcg)
4595	return;
4596	/*
4597	* Even if newpage->mapping was NULL before starting replacement,
4598	* the newpage may be on LRU(or pagevec for LRU) already. We lock
4599	* LRU while we overwrite pc->mem_cgroup.
4600	*/
4601	__mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
4602	}
4603
4604	#ifdef CONFIG_DEBUG_VM
4605	static struct page_cgroup lookup_page_cgroup_used(struct page page)
4606	{
4607	struct page_cgroup *pc;
4608
4609	pc = lookup_page_cgroup(page);
4610	/*
4611	* Can be NULL while feeding pages into the page allocator for
4612	* the first time, i.e. during boot or memory hotplug;
4613	* or when mem_cgroup_disabled().
4614	*/
4615	if (likely(pc) && PageCgroupUsed(pc))
4616	return pc;
4617	return NULL;
4618	}
4619
4620	bool mem_cgroup_bad_page_check(struct page *page)
4621	{
4622	if (mem_cgroup_disabled())
4623	return false;
4624
4625	return lookup_page_cgroup_used(page) != NULL;
4626	}
4627
4628	void mem_cgroup_print_bad_page(struct page *page)
4629	{
4630	struct page_cgroup *pc;
4631
4632	pc = lookup_page_cgroup_used(page);
4633	if (pc) {
4634	pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
4635	pc, pc->flags, pc->mem_cgroup);
4636	}
4637	}
4638	#endif
4639
4640	static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
4641	unsigned long long val)
4642	{
4643	int retry_count;
4644	u64 memswlimit, memlimit;
4645	int ret = 0;
4646	int children = mem_cgroup_count_children(memcg);
4647	u64 curusage, oldusage;
4648	int enlarge;
4649
4650	/*
4651	* For keeping hierarchical_reclaim simple, how long we should retry
4652	* is depends on callers. We set our retry-count to be function
4653	* of # of children which we should visit in this loop.
4654	*/
4655	retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
4656
4657	oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4658
4659	enlarge = 0;
4660	while (retry_count) {
4661	if (signal_pending(current)) {
4662	ret = -EINTR;
4663	break;
4664	}
4665	/*
4666	* Rather than hide all in some function, I do this in
4667	* open coded manner. You see what this really does.
4668	* We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4669	*/
4670	mutex_lock(&set_limit_mutex);
4671	memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4672	if (memswlimit < val) {
4673	ret = -EINVAL;
4674	mutex_unlock(&set_limit_mutex);
4675	break;
4676	}
4677
4678	memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4679	if (memlimit < val)
4680	enlarge = 1;
4681
4682	ret = res_counter_set_limit(&memcg->res, val);
4683	if (!ret) {
4684	if (memswlimit == val)
4685	memcg->memsw_is_minimum = true;
4686	else
4687	memcg->memsw_is_minimum = false;
4688	}
4689	mutex_unlock(&set_limit_mutex);
4690
4691	if (!ret)
4692	break;
4693
4694	mem_cgroup_reclaim(memcg, GFP_KERNEL,
4695	MEM_CGROUP_RECLAIM_SHRINK);
4696	curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4697	/* Usage is reduced ? */
4698	if (curusage >= oldusage)
4699	retry_count--;
4700	else
4701	oldusage = curusage;
4702	}
4703	if (!ret && enlarge)
4704	memcg_oom_recover(memcg);
4705
4706	return ret;
4707	}
4708
4709	static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
4710	unsigned long long val)
4711	{
4712	int retry_count;
4713	u64 memlimit, memswlimit, oldusage, curusage;
4714	int children = mem_cgroup_count_children(memcg);
4715	int ret = -EBUSY;
4716	int enlarge = 0;
4717
4718	/* see mem_cgroup_resize_res_limit */
4719	retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
4720	oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4721	while (retry_count) {
4722	if (signal_pending(current)) {
4723	ret = -EINTR;
4724	break;
4725	}
4726	/*
4727	* Rather than hide all in some function, I do this in
4728	* open coded manner. You see what this really does.
4729	* We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4730	*/
4731	mutex_lock(&set_limit_mutex);
4732	memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4733	if (memlimit > val) {
4734	ret = -EINVAL;
4735	mutex_unlock(&set_limit_mutex);
4736	break;
4737	}
4738	memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4739	if (memswlimit < val)
4740	enlarge = 1;
4741	ret = res_counter_set_limit(&memcg->memsw, val);
4742	if (!ret) {
4743	if (memlimit == val)
4744	memcg->memsw_is_minimum = true;
4745	else
4746	memcg->memsw_is_minimum = false;
4747	}
4748	mutex_unlock(&set_limit_mutex);
4749
4750	if (!ret)
4751	break;
4752
4753	mem_cgroup_reclaim(memcg, GFP_KERNEL,
4754	MEM_CGROUP_RECLAIM_NOSWAP \|
4755	MEM_CGROUP_RECLAIM_SHRINK);
4756	curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4757	/* Usage is reduced ? */
4758	if (curusage >= oldusage)
4759	retry_count--;
4760	else
4761	oldusage = curusage;
4762	}
4763	if (!ret && enlarge)
4764	memcg_oom_recover(memcg);
4765	return ret;
4766	}
4767
4768	unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4769	gfp_t gfp_mask,
4770	unsigned long *total_scanned)
4771	{
4772	unsigned long nr_reclaimed = 0;
4773	struct mem_cgroup_per_zone mz, next_mz = NULL;
4774	unsigned long reclaimed;
4775	int loop = 0;
4776	struct mem_cgroup_tree_per_zone *mctz;
4777	unsigned long long excess;
4778	unsigned long nr_scanned;
4779
4780	if (order > 0)
4781	return 0;
4782
4783	mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4784	/*
4785	* This loop can run a while, specially if mem_cgroup's continuously
4786	* keep exceeding their soft limit and putting the system under
4787	* pressure
4788	*/
4789	do {
4790	if (next_mz)
4791	mz = next_mz;
4792	else
4793	mz = mem_cgroup_largest_soft_limit_node(mctz);
4794	if (!mz)
4795	break;
4796
4797	nr_scanned = 0;
4798	reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4799	gfp_mask, &nr_scanned);
4800	nr_reclaimed += reclaimed;
4801	*total_scanned += nr_scanned;
4802	spin_lock(&mctz->lock);
4803
4804	/*
4805	* If we failed to reclaim anything from this memory cgroup
4806	* it is time to move on to the next cgroup
4807	*/
4808	next_mz = NULL;
4809	if (!reclaimed) {
4810	do {
4811	/*
4812	* Loop until we find yet another one.
4813	*
4814	* By the time we get the soft_limit lock
4815	* again, someone might have aded the
4816	* group back on the RB tree. Iterate to
4817	* make sure we get a different mem.
4818	* mem_cgroup_largest_soft_limit_node returns
4819	* NULL if no other cgroup is present on
4820	* the tree
4821	*/
4822	next_mz =
4823	__mem_cgroup_largest_soft_limit_node(mctz);
4824	if (next_mz == mz)
4825	css_put(&next_mz->memcg->css);
4826	else /* next_mz == NULL or other memcg */
4827	break;
4828	} while (1);
4829	}
4830	__mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
4831	excess = res_counter_soft_limit_excess(&mz->memcg->res);
4832	/*
4833	* One school of thought says that we should not add
4834	* back the node to the tree if reclaim returns 0.
4835	* But our reclaim could return 0, simply because due
4836	* to priority we are exposing a smaller subset of
4837	* memory to reclaim from. Consider this as a longer
4838	* term TODO.
4839	*/
4840	/* If excess == 0, no tree ops */
4841	__mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4842	spin_unlock(&mctz->lock);
4843	css_put(&mz->memcg->css);
4844	loop++;
4845	/*
4846	* Could not reclaim anything and there are no more
4847	* mem cgroups to try or we seem to be looping without
4848	* reclaiming anything.
4849	*/
4850	if (!nr_reclaimed &&
4851	(next_mz == NULL \|\|
4852	loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
4853	break;
4854	} while (!nr_reclaimed);
4855	if (next_mz)
4856	css_put(&next_mz->memcg->css);
4857	return nr_reclaimed;
4858	}
4859
4860	/**
4861	* mem_cgroup_force_empty_list - clears LRU of a group
4862	* @memcg: group to clear
4863	* @node: NUMA node
4864	* @zid: zone id
4865	* @lru: lru to to clear
4866	*
4867	* Traverse a specified page_cgroup list and try to drop them all. This doesn't
4868	* reclaim the pages page themselves - pages are moved to the parent (or root)
4869	* group.
4870	*/
4871	static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
4872	int node, int zid, enum lru_list lru)
4873	{
4874	struct lruvec *lruvec;
4875	unsigned long flags;
4876	struct list_head *list;
4877	struct page *busy;
4878	struct zone *zone;
4879
4880	zone = &NODE_DATA(node)->node_zones[zid];
4881	lruvec = mem_cgroup_zone_lruvec(zone, memcg);
4882	list = &lruvec->lists[lru];
4883
4884	busy = NULL;
4885	do {
4886	struct page_cgroup *pc;
4887	struct page *page;
4888
4889	spin_lock_irqsave(&zone->lru_lock, flags);
4890	if (list_empty(list)) {
4891	spin_unlock_irqrestore(&zone->lru_lock, flags);
4892	break;
4893	}
4894	page = list_entry(list->prev, struct page, lru);
4895	if (busy == page) {
4896	list_move(&page->lru, list);
4897	busy = NULL;
4898	spin_unlock_irqrestore(&zone->lru_lock, flags);
4899	continue;
4900	}
4901	spin_unlock_irqrestore(&zone->lru_lock, flags);
4902
4903	pc = lookup_page_cgroup(page);
4904
4905	if (mem_cgroup_move_parent(page, pc, memcg)) {
4906	/* found lock contention or "pc" is obsolete. */
4907	busy = page;
4908	cond_resched();
4909	} else
4910	busy = NULL;
4911	} while (!list_empty(list));
4912	}
4913
4914	/*
4915	* make mem_cgroup's charge to be 0 if there is no task by moving
4916	* all the charges and pages to the parent.
4917	* This enables deleting this mem_cgroup.
4918	*
4919	* Caller is responsible for holding css reference on the memcg.
4920	*/
4921	static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
4922	{
4923	int node, zid;
4924	u64 usage;
4925
4926	do {
4927	/* This is for making all used pages to be on LRU. */
4928	lru_add_drain_all();
4929	drain_all_stock_sync(memcg);
4930	mem_cgroup_start_move(memcg);
4931	for_each_node_state(node, N_MEMORY) {
4932	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
4933	enum lru_list lru;
4934	for_each_lru(lru) {
4935	mem_cgroup_force_empty_list(memcg,
4936	node, zid, lru);
4937	}
4938	}
4939	}
4940	mem_cgroup_end_move(memcg);
4941	memcg_oom_recover(memcg);
4942	cond_resched();
4943
4944	/*
4945	* Kernel memory may not necessarily be trackable to a specific
4946	* process. So they are not migrated, and therefore we can't
4947	* expect their value to drop to 0 here.
4948	* Having res filled up with kmem only is enough.
4949	*
4950	* This is a safety check because mem_cgroup_force_empty_list
4951	* could have raced with mem_cgroup_replace_page_cache callers
4952	* so the lru seemed empty but the page could have been added
4953	* right after the check. RES_USAGE should be safe as we always
4954	* charge before adding to the LRU.
4955	*/
4956	usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
4957	res_counter_read_u64(&memcg->kmem, RES_USAGE);
4958	} while (usage > 0);
4959	}
4960
4961	static inline bool memcg_has_children(struct mem_cgroup *memcg)
4962	{
4963	lockdep_assert_held(&memcg_create_mutex);
4964	/*
4965	* The lock does not prevent addition or deletion to the list
4966	* of children, but it prevents a new child from being
4967	* initialized based on this parent in css_online(), so it's
4968	* enough to decide whether hierarchically inherited
4969	* attributes can still be changed or not.
4970	*/
4971	return memcg->use_hierarchy &&
4972	!list_empty(&memcg->css.cgroup->children);
4973	}
4974
4975	/*
4976	* Reclaims as many pages from the given memcg as possible and moves
4977	* the rest to the parent.
4978	*
4979	* Caller is responsible for holding css reference for memcg.
4980	*/
4981	static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
4982	{
4983	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
4984	struct cgroup *cgrp = memcg->css.cgroup;
4985
4986	/* returns EBUSY if there is a task or if we come here twice. */
4987	if (cgroup_task_count(cgrp) \|\| !list_empty(&cgrp->children))
4988	return -EBUSY;
4989
4990	/* we call try-to-free pages for make this cgroup empty */
4991	lru_add_drain_all();
4992	/* try to free all pages in this cgroup */
4993	while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
4994	int progress;
4995
4996	if (signal_pending(current))
4997	return -EINTR;
4998
4999	progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
5000	false);
5001	if (!progress) {
5002	nr_retries--;
5003	/* maybe some writeback is necessary */
5004	congestion_wait(BLK_RW_ASYNC, HZ/10);
5005	}
5006
5007	}
5008	lru_add_drain();
5009	mem_cgroup_reparent_charges(memcg);
5010
5011	return 0;
5012	}
5013
5014	static int mem_cgroup_force_empty_write(struct cgroup_subsys_state *css,
5015	unsigned int event)
5016	{
5017	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5018
5019	if (mem_cgroup_is_root(memcg))
5020	return -EINVAL;
5021	return mem_cgroup_force_empty(memcg);
5022	}
5023
5024	static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
5025	struct cftype *cft)
5026	{
5027	return mem_cgroup_from_css(css)->use_hierarchy;
5028	}
5029
5030	static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
5031	struct cftype *cft, u64 val)
5032	{
5033	int retval = 0;
5034	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5035	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(css_parent(&memcg->css));
5036
5037	mutex_lock(&memcg_create_mutex);
5038
5039	if (memcg->use_hierarchy == val)
5040	goto out;
5041
5042	/*
5043	* If parent's use_hierarchy is set, we can't make any modifications
5044	* in the child subtrees. If it is unset, then the change can
5045	* occur, provided the current cgroup has no children.
5046	*
5047	* For the root cgroup, parent_mem is NULL, we allow value to be
5048	* set if there are no children.
5049	*/
5050	if ((!parent_memcg \|\| !parent_memcg->use_hierarchy) &&
5051	(val == 1 \|\| val == 0)) {
5052	if (list_empty(&memcg->css.cgroup->children))
5053	memcg->use_hierarchy = val;
5054	else
5055	retval = -EBUSY;
5056	} else
5057	retval = -EINVAL;
5058
5059	out:
5060	mutex_unlock(&memcg_create_mutex);
5061
5062	return retval;
5063	}
5064
5065
5066	static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
5067	enum mem_cgroup_stat_index idx)
5068	{
5069	struct mem_cgroup *iter;
5070	long val = 0;
5071
5072	/* Per-cpu values can be negative, use a signed accumulator */
5073	for_each_mem_cgroup_tree(iter, memcg)
5074	val += mem_cgroup_read_stat(iter, idx);
5075
5076	if (val < 0) /* race ? */
5077	val = 0;
5078	return val;
5079	}
5080
5081	static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
5082	{
5083	u64 val;
5084
5085	if (!mem_cgroup_is_root(memcg)) {
5086	if (!swap)
5087	return res_counter_read_u64(&memcg->res, RES_USAGE);
5088	else
5089	return res_counter_read_u64(&memcg->memsw, RES_USAGE);
5090	}
5091
5092	/*
5093	* Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
5094	* as well as in MEM_CGROUP_STAT_RSS_HUGE.
5095	*/
5096	val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
5097	val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
5098
5099	if (swap)
5100	val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
5101
5102	return val << PAGE_SHIFT;
5103	}
5104
5105	static ssize_t mem_cgroup_read(struct cgroup_subsys_state *css,
5106	struct cftype cft, struct file file,
5107	char __user buf, size_t nbytes, loff_t ppos)
5108	{
5109	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5110	char str[64];
5111	u64 val;
5112	int name, len;
5113	enum res_type type;
5114
5115	type = MEMFILE_TYPE(cft->private);
5116	name = MEMFILE_ATTR(cft->private);
5117
5118	switch (type) {
5119	case _MEM:
5120	if (name == RES_USAGE)
5121	val = mem_cgroup_usage(memcg, false);
5122	else
5123	val = res_counter_read_u64(&memcg->res, name);
5124	break;
5125	case _MEMSWAP:
5126	if (name == RES_USAGE)
5127	val = mem_cgroup_usage(memcg, true);
5128	else
5129	val = res_counter_read_u64(&memcg->memsw, name);
5130	break;
5131	case _KMEM:
5132	val = res_counter_read_u64(&memcg->kmem, name);
5133	break;
5134	default:
5135	BUG();
5136	}
5137
5138	len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
5139	return simple_read_from_buffer(buf, nbytes, ppos, str, len);
5140	}
5141
5142	static int memcg_update_kmem_limit(struct cgroup_subsys_state *css, u64 val)
5143	{
5144	int ret = -EINVAL;
5145	#ifdef CONFIG_MEMCG_KMEM
5146	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5147	/*
5148	* For simplicity, we won't allow this to be disabled. It also can't
5149	* be changed if the cgroup has children already, or if tasks had
5150	* already joined.
5151	*
5152	* If tasks join before we set the limit, a person looking at
5153	* kmem.usage_in_bytes will have no way to determine when it took
5154	* place, which makes the value quite meaningless.
5155	*
5156	* After it first became limited, changes in the value of the limit are
5157	* of course permitted.
5158	*/
5159	mutex_lock(&memcg_create_mutex);
5160	mutex_lock(&set_limit_mutex);
5161	if (!memcg->kmem_account_flags && val != RES_COUNTER_MAX) {
5162	if (cgroup_task_count(css->cgroup) \|\| memcg_has_children(memcg)) {
5163	ret = -EBUSY;
5164	goto out;
5165	}
5166	ret = res_counter_set_limit(&memcg->kmem, val);
5167	VM_BUG_ON(ret);
5168
5169	ret = memcg_update_cache_sizes(memcg);
5170	if (ret) {
5171	res_counter_set_limit(&memcg->kmem, RES_COUNTER_MAX);
5172	goto out;
5173	}
5174	static_key_slow_inc(&memcg_kmem_enabled_key);
5175	/*
5176	* setting the active bit after the inc will guarantee no one
5177	* starts accounting before all call sites are patched
5178	*/
5179	memcg_kmem_set_active(memcg);
5180	} else
5181	ret = res_counter_set_limit(&memcg->kmem, val);
5182	out:
5183	mutex_unlock(&set_limit_mutex);
5184	mutex_unlock(&memcg_create_mutex);
5185	#endif
5186	return ret;
5187	}
5188
5189	#ifdef CONFIG_MEMCG_KMEM
5190	static int memcg_propagate_kmem(struct mem_cgroup *memcg)
5191	{
5192	int ret = 0;
5193	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5194	if (!parent)
5195	goto out;
5196
5197	memcg->kmem_account_flags = parent->kmem_account_flags;
5198	/*
5199	* When that happen, we need to disable the static branch only on those
5200	* memcgs that enabled it. To achieve this, we would be forced to
5201	* complicate the code by keeping track of which memcgs were the ones
5202	* that actually enabled limits, and which ones got it from its
5203	* parents.
5204	*
5205	* It is a lot simpler just to do static_key_slow_inc() on every child
5206	* that is accounted.
5207	*/
5208	if (!memcg_kmem_is_active(memcg))
5209	goto out;
5210
5211	/*
5212	* __mem_cgroup_free() will issue static_key_slow_dec() because this
5213	* memcg is active already. If the later initialization fails then the
5214	* cgroup core triggers the cleanup so we do not have to do it here.
5215	*/
5216	static_key_slow_inc(&memcg_kmem_enabled_key);
5217
5218	mutex_lock(&set_limit_mutex);
5219	memcg_stop_kmem_account();
5220	ret = memcg_update_cache_sizes(memcg);
5221	memcg_resume_kmem_account();
5222	mutex_unlock(&set_limit_mutex);
5223	out:
5224	return ret;
5225	}
5226	#endif /* CONFIG_MEMCG_KMEM */
5227
5228	/*
5229	* The user of this function is...
5230	* RES_LIMIT.
5231	*/
5232	static int mem_cgroup_write(struct cgroup_subsys_state css, struct cftype cft,
5233	const char *buffer)
5234	{
5235	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5236	enum res_type type;
5237	int name;
5238	unsigned long long val;
5239	int ret;
5240
5241	type = MEMFILE_TYPE(cft->private);
5242	name = MEMFILE_ATTR(cft->private);
5243
5244	switch (name) {
5245	case RES_LIMIT:
5246	if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
5247	ret = -EINVAL;
5248	break;
5249	}
5250	/* This function does all necessary parse...reuse it */
5251	ret = res_counter_memparse_write_strategy(buffer, &val);
5252	if (ret)
5253	break;
5254	if (type == _MEM)
5255	ret = mem_cgroup_resize_limit(memcg, val);
5256	else if (type == _MEMSWAP)
5257	ret = mem_cgroup_resize_memsw_limit(memcg, val);
5258	else if (type == _KMEM)
5259	ret = memcg_update_kmem_limit(css, val);
5260	else
5261	return -EINVAL;
5262	break;
5263	case RES_SOFT_LIMIT:
5264	ret = res_counter_memparse_write_strategy(buffer, &val);
5265	if (ret)
5266	break;
5267	/*
5268	* For memsw, soft limits are hard to implement in terms
5269	* of semantics, for now, we support soft limits for
5270	* control without swap
5271	*/
5272	if (type == _MEM)
5273	ret = res_counter_set_soft_limit(&memcg->res, val);
5274	else
5275	ret = -EINVAL;
5276	break;
5277	default:
5278	ret = -EINVAL; /* should be BUG() ? */
5279	break;
5280	}
5281	return ret;
5282	}
5283
5284	static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
5285	unsigned long long mem_limit, unsigned long long memsw_limit)
5286	{
5287	unsigned long long min_limit, min_memsw_limit, tmp;
5288
5289	min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
5290	min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5291	if (!memcg->use_hierarchy)
5292	goto out;
5293
5294	while (css_parent(&memcg->css)) {
5295	memcg = mem_cgroup_from_css(css_parent(&memcg->css));
5296	if (!memcg->use_hierarchy)
5297	break;
5298	tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
5299	min_limit = min(min_limit, tmp);
5300	tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5301	min_memsw_limit = min(min_memsw_limit, tmp);
5302	}
5303	out:
5304	*mem_limit = min_limit;
5305	*memsw_limit = min_memsw_limit;
5306	}
5307
5308	static int mem_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
5309	{
5310	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5311	int name;
5312	enum res_type type;
5313
5314	type = MEMFILE_TYPE(event);
5315	name = MEMFILE_ATTR(event);
5316
5317	switch (name) {
5318	case RES_MAX_USAGE:
5319	if (type == _MEM)
5320	res_counter_reset_max(&memcg->res);
5321	else if (type == _MEMSWAP)
5322	res_counter_reset_max(&memcg->memsw);
5323	else if (type == _KMEM)
5324	res_counter_reset_max(&memcg->kmem);
5325	else
5326	return -EINVAL;
5327	break;
5328	case RES_FAILCNT:
5329	if (type == _MEM)
5330	res_counter_reset_failcnt(&memcg->res);
5331	else if (type == _MEMSWAP)
5332	res_counter_reset_failcnt(&memcg->memsw);
5333	else if (type == _KMEM)
5334	res_counter_reset_failcnt(&memcg->kmem);
5335	else
5336	return -EINVAL;
5337	break;
5338	}
5339
5340	return 0;
5341	}
5342
5343	static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
5344	struct cftype *cft)
5345	{
5346	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
5347	}
5348
5349	#ifdef CONFIG_MMU
5350	static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
5351	struct cftype *cft, u64 val)
5352	{
5353	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5354
5355	if (val >= (1 << NR_MOVE_TYPE))
5356	return -EINVAL;
5357
5358	/*
5359	* No kind of locking is needed in here, because ->can_attach() will
5360	* check this value once in the beginning of the process, and then carry
5361	* on with stale data. This means that changes to this value will only
5362	* affect task migrations starting after the change.
5363	*/
5364	memcg->move_charge_at_immigrate = val;
5365	return 0;
5366	}
5367	#else
5368	static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
5369	struct cftype *cft, u64 val)
5370	{
5371	return -ENOSYS;
5372	}
5373	#endif
5374
5375	#ifdef CONFIG_NUMA
5376	static int memcg_numa_stat_show(struct cgroup_subsys_state *css,
5377	struct cftype cft, struct seq_file m)
5378	{
5379	int nid;
5380	unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
5381	unsigned long node_nr;
5382	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5383
5384	total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
5385	seq_printf(m, "total=%lu", total_nr);
5386	for_each_node_state(nid, N_MEMORY) {
5387	node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
5388	seq_printf(m, " N%d=%lu", nid, node_nr);
5389	}
5390	seq_putc(m, '\n');
5391
5392	file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
5393	seq_printf(m, "file=%lu", file_nr);
5394	for_each_node_state(nid, N_MEMORY) {
5395	node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5396	LRU_ALL_FILE);
5397	seq_printf(m, " N%d=%lu", nid, node_nr);
5398	}
5399	seq_putc(m, '\n');
5400
5401	anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
5402	seq_printf(m, "anon=%lu", anon_nr);
5403	for_each_node_state(nid, N_MEMORY) {
5404	node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5405	LRU_ALL_ANON);
5406	seq_printf(m, " N%d=%lu", nid, node_nr);
5407	}
5408	seq_putc(m, '\n');
5409
5410	unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
5411	seq_printf(m, "unevictable=%lu", unevictable_nr);
5412	for_each_node_state(nid, N_MEMORY) {
5413	node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5414	BIT(LRU_UNEVICTABLE));
5415	seq_printf(m, " N%d=%lu", nid, node_nr);
5416	}
5417	seq_putc(m, '\n');
5418	return 0;
5419	}
5420	#endif /* CONFIG_NUMA */
5421
5422	static inline void mem_cgroup_lru_names_not_uptodate(void)
5423	{
5424	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
5425	}
5426
5427	static int memcg_stat_show(struct cgroup_subsys_state css, struct cftype cft,
5428	struct seq_file *m)
5429	{
5430	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5431	struct mem_cgroup *mi;
5432	unsigned int i;
5433
5434	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5435	if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5436	continue;
5437	seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
5438	mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
5439	}
5440
5441	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
5442	seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
5443	mem_cgroup_read_events(memcg, i));
5444
5445	for (i = 0; i < NR_LRU_LISTS; i++)
5446	seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
5447	mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
5448
5449	/* Hierarchical information */
5450	{
5451	unsigned long long limit, memsw_limit;
5452	memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
5453	seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
5454	if (do_swap_account)
5455	seq_printf(m, "hierarchical_memsw_limit %llu\n",
5456	memsw_limit);
5457	}
5458
5459	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5460	long long val = 0;
5461
5462	if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5463	continue;
5464	for_each_mem_cgroup_tree(mi, memcg)
5465	val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
5466	seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
5467	}
5468
5469	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
5470	unsigned long long val = 0;
5471
5472	for_each_mem_cgroup_tree(mi, memcg)
5473	val += mem_cgroup_read_events(mi, i);
5474	seq_printf(m, "total_%s %llu\n",
5475	mem_cgroup_events_names[i], val);
5476	}
5477
5478	for (i = 0; i < NR_LRU_LISTS; i++) {
5479	unsigned long long val = 0;
5480
5481	for_each_mem_cgroup_tree(mi, memcg)
5482	val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
5483	seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
5484	}
5485
5486	#ifdef CONFIG_DEBUG_VM
5487	{
5488	int nid, zid;
5489	struct mem_cgroup_per_zone *mz;
5490	struct zone_reclaim_stat *rstat;
5491	unsigned long recent_rotated[2] = {0, 0};
5492	unsigned long recent_scanned[2] = {0, 0};
5493
5494	for_each_online_node(nid)
5495	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
5496	mz = mem_cgroup_zoneinfo(memcg, nid, zid);
5497	rstat = &mz->lruvec.reclaim_stat;
5498
5499	recent_rotated[0] += rstat->recent_rotated[0];
5500	recent_rotated[1] += rstat->recent_rotated[1];
5501	recent_scanned[0] += rstat->recent_scanned[0];
5502	recent_scanned[1] += rstat->recent_scanned[1];
5503	}
5504	seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
5505	seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
5506	seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
5507	seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
5508	}
5509	#endif
5510
5511	return 0;
5512	}
5513
5514	static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
5515	struct cftype *cft)
5516	{
5517	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5518
5519	return mem_cgroup_swappiness(memcg);
5520	}
5521
5522	static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
5523	struct cftype *cft, u64 val)
5524	{
5525	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5526	struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
5527
5528	if (val > 100 \|\| !parent)
5529	return -EINVAL;
5530
5531	mutex_lock(&memcg_create_mutex);
5532
5533	/* If under hierarchy, only empty-root can set this value */
5534	if ((parent->use_hierarchy) \|\| memcg_has_children(memcg)) {
5535	mutex_unlock(&memcg_create_mutex);
5536	return -EINVAL;
5537	}
5538
5539	memcg->swappiness = val;
5540
5541	mutex_unlock(&memcg_create_mutex);
5542
5543	return 0;
5544	}
5545
5546	static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
5547	{
5548	struct mem_cgroup_threshold_ary *t;
5549	u64 usage;
5550	int i;
5551
5552	rcu_read_lock();
5553	if (!swap)
5554	t = rcu_dereference(memcg->thresholds.primary);
5555	else
5556	t = rcu_dereference(memcg->memsw_thresholds.primary);
5557
5558	if (!t)
5559	goto unlock;
5560
5561	usage = mem_cgroup_usage(memcg, swap);
5562
5563	/*
5564	* current_threshold points to threshold just below or equal to usage.
5565	* If it's not true, a threshold was crossed after last
5566	* call of __mem_cgroup_threshold().
5567	*/
5568	i = t->current_threshold;
5569
5570	/*
5571	* Iterate backward over array of thresholds starting from
5572	* current_threshold and check if a threshold is crossed.
5573	* If none of thresholds below usage is crossed, we read
5574	* only one element of the array here.
5575	*/
5576	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
5577	eventfd_signal(t->entries[i].eventfd, 1);
5578
5579	/* i = current_threshold + 1 */
5580	i++;
5581
5582	/*
5583	* Iterate forward over array of thresholds starting from
5584	* current_threshold+1 and check if a threshold is crossed.
5585	* If none of thresholds above usage is crossed, we read
5586	* only one element of the array here.
5587	*/
5588	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
5589	eventfd_signal(t->entries[i].eventfd, 1);
5590
5591	/* Update current_threshold */
5592	t->current_threshold = i - 1;
5593	unlock:
5594	rcu_read_unlock();
5595	}
5596
5597	static void mem_cgroup_threshold(struct mem_cgroup *memcg)
5598	{
5599	while (memcg) {
5600	__mem_cgroup_threshold(memcg, false);
5601	if (do_swap_account)
5602	__mem_cgroup_threshold(memcg, true);
5603
5604	memcg = parent_mem_cgroup(memcg);
5605	}
5606	}
5607
5608	static int compare_thresholds(const void a, const void b)
5609	{
5610	const struct mem_cgroup_threshold *_a = a;
5611	const struct mem_cgroup_threshold *_b = b;
5612
5613	if (_a->threshold > _b->threshold)
5614	return 1;
5615
5616	if (_a->threshold < _b->threshold)
5617	return -1;
5618
5619	return 0;
5620	}
5621
5622	static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
5623	{
5624	struct mem_cgroup_eventfd_list *ev;
5625
5626	list_for_each_entry(ev, &memcg->oom_notify, list)
5627	eventfd_signal(ev->eventfd, 1);
5628	return 0;
5629	}
5630
5631	static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
5632	{
5633	struct mem_cgroup *iter;
5634
5635	for_each_mem_cgroup_tree(iter, memcg)
5636	mem_cgroup_oom_notify_cb(iter);
5637	}
5638
5639	static int mem_cgroup_usage_register_event(struct cgroup_subsys_state *css,
5640	struct cftype cft, struct eventfd_ctx eventfd, const char *args)
5641	{
5642	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5643	struct mem_cgroup_thresholds *thresholds;
5644	struct mem_cgroup_threshold_ary *new;
5645	enum res_type type = MEMFILE_TYPE(cft->private);
5646	u64 threshold, usage;
5647	int i, size, ret;
5648
5649	ret = res_counter_memparse_write_strategy(args, &threshold);
5650	if (ret)
5651	return ret;
5652
5653	mutex_lock(&memcg->thresholds_lock);
5654
5655	if (type == _MEM)
5656	thresholds = &memcg->thresholds;
5657	else if (type == _MEMSWAP)
5658	thresholds = &memcg->memsw_thresholds;
5659	else
5660	BUG();
5661
5662	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5663
5664	/* Check if a threshold crossed before adding a new one */
5665	if (thresholds->primary)
5666	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
5667
5668	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
5669
5670	/* Allocate memory for new array of thresholds */
5671	new = kmalloc(sizeof(new) + size sizeof(struct mem_cgroup_threshold),
5672	GFP_KERNEL);
5673	if (!new) {
5674	ret = -ENOMEM;
5675	goto unlock;
5676	}
5677	new->size = size;
5678
5679	/* Copy thresholds (if any) to new array */
5680	if (thresholds->primary) {
5681	memcpy(new->entries, thresholds->primary->entries, (size - 1) *
5682	sizeof(struct mem_cgroup_threshold));
5683	}
5684
5685	/* Add new threshold */
5686	new->entries[size - 1].eventfd = eventfd;
5687	new->entries[size - 1].threshold = threshold;
5688
5689	/* Sort thresholds. Registering of new threshold isn't time-critical */
5690	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
5691	compare_thresholds, NULL);
5692
5693	/* Find current threshold */
5694	new->current_threshold = -1;
5695	for (i = 0; i < size; i++) {
5696	if (new->entries[i].threshold <= usage) {
5697	/*
5698	* new->current_threshold will not be used until
5699	* rcu_assign_pointer(), so it's safe to increment
5700	* it here.
5701	*/
5702	++new->current_threshold;
5703	} else
5704	break;
5705	}
5706
5707	/* Free old spare buffer and save old primary buffer as spare */
5708	kfree(thresholds->spare);
5709	thresholds->spare = thresholds->primary;
5710
5711	rcu_assign_pointer(thresholds->primary, new);
5712
5713	/* To be sure that nobody uses thresholds */
5714	synchronize_rcu();
5715
5716	unlock:
5717	mutex_unlock(&memcg->thresholds_lock);
5718
5719	return ret;
5720	}
5721
5722	static void mem_cgroup_usage_unregister_event(struct cgroup_subsys_state *css,
5723	struct cftype cft, struct eventfd_ctx eventfd)
5724	{
5725	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5726	struct mem_cgroup_thresholds *thresholds;
5727	struct mem_cgroup_threshold_ary *new;
5728	enum res_type type = MEMFILE_TYPE(cft->private);
5729	u64 usage;
5730	int i, j, size;
5731
5732	mutex_lock(&memcg->thresholds_lock);
5733	if (type == _MEM)
5734	thresholds = &memcg->thresholds;
5735	else if (type == _MEMSWAP)
5736	thresholds = &memcg->memsw_thresholds;
5737	else
5738	BUG();
5739
5740	if (!thresholds->primary)
5741	goto unlock;
5742
5743	usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5744
5745	/* Check if a threshold crossed before removing */
5746	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
5747
5748	/* Calculate new number of threshold */
5749	size = 0;
5750	for (i = 0; i < thresholds->primary->size; i++) {
5751	if (thresholds->primary->entries[i].eventfd != eventfd)
5752	size++;
5753	}
5754
5755	new = thresholds->spare;
5756
5757	/* Set thresholds array to NULL if we don't have thresholds */
5758	if (!size) {
5759	kfree(new);
5760	new = NULL;
5761	goto swap_buffers;
5762	}
5763
5764	new->size = size;
5765
5766	/* Copy thresholds and find current threshold */
5767	new->current_threshold = -1;
5768	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
5769	if (thresholds->primary->entries[i].eventfd == eventfd)
5770	continue;
5771
5772	new->entries[j] = thresholds->primary->entries[i];
5773	if (new->entries[j].threshold <= usage) {
5774	/*
5775	* new->current_threshold will not be used
5776	* until rcu_assign_pointer(), so it's safe to increment
5777	* it here.
5778	*/
5779	++new->current_threshold;
5780	}
5781	j++;
5782	}
5783
5784	swap_buffers:
5785	/* Swap primary and spare array */
5786	thresholds->spare = thresholds->primary;
5787	/* If all events are unregistered, free the spare array */
5788	if (!new) {
5789	kfree(thresholds->spare);
5790	thresholds->spare = NULL;
5791	}
5792
5793	rcu_assign_pointer(thresholds->primary, new);
5794
5795	/* To be sure that nobody uses thresholds */
5796	synchronize_rcu();
5797	unlock:
5798	mutex_unlock(&memcg->thresholds_lock);
5799	}
5800
5801	static int mem_cgroup_oom_register_event(struct cgroup_subsys_state *css,
5802	struct cftype cft, struct eventfd_ctx eventfd, const char *args)
5803	{
5804	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5805	struct mem_cgroup_eventfd_list *event;
5806	enum res_type type = MEMFILE_TYPE(cft->private);
5807
5808	BUG_ON(type != _OOM_TYPE);
5809	event = kmalloc(sizeof(*event), GFP_KERNEL);
5810	if (!event)
5811	return -ENOMEM;
5812
5813	spin_lock(&memcg_oom_lock);
5814
5815	event->eventfd = eventfd;
5816	list_add(&event->list, &memcg->oom_notify);
5817
5818	/* already in OOM ? */
5819	if (atomic_read(&memcg->under_oom))
5820	eventfd_signal(eventfd, 1);
5821	spin_unlock(&memcg_oom_lock);
5822
5823	return 0;
5824	}
5825
5826	static void mem_cgroup_oom_unregister_event(struct cgroup_subsys_state *css,
5827	struct cftype cft, struct eventfd_ctx eventfd)
5828	{
5829	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5830	struct mem_cgroup_eventfd_list ev, tmp;
5831	enum res_type type = MEMFILE_TYPE(cft->private);
5832
5833	BUG_ON(type != _OOM_TYPE);
5834
5835	spin_lock(&memcg_oom_lock);
5836
5837	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
5838	if (ev->eventfd == eventfd) {
5839	list_del(&ev->list);
5840	kfree(ev);
5841	}
5842	}
5843
5844	spin_unlock(&memcg_oom_lock);
5845	}
5846
5847	static int mem_cgroup_oom_control_read(struct cgroup_subsys_state *css,
5848	struct cftype cft, struct cgroup_map_cb cb)
5849	{
5850	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5851
5852	cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
5853
5854	if (atomic_read(&memcg->under_oom))
5855	cb->fill(cb, "under_oom", 1);
5856	else
5857	cb->fill(cb, "under_oom", 0);
5858	return 0;
5859	}
5860
5861	static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
5862	struct cftype *cft, u64 val)
5863	{
5864	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5865	struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
5866
5867	/* cannot set to root cgroup and only 0 and 1 are allowed */
5868	if (!parent \|\| !((val == 0) \|\| (val == 1)))
5869	return -EINVAL;
5870
5871	mutex_lock(&memcg_create_mutex);
5872	/* oom-kill-disable is a flag for subhierarchy. */
5873	if ((parent->use_hierarchy) \|\| memcg_has_children(memcg)) {
5874	mutex_unlock(&memcg_create_mutex);
5875	return -EINVAL;
5876	}
5877	memcg->oom_kill_disable = val;
5878	if (!val)
5879	memcg_oom_recover(memcg);
5880	mutex_unlock(&memcg_create_mutex);
5881	return 0;
5882	}
5883
5884	#ifdef CONFIG_MEMCG_KMEM
5885	static int memcg_init_kmem(struct mem_cgroup memcg, struct cgroup_subsys ss)
5886	{
5887	int ret;
5888
5889	memcg->kmemcg_id = -1;
5890	ret = memcg_propagate_kmem(memcg);
5891	if (ret)
5892	return ret;
5893
5894	return mem_cgroup_sockets_init(memcg, ss);
5895	}
5896
5897	static void memcg_destroy_kmem(struct mem_cgroup *memcg)
5898	{
5899	mem_cgroup_sockets_destroy(memcg);
5900	}
5901
5902	static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
5903	{
5904	if (!memcg_kmem_is_active(memcg))
5905	return;
5906
5907	/*
5908	* kmem charges can outlive the cgroup. In the case of slab
5909	* pages, for instance, a page contain objects from various
5910	* processes. As we prevent from taking a reference for every
5911	* such allocation we have to be careful when doing uncharge
5912	* (see memcg_uncharge_kmem) and here during offlining.
5913	*
5914	* The idea is that that only the _last_ uncharge which sees
5915	* the dead memcg will drop the last reference. An additional
5916	* reference is taken here before the group is marked dead
5917	* which is then paired with css_put during uncharge resp. here.
5918	*
5919	* Although this might sound strange as this path is called from
5920	* css_offline() when the referencemight have dropped down to 0
5921	* and shouldn't be incremented anymore (css_tryget would fail)
5922	* we do not have other options because of the kmem allocations
5923	* lifetime.
5924	*/
5925	css_get(&memcg->css);