Root/mm/memcontrol.c

1/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
6 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
9 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
13 * Kernel Memory Controller
14 * Copyright (C) 2012 Parallels Inc. and Google Inc.
15 * Authors: Glauber Costa and Suleiman Souhlal
16 *
17 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License as published by
19 * the Free Software Foundation; either version 2 of the License, or
20 * (at your option) any later version.
21 *
22 * This program is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 */
27
28#include <linux/res_counter.h>
29#include <linux/memcontrol.h>
30#include <linux/cgroup.h>
31#include <linux/mm.h>
32#include <linux/hugetlb.h>
33#include <linux/pagemap.h>
34#include <linux/smp.h>
35#include <linux/page-flags.h>
36#include <linux/backing-dev.h>
37#include <linux/bit_spinlock.h>
38#include <linux/rcupdate.h>
39#include <linux/limits.h>
40#include <linux/export.h>
41#include <linux/mutex.h>
42#include <linux/rbtree.h>
43#include <linux/slab.h>
44#include <linux/swap.h>
45#include <linux/swapops.h>
46#include <linux/spinlock.h>
47#include <linux/eventfd.h>
48#include <linux/sort.h>
49#include <linux/fs.h>
50#include <linux/seq_file.h>
51#include <linux/vmalloc.h>
52#include <linux/mm_inline.h>
53#include <linux/page_cgroup.h>
54#include <linux/cpu.h>
55#include <linux/oom.h>
56#include "internal.h"
57#include <net/sock.h>
58#include <net/ip.h>
59#include <net/tcp_memcontrol.h>
60
61#include <asm/uaccess.h>
62
63#include <trace/events/vmscan.h>
64
65struct cgroup_subsys mem_cgroup_subsys __read_mostly;
66EXPORT_SYMBOL(mem_cgroup_subsys);
67
68#define MEM_CGROUP_RECLAIM_RETRIES 5
69static struct mem_cgroup *root_mem_cgroup __read_mostly;
70
71#ifdef CONFIG_MEMCG_SWAP
72/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
73int do_swap_account __read_mostly;
74
75/* for remember boot option*/
76#ifdef CONFIG_MEMCG_SWAP_ENABLED
77static int really_do_swap_account __initdata = 1;
78#else
79static int really_do_swap_account __initdata = 0;
80#endif
81
82#else
83#define do_swap_account 0
84#endif
85
86
87/*
88 * Statistics for memory cgroup.
89 */
90enum mem_cgroup_stat_index {
91    /*
92     * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
93     */
94    MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
95    MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
96    MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
97    MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
98    MEM_CGROUP_STAT_NSTATS,
99};
100
101static const char * const mem_cgroup_stat_names[] = {
102    "cache",
103    "rss",
104    "mapped_file",
105    "swap",
106};
107
108enum mem_cgroup_events_index {
109    MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
110    MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
111    MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
112    MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
113    MEM_CGROUP_EVENTS_NSTATS,
114};
115
116static const char * const mem_cgroup_events_names[] = {
117    "pgpgin",
118    "pgpgout",
119    "pgfault",
120    "pgmajfault",
121};
122
123static const char * const mem_cgroup_lru_names[] = {
124    "inactive_anon",
125    "active_anon",
126    "inactive_file",
127    "active_file",
128    "unevictable",
129};
130
131/*
132 * Per memcg event counter is incremented at every pagein/pageout. With THP,
133 * it will be incremated by the number of pages. This counter is used for
134 * for trigger some periodic events. This is straightforward and better
135 * than using jiffies etc. to handle periodic memcg event.
136 */
137enum mem_cgroup_events_target {
138    MEM_CGROUP_TARGET_THRESH,
139    MEM_CGROUP_TARGET_SOFTLIMIT,
140    MEM_CGROUP_TARGET_NUMAINFO,
141    MEM_CGROUP_NTARGETS,
142};
143#define THRESHOLDS_EVENTS_TARGET 128
144#define SOFTLIMIT_EVENTS_TARGET 1024
145#define NUMAINFO_EVENTS_TARGET 1024
146
147struct mem_cgroup_stat_cpu {
148    long count[MEM_CGROUP_STAT_NSTATS];
149    unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
150    unsigned long nr_page_events;
151    unsigned long targets[MEM_CGROUP_NTARGETS];
152};
153
154struct mem_cgroup_reclaim_iter {
155    /* css_id of the last scanned hierarchy member */
156    int position;
157    /* scan generation, increased every round-trip */
158    unsigned int generation;
159};
160
161/*
162 * per-zone information in memory controller.
163 */
164struct mem_cgroup_per_zone {
165    struct lruvec lruvec;
166    unsigned long lru_size[NR_LRU_LISTS];
167
168    struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
169
170    struct rb_node tree_node; /* RB tree node */
171    unsigned long long usage_in_excess;/* Set to the value by which */
172                        /* the soft limit is exceeded*/
173    bool on_tree;
174    struct mem_cgroup *memcg; /* Back pointer, we cannot */
175                        /* use container_of */
176};
177
178struct mem_cgroup_per_node {
179    struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
180};
181
182struct mem_cgroup_lru_info {
183    struct mem_cgroup_per_node *nodeinfo[0];
184};
185
186/*
187 * Cgroups above their limits are maintained in a RB-Tree, independent of
188 * their hierarchy representation
189 */
190
191struct mem_cgroup_tree_per_zone {
192    struct rb_root rb_root;
193    spinlock_t lock;
194};
195
196struct mem_cgroup_tree_per_node {
197    struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
198};
199
200struct mem_cgroup_tree {
201    struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
202};
203
204static struct mem_cgroup_tree soft_limit_tree __read_mostly;
205
206struct mem_cgroup_threshold {
207    struct eventfd_ctx *eventfd;
208    u64 threshold;
209};
210
211/* For threshold */
212struct mem_cgroup_threshold_ary {
213    /* An array index points to threshold just below or equal to usage. */
214    int current_threshold;
215    /* Size of entries[] */
216    unsigned int size;
217    /* Array of thresholds */
218    struct mem_cgroup_threshold entries[0];
219};
220
221struct mem_cgroup_thresholds {
222    /* Primary thresholds array */
223    struct mem_cgroup_threshold_ary *primary;
224    /*
225     * Spare threshold array.
226     * This is needed to make mem_cgroup_unregister_event() "never fail".
227     * It must be able to store at least primary->size - 1 entries.
228     */
229    struct mem_cgroup_threshold_ary *spare;
230};
231
232/* for OOM */
233struct mem_cgroup_eventfd_list {
234    struct list_head list;
235    struct eventfd_ctx *eventfd;
236};
237
238static void mem_cgroup_threshold(struct mem_cgroup *memcg);
239static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
240
241/*
242 * The memory controller data structure. The memory controller controls both
243 * page cache and RSS per cgroup. We would eventually like to provide
244 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
245 * to help the administrator determine what knobs to tune.
246 *
247 * TODO: Add a water mark for the memory controller. Reclaim will begin when
248 * we hit the water mark. May be even add a low water mark, such that
249 * no reclaim occurs from a cgroup at it's low water mark, this is
250 * a feature that will be implemented much later in the future.
251 */
252struct mem_cgroup {
253    struct cgroup_subsys_state css;
254    /*
255     * the counter to account for memory usage
256     */
257    struct res_counter res;
258
259    union {
260        /*
261         * the counter to account for mem+swap usage.
262         */
263        struct res_counter memsw;
264
265        /*
266         * rcu_freeing is used only when freeing struct mem_cgroup,
267         * so put it into a union to avoid wasting more memory.
268         * It must be disjoint from the css field. It could be
269         * in a union with the res field, but res plays a much
270         * larger part in mem_cgroup life than memsw, and might
271         * be of interest, even at time of free, when debugging.
272         * So share rcu_head with the less interesting memsw.
273         */
274        struct rcu_head rcu_freeing;
275        /*
276         * We also need some space for a worker in deferred freeing.
277         * By the time we call it, rcu_freeing is no longer in use.
278         */
279        struct work_struct work_freeing;
280    };
281
282    /*
283     * the counter to account for kernel memory usage.
284     */
285    struct res_counter kmem;
286    /*
287     * Should the accounting and control be hierarchical, per subtree?
288     */
289    bool use_hierarchy;
290    unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
291
292    bool oom_lock;
293    atomic_t under_oom;
294
295    atomic_t refcnt;
296
297    int swappiness;
298    /* OOM-Killer disable */
299    int oom_kill_disable;
300
301    /* set when res.limit == memsw.limit */
302    bool memsw_is_minimum;
303
304    /* protect arrays of thresholds */
305    struct mutex thresholds_lock;
306
307    /* thresholds for memory usage. RCU-protected */
308    struct mem_cgroup_thresholds thresholds;
309
310    /* thresholds for mem+swap usage. RCU-protected */
311    struct mem_cgroup_thresholds memsw_thresholds;
312
313    /* For oom notifier event fd */
314    struct list_head oom_notify;
315
316    /*
317     * Should we move charges of a task when a task is moved into this
318     * mem_cgroup ? And what type of charges should we move ?
319     */
320    unsigned long move_charge_at_immigrate;
321    /*
322     * set > 0 if pages under this cgroup are moving to other cgroup.
323     */
324    atomic_t moving_account;
325    /* taken only while moving_account > 0 */
326    spinlock_t move_lock;
327    /*
328     * percpu counter.
329     */
330    struct mem_cgroup_stat_cpu __percpu *stat;
331    /*
332     * used when a cpu is offlined or other synchronizations
333     * See mem_cgroup_read_stat().
334     */
335    struct mem_cgroup_stat_cpu nocpu_base;
336    spinlock_t pcp_counter_lock;
337
338#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
339    struct tcp_memcontrol tcp_mem;
340#endif
341#if defined(CONFIG_MEMCG_KMEM)
342    /* analogous to slab_common's slab_caches list. per-memcg */
343    struct list_head memcg_slab_caches;
344    /* Not a spinlock, we can take a lot of time walking the list */
345    struct mutex slab_caches_mutex;
346        /* Index in the kmem_cache->memcg_params->memcg_caches array */
347    int kmemcg_id;
348#endif
349
350    int last_scanned_node;
351#if MAX_NUMNODES > 1
352    nodemask_t scan_nodes;
353    atomic_t numainfo_events;
354    atomic_t numainfo_updating;
355#endif
356    /*
357     * Per cgroup active and inactive list, similar to the
358     * per zone LRU lists.
359     *
360     * WARNING: This has to be the last element of the struct. Don't
361     * add new fields after this point.
362     */
363    struct mem_cgroup_lru_info info;
364};
365
366static size_t memcg_size(void)
367{
368    return sizeof(struct mem_cgroup) +
369        nr_node_ids * sizeof(struct mem_cgroup_per_node);
370}
371
372/* internal only representation about the status of kmem accounting. */
373enum {
374    KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
375    KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
376    KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
377};
378
379/* We account when limit is on, but only after call sites are patched */
380#define KMEM_ACCOUNTED_MASK \
381        ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
382
383#ifdef CONFIG_MEMCG_KMEM
384static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
385{
386    set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
387}
388
389static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
390{
391    return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
392}
393
394static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
395{
396    set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
397}
398
399static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
400{
401    clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
402}
403
404static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
405{
406    if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
407        set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
408}
409
410static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
411{
412    return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
413                  &memcg->kmem_account_flags);
414}
415#endif
416
417/* Stuffs for move charges at task migration. */
418/*
419 * Types of charges to be moved. "move_charge_at_immitgrate" and
420 * "immigrate_flags" are treated as a left-shifted bitmap of these types.
421 */
422enum move_type {
423    MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
424    MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
425    NR_MOVE_TYPE,
426};
427
428/* "mc" and its members are protected by cgroup_mutex */
429static struct move_charge_struct {
430    spinlock_t lock; /* for from, to */
431    struct mem_cgroup *from;
432    struct mem_cgroup *to;
433    unsigned long immigrate_flags;
434    unsigned long precharge;
435    unsigned long moved_charge;
436    unsigned long moved_swap;
437    struct task_struct *moving_task; /* a task moving charges */
438    wait_queue_head_t waitq; /* a waitq for other context */
439} mc = {
440    .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
441    .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
442};
443
444static bool move_anon(void)
445{
446    return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
447}
448
449static bool move_file(void)
450{
451    return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
452}
453
454/*
455 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
456 * limit reclaim to prevent infinite loops, if they ever occur.
457 */
458#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
459#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
460
461enum charge_type {
462    MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
463    MEM_CGROUP_CHARGE_TYPE_ANON,
464    MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
465    MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
466    NR_CHARGE_TYPE,
467};
468
469/* for encoding cft->private value on file */
470enum res_type {
471    _MEM,
472    _MEMSWAP,
473    _OOM_TYPE,
474    _KMEM,
475};
476
477#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
478#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
479#define MEMFILE_ATTR(val) ((val) & 0xffff)
480/* Used for OOM nofiier */
481#define OOM_CONTROL (0)
482
483/*
484 * Reclaim flags for mem_cgroup_hierarchical_reclaim
485 */
486#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
487#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
488#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
489#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
490
491/*
492 * The memcg_create_mutex will be held whenever a new cgroup is created.
493 * As a consequence, any change that needs to protect against new child cgroups
494 * appearing has to hold it as well.
495 */
496static DEFINE_MUTEX(memcg_create_mutex);
497
498static void mem_cgroup_get(struct mem_cgroup *memcg);
499static void mem_cgroup_put(struct mem_cgroup *memcg);
500
501static inline
502struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
503{
504    return container_of(s, struct mem_cgroup, css);
505}
506
507static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
508{
509    return (memcg == root_mem_cgroup);
510}
511
512/* Writing them here to avoid exposing memcg's inner layout */
513#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
514
515void sock_update_memcg(struct sock *sk)
516{
517    if (mem_cgroup_sockets_enabled) {
518        struct mem_cgroup *memcg;
519        struct cg_proto *cg_proto;
520
521        BUG_ON(!sk->sk_prot->proto_cgroup);
522
523        /* Socket cloning can throw us here with sk_cgrp already
524         * filled. It won't however, necessarily happen from
525         * process context. So the test for root memcg given
526         * the current task's memcg won't help us in this case.
527         *
528         * Respecting the original socket's memcg is a better
529         * decision in this case.
530         */
531        if (sk->sk_cgrp) {
532            BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
533            mem_cgroup_get(sk->sk_cgrp->memcg);
534            return;
535        }
536
537        rcu_read_lock();
538        memcg = mem_cgroup_from_task(current);
539        cg_proto = sk->sk_prot->proto_cgroup(memcg);
540        if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) {
541            mem_cgroup_get(memcg);
542            sk->sk_cgrp = cg_proto;
543        }
544        rcu_read_unlock();
545    }
546}
547EXPORT_SYMBOL(sock_update_memcg);
548
549void sock_release_memcg(struct sock *sk)
550{
551    if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
552        struct mem_cgroup *memcg;
553        WARN_ON(!sk->sk_cgrp->memcg);
554        memcg = sk->sk_cgrp->memcg;
555        mem_cgroup_put(memcg);
556    }
557}
558
559struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
560{
561    if (!memcg || mem_cgroup_is_root(memcg))
562        return NULL;
563
564    return &memcg->tcp_mem.cg_proto;
565}
566EXPORT_SYMBOL(tcp_proto_cgroup);
567
568static void disarm_sock_keys(struct mem_cgroup *memcg)
569{
570    if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
571        return;
572    static_key_slow_dec(&memcg_socket_limit_enabled);
573}
574#else
575static void disarm_sock_keys(struct mem_cgroup *memcg)
576{
577}
578#endif
579
580#ifdef CONFIG_MEMCG_KMEM
581/*
582 * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
583 * There are two main reasons for not using the css_id for this:
584 * 1) this works better in sparse environments, where we have a lot of memcgs,
585 * but only a few kmem-limited. Or also, if we have, for instance, 200
586 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
587 * 200 entry array for that.
588 *
589 * 2) In order not to violate the cgroup API, we would like to do all memory
590 * allocation in ->create(). At that point, we haven't yet allocated the
591 * css_id. Having a separate index prevents us from messing with the cgroup
592 * core for this
593 *
594 * The current size of the caches array is stored in
595 * memcg_limited_groups_array_size. It will double each time we have to
596 * increase it.
597 */
598static DEFINE_IDA(kmem_limited_groups);
599int memcg_limited_groups_array_size;
600
601/*
602 * MIN_SIZE is different than 1, because we would like to avoid going through
603 * the alloc/free process all the time. In a small machine, 4 kmem-limited
604 * cgroups is a reasonable guess. In the future, it could be a parameter or
605 * tunable, but that is strictly not necessary.
606 *
607 * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
608 * this constant directly from cgroup, but it is understandable that this is
609 * better kept as an internal representation in cgroup.c. In any case, the
610 * css_id space is not getting any smaller, and we don't have to necessarily
611 * increase ours as well if it increases.
612 */
613#define MEMCG_CACHES_MIN_SIZE 4
614#define MEMCG_CACHES_MAX_SIZE 65535
615
616/*
617 * A lot of the calls to the cache allocation functions are expected to be
618 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
619 * conditional to this static branch, we'll have to allow modules that does
620 * kmem_cache_alloc and the such to see this symbol as well
621 */
622struct static_key memcg_kmem_enabled_key;
623EXPORT_SYMBOL(memcg_kmem_enabled_key);
624
625static void disarm_kmem_keys(struct mem_cgroup *memcg)
626{
627    if (memcg_kmem_is_active(memcg)) {
628        static_key_slow_dec(&memcg_kmem_enabled_key);
629        ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
630    }
631    /*
632     * This check can't live in kmem destruction function,
633     * since the charges will outlive the cgroup
634     */
635    WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
636}
637#else
638static void disarm_kmem_keys(struct mem_cgroup *memcg)
639{
640}
641#endif /* CONFIG_MEMCG_KMEM */
642
643static void disarm_static_keys(struct mem_cgroup *memcg)
644{
645    disarm_sock_keys(memcg);
646    disarm_kmem_keys(memcg);
647}
648
649static void drain_all_stock_async(struct mem_cgroup *memcg);
650
651static struct mem_cgroup_per_zone *
652mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
653{
654    VM_BUG_ON((unsigned)nid >= nr_node_ids);
655    return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
656}
657
658struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
659{
660    return &memcg->css;
661}
662
663static struct mem_cgroup_per_zone *
664page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
665{
666    int nid = page_to_nid(page);
667    int zid = page_zonenum(page);
668
669    return mem_cgroup_zoneinfo(memcg, nid, zid);
670}
671
672static struct mem_cgroup_tree_per_zone *
673soft_limit_tree_node_zone(int nid, int zid)
674{
675    return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
676}
677
678static struct mem_cgroup_tree_per_zone *
679soft_limit_tree_from_page(struct page *page)
680{
681    int nid = page_to_nid(page);
682    int zid = page_zonenum(page);
683
684    return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
685}
686
687static void
688__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
689                struct mem_cgroup_per_zone *mz,
690                struct mem_cgroup_tree_per_zone *mctz,
691                unsigned long long new_usage_in_excess)
692{
693    struct rb_node **p = &mctz->rb_root.rb_node;
694    struct rb_node *parent = NULL;
695    struct mem_cgroup_per_zone *mz_node;
696
697    if (mz->on_tree)
698        return;
699
700    mz->usage_in_excess = new_usage_in_excess;
701    if (!mz->usage_in_excess)
702        return;
703    while (*p) {
704        parent = *p;
705        mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
706                    tree_node);
707        if (mz->usage_in_excess < mz_node->usage_in_excess)
708            p = &(*p)->rb_left;
709        /*
710         * We can't avoid mem cgroups that are over their soft
711         * limit by the same amount
712         */
713        else if (mz->usage_in_excess >= mz_node->usage_in_excess)
714            p = &(*p)->rb_right;
715    }
716    rb_link_node(&mz->tree_node, parent, p);
717    rb_insert_color(&mz->tree_node, &mctz->rb_root);
718    mz->on_tree = true;
719}
720
721static void
722__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
723                struct mem_cgroup_per_zone *mz,
724                struct mem_cgroup_tree_per_zone *mctz)
725{
726    if (!mz->on_tree)
727        return;
728    rb_erase(&mz->tree_node, &mctz->rb_root);
729    mz->on_tree = false;
730}
731
732static void
733mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
734                struct mem_cgroup_per_zone *mz,
735                struct mem_cgroup_tree_per_zone *mctz)
736{
737    spin_lock(&mctz->lock);
738    __mem_cgroup_remove_exceeded(memcg, mz, mctz);
739    spin_unlock(&mctz->lock);
740}
741
742
743static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
744{
745    unsigned long long excess;
746    struct mem_cgroup_per_zone *mz;
747    struct mem_cgroup_tree_per_zone *mctz;
748    int nid = page_to_nid(page);
749    int zid = page_zonenum(page);
750    mctz = soft_limit_tree_from_page(page);
751
752    /*
753     * Necessary to update all ancestors when hierarchy is used.
754     * because their event counter is not touched.
755     */
756    for (; memcg; memcg = parent_mem_cgroup(memcg)) {
757        mz = mem_cgroup_zoneinfo(memcg, nid, zid);
758        excess = res_counter_soft_limit_excess(&memcg->res);
759        /*
760         * We have to update the tree if mz is on RB-tree or
761         * mem is over its softlimit.
762         */
763        if (excess || mz->on_tree) {
764            spin_lock(&mctz->lock);
765            /* if on-tree, remove it */
766            if (mz->on_tree)
767                __mem_cgroup_remove_exceeded(memcg, mz, mctz);
768            /*
769             * Insert again. mz->usage_in_excess will be updated.
770             * If excess is 0, no tree ops.
771             */
772            __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
773            spin_unlock(&mctz->lock);
774        }
775    }
776}
777
778static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
779{
780    int node, zone;
781    struct mem_cgroup_per_zone *mz;
782    struct mem_cgroup_tree_per_zone *mctz;
783
784    for_each_node(node) {
785        for (zone = 0; zone < MAX_NR_ZONES; zone++) {
786            mz = mem_cgroup_zoneinfo(memcg, node, zone);
787            mctz = soft_limit_tree_node_zone(node, zone);
788            mem_cgroup_remove_exceeded(memcg, mz, mctz);
789        }
790    }
791}
792
793static struct mem_cgroup_per_zone *
794__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
795{
796    struct rb_node *rightmost = NULL;
797    struct mem_cgroup_per_zone *mz;
798
799retry:
800    mz = NULL;
801    rightmost = rb_last(&mctz->rb_root);
802    if (!rightmost)
803        goto done; /* Nothing to reclaim from */
804
805    mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
806    /*
807     * Remove the node now but someone else can add it back,
808     * we will to add it back at the end of reclaim to its correct
809     * position in the tree.
810     */
811    __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
812    if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
813        !css_tryget(&mz->memcg->css))
814        goto retry;
815done:
816    return mz;
817}
818
819static struct mem_cgroup_per_zone *
820mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
821{
822    struct mem_cgroup_per_zone *mz;
823
824    spin_lock(&mctz->lock);
825    mz = __mem_cgroup_largest_soft_limit_node(mctz);
826    spin_unlock(&mctz->lock);
827    return mz;
828}
829
830/*
831 * Implementation Note: reading percpu statistics for memcg.
832 *
833 * Both of vmstat[] and percpu_counter has threshold and do periodic
834 * synchronization to implement "quick" read. There are trade-off between
835 * reading cost and precision of value. Then, we may have a chance to implement
836 * a periodic synchronizion of counter in memcg's counter.
837 *
838 * But this _read() function is used for user interface now. The user accounts
839 * memory usage by memory cgroup and he _always_ requires exact value because
840 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
841 * have to visit all online cpus and make sum. So, for now, unnecessary
842 * synchronization is not implemented. (just implemented for cpu hotplug)
843 *
844 * If there are kernel internal actions which can make use of some not-exact
845 * value, and reading all cpu value can be performance bottleneck in some
846 * common workload, threashold and synchonization as vmstat[] should be
847 * implemented.
848 */
849static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
850                 enum mem_cgroup_stat_index idx)
851{
852    long val = 0;
853    int cpu;
854
855    get_online_cpus();
856    for_each_online_cpu(cpu)
857        val += per_cpu(memcg->stat->count[idx], cpu);
858#ifdef CONFIG_HOTPLUG_CPU
859    spin_lock(&memcg->pcp_counter_lock);
860    val += memcg->nocpu_base.count[idx];
861    spin_unlock(&memcg->pcp_counter_lock);
862#endif
863    put_online_cpus();
864    return val;
865}
866
867static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
868                     bool charge)
869{
870    int val = (charge) ? 1 : -1;
871    this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
872}
873
874static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
875                        enum mem_cgroup_events_index idx)
876{
877    unsigned long val = 0;
878    int cpu;
879
880    for_each_online_cpu(cpu)
881        val += per_cpu(memcg->stat->events[idx], cpu);
882#ifdef CONFIG_HOTPLUG_CPU
883    spin_lock(&memcg->pcp_counter_lock);
884    val += memcg->nocpu_base.events[idx];
885    spin_unlock(&memcg->pcp_counter_lock);
886#endif
887    return val;
888}
889
890static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
891                     bool anon, int nr_pages)
892{
893    preempt_disable();
894
895    /*
896     * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
897     * counted as CACHE even if it's on ANON LRU.
898     */
899    if (anon)
900        __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
901                nr_pages);
902    else
903        __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
904                nr_pages);
905
906    /* pagein of a big page is an event. So, ignore page size */
907    if (nr_pages > 0)
908        __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
909    else {
910        __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
911        nr_pages = -nr_pages; /* for event */
912    }
913
914    __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
915
916    preempt_enable();
917}
918
919unsigned long
920mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
921{
922    struct mem_cgroup_per_zone *mz;
923
924    mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
925    return mz->lru_size[lru];
926}
927
928static unsigned long
929mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
930            unsigned int lru_mask)
931{
932    struct mem_cgroup_per_zone *mz;
933    enum lru_list lru;
934    unsigned long ret = 0;
935
936    mz = mem_cgroup_zoneinfo(memcg, nid, zid);
937
938    for_each_lru(lru) {
939        if (BIT(lru) & lru_mask)
940            ret += mz->lru_size[lru];
941    }
942    return ret;
943}
944
945static unsigned long
946mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
947            int nid, unsigned int lru_mask)
948{
949    u64 total = 0;
950    int zid;
951
952    for (zid = 0; zid < MAX_NR_ZONES; zid++)
953        total += mem_cgroup_zone_nr_lru_pages(memcg,
954                        nid, zid, lru_mask);
955
956    return total;
957}
958
959static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
960            unsigned int lru_mask)
961{
962    int nid;
963    u64 total = 0;
964
965    for_each_node_state(nid, N_MEMORY)
966        total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
967    return total;
968}
969
970static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
971                       enum mem_cgroup_events_target target)
972{
973    unsigned long val, next;
974
975    val = __this_cpu_read(memcg->stat->nr_page_events);
976    next = __this_cpu_read(memcg->stat->targets[target]);
977    /* from time_after() in jiffies.h */
978    if ((long)next - (long)val < 0) {
979        switch (target) {
980        case MEM_CGROUP_TARGET_THRESH:
981            next = val + THRESHOLDS_EVENTS_TARGET;
982            break;
983        case MEM_CGROUP_TARGET_SOFTLIMIT:
984            next = val + SOFTLIMIT_EVENTS_TARGET;
985            break;
986        case MEM_CGROUP_TARGET_NUMAINFO:
987            next = val + NUMAINFO_EVENTS_TARGET;
988            break;
989        default:
990            break;
991        }
992        __this_cpu_write(memcg->stat->targets[target], next);
993        return true;
994    }
995    return false;
996}
997
998/*
999 * Check events in order.
1000 *
1001 */
1002static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
1003{
1004    preempt_disable();
1005    /* threshold event is triggered in finer grain than soft limit */
1006    if (unlikely(mem_cgroup_event_ratelimit(memcg,
1007                        MEM_CGROUP_TARGET_THRESH))) {
1008        bool do_softlimit;
1009        bool do_numainfo __maybe_unused;
1010
1011        do_softlimit = mem_cgroup_event_ratelimit(memcg,
1012                        MEM_CGROUP_TARGET_SOFTLIMIT);
1013#if MAX_NUMNODES > 1
1014        do_numainfo = mem_cgroup_event_ratelimit(memcg,
1015                        MEM_CGROUP_TARGET_NUMAINFO);
1016#endif
1017        preempt_enable();
1018
1019        mem_cgroup_threshold(memcg);
1020        if (unlikely(do_softlimit))
1021            mem_cgroup_update_tree(memcg, page);
1022#if MAX_NUMNODES > 1
1023        if (unlikely(do_numainfo))
1024            atomic_inc(&memcg->numainfo_events);
1025#endif
1026    } else
1027        preempt_enable();
1028}
1029
1030struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
1031{
1032    return mem_cgroup_from_css(
1033        cgroup_subsys_state(cont, mem_cgroup_subsys_id));
1034}
1035
1036struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
1037{
1038    /*
1039     * mm_update_next_owner() may clear mm->owner to NULL
1040     * if it races with swapoff, page migration, etc.
1041     * So this can be called with p == NULL.
1042     */
1043    if (unlikely(!p))
1044        return NULL;
1045
1046    return mem_cgroup_from_css(task_subsys_state(p, mem_cgroup_subsys_id));
1047}
1048
1049struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
1050{
1051    struct mem_cgroup *memcg = NULL;
1052
1053    if (!mm)
1054        return NULL;
1055    /*
1056     * Because we have no locks, mm->owner's may be being moved to other
1057     * cgroup. We use css_tryget() here even if this looks
1058     * pessimistic (rather than adding locks here).
1059     */
1060    rcu_read_lock();
1061    do {
1062        memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1063        if (unlikely(!memcg))
1064            break;
1065    } while (!css_tryget(&memcg->css));
1066    rcu_read_unlock();
1067    return memcg;
1068}
1069
1070/**
1071 * mem_cgroup_iter - iterate over memory cgroup hierarchy
1072 * @root: hierarchy root
1073 * @prev: previously returned memcg, NULL on first invocation
1074 * @reclaim: cookie for shared reclaim walks, NULL for full walks
1075 *
1076 * Returns references to children of the hierarchy below @root, or
1077 * @root itself, or %NULL after a full round-trip.
1078 *
1079 * Caller must pass the return value in @prev on subsequent
1080 * invocations for reference counting, or use mem_cgroup_iter_break()
1081 * to cancel a hierarchy walk before the round-trip is complete.
1082 *
1083 * Reclaimers can specify a zone and a priority level in @reclaim to
1084 * divide up the memcgs in the hierarchy among all concurrent
1085 * reclaimers operating on the same zone and priority.
1086 */
1087struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1088                   struct mem_cgroup *prev,
1089                   struct mem_cgroup_reclaim_cookie *reclaim)
1090{
1091    struct mem_cgroup *memcg = NULL;
1092    int id = 0;
1093
1094    if (mem_cgroup_disabled())
1095        return NULL;
1096
1097    if (!root)
1098        root = root_mem_cgroup;
1099
1100    if (prev && !reclaim)
1101        id = css_id(&prev->css);
1102
1103    if (prev && prev != root)
1104        css_put(&prev->css);
1105
1106    if (!root->use_hierarchy && root != root_mem_cgroup) {
1107        if (prev)
1108            return NULL;
1109        return root;
1110    }
1111
1112    while (!memcg) {
1113        struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
1114        struct cgroup_subsys_state *css;
1115
1116        if (reclaim) {
1117            int nid = zone_to_nid(reclaim->zone);
1118            int zid = zone_idx(reclaim->zone);
1119            struct mem_cgroup_per_zone *mz;
1120
1121            mz = mem_cgroup_zoneinfo(root, nid, zid);
1122            iter = &mz->reclaim_iter[reclaim->priority];
1123            if (prev && reclaim->generation != iter->generation)
1124                return NULL;
1125            id = iter->position;
1126        }
1127
1128        rcu_read_lock();
1129        css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
1130        if (css) {
1131            if (css == &root->css || css_tryget(css))
1132                memcg = mem_cgroup_from_css(css);
1133        } else
1134            id = 0;
1135        rcu_read_unlock();
1136
1137        if (reclaim) {
1138            iter->position = id;
1139            if (!css)
1140                iter->generation++;
1141            else if (!prev && memcg)
1142                reclaim->generation = iter->generation;
1143        }
1144
1145        if (prev && !css)
1146            return NULL;
1147    }
1148    return memcg;
1149}
1150
1151/**
1152 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1153 * @root: hierarchy root
1154 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1155 */
1156void mem_cgroup_iter_break(struct mem_cgroup *root,
1157               struct mem_cgroup *prev)
1158{
1159    if (!root)
1160        root = root_mem_cgroup;
1161    if (prev && prev != root)
1162        css_put(&prev->css);
1163}
1164
1165/*
1166 * Iteration constructs for visiting all cgroups (under a tree). If
1167 * loops are exited prematurely (break), mem_cgroup_iter_break() must
1168 * be used for reference counting.
1169 */
1170#define for_each_mem_cgroup_tree(iter, root) \
1171    for (iter = mem_cgroup_iter(root, NULL, NULL); \
1172         iter != NULL; \
1173         iter = mem_cgroup_iter(root, iter, NULL))
1174
1175#define for_each_mem_cgroup(iter) \
1176    for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
1177         iter != NULL; \
1178         iter = mem_cgroup_iter(NULL, iter, NULL))
1179
1180void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
1181{
1182    struct mem_cgroup *memcg;
1183
1184    rcu_read_lock();
1185    memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1186    if (unlikely(!memcg))
1187        goto out;
1188
1189    switch (idx) {
1190    case PGFAULT:
1191        this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1192        break;
1193    case PGMAJFAULT:
1194        this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
1195        break;
1196    default:
1197        BUG();
1198    }
1199out:
1200    rcu_read_unlock();
1201}
1202EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
1203
1204/**
1205 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1206 * @zone: zone of the wanted lruvec
1207 * @memcg: memcg of the wanted lruvec
1208 *
1209 * Returns the lru list vector holding pages for the given @zone and
1210 * @mem. This can be the global zone lruvec, if the memory controller
1211 * is disabled.
1212 */
1213struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
1214                      struct mem_cgroup *memcg)
1215{
1216    struct mem_cgroup_per_zone *mz;
1217    struct lruvec *lruvec;
1218
1219    if (mem_cgroup_disabled()) {
1220        lruvec = &zone->lruvec;
1221        goto out;
1222    }
1223
1224    mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
1225    lruvec = &mz->lruvec;
1226out:
1227    /*
1228     * Since a node can be onlined after the mem_cgroup was created,
1229     * we have to be prepared to initialize lruvec->zone here;
1230     * and if offlined then reonlined, we need to reinitialize it.
1231     */
1232    if (unlikely(lruvec->zone != zone))
1233        lruvec->zone = zone;
1234    return lruvec;
1235}
1236
1237/*
1238 * Following LRU functions are allowed to be used without PCG_LOCK.
1239 * Operations are called by routine of global LRU independently from memcg.
1240 * What we have to take care of here is validness of pc->mem_cgroup.
1241 *
1242 * Changes to pc->mem_cgroup happens when
1243 * 1. charge
1244 * 2. moving account
1245 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
1246 * It is added to LRU before charge.
1247 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
1248 * When moving account, the page is not on LRU. It's isolated.
1249 */
1250
1251/**
1252 * mem_cgroup_page_lruvec - return lruvec for adding an lru page
1253 * @page: the page
1254 * @zone: zone of the page
1255 */
1256struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
1257{
1258    struct mem_cgroup_per_zone *mz;
1259    struct mem_cgroup *memcg;
1260    struct page_cgroup *pc;
1261    struct lruvec *lruvec;
1262
1263    if (mem_cgroup_disabled()) {
1264        lruvec = &zone->lruvec;
1265        goto out;
1266    }
1267
1268    pc = lookup_page_cgroup(page);
1269    memcg = pc->mem_cgroup;
1270
1271    /*
1272     * Surreptitiously switch any uncharged offlist page to root:
1273     * an uncharged page off lru does nothing to secure
1274     * its former mem_cgroup from sudden removal.
1275     *
1276     * Our caller holds lru_lock, and PageCgroupUsed is updated
1277     * under page_cgroup lock: between them, they make all uses
1278     * of pc->mem_cgroup safe.
1279     */
1280    if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
1281        pc->mem_cgroup = memcg = root_mem_cgroup;
1282
1283    mz = page_cgroup_zoneinfo(memcg, page);
1284    lruvec = &mz->lruvec;
1285out:
1286    /*
1287     * Since a node can be onlined after the mem_cgroup was created,
1288     * we have to be prepared to initialize lruvec->zone here;
1289     * and if offlined then reonlined, we need to reinitialize it.
1290     */
1291    if (unlikely(lruvec->zone != zone))
1292        lruvec->zone = zone;
1293    return lruvec;
1294}
1295
1296/**
1297 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1298 * @lruvec: mem_cgroup per zone lru vector
1299 * @lru: index of lru list the page is sitting on
1300 * @nr_pages: positive when adding or negative when removing
1301 *
1302 * This function must be called when a page is added to or removed from an
1303 * lru list.
1304 */
1305void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1306                int nr_pages)
1307{
1308    struct mem_cgroup_per_zone *mz;
1309    unsigned long *lru_size;
1310
1311    if (mem_cgroup_disabled())
1312        return;
1313
1314    mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1315    lru_size = mz->lru_size + lru;
1316    *lru_size += nr_pages;
1317    VM_BUG_ON((long)(*lru_size) < 0);
1318}
1319
1320/*
1321 * Checks whether given mem is same or in the root_mem_cgroup's
1322 * hierarchy subtree
1323 */
1324bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1325                  struct mem_cgroup *memcg)
1326{
1327    if (root_memcg == memcg)
1328        return true;
1329    if (!root_memcg->use_hierarchy || !memcg)
1330        return false;
1331    return css_is_ancestor(&memcg->css, &root_memcg->css);
1332}
1333
1334static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1335                       struct mem_cgroup *memcg)
1336{
1337    bool ret;
1338
1339    rcu_read_lock();
1340    ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
1341    rcu_read_unlock();
1342    return ret;
1343}
1344
1345int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
1346{
1347    int ret;
1348    struct mem_cgroup *curr = NULL;
1349    struct task_struct *p;
1350
1351    p = find_lock_task_mm(task);
1352    if (p) {
1353        curr = try_get_mem_cgroup_from_mm(p->mm);
1354        task_unlock(p);
1355    } else {
1356        /*
1357         * All threads may have already detached their mm's, but the oom
1358         * killer still needs to detect if they have already been oom
1359         * killed to prevent needlessly killing additional tasks.
1360         */
1361        task_lock(task);
1362        curr = mem_cgroup_from_task(task);
1363        if (curr)
1364            css_get(&curr->css);
1365        task_unlock(task);
1366    }
1367    if (!curr)
1368        return 0;
1369    /*
1370     * We should check use_hierarchy of "memcg" not "curr". Because checking
1371     * use_hierarchy of "curr" here make this function true if hierarchy is
1372     * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
1373     * hierarchy(even if use_hierarchy is disabled in "memcg").
1374     */
1375    ret = mem_cgroup_same_or_subtree(memcg, curr);
1376    css_put(&curr->css);
1377    return ret;
1378}
1379
1380int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
1381{
1382    unsigned long inactive_ratio;
1383    unsigned long inactive;
1384    unsigned long active;
1385    unsigned long gb;
1386
1387    inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1388    active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
1389
1390    gb = (inactive + active) >> (30 - PAGE_SHIFT);
1391    if (gb)
1392        inactive_ratio = int_sqrt(10 * gb);
1393    else
1394        inactive_ratio = 1;
1395
1396    return inactive * inactive_ratio < active;
1397}
1398
1399#define mem_cgroup_from_res_counter(counter, member) \
1400    container_of(counter, struct mem_cgroup, member)
1401
1402/**
1403 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1404 * @memcg: the memory cgroup
1405 *
1406 * Returns the maximum amount of memory @mem can be charged with, in
1407 * pages.
1408 */
1409static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1410{
1411    unsigned long long margin;
1412
1413    margin = res_counter_margin(&memcg->res);
1414    if (do_swap_account)
1415        margin = min(margin, res_counter_margin(&memcg->memsw));
1416    return margin >> PAGE_SHIFT;
1417}
1418
1419int mem_cgroup_swappiness(struct mem_cgroup *memcg)
1420{
1421    struct cgroup *cgrp = memcg->css.cgroup;
1422
1423    /* root ? */
1424    if (cgrp->parent == NULL)
1425        return vm_swappiness;
1426
1427    return memcg->swappiness;
1428}
1429
1430/*
1431 * memcg->moving_account is used for checking possibility that some thread is
1432 * calling move_account(). When a thread on CPU-A starts moving pages under
1433 * a memcg, other threads should check memcg->moving_account under
1434 * rcu_read_lock(), like this:
1435 *
1436 * CPU-A CPU-B
1437 * rcu_read_lock()
1438 * memcg->moving_account+1 if (memcg->mocing_account)
1439 * take heavy locks.
1440 * synchronize_rcu() update something.
1441 * rcu_read_unlock()
1442 * start move here.
1443 */
1444
1445/* for quick checking without looking up memcg */
1446atomic_t memcg_moving __read_mostly;
1447
1448static void mem_cgroup_start_move(struct mem_cgroup *memcg)
1449{
1450    atomic_inc(&memcg_moving);
1451    atomic_inc(&memcg->moving_account);
1452    synchronize_rcu();
1453}
1454
1455static void mem_cgroup_end_move(struct mem_cgroup *memcg)
1456{
1457    /*
1458     * Now, mem_cgroup_clear_mc() may call this function with NULL.
1459     * We check NULL in callee rather than caller.
1460     */
1461    if (memcg) {
1462        atomic_dec(&memcg_moving);
1463        atomic_dec(&memcg->moving_account);
1464    }
1465}
1466
1467/*
1468 * 2 routines for checking "mem" is under move_account() or not.
1469 *
1470 * mem_cgroup_stolen() - checking whether a cgroup is mc.from or not. This
1471 * is used for avoiding races in accounting. If true,
1472 * pc->mem_cgroup may be overwritten.
1473 *
1474 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1475 * under hierarchy of moving cgroups. This is for
1476 * waiting at hith-memory prressure caused by "move".
1477 */
1478
1479static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
1480{
1481    VM_BUG_ON(!rcu_read_lock_held());
1482    return atomic_read(&memcg->moving_account) > 0;
1483}
1484
1485static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1486{
1487    struct mem_cgroup *from;
1488    struct mem_cgroup *to;
1489    bool ret = false;
1490    /*
1491     * Unlike task_move routines, we access mc.to, mc.from not under
1492     * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1493     */
1494    spin_lock(&mc.lock);
1495    from = mc.from;
1496    to = mc.to;
1497    if (!from)
1498        goto unlock;
1499
1500    ret = mem_cgroup_same_or_subtree(memcg, from)
1501        || mem_cgroup_same_or_subtree(memcg, to);
1502unlock:
1503    spin_unlock(&mc.lock);
1504    return ret;
1505}
1506
1507static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1508{
1509    if (mc.moving_task && current != mc.moving_task) {
1510        if (mem_cgroup_under_move(memcg)) {
1511            DEFINE_WAIT(wait);
1512            prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1513            /* moving charge context might have finished. */
1514            if (mc.moving_task)
1515                schedule();
1516            finish_wait(&mc.waitq, &wait);
1517            return true;
1518        }
1519    }
1520    return false;
1521}
1522
1523/*
1524 * Take this lock when
1525 * - a code tries to modify page's memcg while it's USED.
1526 * - a code tries to modify page state accounting in a memcg.
1527 * see mem_cgroup_stolen(), too.
1528 */
1529static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
1530                  unsigned long *flags)
1531{
1532    spin_lock_irqsave(&memcg->move_lock, *flags);
1533}
1534
1535static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
1536                unsigned long *flags)
1537{
1538    spin_unlock_irqrestore(&memcg->move_lock, *flags);
1539}
1540
1541#define K(x) ((x) << (PAGE_SHIFT-10))
1542/**
1543 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1544 * @memcg: The memory cgroup that went over limit
1545 * @p: Task that is going to be killed
1546 *
1547 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1548 * enabled
1549 */
1550void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1551{
1552    struct cgroup *task_cgrp;
1553    struct cgroup *mem_cgrp;
1554    /*
1555     * Need a buffer in BSS, can't rely on allocations. The code relies
1556     * on the assumption that OOM is serialized for memory controller.
1557     * If this assumption is broken, revisit this code.
1558     */
1559    static char memcg_name[PATH_MAX];
1560    int ret;
1561    struct mem_cgroup *iter;
1562    unsigned int i;
1563
1564    if (!p)
1565        return;
1566
1567    rcu_read_lock();
1568
1569    mem_cgrp = memcg->css.cgroup;
1570    task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1571
1572    ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1573    if (ret < 0) {
1574        /*
1575         * Unfortunately, we are unable to convert to a useful name
1576         * But we'll still print out the usage information
1577         */
1578        rcu_read_unlock();
1579        goto done;
1580    }
1581    rcu_read_unlock();
1582
1583    pr_info("Task in %s killed", memcg_name);
1584
1585    rcu_read_lock();
1586    ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1587    if (ret < 0) {
1588        rcu_read_unlock();
1589        goto done;
1590    }
1591    rcu_read_unlock();
1592
1593    /*
1594     * Continues from above, so we don't need an KERN_ level
1595     */
1596    pr_cont(" as a result of limit of %s\n", memcg_name);
1597done:
1598
1599    pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
1600        res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1601        res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1602        res_counter_read_u64(&memcg->res, RES_FAILCNT));
1603    pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
1604        res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1605        res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1606        res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1607    pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1608        res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
1609        res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
1610        res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
1611
1612    for_each_mem_cgroup_tree(iter, memcg) {
1613        pr_info("Memory cgroup stats");
1614
1615        rcu_read_lock();
1616        ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
1617        if (!ret)
1618            pr_cont(" for %s", memcg_name);
1619        rcu_read_unlock();
1620        pr_cont(":");
1621
1622        for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1623            if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1624                continue;
1625            pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
1626                K(mem_cgroup_read_stat(iter, i)));
1627        }
1628
1629        for (i = 0; i < NR_LRU_LISTS; i++)
1630            pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1631                K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1632
1633        pr_cont("\n");
1634    }
1635}
1636
1637/*
1638 * This function returns the number of memcg under hierarchy tree. Returns
1639 * 1(self count) if no children.
1640 */
1641static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1642{
1643    int num = 0;
1644    struct mem_cgroup *iter;
1645
1646    for_each_mem_cgroup_tree(iter, memcg)
1647        num++;
1648    return num;
1649}
1650
1651/*
1652 * Return the memory (and swap, if configured) limit for a memcg.
1653 */
1654static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1655{
1656    u64 limit;
1657
1658    limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1659
1660    /*
1661     * Do not consider swap space if we cannot swap due to swappiness
1662     */
1663    if (mem_cgroup_swappiness(memcg)) {
1664        u64 memsw;
1665
1666        limit += total_swap_pages << PAGE_SHIFT;
1667        memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1668
1669        /*
1670         * If memsw is finite and limits the amount of swap space
1671         * available to this memcg, return that limit.
1672         */
1673        limit = min(limit, memsw);
1674    }
1675
1676    return limit;
1677}
1678
1679static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1680                     int order)
1681{
1682    struct mem_cgroup *iter;
1683    unsigned long chosen_points = 0;
1684    unsigned long totalpages;
1685    unsigned int points = 0;
1686    struct task_struct *chosen = NULL;
1687
1688    /*
1689     * If current has a pending SIGKILL, then automatically select it. The
1690     * goal is to allow it to allocate so that it may quickly exit and free
1691     * its memory.
1692     */
1693    if (fatal_signal_pending(current)) {
1694        set_thread_flag(TIF_MEMDIE);
1695        return;
1696    }
1697
1698    check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
1699    totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
1700    for_each_mem_cgroup_tree(iter, memcg) {
1701        struct cgroup *cgroup = iter->css.cgroup;
1702        struct cgroup_iter it;
1703        struct task_struct *task;
1704
1705        cgroup_iter_start(cgroup, &it);
1706        while ((task = cgroup_iter_next(cgroup, &it))) {
1707            switch (oom_scan_process_thread(task, totalpages, NULL,
1708                            false)) {
1709            case OOM_SCAN_SELECT:
1710                if (chosen)
1711                    put_task_struct(chosen);
1712                chosen = task;
1713                chosen_points = ULONG_MAX;
1714                get_task_struct(chosen);
1715                /* fall through */
1716            case OOM_SCAN_CONTINUE:
1717                continue;
1718            case OOM_SCAN_ABORT:
1719                cgroup_iter_end(cgroup, &it);
1720                mem_cgroup_iter_break(memcg, iter);
1721                if (chosen)
1722                    put_task_struct(chosen);
1723                return;
1724            case OOM_SCAN_OK:
1725                break;
1726            };
1727            points = oom_badness(task, memcg, NULL, totalpages);
1728            if (points > chosen_points) {
1729                if (chosen)
1730                    put_task_struct(chosen);
1731                chosen = task;
1732                chosen_points = points;
1733                get_task_struct(chosen);
1734            }
1735        }
1736        cgroup_iter_end(cgroup, &it);
1737    }
1738
1739    if (!chosen)
1740        return;
1741    points = chosen_points * 1000 / totalpages;
1742    oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1743             NULL, "Memory cgroup out of memory");
1744}
1745
1746static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
1747                    gfp_t gfp_mask,
1748                    unsigned long flags)
1749{
1750    unsigned long total = 0;
1751    bool noswap = false;
1752    int loop;
1753
1754    if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
1755        noswap = true;
1756    if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
1757        noswap = true;
1758
1759    for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
1760        if (loop)
1761            drain_all_stock_async(memcg);
1762        total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
1763        /*
1764         * Allow limit shrinkers, which are triggered directly
1765         * by userspace, to catch signals and stop reclaim
1766         * after minimal progress, regardless of the margin.
1767         */
1768        if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
1769            break;
1770        if (mem_cgroup_margin(memcg))
1771            break;
1772        /*
1773         * If nothing was reclaimed after two attempts, there
1774         * may be no reclaimable pages in this hierarchy.
1775         */
1776        if (loop && !total)
1777            break;
1778    }
1779    return total;
1780}
1781
1782/**
1783 * test_mem_cgroup_node_reclaimable
1784 * @memcg: the target memcg
1785 * @nid: the node ID to be checked.
1786 * @noswap : specify true here if the user wants flle only information.
1787 *
1788 * This function returns whether the specified memcg contains any
1789 * reclaimable pages on a node. Returns true if there are any reclaimable
1790 * pages in the node.
1791 */
1792static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1793        int nid, bool noswap)
1794{
1795    if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1796        return true;
1797    if (noswap || !total_swap_pages)
1798        return false;
1799    if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1800        return true;
1801    return false;
1802
1803}
1804#if MAX_NUMNODES > 1
1805
1806/*
1807 * Always updating the nodemask is not very good - even if we have an empty
1808 * list or the wrong list here, we can start from some node and traverse all
1809 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1810 *
1811 */
1812static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1813{
1814    int nid;
1815    /*
1816     * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1817     * pagein/pageout changes since the last update.
1818     */
1819    if (!atomic_read(&memcg->numainfo_events))
1820        return;
1821    if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1822        return;
1823
1824    /* make a nodemask where this memcg uses memory from */
1825    memcg->scan_nodes = node_states[N_MEMORY];
1826
1827    for_each_node_mask(nid, node_states[N_MEMORY]) {
1828
1829        if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1830            node_clear(nid, memcg->scan_nodes);
1831    }
1832
1833    atomic_set(&memcg->numainfo_events, 0);
1834    atomic_set(&memcg->numainfo_updating, 0);
1835}
1836
1837/*
1838 * Selecting a node where we start reclaim from. Because what we need is just
1839 * reducing usage counter, start from anywhere is O,K. Considering
1840 * memory reclaim from current node, there are pros. and cons.
1841 *
1842 * Freeing memory from current node means freeing memory from a node which
1843 * we'll use or we've used. So, it may make LRU bad. And if several threads
1844 * hit limits, it will see a contention on a node. But freeing from remote
1845 * node means more costs for memory reclaim because of memory latency.
1846 *
1847 * Now, we use round-robin. Better algorithm is welcomed.
1848 */
1849int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1850{
1851    int node;
1852
1853    mem_cgroup_may_update_nodemask(memcg);
1854    node = memcg->last_scanned_node;
1855
1856    node = next_node(node, memcg->scan_nodes);
1857    if (node == MAX_NUMNODES)
1858        node = first_node(memcg->scan_nodes);
1859    /*
1860     * We call this when we hit limit, not when pages are added to LRU.
1861     * No LRU may hold pages because all pages are UNEVICTABLE or
1862     * memcg is too small and all pages are not on LRU. In that case,
1863     * we use curret node.
1864     */
1865    if (unlikely(node == MAX_NUMNODES))
1866        node = numa_node_id();
1867
1868    memcg->last_scanned_node = node;
1869    return node;
1870}
1871
1872/*
1873 * Check all nodes whether it contains reclaimable pages or not.
1874 * For quick scan, we make use of scan_nodes. This will allow us to skip
1875 * unused nodes. But scan_nodes is lazily updated and may not cotain
1876 * enough new information. We need to do double check.
1877 */
1878static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1879{
1880    int nid;
1881
1882    /*
1883     * quick check...making use of scan_node.
1884     * We can skip unused nodes.
1885     */
1886    if (!nodes_empty(memcg->scan_nodes)) {
1887        for (nid = first_node(memcg->scan_nodes);
1888             nid < MAX_NUMNODES;
1889             nid = next_node(nid, memcg->scan_nodes)) {
1890
1891            if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1892                return true;
1893        }
1894    }
1895    /*
1896     * Check rest of nodes.
1897     */
1898    for_each_node_state(nid, N_MEMORY) {
1899        if (node_isset(nid, memcg->scan_nodes))
1900            continue;
1901        if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1902            return true;
1903    }
1904    return false;
1905}
1906
1907#else
1908int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1909{
1910    return 0;
1911}
1912
1913static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1914{
1915    return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
1916}
1917#endif
1918
1919static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1920                   struct zone *zone,
1921                   gfp_t gfp_mask,
1922                   unsigned long *total_scanned)
1923{
1924    struct mem_cgroup *victim = NULL;
1925    int total = 0;
1926    int loop = 0;
1927    unsigned long excess;
1928    unsigned long nr_scanned;
1929    struct mem_cgroup_reclaim_cookie reclaim = {
1930        .zone = zone,
1931        .priority = 0,
1932    };
1933
1934    excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
1935
1936    while (1) {
1937        victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1938        if (!victim) {
1939            loop++;
1940            if (loop >= 2) {
1941                /*
1942                 * If we have not been able to reclaim
1943                 * anything, it might because there are
1944                 * no reclaimable pages under this hierarchy
1945                 */
1946                if (!total)
1947                    break;
1948                /*
1949                 * We want to do more targeted reclaim.
1950                 * excess >> 2 is not to excessive so as to
1951                 * reclaim too much, nor too less that we keep
1952                 * coming back to reclaim from this cgroup
1953                 */
1954                if (total >= (excess >> 2) ||
1955                    (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1956                    break;
1957            }
1958            continue;
1959        }
1960        if (!mem_cgroup_reclaimable(victim, false))
1961            continue;
1962        total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
1963                             zone, &nr_scanned);
1964        *total_scanned += nr_scanned;
1965        if (!res_counter_soft_limit_excess(&root_memcg->res))
1966            break;
1967    }
1968    mem_cgroup_iter_break(root_memcg, victim);
1969    return total;
1970}
1971
1972/*
1973 * Check OOM-Killer is already running under our hierarchy.
1974 * If someone is running, return false.
1975 * Has to be called with memcg_oom_lock
1976 */
1977static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
1978{
1979    struct mem_cgroup *iter, *failed = NULL;
1980
1981    for_each_mem_cgroup_tree(iter, memcg) {
1982        if (iter->oom_lock) {
1983            /*
1984             * this subtree of our hierarchy is already locked
1985             * so we cannot give a lock.
1986             */
1987            failed = iter;
1988            mem_cgroup_iter_break(memcg, iter);
1989            break;
1990        } else
1991            iter->oom_lock = true;
1992    }
1993
1994    if (!failed)
1995        return true;
1996
1997    /*
1998     * OK, we failed to lock the whole subtree so we have to clean up
1999     * what we set up to the failing subtree
2000     */
2001    for_each_mem_cgroup_tree(iter, memcg) {
2002        if (iter == failed) {
2003            mem_cgroup_iter_break(memcg, iter);
2004            break;
2005        }
2006        iter->oom_lock = false;
2007    }
2008    return false;
2009}
2010
2011/*
2012 * Has to be called with memcg_oom_lock
2013 */
2014static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
2015{
2016    struct mem_cgroup *iter;
2017
2018    for_each_mem_cgroup_tree(iter, memcg)
2019        iter->oom_lock = false;
2020    return 0;
2021}
2022
2023static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
2024{
2025    struct mem_cgroup *iter;
2026
2027    for_each_mem_cgroup_tree(iter, memcg)
2028        atomic_inc(&iter->under_oom);
2029}
2030
2031static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
2032{
2033    struct mem_cgroup *iter;
2034
2035    /*
2036     * When a new child is created while the hierarchy is under oom,
2037     * mem_cgroup_oom_lock() may not be called. We have to use
2038     * atomic_add_unless() here.
2039     */
2040    for_each_mem_cgroup_tree(iter, memcg)
2041        atomic_add_unless(&iter->under_oom, -1, 0);
2042}
2043
2044static DEFINE_SPINLOCK(memcg_oom_lock);
2045static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2046
2047struct oom_wait_info {
2048    struct mem_cgroup *memcg;
2049    wait_queue_t wait;
2050};
2051
2052static int memcg_oom_wake_function(wait_queue_t *wait,
2053    unsigned mode, int sync, void *arg)
2054{
2055    struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
2056    struct mem_cgroup *oom_wait_memcg;
2057    struct oom_wait_info *oom_wait_info;
2058
2059    oom_wait_info = container_of(wait, struct oom_wait_info, wait);
2060    oom_wait_memcg = oom_wait_info->memcg;
2061
2062    /*
2063     * Both of oom_wait_info->memcg and wake_memcg are stable under us.
2064     * Then we can use css_is_ancestor without taking care of RCU.
2065     */
2066    if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
2067        && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
2068        return 0;
2069    return autoremove_wake_function(wait, mode, sync, arg);
2070}
2071
2072static void memcg_wakeup_oom(struct mem_cgroup *memcg)
2073{
2074    /* for filtering, pass "memcg" as argument. */
2075    __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
2076}
2077
2078static void memcg_oom_recover(struct mem_cgroup *memcg)
2079{
2080    if (memcg && atomic_read(&memcg->under_oom))
2081        memcg_wakeup_oom(memcg);
2082}
2083
2084/*
2085 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
2086 */
2087static bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask,
2088                  int order)
2089{
2090    struct oom_wait_info owait;
2091    bool locked, need_to_kill;
2092
2093    owait.memcg = memcg;
2094    owait.wait.flags = 0;
2095    owait.wait.func = memcg_oom_wake_function;
2096    owait.wait.private = current;
2097    INIT_LIST_HEAD(&owait.wait.task_list);
2098    need_to_kill = true;
2099    mem_cgroup_mark_under_oom(memcg);
2100
2101    /* At first, try to OOM lock hierarchy under memcg.*/
2102    spin_lock(&memcg_oom_lock);
2103    locked = mem_cgroup_oom_lock(memcg);
2104    /*
2105     * Even if signal_pending(), we can't quit charge() loop without
2106     * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
2107     * under OOM is always welcomed, use TASK_KILLABLE here.
2108     */
2109    prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
2110    if (!locked || memcg->oom_kill_disable)
2111        need_to_kill = false;
2112    if (locked)
2113        mem_cgroup_oom_notify(memcg);
2114    spin_unlock(&memcg_oom_lock);
2115
2116    if (need_to_kill) {
2117        finish_wait(&memcg_oom_waitq, &owait.wait);
2118        mem_cgroup_out_of_memory(memcg, mask, order);
2119    } else {
2120        schedule();
2121        finish_wait(&memcg_oom_waitq, &owait.wait);
2122    }
2123    spin_lock(&memcg_oom_lock);
2124    if (locked)
2125        mem_cgroup_oom_unlock(memcg);
2126    memcg_wakeup_oom(memcg);
2127    spin_unlock(&memcg_oom_lock);
2128
2129    mem_cgroup_unmark_under_oom(memcg);
2130
2131    if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
2132        return false;
2133    /* Give chance to dying process */
2134    schedule_timeout_uninterruptible(1);
2135    return true;
2136}
2137
2138/*
2139 * Currently used to update mapped file statistics, but the routine can be
2140 * generalized to update other statistics as well.
2141 *
2142 * Notes: Race condition
2143 *
2144 * We usually use page_cgroup_lock() for accessing page_cgroup member but
2145 * it tends to be costly. But considering some conditions, we doesn't need
2146 * to do so _always_.
2147 *
2148 * Considering "charge", lock_page_cgroup() is not required because all
2149 * file-stat operations happen after a page is attached to radix-tree. There
2150 * are no race with "charge".
2151 *
2152 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2153 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2154 * if there are race with "uncharge". Statistics itself is properly handled
2155 * by flags.
2156 *
2157 * Considering "move", this is an only case we see a race. To make the race
2158 * small, we check mm->moving_account and detect there are possibility of race
2159 * If there is, we take a lock.
2160 */
2161
2162void __mem_cgroup_begin_update_page_stat(struct page *page,
2163                bool *locked, unsigned long *flags)
2164{
2165    struct mem_cgroup *memcg;
2166    struct page_cgroup *pc;
2167
2168    pc = lookup_page_cgroup(page);
2169again:
2170    memcg = pc->mem_cgroup;
2171    if (unlikely(!memcg || !PageCgroupUsed(pc)))
2172        return;
2173    /*
2174     * If this memory cgroup is not under account moving, we don't
2175     * need to take move_lock_mem_cgroup(). Because we already hold
2176     * rcu_read_lock(), any calls to move_account will be delayed until
2177     * rcu_read_unlock() if mem_cgroup_stolen() == true.
2178     */
2179    if (!mem_cgroup_stolen(memcg))
2180        return;
2181
2182    move_lock_mem_cgroup(memcg, flags);
2183    if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
2184        move_unlock_mem_cgroup(memcg, flags);
2185        goto again;
2186    }
2187    *locked = true;
2188}
2189
2190void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
2191{
2192    struct page_cgroup *pc = lookup_page_cgroup(page);
2193
2194    /*
2195     * It's guaranteed that pc->mem_cgroup never changes while
2196     * lock is held because a routine modifies pc->mem_cgroup
2197     * should take move_lock_mem_cgroup().
2198     */
2199    move_unlock_mem_cgroup(pc->mem_cgroup, flags);
2200}
2201
2202void mem_cgroup_update_page_stat(struct page *page,
2203                 enum mem_cgroup_page_stat_item idx, int val)
2204{
2205    struct mem_cgroup *memcg;
2206    struct page_cgroup *pc = lookup_page_cgroup(page);
2207    unsigned long uninitialized_var(flags);
2208
2209    if (mem_cgroup_disabled())
2210        return;
2211
2212    memcg = pc->mem_cgroup;
2213    if (unlikely(!memcg || !PageCgroupUsed(pc)))
2214        return;
2215
2216    switch (idx) {
2217    case MEMCG_NR_FILE_MAPPED:
2218        idx = MEM_CGROUP_STAT_FILE_MAPPED;
2219        break;
2220    default:
2221        BUG();
2222    }
2223
2224    this_cpu_add(memcg->stat->count[idx], val);
2225}
2226
2227/*
2228 * size of first charge trial. "32" comes from vmscan.c's magic value.
2229 * TODO: maybe necessary to use big numbers in big irons.
2230 */
2231#define CHARGE_BATCH 32U
2232struct memcg_stock_pcp {
2233    struct mem_cgroup *cached; /* this never be root cgroup */
2234    unsigned int nr_pages;
2235    struct work_struct work;
2236    unsigned long flags;
2237#define FLUSHING_CACHED_CHARGE 0
2238};
2239static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
2240static DEFINE_MUTEX(percpu_charge_mutex);
2241
2242/**
2243 * consume_stock: Try to consume stocked charge on this cpu.
2244 * @memcg: memcg to consume from.
2245 * @nr_pages: how many pages to charge.
2246 *
2247 * The charges will only happen if @memcg matches the current cpu's memcg
2248 * stock, and at least @nr_pages are available in that stock. Failure to
2249 * service an allocation will refill the stock.
2250 *
2251 * returns true if successful, false otherwise.
2252 */
2253static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2254{
2255    struct memcg_stock_pcp *stock;
2256    bool ret = true;
2257
2258    if (nr_pages > CHARGE_BATCH)
2259        return false;
2260
2261    stock = &get_cpu_var(memcg_stock);
2262    if (memcg == stock->cached && stock->nr_pages >= nr_pages)
2263        stock->nr_pages -= nr_pages;
2264    else /* need to call res_counter_charge */
2265        ret = false;
2266    put_cpu_var(memcg_stock);
2267    return ret;
2268}
2269
2270/*
2271 * Returns stocks cached in percpu to res_counter and reset cached information.
2272 */
2273static void drain_stock(struct memcg_stock_pcp *stock)
2274{
2275    struct mem_cgroup *old = stock->cached;
2276
2277    if (stock->nr_pages) {
2278        unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2279
2280        res_counter_uncharge(&old->res, bytes);
2281        if (do_swap_account)
2282            res_counter_uncharge(&old->memsw, bytes);
2283        stock->nr_pages = 0;
2284    }
2285    stock->cached = NULL;
2286}
2287
2288/*
2289 * This must be called under preempt disabled or must be called by
2290 * a thread which is pinned to local cpu.
2291 */
2292static void drain_local_stock(struct work_struct *dummy)
2293{
2294    struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2295    drain_stock(stock);
2296    clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
2297}
2298
2299static void __init memcg_stock_init(void)
2300{
2301    int cpu;
2302
2303    for_each_possible_cpu(cpu) {
2304        struct memcg_stock_pcp *stock =
2305                    &per_cpu(memcg_stock, cpu);
2306        INIT_WORK(&stock->work, drain_local_stock);
2307    }
2308}
2309
2310/*
2311 * Cache charges(val) which is from res_counter, to local per_cpu area.
2312 * This will be consumed by consume_stock() function, later.
2313 */
2314static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
2315{
2316    struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2317
2318    if (stock->cached != memcg) { /* reset if necessary */
2319        drain_stock(stock);
2320        stock->cached = memcg;
2321    }
2322    stock->nr_pages += nr_pages;
2323    put_cpu_var(memcg_stock);
2324}
2325
2326/*
2327 * Drains all per-CPU charge caches for given root_memcg resp. subtree
2328 * of the hierarchy under it. sync flag says whether we should block
2329 * until the work is done.
2330 */
2331static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
2332{
2333    int cpu, curcpu;
2334
2335    /* Notify other cpus that system-wide "drain" is running */
2336    get_online_cpus();
2337    curcpu = get_cpu();
2338    for_each_online_cpu(cpu) {
2339        struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2340        struct mem_cgroup *memcg;
2341
2342        memcg = stock->cached;
2343        if (!memcg || !stock->nr_pages)
2344            continue;
2345        if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
2346            continue;
2347        if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2348            if (cpu == curcpu)
2349                drain_local_stock(&stock->work);
2350            else
2351                schedule_work_on(cpu, &stock->work);
2352        }
2353    }
2354    put_cpu();
2355
2356    if (!sync)
2357        goto out;
2358
2359    for_each_online_cpu(cpu) {
2360        struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
2361        if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
2362            flush_work(&stock->work);
2363    }
2364out:
2365     put_online_cpus();
2366}
2367
2368/*
2369 * Tries to drain stocked charges in other cpus. This function is asynchronous
2370 * and just put a work per cpu for draining localy on each cpu. Caller can
2371 * expects some charges will be back to res_counter later but cannot wait for
2372 * it.
2373 */
2374static void drain_all_stock_async(struct mem_cgroup *root_memcg)
2375{
2376    /*
2377     * If someone calls draining, avoid adding more kworker runs.
2378     */
2379    if (!mutex_trylock(&percpu_charge_mutex))
2380        return;
2381    drain_all_stock(root_memcg, false);
2382    mutex_unlock(&percpu_charge_mutex);
2383}
2384
2385/* This is a synchronous drain interface. */
2386static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
2387{
2388    /* called when force_empty is called */
2389    mutex_lock(&percpu_charge_mutex);
2390    drain_all_stock(root_memcg, true);
2391    mutex_unlock(&percpu_charge_mutex);
2392}
2393
2394/*
2395 * This function drains percpu counter value from DEAD cpu and
2396 * move it to local cpu. Note that this function can be preempted.
2397 */
2398static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
2399{
2400    int i;
2401
2402    spin_lock(&memcg->pcp_counter_lock);
2403    for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
2404        long x = per_cpu(memcg->stat->count[i], cpu);
2405
2406        per_cpu(memcg->stat->count[i], cpu) = 0;
2407        memcg->nocpu_base.count[i] += x;
2408    }
2409    for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2410        unsigned long x = per_cpu(memcg->stat->events[i], cpu);
2411
2412        per_cpu(memcg->stat->events[i], cpu) = 0;
2413        memcg->nocpu_base.events[i] += x;
2414    }
2415    spin_unlock(&memcg->pcp_counter_lock);
2416}
2417
2418static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
2419                    unsigned long action,
2420                    void *hcpu)
2421{
2422    int cpu = (unsigned long)hcpu;
2423    struct memcg_stock_pcp *stock;
2424    struct mem_cgroup *iter;
2425
2426    if (action == CPU_ONLINE)
2427        return NOTIFY_OK;
2428
2429    if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
2430        return NOTIFY_OK;
2431
2432    for_each_mem_cgroup(iter)
2433        mem_cgroup_drain_pcp_counter(iter, cpu);
2434
2435    stock = &per_cpu(memcg_stock, cpu);
2436    drain_stock(stock);
2437    return NOTIFY_OK;
2438}
2439
2440
2441/* See __mem_cgroup_try_charge() for details */
2442enum {
2443    CHARGE_OK, /* success */
2444    CHARGE_RETRY, /* need to retry but retry is not bad */
2445    CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
2446    CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
2447    CHARGE_OOM_DIE, /* the current is killed because of OOM */
2448};
2449
2450static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2451                unsigned int nr_pages, unsigned int min_pages,
2452                bool oom_check)
2453{
2454    unsigned long csize = nr_pages * PAGE_SIZE;
2455    struct mem_cgroup *mem_over_limit;
2456    struct res_counter *fail_res;
2457    unsigned long flags = 0;
2458    int ret;
2459
2460    ret = res_counter_charge(&memcg->res, csize, &fail_res);
2461
2462    if (likely(!ret)) {
2463        if (!do_swap_account)
2464            return CHARGE_OK;
2465        ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
2466        if (likely(!ret))
2467            return CHARGE_OK;
2468
2469        res_counter_uncharge(&memcg->res, csize);
2470        mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2471        flags |= MEM_CGROUP_RECLAIM_NOSWAP;
2472    } else
2473        mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
2474    /*
2475     * Never reclaim on behalf of optional batching, retry with a
2476     * single page instead.
2477     */
2478    if (nr_pages > min_pages)
2479        return CHARGE_RETRY;
2480
2481    if (!(gfp_mask & __GFP_WAIT))
2482        return CHARGE_WOULDBLOCK;
2483
2484    if (gfp_mask & __GFP_NORETRY)
2485        return CHARGE_NOMEM;
2486
2487    ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
2488    if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
2489        return CHARGE_RETRY;
2490    /*
2491     * Even though the limit is exceeded at this point, reclaim
2492     * may have been able to free some pages. Retry the charge
2493     * before killing the task.
2494     *
2495     * Only for regular pages, though: huge pages are rather
2496     * unlikely to succeed so close to the limit, and we fall back
2497     * to regular pages anyway in case of failure.
2498     */
2499    if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
2500        return CHARGE_RETRY;
2501
2502    /*
2503     * At task move, charge accounts can be doubly counted. So, it's
2504     * better to wait until the end of task_move if something is going on.
2505     */
2506    if (mem_cgroup_wait_acct_move(mem_over_limit))
2507        return CHARGE_RETRY;
2508
2509    /* If we don't need to call oom-killer at el, return immediately */
2510    if (!oom_check)
2511        return CHARGE_NOMEM;
2512    /* check OOM */
2513    if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask, get_order(csize)))
2514        return CHARGE_OOM_DIE;
2515
2516    return CHARGE_RETRY;
2517}
2518
2519/*
2520 * __mem_cgroup_try_charge() does
2521 * 1. detect memcg to be charged against from passed *mm and *ptr,
2522 * 2. update res_counter
2523 * 3. call memory reclaim if necessary.
2524 *
2525 * In some special case, if the task is fatal, fatal_signal_pending() or
2526 * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
2527 * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
2528 * as possible without any hazards. 2: all pages should have a valid
2529 * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
2530 * pointer, that is treated as a charge to root_mem_cgroup.
2531 *
2532 * So __mem_cgroup_try_charge() will return
2533 * 0 ... on success, filling *ptr with a valid memcg pointer.
2534 * -ENOMEM ... charge failure because of resource limits.
2535 * -EINTR ... if thread is fatal. *ptr is filled with root_mem_cgroup.
2536 *
2537 * Unlike the exported interface, an "oom" parameter is added. if oom==true,
2538 * the oom-killer can be invoked.
2539 */
2540static int __mem_cgroup_try_charge(struct mm_struct *mm,
2541                   gfp_t gfp_mask,
2542                   unsigned int nr_pages,
2543                   struct mem_cgroup **ptr,
2544                   bool oom)
2545{
2546    unsigned int batch = max(CHARGE_BATCH, nr_pages);
2547    int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2548    struct mem_cgroup *memcg = NULL;
2549    int ret;
2550
2551    /*
2552     * Unlike gloval-vm's OOM-kill, we're not in memory shortage
2553     * in system level. So, allow to go ahead dying process in addition to
2554     * MEMDIE process.
2555     */
2556    if (unlikely(test_thread_flag(TIF_MEMDIE)
2557             || fatal_signal_pending(current)))
2558        goto bypass;
2559
2560    /*
2561     * We always charge the cgroup the mm_struct belongs to.
2562     * The mm_struct's mem_cgroup changes on task migration if the
2563     * thread group leader migrates. It's possible that mm is not
2564     * set, if so charge the root memcg (happens for pagecache usage).
2565     */
2566    if (!*ptr && !mm)
2567        *ptr = root_mem_cgroup;
2568again:
2569    if (*ptr) { /* css should be a valid one */
2570        memcg = *ptr;
2571        if (mem_cgroup_is_root(memcg))
2572            goto done;
2573        if (consume_stock(memcg, nr_pages))
2574            goto done;
2575        css_get(&memcg->css);
2576    } else {
2577        struct task_struct *p;
2578
2579        rcu_read_lock();
2580        p = rcu_dereference(mm->owner);
2581        /*
2582         * Because we don't have task_lock(), "p" can exit.
2583         * In that case, "memcg" can point to root or p can be NULL with
2584         * race with swapoff. Then, we have small risk of mis-accouning.
2585         * But such kind of mis-account by race always happens because
2586         * we don't have cgroup_mutex(). It's overkill and we allo that
2587         * small race, here.
2588         * (*) swapoff at el will charge against mm-struct not against
2589         * task-struct. So, mm->owner can be NULL.
2590         */
2591        memcg = mem_cgroup_from_task(p);
2592        if (!memcg)
2593            memcg = root_mem_cgroup;
2594        if (mem_cgroup_is_root(memcg)) {
2595            rcu_read_unlock();
2596            goto done;
2597        }
2598        if (consume_stock(memcg, nr_pages)) {
2599            /*
2600             * It seems dagerous to access memcg without css_get().
2601             * But considering how consume_stok works, it's not
2602             * necessary. If consume_stock success, some charges
2603             * from this memcg are cached on this cpu. So, we
2604             * don't need to call css_get()/css_tryget() before
2605             * calling consume_stock().
2606             */
2607            rcu_read_unlock();
2608            goto done;
2609        }
2610        /* after here, we may be blocked. we need to get refcnt */
2611        if (!css_tryget(&memcg->css)) {
2612            rcu_read_unlock();
2613            goto again;
2614        }
2615        rcu_read_unlock();
2616    }
2617
2618    do {
2619        bool oom_check;
2620
2621        /* If killed, bypass charge */
2622        if (fatal_signal_pending(current)) {
2623            css_put(&memcg->css);
2624            goto bypass;
2625        }
2626
2627        oom_check = false;
2628        if (oom && !nr_oom_retries) {
2629            oom_check = true;
2630            nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2631        }
2632
2633        ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, nr_pages,
2634            oom_check);
2635        switch (ret) {
2636        case CHARGE_OK:
2637            break;
2638        case CHARGE_RETRY: /* not in OOM situation but retry */
2639            batch = nr_pages;
2640            css_put(&memcg->css);
2641            memcg = NULL;
2642            goto again;
2643        case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
2644            css_put(&memcg->css);
2645            goto nomem;
2646        case CHARGE_NOMEM: /* OOM routine works */
2647            if (!oom) {
2648                css_put(&memcg->css);
2649                goto nomem;
2650            }
2651            /* If oom, we never return -ENOMEM */
2652            nr_oom_retries--;
2653            break;
2654        case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2655            css_put(&memcg->css);
2656            goto bypass;
2657        }
2658    } while (ret != CHARGE_OK);
2659
2660    if (batch > nr_pages)
2661        refill_stock(memcg, batch - nr_pages);
2662    css_put(&memcg->css);
2663done:
2664    *ptr = memcg;
2665    return 0;
2666nomem:
2667    *ptr = NULL;
2668    return -ENOMEM;
2669bypass:
2670    *ptr = root_mem_cgroup;
2671    return -EINTR;
2672}
2673
2674/*
2675 * Somemtimes we have to undo a charge we got by try_charge().
2676 * This function is for that and do uncharge, put css's refcnt.
2677 * gotten by try_charge().
2678 */
2679static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
2680                       unsigned int nr_pages)
2681{
2682    if (!mem_cgroup_is_root(memcg)) {
2683        unsigned long bytes = nr_pages * PAGE_SIZE;
2684
2685        res_counter_uncharge(&memcg->res, bytes);
2686        if (do_swap_account)
2687            res_counter_uncharge(&memcg->memsw, bytes);
2688    }
2689}
2690
2691/*
2692 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
2693 * This is useful when moving usage to parent cgroup.
2694 */
2695static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
2696                    unsigned int nr_pages)
2697{
2698    unsigned long bytes = nr_pages * PAGE_SIZE;
2699
2700    if (mem_cgroup_is_root(memcg))
2701        return;
2702
2703    res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
2704    if (do_swap_account)
2705        res_counter_uncharge_until(&memcg->memsw,
2706                        memcg->memsw.parent, bytes);
2707}
2708
2709/*
2710 * A helper function to get mem_cgroup from ID. must be called under
2711 * rcu_read_lock(). The caller is responsible for calling css_tryget if
2712 * the mem_cgroup is used for charging. (dropping refcnt from swap can be
2713 * called against removed memcg.)
2714 */
2715static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2716{
2717    struct cgroup_subsys_state *css;
2718
2719    /* ID 0 is unused ID */
2720    if (!id)
2721        return NULL;
2722    css = css_lookup(&mem_cgroup_subsys, id);
2723    if (!css)
2724        return NULL;
2725    return mem_cgroup_from_css(css);
2726}
2727
2728struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2729{
2730    struct mem_cgroup *memcg = NULL;
2731    struct page_cgroup *pc;
2732    unsigned short id;
2733    swp_entry_t ent;
2734
2735    VM_BUG_ON(!PageLocked(page));
2736
2737    pc = lookup_page_cgroup(page);
2738    lock_page_cgroup(pc);
2739    if (PageCgroupUsed(pc)) {
2740        memcg = pc->mem_cgroup;
2741        if (memcg && !css_tryget(&memcg->css))
2742            memcg = NULL;
2743    } else if (PageSwapCache(page)) {
2744        ent.val = page_private(page);
2745        id = lookup_swap_cgroup_id(ent);
2746        rcu_read_lock();
2747        memcg = mem_cgroup_lookup(id);
2748        if (memcg && !css_tryget(&memcg->css))
2749            memcg = NULL;
2750        rcu_read_unlock();
2751    }
2752    unlock_page_cgroup(pc);
2753    return memcg;
2754}
2755
2756static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
2757                       struct page *page,
2758                       unsigned int nr_pages,
2759                       enum charge_type ctype,
2760                       bool lrucare)
2761{
2762    struct page_cgroup *pc = lookup_page_cgroup(page);
2763    struct zone *uninitialized_var(zone);
2764    struct lruvec *lruvec;
2765    bool was_on_lru = false;
2766    bool anon;
2767
2768    lock_page_cgroup(pc);
2769    VM_BUG_ON(PageCgroupUsed(pc));
2770    /*
2771     * we don't need page_cgroup_lock about tail pages, becase they are not
2772     * accessed by any other context at this point.
2773     */
2774
2775    /*
2776     * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2777     * may already be on some other mem_cgroup's LRU. Take care of it.
2778     */
2779    if (lrucare) {
2780        zone = page_zone(page);
2781        spin_lock_irq(&zone->lru_lock);
2782        if (PageLRU(page)) {
2783            lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2784            ClearPageLRU(page);
2785            del_page_from_lru_list(page, lruvec, page_lru(page));
2786            was_on_lru = true;
2787        }
2788    }
2789
2790    pc->mem_cgroup = memcg;
2791    /*
2792     * We access a page_cgroup asynchronously without lock_page_cgroup().
2793     * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2794     * is accessed after testing USED bit. To make pc->mem_cgroup visible
2795     * before USED bit, we need memory barrier here.
2796     * See mem_cgroup_add_lru_list(), etc.
2797      */
2798    smp_wmb();
2799    SetPageCgroupUsed(pc);
2800
2801    if (lrucare) {
2802        if (was_on_lru) {
2803            lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
2804            VM_BUG_ON(PageLRU(page));
2805            SetPageLRU(page);
2806            add_page_to_lru_list(page, lruvec, page_lru(page));
2807        }
2808        spin_unlock_irq(&zone->lru_lock);
2809    }
2810
2811    if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
2812        anon = true;
2813    else
2814        anon = false;
2815
2816    mem_cgroup_charge_statistics(memcg, anon, nr_pages);
2817    unlock_page_cgroup(pc);
2818
2819    /*
2820     * "charge_statistics" updated event counter. Then, check it.
2821     * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2822     * if they exceeds softlimit.
2823     */
2824    memcg_check_events(memcg, page);
2825}
2826
2827static DEFINE_MUTEX(set_limit_mutex);
2828
2829#ifdef CONFIG_MEMCG_KMEM
2830static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
2831{
2832    return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
2833        (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
2834}
2835
2836/*
2837 * This is a bit cumbersome, but it is rarely used and avoids a backpointer
2838 * in the memcg_cache_params struct.
2839 */
2840static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
2841{
2842    struct kmem_cache *cachep;
2843
2844    VM_BUG_ON(p->is_root_cache);
2845    cachep = p->root_cache;
2846    return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
2847}
2848
2849#ifdef CONFIG_SLABINFO
2850static int mem_cgroup_slabinfo_read(struct cgroup *cont, struct cftype *cft,
2851                    struct seq_file *m)
2852{
2853    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
2854    struct memcg_cache_params *params;
2855
2856    if (!memcg_can_account_kmem(memcg))
2857        return -EIO;
2858
2859    print_slabinfo_header(m);
2860
2861    mutex_lock(&memcg->slab_caches_mutex);
2862    list_for_each_entry(params, &memcg->memcg_slab_caches, list)
2863        cache_show(memcg_params_to_cache(params), m);
2864    mutex_unlock(&memcg->slab_caches_mutex);
2865
2866    return 0;
2867}
2868#endif
2869
2870static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
2871{
2872    struct res_counter *fail_res;
2873    struct mem_cgroup *_memcg;
2874    int ret = 0;
2875    bool may_oom;
2876
2877    ret = res_counter_charge(&memcg->kmem, size, &fail_res);
2878    if (ret)
2879        return ret;
2880
2881    /*
2882     * Conditions under which we can wait for the oom_killer. Those are
2883     * the same conditions tested by the core page allocator
2884     */
2885    may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);
2886
2887    _memcg = memcg;
2888    ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
2889                      &_memcg, may_oom);
2890
2891    if (ret == -EINTR) {
2892        /*
2893         * __mem_cgroup_try_charge() chosed to bypass to root due to
2894         * OOM kill or fatal signal. Since our only options are to
2895         * either fail the allocation or charge it to this cgroup, do
2896         * it as a temporary condition. But we can't fail. From a
2897         * kmem/slab perspective, the cache has already been selected,
2898         * by mem_cgroup_kmem_get_cache(), so it is too late to change
2899         * our minds.
2900         *
2901         * This condition will only trigger if the task entered
2902         * memcg_charge_kmem in a sane state, but was OOM-killed during
2903         * __mem_cgroup_try_charge() above. Tasks that were already
2904         * dying when the allocation triggers should have been already
2905         * directed to the root cgroup in memcontrol.h
2906         */
2907        res_counter_charge_nofail(&memcg->res, size, &fail_res);
2908        if (do_swap_account)
2909            res_counter_charge_nofail(&memcg->memsw, size,
2910                          &fail_res);
2911        ret = 0;
2912    } else if (ret)
2913        res_counter_uncharge(&memcg->kmem, size);
2914
2915    return ret;
2916}
2917
2918static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
2919{
2920    res_counter_uncharge(&memcg->res, size);
2921    if (do_swap_account)
2922        res_counter_uncharge(&memcg->memsw, size);
2923
2924    /* Not down to 0 */
2925    if (res_counter_uncharge(&memcg->kmem, size))
2926        return;
2927
2928    if (memcg_kmem_test_and_clear_dead(memcg))
2929        mem_cgroup_put(memcg);
2930}
2931
2932void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
2933{
2934    if (!memcg)
2935        return;
2936
2937    mutex_lock(&memcg->slab_caches_mutex);
2938    list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
2939    mutex_unlock(&memcg->slab_caches_mutex);
2940}
2941
2942/*
2943 * helper for acessing a memcg's index. It will be used as an index in the
2944 * child cache array in kmem_cache, and also to derive its name. This function
2945 * will return -1 when this is not a kmem-limited memcg.
2946 */
2947int memcg_cache_id(struct mem_cgroup *memcg)
2948{
2949    return memcg ? memcg->kmemcg_id : -1;
2950}
2951
2952/*
2953 * This ends up being protected by the set_limit mutex, during normal
2954 * operation, because that is its main call site.
2955 *
2956 * But when we create a new cache, we can call this as well if its parent
2957 * is kmem-limited. That will have to hold set_limit_mutex as well.
2958 */
2959int memcg_update_cache_sizes(struct mem_cgroup *memcg)
2960{
2961    int num, ret;
2962
2963    num = ida_simple_get(&kmem_limited_groups,
2964                0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2965    if (num < 0)
2966        return num;
2967    /*
2968     * After this point, kmem_accounted (that we test atomically in
2969     * the beginning of this conditional), is no longer 0. This
2970     * guarantees only one process will set the following boolean
2971     * to true. We don't need test_and_set because we're protected
2972     * by the set_limit_mutex anyway.
2973     */
2974    memcg_kmem_set_activated(memcg);
2975
2976    ret = memcg_update_all_caches(num+1);
2977    if (ret) {
2978        ida_simple_remove(&kmem_limited_groups, num);
2979        memcg_kmem_clear_activated(memcg);
2980        return ret;
2981    }
2982
2983    memcg->kmemcg_id = num;
2984    INIT_LIST_HEAD(&memcg->memcg_slab_caches);
2985    mutex_init(&memcg->slab_caches_mutex);
2986    return 0;
2987}
2988
2989static size_t memcg_caches_array_size(int num_groups)
2990{
2991    ssize_t size;
2992    if (num_groups <= 0)
2993        return 0;
2994
2995    size = 2 * num_groups;
2996    if (size < MEMCG_CACHES_MIN_SIZE)
2997        size = MEMCG_CACHES_MIN_SIZE;
2998    else if (size > MEMCG_CACHES_MAX_SIZE)
2999        size = MEMCG_CACHES_MAX_SIZE;
3000
3001    return size;
3002}
3003
3004/*
3005 * We should update the current array size iff all caches updates succeed. This
3006 * can only be done from the slab side. The slab mutex needs to be held when
3007 * calling this.
3008 */
3009void memcg_update_array_size(int num)
3010{
3011    if (num > memcg_limited_groups_array_size)
3012        memcg_limited_groups_array_size = memcg_caches_array_size(num);
3013}
3014
3015static void kmem_cache_destroy_work_func(struct work_struct *w);
3016
3017int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
3018{
3019    struct memcg_cache_params *cur_params = s->memcg_params;
3020
3021    VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
3022
3023    if (num_groups > memcg_limited_groups_array_size) {
3024        int i;
3025        ssize_t size = memcg_caches_array_size(num_groups);
3026
3027        size *= sizeof(void *);
3028        size += sizeof(struct memcg_cache_params);
3029
3030        s->memcg_params = kzalloc(size, GFP_KERNEL);
3031        if (!s->memcg_params) {
3032            s->memcg_params = cur_params;
3033            return -ENOMEM;
3034        }
3035
3036        INIT_WORK(&s->memcg_params->destroy,
3037                kmem_cache_destroy_work_func);
3038        s->memcg_params->is_root_cache = true;
3039
3040        /*
3041         * There is the chance it will be bigger than
3042         * memcg_limited_groups_array_size, if we failed an allocation
3043         * in a cache, in which case all caches updated before it, will
3044         * have a bigger array.
3045         *
3046         * But if that is the case, the data after
3047         * memcg_limited_groups_array_size is certainly unused
3048         */
3049        for (i = 0; i < memcg_limited_groups_array_size; i++) {
3050            if (!cur_params->memcg_caches[i])
3051                continue;
3052            s->memcg_params->memcg_caches[i] =
3053                        cur_params->memcg_caches[i];
3054        }
3055
3056        /*
3057         * Ideally, we would wait until all caches succeed, and only
3058         * then free the old one. But this is not worth the extra
3059         * pointer per-cache we'd have to have for this.
3060         *
3061         * It is not a big deal if some caches are left with a size
3062         * bigger than the others. And all updates will reset this
3063         * anyway.
3064         */
3065        kfree(cur_params);
3066    }
3067    return 0;
3068}
3069
3070int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
3071             struct kmem_cache *root_cache)
3072{
3073    size_t size = sizeof(struct memcg_cache_params);
3074
3075    if (!memcg_kmem_enabled())
3076        return 0;
3077
3078    if (!memcg)
3079        size += memcg_limited_groups_array_size * sizeof(void *);
3080
3081    s->memcg_params = kzalloc(size, GFP_KERNEL);
3082    if (!s->memcg_params)
3083        return -ENOMEM;
3084
3085    INIT_WORK(&s->memcg_params->destroy,
3086            kmem_cache_destroy_work_func);
3087    if (memcg) {
3088        s->memcg_params->memcg = memcg;
3089        s->memcg_params->root_cache = root_cache;
3090    } else
3091        s->memcg_params->is_root_cache = true;
3092
3093    return 0;
3094}
3095
3096void memcg_release_cache(struct kmem_cache *s)
3097{
3098    struct kmem_cache *root;
3099    struct mem_cgroup *memcg;
3100    int id;
3101
3102    /*
3103     * This happens, for instance, when a root cache goes away before we
3104     * add any memcg.
3105     */
3106    if (!s->memcg_params)
3107        return;
3108
3109    if (s->memcg_params->is_root_cache)
3110        goto out;
3111
3112    memcg = s->memcg_params->memcg;
3113    id = memcg_cache_id(memcg);
3114
3115    root = s->memcg_params->root_cache;
3116    root->memcg_params->memcg_caches[id] = NULL;
3117    mem_cgroup_put(memcg);
3118
3119    mutex_lock(&memcg->slab_caches_mutex);
3120    list_del(&s->memcg_params->list);
3121    mutex_unlock(&memcg->slab_caches_mutex);
3122
3123out:
3124    kfree(s->memcg_params);
3125}
3126
3127/*
3128 * During the creation a new cache, we need to disable our accounting mechanism
3129 * altogether. This is true even if we are not creating, but rather just
3130 * enqueing new caches to be created.
3131 *
3132 * This is because that process will trigger allocations; some visible, like
3133 * explicit kmallocs to auxiliary data structures, name strings and internal
3134 * cache structures; some well concealed, like INIT_WORK() that can allocate
3135 * objects during debug.
3136 *
3137 * If any allocation happens during memcg_kmem_get_cache, we will recurse back
3138 * to it. This may not be a bounded recursion: since the first cache creation
3139 * failed to complete (waiting on the allocation), we'll just try to create the
3140 * cache again, failing at the same point.
3141 *
3142 * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
3143 * memcg_kmem_skip_account. So we enclose anything that might allocate memory
3144 * inside the following two functions.
3145 */
3146static inline void memcg_stop_kmem_account(void)
3147{
3148    VM_BUG_ON(!current->mm);
3149    current->memcg_kmem_skip_account++;
3150}
3151
3152static inline void memcg_resume_kmem_account(void)
3153{
3154    VM_BUG_ON(!current->mm);
3155    current->memcg_kmem_skip_account--;
3156}
3157
3158static void kmem_cache_destroy_work_func(struct work_struct *w)
3159{
3160    struct kmem_cache *cachep;
3161    struct memcg_cache_params *p;
3162
3163    p = container_of(w, struct memcg_cache_params, destroy);
3164
3165    cachep = memcg_params_to_cache(p);
3166
3167    /*
3168     * If we get down to 0 after shrink, we could delete right away.
3169     * However, memcg_release_pages() already puts us back in the workqueue
3170     * in that case. If we proceed deleting, we'll get a dangling
3171     * reference, and removing the object from the workqueue in that case
3172     * is unnecessary complication. We are not a fast path.
3173     *
3174     * Note that this case is fundamentally different from racing with
3175     * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
3176     * kmem_cache_shrink, not only we would be reinserting a dead cache
3177     * into the queue, but doing so from inside the worker racing to
3178     * destroy it.
3179     *
3180     * So if we aren't down to zero, we'll just schedule a worker and try
3181     * again
3182     */
3183    if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
3184        kmem_cache_shrink(cachep);
3185        if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
3186            return;
3187    } else
3188        kmem_cache_destroy(cachep);
3189}
3190
3191void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
3192{
3193    if (!cachep->memcg_params->dead)
3194        return;
3195
3196    /*
3197     * There are many ways in which we can get here.
3198     *
3199     * We can get to a memory-pressure situation while the delayed work is
3200     * still pending to run. The vmscan shrinkers can then release all
3201     * cache memory and get us to destruction. If this is the case, we'll
3202     * be executed twice, which is a bug (the second time will execute over
3203     * bogus data). In this case, cancelling the work should be fine.
3204     *
3205     * But we can also get here from the worker itself, if
3206     * kmem_cache_shrink is enough to shake all the remaining objects and
3207     * get the page count to 0. In this case, we'll deadlock if we try to
3208     * cancel the work (the worker runs with an internal lock held, which
3209     * is the same lock we would hold for cancel_work_sync().)
3210     *
3211     * Since we can't possibly know who got us here, just refrain from
3212     * running if there is already work pending
3213     */
3214    if (work_pending(&cachep->memcg_params->destroy))
3215        return;
3216    /*
3217     * We have to defer the actual destroying to a workqueue, because
3218     * we might currently be in a context that cannot sleep.
3219     */
3220    schedule_work(&cachep->memcg_params->destroy);
3221}
3222
3223static char *memcg_cache_name(struct mem_cgroup *memcg, struct kmem_cache *s)
3224{
3225    char *name;
3226    struct dentry *dentry;
3227
3228    rcu_read_lock();
3229    dentry = rcu_dereference(memcg->css.cgroup->dentry);
3230    rcu_read_unlock();
3231
3232    BUG_ON(dentry == NULL);
3233
3234    name = kasprintf(GFP_KERNEL, "%s(%d:%s)", s->name,
3235             memcg_cache_id(memcg), dentry->d_name.name);
3236
3237    return name;
3238}
3239
3240static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
3241                     struct kmem_cache *s)
3242{
3243    char *name;
3244    struct kmem_cache *new;
3245
3246    name = memcg_cache_name(memcg, s);
3247    if (!name)
3248        return NULL;
3249
3250    new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,
3251                      (s->flags & ~SLAB_PANIC), s->ctor, s);
3252
3253    if (new)
3254        new->allocflags |= __GFP_KMEMCG;
3255
3256    kfree(name);
3257    return new;
3258}
3259
3260/*
3261 * This lock protects updaters, not readers. We want readers to be as fast as
3262 * they can, and they will either see NULL or a valid cache value. Our model
3263 * allow them to see NULL, in which case the root memcg will be selected.
3264 *
3265 * We need this lock because multiple allocations to the same cache from a non
3266 * will span more than one worker. Only one of them can create the cache.
3267 */
3268static DEFINE_MUTEX(memcg_cache_mutex);
3269static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
3270                          struct kmem_cache *cachep)
3271{
3272    struct kmem_cache *new_cachep;
3273    int idx;
3274
3275    BUG_ON(!memcg_can_account_kmem(memcg));
3276
3277    idx = memcg_cache_id(memcg);
3278
3279    mutex_lock(&memcg_cache_mutex);
3280    new_cachep = cachep->memcg_params->memcg_caches[idx];
3281    if (new_cachep)
3282        goto out;
3283
3284    new_cachep = kmem_cache_dup(memcg, cachep);
3285    if (new_cachep == NULL) {
3286        new_cachep = cachep;
3287        goto out;
3288    }
3289
3290    mem_cgroup_get(memcg);
3291    atomic_set(&new_cachep->memcg_params->nr_pages , 0);
3292
3293    cachep->memcg_params->memcg_caches[idx] = new_cachep;
3294    /*
3295     * the readers won't lock, make sure everybody sees the updated value,
3296     * so they won't put stuff in the queue again for no reason
3297     */
3298    wmb();
3299out:
3300    mutex_unlock(&memcg_cache_mutex);
3301    return new_cachep;
3302}
3303
3304void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
3305{
3306    struct kmem_cache *c;
3307    int i;
3308
3309    if (!s->memcg_params)
3310        return;
3311    if (!s->memcg_params->is_root_cache)
3312        return;
3313
3314    /*
3315     * If the cache is being destroyed, we trust that there is no one else
3316     * requesting objects from it. Even if there are, the sanity checks in
3317     * kmem_cache_destroy should caught this ill-case.
3318     *
3319     * Still, we don't want anyone else freeing memcg_caches under our
3320     * noses, which can happen if a new memcg comes to life. As usual,
3321     * we'll take the set_limit_mutex to protect ourselves against this.
3322     */
3323    mutex_lock(&set_limit_mutex);
3324    for (i = 0; i < memcg_limited_groups_array_size; i++) {
3325        c = s->memcg_params->memcg_caches[i];
3326        if (!c)
3327            continue;
3328
3329        /*
3330         * We will now manually delete the caches, so to avoid races
3331         * we need to cancel all pending destruction workers and
3332         * proceed with destruction ourselves.
3333         *
3334         * kmem_cache_destroy() will call kmem_cache_shrink internally,
3335         * and that could spawn the workers again: it is likely that
3336         * the cache still have active pages until this very moment.
3337         * This would lead us back to mem_cgroup_destroy_cache.
3338         *
3339         * But that will not execute at all if the "dead" flag is not
3340         * set, so flip it down to guarantee we are in control.
3341         */
3342        c->memcg_params->dead = false;
3343        cancel_work_sync(&c->memcg_params->destroy);
3344        kmem_cache_destroy(c);
3345    }
3346    mutex_unlock(&set_limit_mutex);
3347}
3348
3349struct create_work {
3350    struct mem_cgroup *memcg;
3351    struct kmem_cache *cachep;
3352    struct work_struct work;
3353};
3354
3355static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3356{
3357    struct kmem_cache *cachep;
3358    struct memcg_cache_params *params;
3359
3360    if (!memcg_kmem_is_active(memcg))
3361        return;
3362
3363    mutex_lock(&memcg->slab_caches_mutex);
3364    list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
3365        cachep = memcg_params_to_cache(params);
3366        cachep->memcg_params->dead = true;
3367        schedule_work(&cachep->memcg_params->destroy);
3368    }
3369    mutex_unlock(&memcg->slab_caches_mutex);
3370}
3371
3372static void memcg_create_cache_work_func(struct work_struct *w)
3373{
3374    struct create_work *cw;
3375
3376    cw = container_of(w, struct create_work, work);
3377    memcg_create_kmem_cache(cw->memcg, cw->cachep);
3378    /* Drop the reference gotten when we enqueued. */
3379    css_put(&cw->memcg->css);
3380    kfree(cw);
3381}
3382
3383/*
3384 * Enqueue the creation of a per-memcg kmem_cache.
3385 * Called with rcu_read_lock.
3386 */
3387static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3388                     struct kmem_cache *cachep)
3389{
3390    struct create_work *cw;
3391
3392    cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
3393    if (cw == NULL)
3394        return;
3395
3396    /* The corresponding put will be done in the workqueue. */
3397    if (!css_tryget(&memcg->css)) {
3398        kfree(cw);
3399        return;
3400    }
3401
3402    cw->memcg = memcg;
3403    cw->cachep = cachep;
3404
3405    INIT_WORK(&cw->work, memcg_create_cache_work_func);
3406    schedule_work(&cw->work);
3407}
3408
3409static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3410                       struct kmem_cache *cachep)
3411{
3412    /*
3413     * We need to stop accounting when we kmalloc, because if the
3414     * corresponding kmalloc cache is not yet created, the first allocation
3415     * in __memcg_create_cache_enqueue will recurse.
3416     *
3417     * However, it is better to enclose the whole function. Depending on
3418     * the debugging options enabled, INIT_WORK(), for instance, can
3419     * trigger an allocation. This too, will make us recurse. Because at
3420     * this point we can't allow ourselves back into memcg_kmem_get_cache,
3421     * the safest choice is to do it like this, wrapping the whole function.
3422     */
3423    memcg_stop_kmem_account();
3424    __memcg_create_cache_enqueue(memcg, cachep);
3425    memcg_resume_kmem_account();
3426}
3427/*
3428 * Return the kmem_cache we're supposed to use for a slab allocation.
3429 * We try to use the current memcg's version of the cache.
3430 *
3431 * If the cache does not exist yet, if we are the first user of it,
3432 * we either create it immediately, if possible, or create it asynchronously
3433 * in a workqueue.
3434 * In the latter case, we will let the current allocation go through with
3435 * the original cache.
3436 *
3437 * Can't be called in interrupt context or from kernel threads.
3438 * This function needs to be called with rcu_read_lock() held.
3439 */
3440struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
3441                      gfp_t gfp)
3442{
3443    struct mem_cgroup *memcg;
3444    int idx;
3445
3446    VM_BUG_ON(!cachep->memcg_params);
3447    VM_BUG_ON(!cachep->memcg_params->is_root_cache);
3448
3449    if (!current->mm || current->memcg_kmem_skip_account)
3450        return cachep;
3451
3452    rcu_read_lock();
3453    memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
3454    rcu_read_unlock();
3455
3456    if (!memcg_can_account_kmem(memcg))
3457        return cachep;
3458
3459    idx = memcg_cache_id(memcg);
3460
3461    /*
3462     * barrier to mare sure we're always seeing the up to date value. The
3463     * code updating memcg_caches will issue a write barrier to match this.
3464     */
3465    read_barrier_depends();
3466    if (unlikely(cachep->memcg_params->memcg_caches[idx] == NULL)) {
3467        /*
3468         * If we are in a safe context (can wait, and not in interrupt
3469         * context), we could be be predictable and return right away.
3470         * This would guarantee that the allocation being performed
3471         * already belongs in the new cache.
3472         *
3473         * However, there are some clashes that can arrive from locking.
3474         * For instance, because we acquire the slab_mutex while doing
3475         * kmem_cache_dup, this means no further allocation could happen
3476         * with the slab_mutex held.
3477         *
3478         * Also, because cache creation issue get_online_cpus(), this
3479         * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
3480         * that ends up reversed during cpu hotplug. (cpuset allocates
3481         * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
3482         * better to defer everything.
3483         */
3484        memcg_create_cache_enqueue(memcg, cachep);
3485        return cachep;
3486    }
3487
3488    return cachep->memcg_params->memcg_caches[idx];
3489}
3490EXPORT_SYMBOL(__memcg_kmem_get_cache);
3491
3492/*
3493 * We need to verify if the allocation against current->mm->owner's memcg is
3494 * possible for the given order. But the page is not allocated yet, so we'll
3495 * need a further commit step to do the final arrangements.
3496 *
3497 * It is possible for the task to switch cgroups in this mean time, so at
3498 * commit time, we can't rely on task conversion any longer. We'll then use
3499 * the handle argument to return to the caller which cgroup we should commit
3500 * against. We could also return the memcg directly and avoid the pointer
3501 * passing, but a boolean return value gives better semantics considering
3502 * the compiled-out case as well.
3503 *
3504 * Returning true means the allocation is possible.
3505 */
3506bool
3507__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
3508{
3509    struct mem_cgroup *memcg;
3510    int ret;
3511
3512    *_memcg = NULL;
3513    memcg = try_get_mem_cgroup_from_mm(current->mm);
3514
3515    /*
3516     * very rare case described in mem_cgroup_from_task. Unfortunately there
3517     * isn't much we can do without complicating this too much, and it would
3518     * be gfp-dependent anyway. Just let it go
3519     */
3520    if (unlikely(!memcg))
3521        return true;
3522
3523    if (!memcg_can_account_kmem(memcg)) {
3524        css_put(&memcg->css);
3525        return true;
3526    }
3527
3528    ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
3529    if (!ret)
3530        *_memcg = memcg;
3531
3532    css_put(&memcg->css);
3533    return (ret == 0);
3534}
3535
3536void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
3537                  int order)
3538{
3539    struct page_cgroup *pc;
3540
3541    VM_BUG_ON(mem_cgroup_is_root(memcg));
3542
3543    /* The page allocation failed. Revert */
3544    if (!page) {
3545        memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3546        return;
3547    }
3548
3549    pc = lookup_page_cgroup(page);
3550    lock_page_cgroup(pc);
3551    pc->mem_cgroup = memcg;
3552    SetPageCgroupUsed(pc);
3553    unlock_page_cgroup(pc);
3554}
3555
3556void __memcg_kmem_uncharge_pages(struct page *page, int order)
3557{
3558    struct mem_cgroup *memcg = NULL;
3559    struct page_cgroup *pc;
3560
3561
3562    pc = lookup_page_cgroup(page);
3563    /*
3564     * Fast unlocked return. Theoretically might have changed, have to
3565     * check again after locking.
3566     */
3567    if (!PageCgroupUsed(pc))
3568        return;
3569
3570    lock_page_cgroup(pc);
3571    if (PageCgroupUsed(pc)) {
3572        memcg = pc->mem_cgroup;
3573        ClearPageCgroupUsed(pc);
3574    }
3575    unlock_page_cgroup(pc);
3576
3577    /*
3578     * We trust that only if there is a memcg associated with the page, it
3579     * is a valid allocation
3580     */
3581    if (!memcg)
3582        return;
3583
3584    VM_BUG_ON(mem_cgroup_is_root(memcg));
3585    memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
3586}
3587#else
3588static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3589{
3590}
3591#endif /* CONFIG_MEMCG_KMEM */
3592
3593#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3594
3595#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
3596/*
3597 * Because tail pages are not marked as "used", set it. We're under
3598 * zone->lru_lock, 'splitting on pmd' and compound_lock.
3599 * charge/uncharge will be never happen and move_account() is done under
3600 * compound_lock(), so we don't have to take care of races.
3601 */
3602void mem_cgroup_split_huge_fixup(struct page *head)
3603{
3604    struct page_cgroup *head_pc = lookup_page_cgroup(head);
3605    struct page_cgroup *pc;
3606    int i;
3607
3608    if (mem_cgroup_disabled())
3609        return;
3610    for (i = 1; i < HPAGE_PMD_NR; i++) {
3611        pc = head_pc + i;
3612        pc->mem_cgroup = head_pc->mem_cgroup;
3613        smp_wmb();/* see __commit_charge() */
3614        pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
3615    }
3616}
3617#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3618
3619/**
3620 * mem_cgroup_move_account - move account of the page
3621 * @page: the page
3622 * @nr_pages: number of regular pages (>1 for huge pages)
3623 * @pc: page_cgroup of the page.
3624 * @from: mem_cgroup which the page is moved from.
3625 * @to: mem_cgroup which the page is moved to. @from != @to.
3626 *
3627 * The caller must confirm following.
3628 * - page is not on LRU (isolate_page() is useful.)
3629 * - compound_lock is held when nr_pages > 1
3630 *
3631 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
3632 * from old cgroup.
3633 */
3634static int mem_cgroup_move_account(struct page *page,
3635                   unsigned int nr_pages,
3636                   struct page_cgroup *pc,
3637                   struct mem_cgroup *from,
3638                   struct mem_cgroup *to)
3639{
3640    unsigned long flags;
3641    int ret;
3642    bool anon = PageAnon(page);
3643
3644    VM_BUG_ON(from == to);
3645    VM_BUG_ON(PageLRU(page));
3646    /*
3647     * The page is isolated from LRU. So, collapse function
3648     * will not handle this page. But page splitting can happen.
3649     * Do this check under compound_page_lock(). The caller should
3650     * hold it.
3651     */
3652    ret = -EBUSY;
3653    if (nr_pages > 1 && !PageTransHuge(page))
3654        goto out;
3655
3656    lock_page_cgroup(pc);
3657
3658    ret = -EINVAL;
3659    if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
3660        goto unlock;
3661
3662    move_lock_mem_cgroup(from, &flags);
3663
3664    if (!anon && page_mapped(page)) {
3665        /* Update mapped_file data for mem_cgroup */
3666        preempt_disable();
3667        __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
3668        __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
3669        preempt_enable();
3670    }
3671    mem_cgroup_charge_statistics(from, anon, -nr_pages);
3672
3673    /* caller should have done css_get */
3674    pc->mem_cgroup = to;
3675    mem_cgroup_charge_statistics(to, anon, nr_pages);
3676    move_unlock_mem_cgroup(from, &flags);
3677    ret = 0;
3678unlock:
3679    unlock_page_cgroup(pc);
3680    /*
3681     * check events
3682     */
3683    memcg_check_events(to, page);
3684    memcg_check_events(from, page);
3685out:
3686    return ret;
3687}
3688
3689/**
3690 * mem_cgroup_move_parent - moves page to the parent group
3691 * @page: the page to move
3692 * @pc: page_cgroup of the page
3693 * @child: page's cgroup
3694 *
3695 * move charges to its parent or the root cgroup if the group has no
3696 * parent (aka use_hierarchy==0).
3697 * Although this might fail (get_page_unless_zero, isolate_lru_page or
3698 * mem_cgroup_move_account fails) the failure is always temporary and
3699 * it signals a race with a page removal/uncharge or migration. In the
3700 * first case the page is on the way out and it will vanish from the LRU
3701 * on the next attempt and the call should be retried later.
3702 * Isolation from the LRU fails only if page has been isolated from
3703 * the LRU since we looked at it and that usually means either global
3704 * reclaim or migration going on. The page will either get back to the
3705 * LRU or vanish.
3706 * Finaly mem_cgroup_move_account fails only if the page got uncharged
3707 * (!PageCgroupUsed) or moved to a different group. The page will
3708 * disappear in the next attempt.
3709 */
3710static int mem_cgroup_move_parent(struct page *page,
3711                  struct page_cgroup *pc,
3712                  struct mem_cgroup *child)
3713{
3714    struct mem_cgroup *parent;
3715    unsigned int nr_pages;
3716    unsigned long uninitialized_var(flags);
3717    int ret;
3718
3719    VM_BUG_ON(mem_cgroup_is_root(child));
3720
3721    ret = -EBUSY;
3722    if (!get_page_unless_zero(page))
3723        goto out;
3724    if (isolate_lru_page(page))
3725        goto put;
3726
3727    nr_pages = hpage_nr_pages(page);
3728
3729    parent = parent_mem_cgroup(child);
3730    /*
3731     * If no parent, move charges to root cgroup.
3732     */
3733    if (!parent)
3734        parent = root_mem_cgroup;
3735
3736    if (nr_pages > 1) {
3737        VM_BUG_ON(!PageTransHuge(page));
3738        flags = compound_lock_irqsave(page);
3739    }
3740
3741    ret = mem_cgroup_move_account(page, nr_pages,
3742                pc, child, parent);
3743    if (!ret)
3744        __mem_cgroup_cancel_local_charge(child, nr_pages);
3745
3746    if (nr_pages > 1)
3747        compound_unlock_irqrestore(page, flags);
3748    putback_lru_page(page);
3749put:
3750    put_page(page);
3751out:
3752    return ret;
3753}
3754
3755/*
3756 * Charge the memory controller for page usage.
3757 * Return
3758 * 0 if the charge was successful
3759 * < 0 if the cgroup is over its limit
3760 */
3761static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
3762                gfp_t gfp_mask, enum charge_type ctype)
3763{
3764    struct mem_cgroup *memcg = NULL;
3765    unsigned int nr_pages = 1;
3766    bool oom = true;
3767    int ret;
3768
3769    if (PageTransHuge(page)) {
3770        nr_pages <<= compound_order(page);
3771        VM_BUG_ON(!PageTransHuge(page));
3772        /*
3773         * Never OOM-kill a process for a huge page. The
3774         * fault handler will fall back to regular pages.
3775         */
3776        oom = false;
3777    }
3778
3779    ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
3780    if (ret == -ENOMEM)
3781        return ret;
3782    __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
3783    return 0;
3784}
3785
3786int mem_cgroup_newpage_charge(struct page *page,
3787                  struct mm_struct *mm, gfp_t gfp_mask)
3788{
3789    if (mem_cgroup_disabled())
3790        return 0;
3791    VM_BUG_ON(page_mapped(page));
3792    VM_BUG_ON(page->mapping && !PageAnon(page));
3793    VM_BUG_ON(!mm);
3794    return mem_cgroup_charge_common(page, mm, gfp_mask,
3795                    MEM_CGROUP_CHARGE_TYPE_ANON);
3796}
3797
3798/*
3799 * While swap-in, try_charge -> commit or cancel, the page is locked.
3800 * And when try_charge() successfully returns, one refcnt to memcg without
3801 * struct page_cgroup is acquired. This refcnt will be consumed by
3802 * "commit()" or removed by "cancel()"
3803 */
3804static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
3805                      struct page *page,
3806                      gfp_t mask,
3807                      struct mem_cgroup **memcgp)
3808{
3809    struct mem_cgroup *memcg;
3810    struct page_cgroup *pc;
3811    int ret;
3812
3813    pc = lookup_page_cgroup(page);
3814    /*
3815     * Every swap fault against a single page tries to charge the
3816     * page, bail as early as possible. shmem_unuse() encounters
3817     * already charged pages, too. The USED bit is protected by
3818     * the page lock, which serializes swap cache removal, which
3819     * in turn serializes uncharging.
3820     */
3821    if (PageCgroupUsed(pc))
3822        return 0;
3823    if (!do_swap_account)
3824        goto charge_cur_mm;
3825    memcg = try_get_mem_cgroup_from_page(page);
3826    if (!memcg)
3827        goto charge_cur_mm;
3828    *memcgp = memcg;
3829    ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
3830    css_put(&memcg->css);
3831    if (ret == -EINTR)
3832        ret = 0;
3833    return ret;
3834charge_cur_mm:
3835    ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
3836    if (ret == -EINTR)
3837        ret = 0;
3838    return ret;
3839}
3840
3841int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
3842                 gfp_t gfp_mask, struct mem_cgroup **memcgp)
3843{
3844    *memcgp = NULL;
3845    if (mem_cgroup_disabled())
3846        return 0;
3847    /*
3848     * A racing thread's fault, or swapoff, may have already
3849     * updated the pte, and even removed page from swap cache: in
3850     * those cases unuse_pte()'s pte_same() test will fail; but
3851     * there's also a KSM case which does need to charge the page.
3852     */
3853    if (!PageSwapCache(page)) {
3854        int ret;
3855
3856        ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
3857        if (ret == -EINTR)
3858            ret = 0;
3859        return ret;
3860    }
3861    return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
3862}
3863
3864void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
3865{
3866    if (mem_cgroup_disabled())
3867        return;
3868    if (!memcg)
3869        return;
3870    __mem_cgroup_cancel_charge(memcg, 1);
3871}
3872
3873static void
3874__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
3875                    enum charge_type ctype)
3876{
3877    if (mem_cgroup_disabled())
3878        return;
3879    if (!memcg)
3880        return;
3881
3882    __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
3883    /*
3884     * Now swap is on-memory. This means this page may be
3885     * counted both as mem and swap....double count.
3886     * Fix it by uncharging from memsw. Basically, this SwapCache is stable
3887     * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
3888     * may call delete_from_swap_cache() before reach here.
3889     */
3890    if (do_swap_account && PageSwapCache(page)) {
3891        swp_entry_t ent = {.val = page_private(page)};
3892        mem_cgroup_uncharge_swap(ent);
3893    }
3894}
3895
3896void mem_cgroup_commit_charge_swapin(struct page *page,
3897                     struct mem_cgroup *memcg)
3898{
3899    __mem_cgroup_commit_charge_swapin(page, memcg,
3900                      MEM_CGROUP_CHARGE_TYPE_ANON);
3901}
3902
3903int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
3904                gfp_t gfp_mask)
3905{
3906    struct mem_cgroup *memcg = NULL;
3907    enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
3908    int ret;
3909
3910    if (mem_cgroup_disabled())
3911        return 0;
3912    if (PageCompound(page))
3913        return 0;
3914
3915    if (!PageSwapCache(page))
3916        ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
3917    else { /* page is swapcache/shmem */
3918        ret = __mem_cgroup_try_charge_swapin(mm, page,
3919                             gfp_mask, &memcg);
3920        if (!ret)
3921            __mem_cgroup_commit_charge_swapin(page, memcg, type);
3922    }
3923    return ret;
3924}
3925
3926static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
3927                   unsigned int nr_pages,
3928                   const enum charge_type ctype)
3929{
3930    struct memcg_batch_info *batch = NULL;
3931    bool uncharge_memsw = true;
3932
3933    /* If swapout, usage of swap doesn't decrease */
3934    if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
3935        uncharge_memsw = false;
3936
3937    batch = &current->memcg_batch;
3938    /*
3939     * In usual, we do css_get() when we remember memcg pointer.
3940     * But in this case, we keep res->usage until end of a series of
3941     * uncharges. Then, it's ok to ignore memcg's refcnt.
3942     */
3943    if (!batch->memcg)
3944        batch->memcg = memcg;
3945    /*
3946     * do_batch > 0 when unmapping pages or inode invalidate/truncate.
3947     * In those cases, all pages freed continuously can be expected to be in
3948     * the same cgroup and we have chance to coalesce uncharges.
3949     * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
3950     * because we want to do uncharge as soon as possible.
3951     */
3952
3953    if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
3954        goto direct_uncharge;
3955
3956    if (nr_pages > 1)
3957        goto direct_uncharge;
3958
3959    /*
3960     * In typical case, batch->memcg == mem. This means we can
3961     * merge a series of uncharges to an uncharge of res_counter.
3962     * If not, we uncharge res_counter ony by one.
3963     */
3964    if (batch->memcg != memcg)
3965        goto direct_uncharge;
3966    /* remember freed charge and uncharge it later */
3967    batch->nr_pages++;
3968    if (uncharge_memsw)
3969        batch->memsw_nr_pages++;
3970    return;
3971direct_uncharge:
3972    res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
3973    if (uncharge_memsw)
3974        res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
3975    if (unlikely(batch->memcg != memcg))
3976        memcg_oom_recover(memcg);
3977}
3978
3979/*
3980 * uncharge if !page_mapped(page)
3981 */
3982static struct mem_cgroup *
3983__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
3984                 bool end_migration)
3985{
3986    struct mem_cgroup *memcg = NULL;
3987    unsigned int nr_pages = 1;
3988    struct page_cgroup *pc;
3989    bool anon;
3990
3991    if (mem_cgroup_disabled())
3992        return NULL;
3993
3994    VM_BUG_ON(PageSwapCache(page));
3995
3996    if (PageTransHuge(page)) {
3997        nr_pages <<= compound_order(page);
3998        VM_BUG_ON(!PageTransHuge(page));
3999    }
4000    /*
4001     * Check if our page_cgroup is valid
4002     */
4003    pc = lookup_page_cgroup(page);
4004    if (unlikely(!PageCgroupUsed(pc)))
4005        return NULL;
4006
4007    lock_page_cgroup(pc);
4008
4009    memcg = pc->mem_cgroup;
4010
4011    if (!PageCgroupUsed(pc))
4012        goto unlock_out;
4013
4014    anon = PageAnon(page);
4015
4016    switch (ctype) {
4017    case MEM_CGROUP_CHARGE_TYPE_ANON:
4018        /*
4019         * Generally PageAnon tells if it's the anon statistics to be
4020         * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
4021         * used before page reached the stage of being marked PageAnon.
4022         */
4023        anon = true;
4024        /* fallthrough */
4025    case MEM_CGROUP_CHARGE_TYPE_DROP:
4026        /* See mem_cgroup_prepare_migration() */
4027        if (page_mapped(page))
4028            goto unlock_out;
4029        /*
4030         * Pages under migration may not be uncharged. But
4031         * end_migration() /must/ be the one uncharging the
4032         * unused post-migration page and so it has to call
4033         * here with the migration bit still set. See the
4034         * res_counter handling below.
4035         */
4036        if (!end_migration && PageCgroupMigration(pc))
4037            goto unlock_out;
4038        break;
4039    case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
4040        if (!PageAnon(page)) { /* Shared memory */
4041            if (page->mapping && !page_is_file_cache(page))
4042                goto unlock_out;
4043        } else if (page_mapped(page)) /* Anon */
4044                goto unlock_out;
4045        break;
4046    default:
4047        break;
4048    }
4049
4050    mem_cgroup_charge_statistics(memcg, anon, -nr_pages);
4051
4052    ClearPageCgroupUsed(pc);
4053    /*
4054     * pc->mem_cgroup is not cleared here. It will be accessed when it's
4055     * freed from LRU. This is safe because uncharged page is expected not
4056     * to be reused (freed soon). Exception is SwapCache, it's handled by
4057     * special functions.
4058     */
4059
4060    unlock_page_cgroup(pc);
4061    /*
4062     * even after unlock, we have memcg->res.usage here and this memcg
4063     * will never be freed.
4064     */
4065    memcg_check_events(memcg, page);
4066    if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
4067        mem_cgroup_swap_statistics(memcg, true);
4068        mem_cgroup_get(memcg);
4069    }
4070    /*
4071     * Migration does not charge the res_counter for the
4072     * replacement page, so leave it alone when phasing out the
4073     * page that is unused after the migration.
4074     */
4075    if (!end_migration && !mem_cgroup_is_root(memcg))
4076        mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
4077
4078    return memcg;
4079
4080unlock_out:
4081    unlock_page_cgroup(pc);
4082    return NULL;
4083}
4084
4085void mem_cgroup_uncharge_page(struct page *page)
4086{
4087    /* early check. */
4088    if (page_mapped(page))
4089        return;
4090    VM_BUG_ON(page->mapping && !PageAnon(page));
4091    if (PageSwapCache(page))
4092        return;
4093    __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
4094}
4095
4096void mem_cgroup_uncharge_cache_page(struct page *page)
4097{
4098    VM_BUG_ON(page_mapped(page));
4099    VM_BUG_ON(page->mapping);
4100    __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
4101}
4102
4103/*
4104 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
4105 * In that cases, pages are freed continuously and we can expect pages
4106 * are in the same memcg. All these calls itself limits the number of
4107 * pages freed at once, then uncharge_start/end() is called properly.
4108 * This may be called prural(2) times in a context,
4109 */
4110
4111void mem_cgroup_uncharge_start(void)
4112{
4113    current->memcg_batch.do_batch++;
4114    /* We can do nest. */
4115    if (current->memcg_batch.do_batch == 1) {
4116        current->memcg_batch.memcg = NULL;
4117        current->memcg_batch.nr_pages = 0;
4118        current->memcg_batch.memsw_nr_pages = 0;
4119    }
4120}
4121
4122void mem_cgroup_uncharge_end(void)
4123{
4124    struct memcg_batch_info *batch = &current->memcg_batch;
4125
4126    if (!batch->do_batch)
4127        return;
4128
4129    batch->do_batch--;
4130    if (batch->do_batch) /* If stacked, do nothing. */
4131        return;
4132
4133    if (!batch->memcg)
4134        return;
4135    /*
4136     * This "batch->memcg" is valid without any css_get/put etc...
4137     * bacause we hide charges behind us.
4138     */
4139    if (batch->nr_pages)
4140        res_counter_uncharge(&batch->memcg->res,
4141                     batch->nr_pages * PAGE_SIZE);
4142    if (batch->memsw_nr_pages)
4143        res_counter_uncharge(&batch->memcg->memsw,
4144                     batch->memsw_nr_pages * PAGE_SIZE);
4145    memcg_oom_recover(batch->memcg);
4146    /* forget this pointer (for sanity check) */
4147    batch->memcg = NULL;
4148}
4149
4150#ifdef CONFIG_SWAP
4151/*
4152 * called after __delete_from_swap_cache() and drop "page" account.
4153 * memcg information is recorded to swap_cgroup of "ent"
4154 */
4155void
4156mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
4157{
4158    struct mem_cgroup *memcg;
4159    int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
4160
4161    if (!swapout) /* this was a swap cache but the swap is unused ! */
4162        ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
4163
4164    memcg = __mem_cgroup_uncharge_common(page, ctype, false);
4165
4166    /*
4167     * record memcg information, if swapout && memcg != NULL,
4168     * mem_cgroup_get() was called in uncharge().
4169     */
4170    if (do_swap_account && swapout && memcg)
4171        swap_cgroup_record(ent, css_id(&memcg->css));
4172}
4173#endif
4174
4175#ifdef CONFIG_MEMCG_SWAP
4176/*
4177 * called from swap_entry_free(). remove record in swap_cgroup and
4178 * uncharge "memsw" account.
4179 */
4180void mem_cgroup_uncharge_swap(swp_entry_t ent)
4181{
4182    struct mem_cgroup *memcg;
4183    unsigned short id;
4184
4185    if (!do_swap_account)
4186        return;
4187
4188    id = swap_cgroup_record(ent, 0);
4189    rcu_read_lock();
4190    memcg = mem_cgroup_lookup(id);
4191    if (memcg) {
4192        /*
4193         * We uncharge this because swap is freed.
4194         * This memcg can be obsolete one. We avoid calling css_tryget
4195         */
4196        if (!mem_cgroup_is_root(memcg))
4197            res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
4198        mem_cgroup_swap_statistics(memcg, false);
4199        mem_cgroup_put(memcg);
4200    }
4201    rcu_read_unlock();
4202}
4203
4204/**
4205 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
4206 * @entry: swap entry to be moved
4207 * @from: mem_cgroup which the entry is moved from
4208 * @to: mem_cgroup which the entry is moved to
4209 *
4210 * It succeeds only when the swap_cgroup's record for this entry is the same
4211 * as the mem_cgroup's id of @from.
4212 *
4213 * Returns 0 on success, -EINVAL on failure.
4214 *
4215 * The caller must have charged to @to, IOW, called res_counter_charge() about
4216 * both res and memsw, and called css_get().
4217 */
4218static int mem_cgroup_move_swap_account(swp_entry_t entry,
4219                struct mem_cgroup *from, struct mem_cgroup *to)
4220{
4221    unsigned short old_id, new_id;
4222
4223    old_id = css_id(&from->css);
4224    new_id = css_id(&to->css);
4225
4226    if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
4227        mem_cgroup_swap_statistics(from, false);
4228        mem_cgroup_swap_statistics(to, true);
4229        /*
4230         * This function is only called from task migration context now.
4231         * It postpones res_counter and refcount handling till the end
4232         * of task migration(mem_cgroup_clear_mc()) for performance
4233         * improvement. But we cannot postpone mem_cgroup_get(to)
4234         * because if the process that has been moved to @to does
4235         * swap-in, the refcount of @to might be decreased to 0.
4236         */
4237        mem_cgroup_get(to);
4238        return 0;
4239    }
4240    return -EINVAL;
4241}
4242#else
4243static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
4244                struct mem_cgroup *from, struct mem_cgroup *to)
4245{
4246    return -EINVAL;
4247}
4248#endif
4249
4250/*
4251 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
4252 * page belongs to.
4253 */
4254void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
4255                  struct mem_cgroup **memcgp)
4256{
4257    struct mem_cgroup *memcg = NULL;
4258    unsigned int nr_pages = 1;
4259    struct page_cgroup *pc;
4260    enum charge_type ctype;
4261
4262    *memcgp = NULL;
4263
4264    if (mem_cgroup_disabled())
4265        return;
4266
4267    if (PageTransHuge(page))
4268        nr_pages <<= compound_order(page);
4269
4270    pc = lookup_page_cgroup(page);
4271    lock_page_cgroup(pc);
4272    if (PageCgroupUsed(pc)) {
4273        memcg = pc->mem_cgroup;
4274        css_get(&memcg->css);
4275        /*
4276         * At migrating an anonymous page, its mapcount goes down
4277         * to 0 and uncharge() will be called. But, even if it's fully
4278         * unmapped, migration may fail and this page has to be
4279         * charged again. We set MIGRATION flag here and delay uncharge
4280         * until end_migration() is called
4281         *
4282         * Corner Case Thinking
4283         * A)
4284         * When the old page was mapped as Anon and it's unmap-and-freed
4285         * while migration was ongoing.
4286         * If unmap finds the old page, uncharge() of it will be delayed
4287         * until end_migration(). If unmap finds a new page, it's
4288         * uncharged when it make mapcount to be 1->0. If unmap code
4289         * finds swap_migration_entry, the new page will not be mapped
4290         * and end_migration() will find it(mapcount==0).
4291         *
4292         * B)
4293         * When the old page was mapped but migraion fails, the kernel
4294         * remaps it. A charge for it is kept by MIGRATION flag even
4295         * if mapcount goes down to 0. We can do remap successfully
4296         * without charging it again.
4297         *
4298         * C)
4299         * The "old" page is under lock_page() until the end of
4300         * migration, so, the old page itself will not be swapped-out.
4301         * If the new page is swapped out before end_migraton, our
4302         * hook to usual swap-out path will catch the event.
4303         */
4304        if (PageAnon(page))
4305            SetPageCgroupMigration(pc);
4306    }
4307    unlock_page_cgroup(pc);
4308    /*
4309     * If the page is not charged at this point,
4310     * we return here.
4311     */
4312    if (!memcg)
4313        return;
4314
4315    *memcgp = memcg;
4316    /*
4317     * We charge new page before it's used/mapped. So, even if unlock_page()
4318     * is called before end_migration, we can catch all events on this new
4319     * page. In the case new page is migrated but not remapped, new page's
4320     * mapcount will be finally 0 and we call uncharge in end_migration().
4321     */
4322    if (PageAnon(page))
4323        ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
4324    else
4325        ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
4326    /*
4327     * The page is committed to the memcg, but it's not actually
4328     * charged to the res_counter since we plan on replacing the
4329     * old one and only one page is going to be left afterwards.
4330     */
4331    __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
4332}
4333
4334/* remove redundant charge if migration failed*/
4335void mem_cgroup_end_migration(struct mem_cgroup *memcg,
4336    struct page *oldpage, struct page *newpage, bool migration_ok)
4337{
4338    struct page *used, *unused;
4339    struct page_cgroup *pc;
4340    bool anon;
4341
4342    if (!memcg)
4343        return;
4344
4345    if (!migration_ok) {
4346        used = oldpage;
4347        unused = newpage;
4348    } else {
4349        used = newpage;
4350        unused = oldpage;
4351    }
4352    anon = PageAnon(used);
4353    __mem_cgroup_uncharge_common(unused,
4354                     anon ? MEM_CGROUP_CHARGE_TYPE_ANON
4355                     : MEM_CGROUP_CHARGE_TYPE_CACHE,
4356                     true);
4357    css_put(&memcg->css);
4358    /*
4359     * We disallowed uncharge of pages under migration because mapcount
4360     * of the page goes down to zero, temporarly.
4361     * Clear the flag and check the page should be charged.
4362     */
4363    pc = lookup_page_cgroup(oldpage);
4364    lock_page_cgroup(pc);
4365    ClearPageCgroupMigration(pc);
4366    unlock_page_cgroup(pc);
4367
4368    /*
4369     * If a page is a file cache, radix-tree replacement is very atomic
4370     * and we can skip this check. When it was an Anon page, its mapcount
4371     * goes down to 0. But because we added MIGRATION flage, it's not
4372     * uncharged yet. There are several case but page->mapcount check
4373     * and USED bit check in mem_cgroup_uncharge_page() will do enough
4374     * check. (see prepare_charge() also)
4375     */
4376    if (anon)
4377        mem_cgroup_uncharge_page(used);
4378}
4379
4380/*
4381 * At replace page cache, newpage is not under any memcg but it's on
4382 * LRU. So, this function doesn't touch res_counter but handles LRU
4383 * in correct way. Both pages are locked so we cannot race with uncharge.
4384 */
4385void mem_cgroup_replace_page_cache(struct page *oldpage,
4386                  struct page *newpage)
4387{
4388    struct mem_cgroup *memcg = NULL;
4389    struct page_cgroup *pc;
4390    enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4391
4392    if (mem_cgroup_disabled())
4393        return;
4394
4395    pc = lookup_page_cgroup(oldpage);
4396    /* fix accounting on old pages */
4397    lock_page_cgroup(pc);
4398    if (PageCgroupUsed(pc)) {
4399        memcg = pc->mem_cgroup;
4400        mem_cgroup_charge_statistics(memcg, false, -1);
4401        ClearPageCgroupUsed(pc);
4402    }
4403    unlock_page_cgroup(pc);
4404
4405    /*
4406     * When called from shmem_replace_page(), in some cases the
4407     * oldpage has already been charged, and in some cases not.
4408     */
4409    if (!memcg)
4410        return;
4411    /*
4412     * Even if newpage->mapping was NULL before starting replacement,
4413     * the newpage may be on LRU(or pagevec for LRU) already. We lock
4414     * LRU while we overwrite pc->mem_cgroup.
4415     */
4416    __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
4417}
4418
4419#ifdef CONFIG_DEBUG_VM
4420static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
4421{
4422    struct page_cgroup *pc;
4423
4424    pc = lookup_page_cgroup(page);
4425    /*
4426     * Can be NULL while feeding pages into the page allocator for
4427     * the first time, i.e. during boot or memory hotplug;
4428     * or when mem_cgroup_disabled().
4429     */
4430    if (likely(pc) && PageCgroupUsed(pc))
4431        return pc;
4432    return NULL;
4433}
4434
4435bool mem_cgroup_bad_page_check(struct page *page)
4436{
4437    if (mem_cgroup_disabled())
4438        return false;
4439
4440    return lookup_page_cgroup_used(page) != NULL;
4441}
4442
4443void mem_cgroup_print_bad_page(struct page *page)
4444{
4445    struct page_cgroup *pc;
4446
4447    pc = lookup_page_cgroup_used(page);
4448    if (pc) {
4449        pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
4450             pc, pc->flags, pc->mem_cgroup);
4451    }
4452}
4453#endif
4454
4455static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
4456                unsigned long long val)
4457{
4458    int retry_count;
4459    u64 memswlimit, memlimit;
4460    int ret = 0;
4461    int children = mem_cgroup_count_children(memcg);
4462    u64 curusage, oldusage;
4463    int enlarge;
4464
4465    /*
4466     * For keeping hierarchical_reclaim simple, how long we should retry
4467     * is depends on callers. We set our retry-count to be function
4468     * of # of children which we should visit in this loop.
4469     */
4470    retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
4471
4472    oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4473
4474    enlarge = 0;
4475    while (retry_count) {
4476        if (signal_pending(current)) {
4477            ret = -EINTR;
4478            break;
4479        }
4480        /*
4481         * Rather than hide all in some function, I do this in
4482         * open coded manner. You see what this really does.
4483         * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4484         */
4485        mutex_lock(&set_limit_mutex);
4486        memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4487        if (memswlimit < val) {
4488            ret = -EINVAL;
4489            mutex_unlock(&set_limit_mutex);
4490            break;
4491        }
4492
4493        memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4494        if (memlimit < val)
4495            enlarge = 1;
4496
4497        ret = res_counter_set_limit(&memcg->res, val);
4498        if (!ret) {
4499            if (memswlimit == val)
4500                memcg->memsw_is_minimum = true;
4501            else
4502                memcg->memsw_is_minimum = false;
4503        }
4504        mutex_unlock(&set_limit_mutex);
4505
4506        if (!ret)
4507            break;
4508
4509        mem_cgroup_reclaim(memcg, GFP_KERNEL,
4510                   MEM_CGROUP_RECLAIM_SHRINK);
4511        curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4512        /* Usage is reduced ? */
4513          if (curusage >= oldusage)
4514            retry_count--;
4515        else
4516            oldusage = curusage;
4517    }
4518    if (!ret && enlarge)
4519        memcg_oom_recover(memcg);
4520
4521    return ret;
4522}
4523
4524static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
4525                    unsigned long long val)
4526{
4527    int retry_count;
4528    u64 memlimit, memswlimit, oldusage, curusage;
4529    int children = mem_cgroup_count_children(memcg);
4530    int ret = -EBUSY;
4531    int enlarge = 0;
4532
4533    /* see mem_cgroup_resize_res_limit */
4534     retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
4535    oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4536    while (retry_count) {
4537        if (signal_pending(current)) {
4538            ret = -EINTR;
4539            break;
4540        }
4541        /*
4542         * Rather than hide all in some function, I do this in
4543         * open coded manner. You see what this really does.
4544         * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
4545         */
4546        mutex_lock(&set_limit_mutex);
4547        memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4548        if (memlimit > val) {
4549            ret = -EINVAL;
4550            mutex_unlock(&set_limit_mutex);
4551            break;
4552        }
4553        memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4554        if (memswlimit < val)
4555            enlarge = 1;
4556        ret = res_counter_set_limit(&memcg->memsw, val);
4557        if (!ret) {
4558            if (memlimit == val)
4559                memcg->memsw_is_minimum = true;
4560            else
4561                memcg->memsw_is_minimum = false;
4562        }
4563        mutex_unlock(&set_limit_mutex);
4564
4565        if (!ret)
4566            break;
4567
4568        mem_cgroup_reclaim(memcg, GFP_KERNEL,
4569                   MEM_CGROUP_RECLAIM_NOSWAP |
4570                   MEM_CGROUP_RECLAIM_SHRINK);
4571        curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
4572        /* Usage is reduced ? */
4573        if (curusage >= oldusage)
4574            retry_count--;
4575        else
4576            oldusage = curusage;
4577    }
4578    if (!ret && enlarge)
4579        memcg_oom_recover(memcg);
4580    return ret;
4581}
4582
4583unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4584                        gfp_t gfp_mask,
4585                        unsigned long *total_scanned)
4586{
4587    unsigned long nr_reclaimed = 0;
4588    struct mem_cgroup_per_zone *mz, *next_mz = NULL;
4589    unsigned long reclaimed;
4590    int loop = 0;
4591    struct mem_cgroup_tree_per_zone *mctz;
4592    unsigned long long excess;
4593    unsigned long nr_scanned;
4594
4595    if (order > 0)
4596        return 0;
4597
4598    mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4599    /*
4600     * This loop can run a while, specially if mem_cgroup's continuously
4601     * keep exceeding their soft limit and putting the system under
4602     * pressure
4603     */
4604    do {
4605        if (next_mz)
4606            mz = next_mz;
4607        else
4608            mz = mem_cgroup_largest_soft_limit_node(mctz);
4609        if (!mz)
4610            break;
4611
4612        nr_scanned = 0;
4613        reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4614                            gfp_mask, &nr_scanned);
4615        nr_reclaimed += reclaimed;
4616        *total_scanned += nr_scanned;
4617        spin_lock(&mctz->lock);
4618
4619        /*
4620         * If we failed to reclaim anything from this memory cgroup
4621         * it is time to move on to the next cgroup
4622         */
4623        next_mz = NULL;
4624        if (!reclaimed) {
4625            do {
4626                /*
4627                 * Loop until we find yet another one.
4628                 *
4629                 * By the time we get the soft_limit lock
4630                 * again, someone might have aded the
4631                 * group back on the RB tree. Iterate to
4632                 * make sure we get a different mem.
4633                 * mem_cgroup_largest_soft_limit_node returns
4634                 * NULL if no other cgroup is present on
4635                 * the tree
4636                 */
4637                next_mz =
4638                __mem_cgroup_largest_soft_limit_node(mctz);
4639                if (next_mz == mz)
4640                    css_put(&next_mz->memcg->css);
4641                else /* next_mz == NULL or other memcg */
4642                    break;
4643            } while (1);
4644        }
4645        __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
4646        excess = res_counter_soft_limit_excess(&mz->memcg->res);
4647        /*
4648         * One school of thought says that we should not add
4649         * back the node to the tree if reclaim returns 0.
4650         * But our reclaim could return 0, simply because due
4651         * to priority we are exposing a smaller subset of
4652         * memory to reclaim from. Consider this as a longer
4653         * term TODO.
4654         */
4655        /* If excess == 0, no tree ops */
4656        __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4657        spin_unlock(&mctz->lock);
4658        css_put(&mz->memcg->css);
4659        loop++;
4660        /*
4661         * Could not reclaim anything and there are no more
4662         * mem cgroups to try or we seem to be looping without
4663         * reclaiming anything.
4664         */
4665        if (!nr_reclaimed &&
4666            (next_mz == NULL ||
4667            loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
4668            break;
4669    } while (!nr_reclaimed);
4670    if (next_mz)
4671        css_put(&next_mz->memcg->css);
4672    return nr_reclaimed;
4673}
4674
4675/**
4676 * mem_cgroup_force_empty_list - clears LRU of a group
4677 * @memcg: group to clear
4678 * @node: NUMA node
4679 * @zid: zone id
4680 * @lru: lru to to clear
4681 *
4682 * Traverse a specified page_cgroup list and try to drop them all. This doesn't
4683 * reclaim the pages page themselves - pages are moved to the parent (or root)
4684 * group.
4685 */
4686static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
4687                int node, int zid, enum lru_list lru)
4688{
4689    struct lruvec *lruvec;
4690    unsigned long flags;
4691    struct list_head *list;
4692    struct page *busy;
4693    struct zone *zone;
4694
4695    zone = &NODE_DATA(node)->node_zones[zid];
4696    lruvec = mem_cgroup_zone_lruvec(zone, memcg);
4697    list = &lruvec->lists[lru];
4698
4699    busy = NULL;
4700    do {
4701        struct page_cgroup *pc;
4702        struct page *page;
4703
4704        spin_lock_irqsave(&zone->lru_lock, flags);
4705        if (list_empty(list)) {
4706            spin_unlock_irqrestore(&zone->lru_lock, flags);
4707            break;
4708        }
4709        page = list_entry(list->prev, struct page, lru);
4710        if (busy == page) {
4711            list_move(&page->lru, list);
4712            busy = NULL;
4713            spin_unlock_irqrestore(&zone->lru_lock, flags);
4714            continue;
4715        }
4716        spin_unlock_irqrestore(&zone->lru_lock, flags);
4717
4718        pc = lookup_page_cgroup(page);
4719
4720        if (mem_cgroup_move_parent(page, pc, memcg)) {
4721            /* found lock contention or "pc" is obsolete. */
4722            busy = page;
4723            cond_resched();
4724        } else
4725            busy = NULL;
4726    } while (!list_empty(list));
4727}
4728
4729/*
4730 * make mem_cgroup's charge to be 0 if there is no task by moving
4731 * all the charges and pages to the parent.
4732 * This enables deleting this mem_cgroup.
4733 *
4734 * Caller is responsible for holding css reference on the memcg.
4735 */
4736static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
4737{
4738    int node, zid;
4739    u64 usage;
4740
4741    do {
4742        /* This is for making all *used* pages to be on LRU. */
4743        lru_add_drain_all();
4744        drain_all_stock_sync(memcg);
4745        mem_cgroup_start_move(memcg);
4746        for_each_node_state(node, N_MEMORY) {
4747            for (zid = 0; zid < MAX_NR_ZONES; zid++) {
4748                enum lru_list lru;
4749                for_each_lru(lru) {
4750                    mem_cgroup_force_empty_list(memcg,
4751                            node, zid, lru);
4752                }
4753            }
4754        }
4755        mem_cgroup_end_move(memcg);
4756        memcg_oom_recover(memcg);
4757        cond_resched();
4758
4759        /*
4760         * Kernel memory may not necessarily be trackable to a specific
4761         * process. So they are not migrated, and therefore we can't
4762         * expect their value to drop to 0 here.
4763         * Having res filled up with kmem only is enough.
4764         *
4765         * This is a safety check because mem_cgroup_force_empty_list
4766         * could have raced with mem_cgroup_replace_page_cache callers
4767         * so the lru seemed empty but the page could have been added
4768         * right after the check. RES_USAGE should be safe as we always
4769         * charge before adding to the LRU.
4770         */
4771        usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
4772            res_counter_read_u64(&memcg->kmem, RES_USAGE);
4773    } while (usage > 0);
4774}
4775
4776/*
4777 * This mainly exists for tests during the setting of set of use_hierarchy.
4778 * Since this is the very setting we are changing, the current hierarchy value
4779 * is meaningless
4780 */
4781static inline bool __memcg_has_children(struct mem_cgroup *memcg)
4782{
4783    struct cgroup *pos;
4784
4785    /* bounce at first found */
4786    cgroup_for_each_child(pos, memcg->css.cgroup)
4787        return true;
4788    return false;
4789}
4790
4791/*
4792 * Must be called with memcg_create_mutex held, unless the cgroup is guaranteed
4793 * to be already dead (as in mem_cgroup_force_empty, for instance). This is
4794 * from mem_cgroup_count_children(), in the sense that we don't really care how
4795 * many children we have; we only need to know if we have any. It also counts
4796 * any memcg without hierarchy as infertile.
4797 */
4798static inline bool memcg_has_children(struct mem_cgroup *memcg)
4799{
4800    return memcg->use_hierarchy && __memcg_has_children(memcg);
4801}
4802
4803/*
4804 * Reclaims as many pages from the given memcg as possible and moves
4805 * the rest to the parent.
4806 *
4807 * Caller is responsible for holding css reference for memcg.
4808 */
4809static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
4810{
4811    int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
4812    struct cgroup *cgrp = memcg->css.cgroup;
4813
4814    /* returns EBUSY if there is a task or if we come here twice. */
4815    if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
4816        return -EBUSY;
4817
4818    /* we call try-to-free pages for make this cgroup empty */
4819    lru_add_drain_all();
4820    /* try to free all pages in this cgroup */
4821    while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
4822        int progress;
4823
4824        if (signal_pending(current))
4825            return -EINTR;
4826
4827        progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
4828                        false);
4829        if (!progress) {
4830            nr_retries--;
4831            /* maybe some writeback is necessary */
4832            congestion_wait(BLK_RW_ASYNC, HZ/10);
4833        }
4834
4835    }
4836    lru_add_drain();
4837    mem_cgroup_reparent_charges(memcg);
4838
4839    return 0;
4840}
4841
4842static int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
4843{
4844    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4845    int ret;
4846
4847    if (mem_cgroup_is_root(memcg))
4848        return -EINVAL;
4849    css_get(&memcg->css);
4850    ret = mem_cgroup_force_empty(memcg);
4851    css_put(&memcg->css);
4852
4853    return ret;
4854}
4855
4856
4857static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
4858{
4859    return mem_cgroup_from_cont(cont)->use_hierarchy;
4860}
4861
4862static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
4863                    u64 val)
4864{
4865    int retval = 0;
4866    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4867    struct cgroup *parent = cont->parent;
4868    struct mem_cgroup *parent_memcg = NULL;
4869
4870    if (parent)
4871        parent_memcg = mem_cgroup_from_cont(parent);
4872
4873    mutex_lock(&memcg_create_mutex);
4874
4875    if (memcg->use_hierarchy == val)
4876        goto out;
4877
4878    /*
4879     * If parent's use_hierarchy is set, we can't make any modifications
4880     * in the child subtrees. If it is unset, then the change can
4881     * occur, provided the current cgroup has no children.
4882     *
4883     * For the root cgroup, parent_mem is NULL, we allow value to be
4884     * set if there are no children.
4885     */
4886    if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
4887                (val == 1 || val == 0)) {
4888        if (!__memcg_has_children(memcg))
4889            memcg->use_hierarchy = val;
4890        else
4891            retval = -EBUSY;
4892    } else
4893        retval = -EINVAL;
4894
4895out:
4896    mutex_unlock(&memcg_create_mutex);
4897
4898    return retval;
4899}
4900
4901
4902static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
4903                           enum mem_cgroup_stat_index idx)
4904{
4905    struct mem_cgroup *iter;
4906    long val = 0;
4907
4908    /* Per-cpu values can be negative, use a signed accumulator */
4909    for_each_mem_cgroup_tree(iter, memcg)
4910        val += mem_cgroup_read_stat(iter, idx);
4911
4912    if (val < 0) /* race ? */
4913        val = 0;
4914    return val;
4915}
4916
4917static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
4918{
4919    u64 val;
4920
4921    if (!mem_cgroup_is_root(memcg)) {
4922        if (!swap)
4923            return res_counter_read_u64(&memcg->res, RES_USAGE);
4924        else
4925            return res_counter_read_u64(&memcg->memsw, RES_USAGE);
4926    }
4927
4928    val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
4929    val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
4930
4931    if (swap)
4932        val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
4933
4934    return val << PAGE_SHIFT;
4935}
4936
4937static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
4938                   struct file *file, char __user *buf,
4939                   size_t nbytes, loff_t *ppos)
4940{
4941    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4942    char str[64];
4943    u64 val;
4944    int name, len;
4945    enum res_type type;
4946
4947    type = MEMFILE_TYPE(cft->private);
4948    name = MEMFILE_ATTR(cft->private);
4949
4950    if (!do_swap_account && type == _MEMSWAP)
4951        return -EOPNOTSUPP;
4952
4953    switch (type) {
4954    case _MEM:
4955        if (name == RES_USAGE)
4956            val = mem_cgroup_usage(memcg, false);
4957        else
4958            val = res_counter_read_u64(&memcg->res, name);
4959        break;
4960    case _MEMSWAP:
4961        if (name == RES_USAGE)
4962            val = mem_cgroup_usage(memcg, true);
4963        else
4964            val = res_counter_read_u64(&memcg->memsw, name);
4965        break;
4966    case _KMEM:
4967        val = res_counter_read_u64(&memcg->kmem, name);
4968        break;
4969    default:
4970        BUG();
4971    }
4972
4973    len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
4974    return simple_read_from_buffer(buf, nbytes, ppos, str, len);
4975}
4976
4977static int memcg_update_kmem_limit(struct cgroup *cont, u64 val)
4978{
4979    int ret = -EINVAL;
4980#ifdef CONFIG_MEMCG_KMEM
4981    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4982    /*
4983     * For simplicity, we won't allow this to be disabled. It also can't
4984     * be changed if the cgroup has children already, or if tasks had
4985     * already joined.
4986     *
4987     * If tasks join before we set the limit, a person looking at
4988     * kmem.usage_in_bytes will have no way to determine when it took
4989     * place, which makes the value quite meaningless.
4990     *
4991     * After it first became limited, changes in the value of the limit are
4992     * of course permitted.
4993     */
4994    mutex_lock(&memcg_create_mutex);
4995    mutex_lock(&set_limit_mutex);
4996    if (!memcg->kmem_account_flags && val != RESOURCE_MAX) {
4997        if (cgroup_task_count(cont) || memcg_has_children(memcg)) {
4998            ret = -EBUSY;
4999            goto out;
5000        }
5001        ret = res_counter_set_limit(&memcg->kmem, val);
5002        VM_BUG_ON(ret);
5003
5004        ret = memcg_update_cache_sizes(memcg);
5005        if (ret) {
5006            res_counter_set_limit(&memcg->kmem, RESOURCE_MAX);
5007            goto out;
5008        }
5009        static_key_slow_inc(&memcg_kmem_enabled_key);
5010        /*
5011         * setting the active bit after the inc will guarantee no one
5012         * starts accounting before all call sites are patched
5013         */
5014        memcg_kmem_set_active(memcg);
5015
5016        /*
5017         * kmem charges can outlive the cgroup. In the case of slab
5018         * pages, for instance, a page contain objects from various
5019         * processes, so it is unfeasible to migrate them away. We
5020         * need to reference count the memcg because of that.
5021         */
5022        mem_cgroup_get(memcg);
5023    } else
5024        ret = res_counter_set_limit(&memcg->kmem, val);
5025out:
5026    mutex_unlock(&set_limit_mutex);
5027    mutex_unlock(&memcg_create_mutex);
5028#endif
5029    return ret;
5030}
5031
5032#ifdef CONFIG_MEMCG_KMEM
5033static int memcg_propagate_kmem(struct mem_cgroup *memcg)
5034{
5035    int ret = 0;
5036    struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5037    if (!parent)
5038        goto out;
5039
5040    memcg->kmem_account_flags = parent->kmem_account_flags;
5041    /*
5042     * When that happen, we need to disable the static branch only on those
5043     * memcgs that enabled it. To achieve this, we would be forced to
5044     * complicate the code by keeping track of which memcgs were the ones
5045     * that actually enabled limits, and which ones got it from its
5046     * parents.
5047     *
5048     * It is a lot simpler just to do static_key_slow_inc() on every child
5049     * that is accounted.
5050     */
5051    if (!memcg_kmem_is_active(memcg))
5052        goto out;
5053
5054    /*
5055     * destroy(), called if we fail, will issue static_key_slow_inc() and
5056     * mem_cgroup_put() if kmem is enabled. We have to either call them
5057     * unconditionally, or clear the KMEM_ACTIVE flag. I personally find
5058     * this more consistent, since it always leads to the same destroy path
5059     */
5060    mem_cgroup_get(memcg);
5061    static_key_slow_inc(&memcg_kmem_enabled_key);
5062
5063    mutex_lock(&set_limit_mutex);
5064    ret = memcg_update_cache_sizes(memcg);
5065    mutex_unlock(&set_limit_mutex);
5066out:
5067    return ret;
5068}
5069#endif /* CONFIG_MEMCG_KMEM */
5070
5071/*
5072 * The user of this function is...
5073 * RES_LIMIT.
5074 */
5075static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
5076                const char *buffer)
5077{
5078    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5079    enum res_type type;
5080    int name;
5081    unsigned long long val;
5082    int ret;
5083
5084    type = MEMFILE_TYPE(cft->private);
5085    name = MEMFILE_ATTR(cft->private);
5086
5087    if (!do_swap_account && type == _MEMSWAP)
5088        return -EOPNOTSUPP;
5089
5090    switch (name) {
5091    case RES_LIMIT:
5092        if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
5093            ret = -EINVAL;
5094            break;
5095        }
5096        /* This function does all necessary parse...reuse it */
5097        ret = res_counter_memparse_write_strategy(buffer, &val);
5098        if (ret)
5099            break;
5100        if (type == _MEM)
5101            ret = mem_cgroup_resize_limit(memcg, val);
5102        else if (type == _MEMSWAP)
5103            ret = mem_cgroup_resize_memsw_limit(memcg, val);
5104        else if (type == _KMEM)
5105            ret = memcg_update_kmem_limit(cont, val);
5106        else
5107            return -EINVAL;
5108        break;
5109    case RES_SOFT_LIMIT:
5110        ret = res_counter_memparse_write_strategy(buffer, &val);
5111        if (ret)
5112            break;
5113        /*
5114         * For memsw, soft limits are hard to implement in terms
5115         * of semantics, for now, we support soft limits for
5116         * control without swap
5117         */
5118        if (type == _MEM)
5119            ret = res_counter_set_soft_limit(&memcg->res, val);
5120        else
5121            ret = -EINVAL;
5122        break;
5123    default:
5124        ret = -EINVAL; /* should be BUG() ? */
5125        break;
5126    }
5127    return ret;
5128}
5129
5130static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
5131        unsigned long long *mem_limit, unsigned long long *memsw_limit)
5132{
5133    struct cgroup *cgroup;
5134    unsigned long long min_limit, min_memsw_limit, tmp;
5135
5136    min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
5137    min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5138    cgroup = memcg->css.cgroup;
5139    if (!memcg->use_hierarchy)
5140        goto out;
5141
5142    while (cgroup->parent) {
5143        cgroup = cgroup->parent;
5144        memcg = mem_cgroup_from_cont(cgroup);
5145        if (!memcg->use_hierarchy)
5146            break;
5147        tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
5148        min_limit = min(min_limit, tmp);
5149        tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5150        min_memsw_limit = min(min_memsw_limit, tmp);
5151    }
5152out:
5153    *mem_limit = min_limit;
5154    *memsw_limit = min_memsw_limit;
5155}
5156
5157static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
5158{
5159    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5160    int name;
5161    enum res_type type;
5162
5163    type = MEMFILE_TYPE(event);
5164    name = MEMFILE_ATTR(event);
5165
5166    if (!do_swap_account && type == _MEMSWAP)
5167        return -EOPNOTSUPP;
5168
5169    switch (name) {
5170    case RES_MAX_USAGE:
5171        if (type == _MEM)
5172            res_counter_reset_max(&memcg->res);
5173        else if (type == _MEMSWAP)
5174            res_counter_reset_max(&memcg->memsw);
5175        else if (type == _KMEM)
5176            res_counter_reset_max(&memcg->kmem);
5177        else
5178            return -EINVAL;
5179        break;
5180    case RES_FAILCNT:
5181        if (type == _MEM)
5182            res_counter_reset_failcnt(&memcg->res);
5183        else if (type == _MEMSWAP)
5184            res_counter_reset_failcnt(&memcg->memsw);
5185        else if (type == _KMEM)
5186            res_counter_reset_failcnt(&memcg->kmem);
5187        else
5188            return -EINVAL;
5189        break;
5190    }
5191
5192    return 0;
5193}
5194
5195static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
5196                    struct cftype *cft)
5197{
5198    return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
5199}
5200
5201#ifdef CONFIG_MMU
5202static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
5203                    struct cftype *cft, u64 val)
5204{
5205    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5206
5207    if (val >= (1 << NR_MOVE_TYPE))
5208        return -EINVAL;
5209
5210    /*
5211     * No kind of locking is needed in here, because ->can_attach() will
5212     * check this value once in the beginning of the process, and then carry
5213     * on with stale data. This means that changes to this value will only
5214     * affect task migrations starting after the change.
5215     */
5216    memcg->move_charge_at_immigrate = val;
5217    return 0;
5218}
5219#else
5220static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
5221                    struct cftype *cft, u64 val)
5222{
5223    return -ENOSYS;
5224}
5225#endif
5226
5227#ifdef CONFIG_NUMA
5228static int memcg_numa_stat_show(struct cgroup *cont, struct cftype *cft,
5229                      struct seq_file *m)
5230{
5231    int nid;
5232    unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
5233    unsigned long node_nr;
5234    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5235
5236    total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
5237    seq_printf(m, "total=%lu", total_nr);
5238    for_each_node_state(nid, N_MEMORY) {
5239        node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
5240        seq_printf(m, " N%d=%lu", nid, node_nr);
5241    }
5242    seq_putc(m, '\n');
5243
5244    file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
5245    seq_printf(m, "file=%lu", file_nr);
5246    for_each_node_state(nid, N_MEMORY) {
5247        node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5248                LRU_ALL_FILE);
5249        seq_printf(m, " N%d=%lu", nid, node_nr);
5250    }
5251    seq_putc(m, '\n');
5252
5253    anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
5254    seq_printf(m, "anon=%lu", anon_nr);
5255    for_each_node_state(nid, N_MEMORY) {
5256        node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5257                LRU_ALL_ANON);
5258        seq_printf(m, " N%d=%lu", nid, node_nr);
5259    }
5260    seq_putc(m, '\n');
5261
5262    unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
5263    seq_printf(m, "unevictable=%lu", unevictable_nr);
5264    for_each_node_state(nid, N_MEMORY) {
5265        node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
5266                BIT(LRU_UNEVICTABLE));
5267        seq_printf(m, " N%d=%lu", nid, node_nr);
5268    }
5269    seq_putc(m, '\n');
5270    return 0;
5271}
5272#endif /* CONFIG_NUMA */
5273
5274static inline void mem_cgroup_lru_names_not_uptodate(void)
5275{
5276    BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
5277}
5278
5279static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
5280                 struct seq_file *m)
5281{
5282    struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
5283    struct mem_cgroup *mi;
5284    unsigned int i;
5285
5286    for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5287        if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5288            continue;
5289        seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
5290               mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
5291    }
5292
5293    for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
5294        seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
5295               mem_cgroup_read_events(memcg, i));
5296
5297    for (i = 0; i < NR_LRU_LISTS; i++)
5298        seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
5299               mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
5300
5301    /* Hierarchical information */
5302    {
5303        unsigned long long limit, memsw_limit;
5304        memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
5305        seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
5306        if (do_swap_account)
5307            seq_printf(m, "hierarchical_memsw_limit %llu\n",
5308                   memsw_limit);
5309    }
5310
5311    for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5312        long long val = 0;
5313
5314        if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
5315            continue;
5316        for_each_mem_cgroup_tree(mi, memcg)
5317            val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
5318        seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
5319    }
5320
5321    for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
5322        unsigned long long val = 0;
5323
5324        for_each_mem_cgroup_tree(mi, memcg)
5325            val += mem_cgroup_read_events(mi, i);
5326        seq_printf(m, "total_%s %llu\n",
5327               mem_cgroup_events_names[i], val);
5328    }
5329
5330    for (i = 0; i < NR_LRU_LISTS; i++) {
5331        unsigned long long val = 0;
5332
5333        for_each_mem_cgroup_tree(mi, memcg)
5334            val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
5335        seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
5336    }
5337
5338#ifdef CONFIG_DEBUG_VM
5339    {
5340        int nid, zid;
5341        struct mem_cgroup_per_zone *mz;
5342        struct zone_reclaim_stat *rstat;
5343        unsigned long recent_rotated[2] = {0, 0};
5344        unsigned long recent_scanned[2] = {0, 0};
5345
5346        for_each_online_node(nid)
5347            for (zid = 0; zid < MAX_NR_ZONES; zid++) {
5348                mz = mem_cgroup_zoneinfo(memcg, nid, zid);
5349                rstat = &mz->lruvec.reclaim_stat;
5350
5351                recent_rotated[0] += rstat->recent_rotated[0];
5352                recent_rotated[1] += rstat->recent_rotated[1];
5353                recent_scanned[0] += rstat->recent_scanned[0];
5354                recent_scanned[1] += rstat->recent_scanned[1];
5355            }
5356        seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
5357        seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
5358        seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
5359        seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
5360    }
5361#endif
5362
5363    return 0;
5364}
5365
5366static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
5367{
5368    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5369
5370    return mem_cgroup_swappiness(memcg);
5371}
5372
5373static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
5374                       u64 val)
5375{
5376    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5377    struct mem_cgroup *parent;
5378
5379    if (val > 100)
5380        return -EINVAL;
5381
5382    if (cgrp->parent == NULL)
5383        return -EINVAL;
5384
5385    parent = mem_cgroup_from_cont(cgrp->parent);
5386
5387    mutex_lock(&memcg_create_mutex);
5388
5389    /* If under hierarchy, only empty-root can set this value */
5390    if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5391        mutex_unlock(&memcg_create_mutex);
5392        return -EINVAL;
5393    }
5394
5395    memcg->swappiness = val;
5396
5397    mutex_unlock(&memcg_create_mutex);
5398
5399    return 0;
5400}
5401
5402static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
5403{
5404    struct mem_cgroup_threshold_ary *t;
5405    u64 usage;
5406    int i;
5407
5408    rcu_read_lock();
5409    if (!swap)
5410        t = rcu_dereference(memcg->thresholds.primary);
5411    else
5412        t = rcu_dereference(memcg->memsw_thresholds.primary);
5413
5414    if (!t)
5415        goto unlock;
5416
5417    usage = mem_cgroup_usage(memcg, swap);
5418
5419    /*
5420     * current_threshold points to threshold just below or equal to usage.
5421     * If it's not true, a threshold was crossed after last
5422     * call of __mem_cgroup_threshold().
5423     */
5424    i = t->current_threshold;
5425
5426    /*
5427     * Iterate backward over array of thresholds starting from
5428     * current_threshold and check if a threshold is crossed.
5429     * If none of thresholds below usage is crossed, we read
5430     * only one element of the array here.
5431     */
5432    for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
5433        eventfd_signal(t->entries[i].eventfd, 1);
5434
5435    /* i = current_threshold + 1 */
5436    i++;
5437
5438    /*
5439     * Iterate forward over array of thresholds starting from
5440     * current_threshold+1 and check if a threshold is crossed.
5441     * If none of thresholds above usage is crossed, we read
5442     * only one element of the array here.
5443     */
5444    for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
5445        eventfd_signal(t->entries[i].eventfd, 1);
5446
5447    /* Update current_threshold */
5448    t->current_threshold = i - 1;
5449unlock:
5450    rcu_read_unlock();
5451}
5452
5453static void mem_cgroup_threshold(struct mem_cgroup *memcg)
5454{
5455    while (memcg) {
5456        __mem_cgroup_threshold(memcg, false);
5457        if (do_swap_account)
5458            __mem_cgroup_threshold(memcg, true);
5459
5460        memcg = parent_mem_cgroup(memcg);
5461    }
5462}
5463
5464static int compare_thresholds(const void *a, const void *b)
5465{
5466    const struct mem_cgroup_threshold *_a = a;
5467    const struct mem_cgroup_threshold *_b = b;
5468
5469    return _a->threshold - _b->threshold;
5470}
5471
5472static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
5473{
5474    struct mem_cgroup_eventfd_list *ev;
5475
5476    list_for_each_entry(ev, &memcg->oom_notify, list)
5477        eventfd_signal(ev->eventfd, 1);
5478    return 0;
5479}
5480
5481static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
5482{
5483    struct mem_cgroup *iter;
5484
5485    for_each_mem_cgroup_tree(iter, memcg)
5486        mem_cgroup_oom_notify_cb(iter);
5487}
5488
5489static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
5490    struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5491{
5492    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5493    struct mem_cgroup_thresholds *thresholds;
5494    struct mem_cgroup_threshold_ary *new;
5495    enum res_type type = MEMFILE_TYPE(cft->private);
5496    u64 threshold, usage;
5497    int i, size, ret;
5498
5499    ret = res_counter_memparse_write_strategy(args, &threshold);
5500    if (ret)
5501        return ret;
5502
5503    mutex_lock(&memcg->thresholds_lock);
5504
5505    if (type == _MEM)
5506        thresholds = &memcg->thresholds;
5507    else if (type == _MEMSWAP)
5508        thresholds = &memcg->memsw_thresholds;
5509    else
5510        BUG();
5511
5512    usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5513
5514    /* Check if a threshold crossed before adding a new one */
5515    if (thresholds->primary)
5516        __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5517
5518    size = thresholds->primary ? thresholds->primary->size + 1 : 1;
5519
5520    /* Allocate memory for new array of thresholds */
5521    new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
5522            GFP_KERNEL);
5523    if (!new) {
5524        ret = -ENOMEM;
5525        goto unlock;
5526    }
5527    new->size = size;
5528
5529    /* Copy thresholds (if any) to new array */
5530    if (thresholds->primary) {
5531        memcpy(new->entries, thresholds->primary->entries, (size - 1) *
5532                sizeof(struct mem_cgroup_threshold));
5533    }
5534
5535    /* Add new threshold */
5536    new->entries[size - 1].eventfd = eventfd;
5537    new->entries[size - 1].threshold = threshold;
5538
5539    /* Sort thresholds. Registering of new threshold isn't time-critical */
5540    sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
5541            compare_thresholds, NULL);
5542
5543    /* Find current threshold */
5544    new->current_threshold = -1;
5545    for (i = 0; i < size; i++) {
5546        if (new->entries[i].threshold <= usage) {
5547            /*
5548             * new->current_threshold will not be used until
5549             * rcu_assign_pointer(), so it's safe to increment
5550             * it here.
5551             */
5552            ++new->current_threshold;
5553        } else
5554            break;
5555    }
5556
5557    /* Free old spare buffer and save old primary buffer as spare */
5558    kfree(thresholds->spare);
5559    thresholds->spare = thresholds->primary;
5560
5561    rcu_assign_pointer(thresholds->primary, new);
5562
5563    /* To be sure that nobody uses thresholds */
5564    synchronize_rcu();
5565
5566unlock:
5567    mutex_unlock(&memcg->thresholds_lock);
5568
5569    return ret;
5570}
5571
5572static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
5573    struct cftype *cft, struct eventfd_ctx *eventfd)
5574{
5575    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5576    struct mem_cgroup_thresholds *thresholds;
5577    struct mem_cgroup_threshold_ary *new;
5578    enum res_type type = MEMFILE_TYPE(cft->private);
5579    u64 usage;
5580    int i, j, size;
5581
5582    mutex_lock(&memcg->thresholds_lock);
5583    if (type == _MEM)
5584        thresholds = &memcg->thresholds;
5585    else if (type == _MEMSWAP)
5586        thresholds = &memcg->memsw_thresholds;
5587    else
5588        BUG();
5589
5590    if (!thresholds->primary)
5591        goto unlock;
5592
5593    usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5594
5595    /* Check if a threshold crossed before removing */
5596    __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5597
5598    /* Calculate new number of threshold */
5599    size = 0;
5600    for (i = 0; i < thresholds->primary->size; i++) {
5601        if (thresholds->primary->entries[i].eventfd != eventfd)
5602            size++;
5603    }
5604
5605    new = thresholds->spare;
5606
5607    /* Set thresholds array to NULL if we don't have thresholds */
5608    if (!size) {
5609        kfree(new);
5610        new = NULL;
5611        goto swap_buffers;
5612    }
5613
5614    new->size = size;
5615
5616    /* Copy thresholds and find current threshold */
5617    new->current_threshold = -1;
5618    for (i = 0, j = 0; i < thresholds->primary->size; i++) {
5619        if (thresholds->primary->entries[i].eventfd == eventfd)
5620            continue;
5621
5622        new->entries[j] = thresholds->primary->entries[i];
5623        if (new->entries[j].threshold <= usage) {
5624            /*
5625             * new->current_threshold will not be used
5626             * until rcu_assign_pointer(), so it's safe to increment
5627             * it here.
5628             */
5629            ++new->current_threshold;
5630        }
5631        j++;
5632    }
5633
5634swap_buffers:
5635    /* Swap primary and spare array */
5636    thresholds->spare = thresholds->primary;
5637    /* If all events are unregistered, free the spare array */
5638    if (!new) {
5639        kfree(thresholds->spare);
5640        thresholds->spare = NULL;
5641    }
5642
5643    rcu_assign_pointer(thresholds->primary, new);
5644
5645    /* To be sure that nobody uses thresholds */
5646    synchronize_rcu();
5647unlock:
5648    mutex_unlock(&memcg->thresholds_lock);
5649}
5650
5651static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
5652    struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5653{
5654    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5655    struct mem_cgroup_eventfd_list *event;
5656    enum res_type type = MEMFILE_TYPE(cft->private);
5657
5658    BUG_ON(type != _OOM_TYPE);
5659    event = kmalloc(sizeof(*event), GFP_KERNEL);
5660    if (!event)
5661        return -ENOMEM;
5662
5663    spin_lock(&memcg_oom_lock);
5664
5665    event->eventfd = eventfd;
5666    list_add(&event->list, &memcg->oom_notify);
5667
5668    /* already in OOM ? */
5669    if (atomic_read(&memcg->under_oom))
5670        eventfd_signal(eventfd, 1);
5671    spin_unlock(&memcg_oom_lock);
5672
5673    return 0;
5674}
5675
5676static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
5677    struct cftype *cft, struct eventfd_ctx *eventfd)
5678{
5679    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5680    struct mem_cgroup_eventfd_list *ev, *tmp;
5681    enum res_type type = MEMFILE_TYPE(cft->private);
5682
5683    BUG_ON(type != _OOM_TYPE);
5684
5685    spin_lock(&memcg_oom_lock);
5686
5687    list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
5688        if (ev->eventfd == eventfd) {
5689            list_del(&ev->list);
5690            kfree(ev);
5691        }
5692    }
5693
5694    spin_unlock(&memcg_oom_lock);
5695}
5696
5697static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
5698    struct cftype *cft, struct cgroup_map_cb *cb)
5699{
5700    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5701
5702    cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
5703
5704    if (atomic_read(&memcg->under_oom))
5705        cb->fill(cb, "under_oom", 1);
5706    else
5707        cb->fill(cb, "under_oom", 0);
5708    return 0;
5709}
5710
5711static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
5712    struct cftype *cft, u64 val)
5713{
5714    struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5715    struct mem_cgroup *parent;
5716
5717    /* cannot set to root cgroup and only 0 and 1 are allowed */
5718    if (!cgrp->parent || !((val == 0) || (val == 1)))
5719        return -EINVAL;
5720
5721    parent = mem_cgroup_from_cont(cgrp->parent);
5722
5723    mutex_lock(&memcg_create_mutex);
5724    /* oom-kill-disable is a flag for subhierarchy. */
5725    if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
5726        mutex_unlock(&memcg_create_mutex);
5727        return -EINVAL;
5728    }
5729    memcg->oom_kill_disable = val;
5730    if (!val)
5731        memcg_oom_recover(memcg);
5732    mutex_unlock(&memcg_create_mutex);
5733    return 0;
5734}
5735
5736#ifdef CONFIG_MEMCG_KMEM
5737static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5738{
5739    int ret;
5740
5741    memcg->kmemcg_id = -1;
5742    ret = memcg_propagate_kmem(memcg);
5743    if (ret)
5744        return ret;
5745
5746    return mem_cgroup_sockets_init(memcg, ss);
5747};
5748
5749static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
5750{
5751    mem_cgroup_sockets_destroy(memcg);
5752
5753    memcg_kmem_mark_dead(memcg);
5754
5755    if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
5756        return;
5757
5758    /*
5759     * Charges already down to 0, undo mem_cgroup_get() done in the charge
5760     * path here, being careful not to race with memcg_uncharge_kmem: it is
5761     * possible that the charges went down to 0 between mark_dead and the
5762     * res_counter read, so in that case, we don't need the put
5763     */
5764    if (memcg_kmem_test_and_clear_dead(memcg))
5765        mem_cgroup_put(memcg);
5766}
5767#else
5768static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
5769{
5770    return 0;
5771}
5772
5773static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
5774{
5775}
5776#endif
5777
5778static struct cftype mem_cgroup_files[] = {
5779    {
5780        .name = "usage_in_bytes",
5781        .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
5782        .read = mem_cgroup_read,
5783        .register_event = mem_cgroup_usage_register_event,
5784        .unregister_event = mem_cgroup_usage_unregister_event,
5785    },
5786    {
5787        .name = "max_usage_in_bytes",
5788        .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
5789        .trigger = mem_cgroup_reset,
5790        .read = mem_cgroup_read,
5791    },
5792    {
5793        .name = "limit_in_bytes",
5794        .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
5795        .write_string = mem_cgroup_write,
5796        .read = mem_cgroup_read,
5797    },
5798    {
5799        .name = "soft_limit_in_bytes",
5800        .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
5801        .write_string = mem_cgroup_write,
5802        .read = mem_cgroup_read,
5803    },
5804    {
5805        .name = "failcnt",
5806        .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
5807        .trigger = mem_cgroup_reset,
5808        .read = mem_cgroup_read,
5809    },
5810    {
5811        .name = "stat",
5812        .read_seq_string = memcg_stat_show,
5813    },
5814    {
5815        .name = "force_empty",
5816        .trigger = mem_cgroup_force_empty_write,
5817    },
5818    {
5819        .name = "use_hierarchy",
5820        .write_u64 = mem_cgroup_hierarchy_write,
5821        .read_u64 = mem_cgroup_hierarchy_read,
5822    },
5823    {
5824        .name = "swappiness",
5825        .read_u64 = mem_cgroup_swappiness_read,
5826        .write_u64 = mem_cgroup_swappiness_write,
5827    },
5828    {
5829        .name = "move_charge_at_immigrate",
5830        .read_u64 = mem_cgroup_move_charge_read,
5831        .write_u64 = mem_cgroup_move_charge_write,
5832    },
5833    {
5834        .name = "oom_control",
5835        .read_map = mem_cgroup_oom_control_read,
5836        .write_u64 = mem_cgroup_oom_control_write,
5837        .register_event = mem_cgroup_oom_register_event,
5838        .unregister_event = mem_cgroup_oom_unregister_event,
5839        .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
5840    },
5841#ifdef CONFIG_NUMA
5842    {
5843        .name = "numa_stat",
5844        .read_seq_string = memcg_numa_stat_show,
5845    },
5846#endif
5847#ifdef CONFIG_MEMCG_KMEM
5848    {
5849        .name = "kmem.limit_in_bytes",
5850        .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
5851        .write_string = mem_cgroup_write,
5852        .read = mem_cgroup_read,
5853    },
5854    {
5855        .name = "kmem.usage_in_bytes",
5856        .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
5857        .read = mem_cgroup_read,
5858    },
5859    {
5860        .name = "kmem.failcnt",
5861        .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
5862        .trigger = mem_cgroup_reset,
5863        .read = mem_cgroup_read,
5864    },
5865    {
5866        .name = "kmem.max_usage_in_bytes",
5867        .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
5868        .trigger = mem_cgroup_reset,
5869        .read = mem_cgroup_read,
5870    },
5871#ifdef CONFIG_SLABINFO
5872    {
5873        .name = "kmem.slabinfo",
5874        .read_seq_string = mem_cgroup_slabinfo_read,
5875    },
5876#endif
5877#endif
5878    { }, /* terminate */
5879};
5880
5881#ifdef CONFIG_MEMCG_SWAP
5882static struct cftype memsw_cgroup_files[] = {
5883    {
5884        .name = "memsw.usage_in_bytes",
5885        .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
5886        .read = mem_cgroup_read,
5887        .register_event = mem_cgroup_usage_register_event,
5888        .unregister_event = mem_cgroup_usage_unregister_event,
5889    },
5890    {
5891        .name = "memsw.max_usage_in_bytes",
5892        .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
5893        .trigger = mem_cgroup_reset,
5894        .read = mem_cgroup_read,
5895    },
5896    {
5897        .name = "memsw.limit_in_bytes",
5898        .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
5899        .write_string = mem_cgroup_write,
5900        .read = mem_cgroup_read,
5901    },
5902    {
5903        .name = "memsw.failcnt",
5904        .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
5905        .trigger = mem_cgroup_reset,
5906        .read = mem_cgroup_read,
5907    },
5908    { }, /* terminate */
5909};
5910#endif
5911static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
5912{
5913    struct mem_cgroup_per_node *pn;
5914    struct mem_cgroup_per_zone *mz;
5915    int zone, tmp = node;
5916    /*
5917     * This routine is called against possible nodes.
5918     * But it's BUG to call kmalloc() against offline node.
5919     *
5920     * TODO: this routine can waste much memory for nodes which will
5921     * never be onlined. It's better to use memory hotplug callback
5922     * function.
5923     */
5924    if (!node_state(node, N_NORMAL_MEMORY))
5925        tmp = -1;
5926    pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
5927    if (!pn)
5928        return 1;
5929
5930    for (zone = 0; zone < MAX_NR_ZONES; zone++) {
5931        mz = &pn->zoneinfo[zone];