Root/mm/mempolicy.c

1/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
21 *
22 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
28 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
33 *
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57   fix mmap readahead to honour policy and enable policy for any page cache
58   object
59   statistics for bigpages
60   global policy for page cache? currently it uses process policy. Requires
61   first item above.
62   handle mremap for shared memory (currently ignored for the policy)
63   grows down?
64   make bind policy root only? It can trigger oom much faster and the
65   kernel is not always grateful with that.
66*/
67
68#include <linux/mempolicy.h>
69#include <linux/mm.h>
70#include <linux/highmem.h>
71#include <linux/hugetlb.h>
72#include <linux/kernel.h>
73#include <linux/sched.h>
74#include <linux/nodemask.h>
75#include <linux/cpuset.h>
76#include <linux/slab.h>
77#include <linux/string.h>
78#include <linux/export.h>
79#include <linux/nsproxy.h>
80#include <linux/interrupt.h>
81#include <linux/init.h>
82#include <linux/compat.h>
83#include <linux/swap.h>
84#include <linux/seq_file.h>
85#include <linux/proc_fs.h>
86#include <linux/migrate.h>
87#include <linux/ksm.h>
88#include <linux/rmap.h>
89#include <linux/security.h>
90#include <linux/syscalls.h>
91#include <linux/ctype.h>
92#include <linux/mm_inline.h>
93
94#include <asm/tlbflush.h>
95#include <asm/uaccess.h>
96#include <linux/random.h>
97
98#include "internal.h"
99
100/* Internal flags */
101#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
102#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
103
104static struct kmem_cache *policy_cache;
105static struct kmem_cache *sn_cache;
106
107/* Highest zone. An specific allocation for a zone below that is not
108   policied. */
109enum zone_type policy_zone = 0;
110
111/*
112 * run-time system-wide default policy => local allocation
113 */
114static struct mempolicy default_policy = {
115    .refcnt = ATOMIC_INIT(1), /* never free it */
116    .mode = MPOL_PREFERRED,
117    .flags = MPOL_F_LOCAL,
118};
119
120static const struct mempolicy_operations {
121    int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
122    /*
123     * If read-side task has no lock to protect task->mempolicy, write-side
124     * task will rebind the task->mempolicy by two step. The first step is
125     * setting all the newly nodes, and the second step is cleaning all the
126     * disallowed nodes. In this way, we can avoid finding no node to alloc
127     * page.
128     * If we have a lock to protect task->mempolicy in read-side, we do
129     * rebind directly.
130     *
131     * step:
132     * MPOL_REBIND_ONCE - do rebind work at once
133     * MPOL_REBIND_STEP1 - set all the newly nodes
134     * MPOL_REBIND_STEP2 - clean all the disallowed nodes
135     */
136    void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
137            enum mpol_rebind_step step);
138} mpol_ops[MPOL_MAX];
139
140/* Check that the nodemask contains at least one populated zone */
141static int is_valid_nodemask(const nodemask_t *nodemask)
142{
143    int nd, k;
144
145    for_each_node_mask(nd, *nodemask) {
146        struct zone *z;
147
148        for (k = 0; k <= policy_zone; k++) {
149            z = &NODE_DATA(nd)->node_zones[k];
150            if (z->present_pages > 0)
151                return 1;
152        }
153    }
154
155    return 0;
156}
157
158static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
159{
160    return pol->flags & MPOL_MODE_FLAGS;
161}
162
163static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
164                   const nodemask_t *rel)
165{
166    nodemask_t tmp;
167    nodes_fold(tmp, *orig, nodes_weight(*rel));
168    nodes_onto(*ret, tmp, *rel);
169}
170
171static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
172{
173    if (nodes_empty(*nodes))
174        return -EINVAL;
175    pol->v.nodes = *nodes;
176    return 0;
177}
178
179static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
180{
181    if (!nodes)
182        pol->flags |= MPOL_F_LOCAL; /* local allocation */
183    else if (nodes_empty(*nodes))
184        return -EINVAL; /* no allowed nodes */
185    else
186        pol->v.preferred_node = first_node(*nodes);
187    return 0;
188}
189
190static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
191{
192    if (!is_valid_nodemask(nodes))
193        return -EINVAL;
194    pol->v.nodes = *nodes;
195    return 0;
196}
197
198/*
199 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
200 * any, for the new policy. mpol_new() has already validated the nodes
201 * parameter with respect to the policy mode and flags. But, we need to
202 * handle an empty nodemask with MPOL_PREFERRED here.
203 *
204 * Must be called holding task's alloc_lock to protect task's mems_allowed
205 * and mempolicy. May also be called holding the mmap_semaphore for write.
206 */
207static int mpol_set_nodemask(struct mempolicy *pol,
208             const nodemask_t *nodes, struct nodemask_scratch *nsc)
209{
210    int ret;
211
212    /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
213    if (pol == NULL)
214        return 0;
215    /* Check N_HIGH_MEMORY */
216    nodes_and(nsc->mask1,
217          cpuset_current_mems_allowed, node_states[N_HIGH_MEMORY]);
218
219    VM_BUG_ON(!nodes);
220    if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
221        nodes = NULL; /* explicit local allocation */
222    else {
223        if (pol->flags & MPOL_F_RELATIVE_NODES)
224            mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
225        else
226            nodes_and(nsc->mask2, *nodes, nsc->mask1);
227
228        if (mpol_store_user_nodemask(pol))
229            pol->w.user_nodemask = *nodes;
230        else
231            pol->w.cpuset_mems_allowed =
232                        cpuset_current_mems_allowed;
233    }
234
235    if (nodes)
236        ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
237    else
238        ret = mpol_ops[pol->mode].create(pol, NULL);
239    return ret;
240}
241
242/*
243 * This function just creates a new policy, does some check and simple
244 * initialization. You must invoke mpol_set_nodemask() to set nodes.
245 */
246static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
247                  nodemask_t *nodes)
248{
249    struct mempolicy *policy;
250
251    pr_debug("setting mode %d flags %d nodes[0] %lx\n",
252         mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
253
254    if (mode == MPOL_DEFAULT) {
255        if (nodes && !nodes_empty(*nodes))
256            return ERR_PTR(-EINVAL);
257        return NULL; /* simply delete any existing policy */
258    }
259    VM_BUG_ON(!nodes);
260
261    /*
262     * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
263     * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
264     * All other modes require a valid pointer to a non-empty nodemask.
265     */
266    if (mode == MPOL_PREFERRED) {
267        if (nodes_empty(*nodes)) {
268            if (((flags & MPOL_F_STATIC_NODES) ||
269                 (flags & MPOL_F_RELATIVE_NODES)))
270                return ERR_PTR(-EINVAL);
271        }
272    } else if (nodes_empty(*nodes))
273        return ERR_PTR(-EINVAL);
274    policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
275    if (!policy)
276        return ERR_PTR(-ENOMEM);
277    atomic_set(&policy->refcnt, 1);
278    policy->mode = mode;
279    policy->flags = flags;
280
281    return policy;
282}
283
284/* Slow path of a mpol destructor. */
285void __mpol_put(struct mempolicy *p)
286{
287    if (!atomic_dec_and_test(&p->refcnt))
288        return;
289    kmem_cache_free(policy_cache, p);
290}
291
292static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
293                enum mpol_rebind_step step)
294{
295}
296
297/*
298 * step:
299 * MPOL_REBIND_ONCE - do rebind work at once
300 * MPOL_REBIND_STEP1 - set all the newly nodes
301 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
302 */
303static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
304                 enum mpol_rebind_step step)
305{
306    nodemask_t tmp;
307
308    if (pol->flags & MPOL_F_STATIC_NODES)
309        nodes_and(tmp, pol->w.user_nodemask, *nodes);
310    else if (pol->flags & MPOL_F_RELATIVE_NODES)
311        mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
312    else {
313        /*
314         * if step == 1, we use ->w.cpuset_mems_allowed to cache the
315         * result
316         */
317        if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
318            nodes_remap(tmp, pol->v.nodes,
319                    pol->w.cpuset_mems_allowed, *nodes);
320            pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
321        } else if (step == MPOL_REBIND_STEP2) {
322            tmp = pol->w.cpuset_mems_allowed;
323            pol->w.cpuset_mems_allowed = *nodes;
324        } else
325            BUG();
326    }
327
328    if (nodes_empty(tmp))
329        tmp = *nodes;
330
331    if (step == MPOL_REBIND_STEP1)
332        nodes_or(pol->v.nodes, pol->v.nodes, tmp);
333    else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
334        pol->v.nodes = tmp;
335    else
336        BUG();
337
338    if (!node_isset(current->il_next, tmp)) {
339        current->il_next = next_node(current->il_next, tmp);
340        if (current->il_next >= MAX_NUMNODES)
341            current->il_next = first_node(tmp);
342        if (current->il_next >= MAX_NUMNODES)
343            current->il_next = numa_node_id();
344    }
345}
346
347static void mpol_rebind_preferred(struct mempolicy *pol,
348                  const nodemask_t *nodes,
349                  enum mpol_rebind_step step)
350{
351    nodemask_t tmp;
352
353    if (pol->flags & MPOL_F_STATIC_NODES) {
354        int node = first_node(pol->w.user_nodemask);
355
356        if (node_isset(node, *nodes)) {
357            pol->v.preferred_node = node;
358            pol->flags &= ~MPOL_F_LOCAL;
359        } else
360            pol->flags |= MPOL_F_LOCAL;
361    } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
362        mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
363        pol->v.preferred_node = first_node(tmp);
364    } else if (!(pol->flags & MPOL_F_LOCAL)) {
365        pol->v.preferred_node = node_remap(pol->v.preferred_node,
366                           pol->w.cpuset_mems_allowed,
367                           *nodes);
368        pol->w.cpuset_mems_allowed = *nodes;
369    }
370}
371
372/*
373 * mpol_rebind_policy - Migrate a policy to a different set of nodes
374 *
375 * If read-side task has no lock to protect task->mempolicy, write-side
376 * task will rebind the task->mempolicy by two step. The first step is
377 * setting all the newly nodes, and the second step is cleaning all the
378 * disallowed nodes. In this way, we can avoid finding no node to alloc
379 * page.
380 * If we have a lock to protect task->mempolicy in read-side, we do
381 * rebind directly.
382 *
383 * step:
384 * MPOL_REBIND_ONCE - do rebind work at once
385 * MPOL_REBIND_STEP1 - set all the newly nodes
386 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
387 */
388static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
389                enum mpol_rebind_step step)
390{
391    if (!pol)
392        return;
393    if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
394        nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
395        return;
396
397    if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
398        return;
399
400    if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
401        BUG();
402
403    if (step == MPOL_REBIND_STEP1)
404        pol->flags |= MPOL_F_REBINDING;
405    else if (step == MPOL_REBIND_STEP2)
406        pol->flags &= ~MPOL_F_REBINDING;
407    else if (step >= MPOL_REBIND_NSTEP)
408        BUG();
409
410    mpol_ops[pol->mode].rebind(pol, newmask, step);
411}
412
413/*
414 * Wrapper for mpol_rebind_policy() that just requires task
415 * pointer, and updates task mempolicy.
416 *
417 * Called with task's alloc_lock held.
418 */
419
420void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
421            enum mpol_rebind_step step)
422{
423    mpol_rebind_policy(tsk->mempolicy, new, step);
424}
425
426/*
427 * Rebind each vma in mm to new nodemask.
428 *
429 * Call holding a reference to mm. Takes mm->mmap_sem during call.
430 */
431
432void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
433{
434    struct vm_area_struct *vma;
435
436    down_write(&mm->mmap_sem);
437    for (vma = mm->mmap; vma; vma = vma->vm_next)
438        mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
439    up_write(&mm->mmap_sem);
440}
441
442static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
443    [MPOL_DEFAULT] = {
444        .rebind = mpol_rebind_default,
445    },
446    [MPOL_INTERLEAVE] = {
447        .create = mpol_new_interleave,
448        .rebind = mpol_rebind_nodemask,
449    },
450    [MPOL_PREFERRED] = {
451        .create = mpol_new_preferred,
452        .rebind = mpol_rebind_preferred,
453    },
454    [MPOL_BIND] = {
455        .create = mpol_new_bind,
456        .rebind = mpol_rebind_nodemask,
457    },
458};
459
460static void migrate_page_add(struct page *page, struct list_head *pagelist,
461                unsigned long flags);
462
463/* Scan through pages checking if pages follow certain conditions. */
464static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
465        unsigned long addr, unsigned long end,
466        const nodemask_t *nodes, unsigned long flags,
467        void *private)
468{
469    pte_t *orig_pte;
470    pte_t *pte;
471    spinlock_t *ptl;
472
473    orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
474    do {
475        struct page *page;
476        int nid;
477
478        if (!pte_present(*pte))
479            continue;
480        page = vm_normal_page(vma, addr, *pte);
481        if (!page)
482            continue;
483        /*
484         * vm_normal_page() filters out zero pages, but there might
485         * still be PageReserved pages to skip, perhaps in a VDSO.
486         * And we cannot move PageKsm pages sensibly or safely yet.
487         */
488        if (PageReserved(page) || PageKsm(page))
489            continue;
490        nid = page_to_nid(page);
491        if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
492            continue;
493
494        if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
495            migrate_page_add(page, private, flags);
496        else
497            break;
498    } while (pte++, addr += PAGE_SIZE, addr != end);
499    pte_unmap_unlock(orig_pte, ptl);
500    return addr != end;
501}
502
503static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
504        unsigned long addr, unsigned long end,
505        const nodemask_t *nodes, unsigned long flags,
506        void *private)
507{
508    pmd_t *pmd;
509    unsigned long next;
510
511    pmd = pmd_offset(pud, addr);
512    do {
513        next = pmd_addr_end(addr, end);
514        split_huge_page_pmd(vma->vm_mm, pmd);
515        if (pmd_none_or_trans_huge_or_clear_bad(pmd))
516            continue;
517        if (check_pte_range(vma, pmd, addr, next, nodes,
518                    flags, private))
519            return -EIO;
520    } while (pmd++, addr = next, addr != end);
521    return 0;
522}
523
524static inline int check_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
525        unsigned long addr, unsigned long end,
526        const nodemask_t *nodes, unsigned long flags,
527        void *private)
528{
529    pud_t *pud;
530    unsigned long next;
531
532    pud = pud_offset(pgd, addr);
533    do {
534        next = pud_addr_end(addr, end);
535        if (pud_none_or_clear_bad(pud))
536            continue;
537        if (check_pmd_range(vma, pud, addr, next, nodes,
538                    flags, private))
539            return -EIO;
540    } while (pud++, addr = next, addr != end);
541    return 0;
542}
543
544static inline int check_pgd_range(struct vm_area_struct *vma,
545        unsigned long addr, unsigned long end,
546        const nodemask_t *nodes, unsigned long flags,
547        void *private)
548{
549    pgd_t *pgd;
550    unsigned long next;
551
552    pgd = pgd_offset(vma->vm_mm, addr);
553    do {
554        next = pgd_addr_end(addr, end);
555        if (pgd_none_or_clear_bad(pgd))
556            continue;
557        if (check_pud_range(vma, pgd, addr, next, nodes,
558                    flags, private))
559            return -EIO;
560    } while (pgd++, addr = next, addr != end);
561    return 0;
562}
563
564/*
565 * Check if all pages in a range are on a set of nodes.
566 * If pagelist != NULL then isolate pages from the LRU and
567 * put them on the pagelist.
568 */
569static struct vm_area_struct *
570check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
571        const nodemask_t *nodes, unsigned long flags, void *private)
572{
573    int err;
574    struct vm_area_struct *first, *vma, *prev;
575
576
577    first = find_vma(mm, start);
578    if (!first)
579        return ERR_PTR(-EFAULT);
580    prev = NULL;
581    for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) {
582        if (!(flags & MPOL_MF_DISCONTIG_OK)) {
583            if (!vma->vm_next && vma->vm_end < end)
584                return ERR_PTR(-EFAULT);
585            if (prev && prev->vm_end < vma->vm_start)
586                return ERR_PTR(-EFAULT);
587        }
588        if (!is_vm_hugetlb_page(vma) &&
589            ((flags & MPOL_MF_STRICT) ||
590             ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
591                vma_migratable(vma)))) {
592            unsigned long endvma = vma->vm_end;
593
594            if (endvma > end)
595                endvma = end;
596            if (vma->vm_start > start)
597                start = vma->vm_start;
598            err = check_pgd_range(vma, start, endvma, nodes,
599                        flags, private);
600            if (err) {
601                first = ERR_PTR(err);
602                break;
603            }
604        }
605        prev = vma;
606    }
607    return first;
608}
609
610/* Step 2: apply policy to a range and do splits. */
611static int mbind_range(struct mm_struct *mm, unsigned long start,
612               unsigned long end, struct mempolicy *new_pol)
613{
614    struct vm_area_struct *next;
615    struct vm_area_struct *prev;
616    struct vm_area_struct *vma;
617    int err = 0;
618    pgoff_t pgoff;
619    unsigned long vmstart;
620    unsigned long vmend;
621
622    vma = find_vma(mm, start);
623    if (!vma || vma->vm_start > start)
624        return -EFAULT;
625
626    prev = vma->vm_prev;
627    if (start > vma->vm_start)
628        prev = vma;
629
630    for (; vma && vma->vm_start < end; prev = vma, vma = next) {
631        next = vma->vm_next;
632        vmstart = max(start, vma->vm_start);
633        vmend = min(end, vma->vm_end);
634
635        if (mpol_equal(vma_policy(vma), new_pol))
636            continue;
637
638        pgoff = vma->vm_pgoff +
639            ((vmstart - vma->vm_start) >> PAGE_SHIFT);
640        prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
641                  vma->anon_vma, vma->vm_file, pgoff,
642                  new_pol);
643        if (prev) {
644            vma = prev;
645            next = vma->vm_next;
646            continue;
647        }
648        if (vma->vm_start != vmstart) {
649            err = split_vma(vma->vm_mm, vma, vmstart, 1);
650            if (err)
651                goto out;
652        }
653        if (vma->vm_end != vmend) {
654            err = split_vma(vma->vm_mm, vma, vmend, 0);
655            if (err)
656                goto out;
657        }
658
659        /*
660         * Apply policy to a single VMA. The reference counting of
661         * policy for vma_policy linkages has already been handled by
662         * vma_merge and split_vma as necessary. If this is a shared
663         * policy then ->set_policy will increment the reference count
664         * for an sp node.
665         */
666        pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
667            vma->vm_start, vma->vm_end, vma->vm_pgoff,
668            vma->vm_ops, vma->vm_file,
669            vma->vm_ops ? vma->vm_ops->set_policy : NULL);
670        if (vma->vm_ops && vma->vm_ops->set_policy) {
671            err = vma->vm_ops->set_policy(vma, new_pol);
672            if (err)
673                goto out;
674        }
675    }
676
677 out:
678    return err;
679}
680
681/*
682 * Update task->flags PF_MEMPOLICY bit: set iff non-default
683 * mempolicy. Allows more rapid checking of this (combined perhaps
684 * with other PF_* flag bits) on memory allocation hot code paths.
685 *
686 * If called from outside this file, the task 'p' should -only- be
687 * a newly forked child not yet visible on the task list, because
688 * manipulating the task flags of a visible task is not safe.
689 *
690 * The above limitation is why this routine has the funny name
691 * mpol_fix_fork_child_flag().
692 *
693 * It is also safe to call this with a task pointer of current,
694 * which the static wrapper mpol_set_task_struct_flag() does,
695 * for use within this file.
696 */
697
698void mpol_fix_fork_child_flag(struct task_struct *p)
699{
700    if (p->mempolicy)
701        p->flags |= PF_MEMPOLICY;
702    else
703        p->flags &= ~PF_MEMPOLICY;
704}
705
706static void mpol_set_task_struct_flag(void)
707{
708    mpol_fix_fork_child_flag(current);
709}
710
711/* Set the process memory policy */
712static long do_set_mempolicy(unsigned short mode, unsigned short flags,
713                 nodemask_t *nodes)
714{
715    struct mempolicy *new, *old;
716    struct mm_struct *mm = current->mm;
717    NODEMASK_SCRATCH(scratch);
718    int ret;
719
720    if (!scratch)
721        return -ENOMEM;
722
723    new = mpol_new(mode, flags, nodes);
724    if (IS_ERR(new)) {
725        ret = PTR_ERR(new);
726        goto out;
727    }
728    /*
729     * prevent changing our mempolicy while show_numa_maps()
730     * is using it.
731     * Note: do_set_mempolicy() can be called at init time
732     * with no 'mm'.
733     */
734    if (mm)
735        down_write(&mm->mmap_sem);
736    task_lock(current);
737    ret = mpol_set_nodemask(new, nodes, scratch);
738    if (ret) {
739        task_unlock(current);
740        if (mm)
741            up_write(&mm->mmap_sem);
742        mpol_put(new);
743        goto out;
744    }
745    old = current->mempolicy;
746    current->mempolicy = new;
747    mpol_set_task_struct_flag();
748    if (new && new->mode == MPOL_INTERLEAVE &&
749        nodes_weight(new->v.nodes))
750        current->il_next = first_node(new->v.nodes);
751    task_unlock(current);
752    if (mm)
753        up_write(&mm->mmap_sem);
754
755    mpol_put(old);
756    ret = 0;
757out:
758    NODEMASK_SCRATCH_FREE(scratch);
759    return ret;
760}
761
762/*
763 * Return nodemask for policy for get_mempolicy() query
764 *
765 * Called with task's alloc_lock held
766 */
767static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
768{
769    nodes_clear(*nodes);
770    if (p == &default_policy)
771        return;
772
773    switch (p->mode) {
774    case MPOL_BIND:
775        /* Fall through */
776    case MPOL_INTERLEAVE:
777        *nodes = p->v.nodes;
778        break;
779    case MPOL_PREFERRED:
780        if (!(p->flags & MPOL_F_LOCAL))
781            node_set(p->v.preferred_node, *nodes);
782        /* else return empty node mask for local allocation */
783        break;
784    default:
785        BUG();
786    }
787}
788
789static int lookup_node(struct mm_struct *mm, unsigned long addr)
790{
791    struct page *p;
792    int err;
793
794    err = get_user_pages(current, mm, addr & PAGE_MASK, 1, 0, 0, &p, NULL);
795    if (err >= 0) {
796        err = page_to_nid(p);
797        put_page(p);
798    }
799    return err;
800}
801
802/* Retrieve NUMA policy */
803static long do_get_mempolicy(int *policy, nodemask_t *nmask,
804                 unsigned long addr, unsigned long flags)
805{
806    int err;
807    struct mm_struct *mm = current->mm;
808    struct vm_area_struct *vma = NULL;
809    struct mempolicy *pol = current->mempolicy;
810
811    if (flags &
812        ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
813        return -EINVAL;
814
815    if (flags & MPOL_F_MEMS_ALLOWED) {
816        if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
817            return -EINVAL;
818        *policy = 0; /* just so it's initialized */
819        task_lock(current);
820        *nmask = cpuset_current_mems_allowed;
821        task_unlock(current);
822        return 0;
823    }
824
825    if (flags & MPOL_F_ADDR) {
826        /*
827         * Do NOT fall back to task policy if the
828         * vma/shared policy at addr is NULL. We
829         * want to return MPOL_DEFAULT in this case.
830         */
831        down_read(&mm->mmap_sem);
832        vma = find_vma_intersection(mm, addr, addr+1);
833        if (!vma) {
834            up_read(&mm->mmap_sem);
835            return -EFAULT;
836        }
837        if (vma->vm_ops && vma->vm_ops->get_policy)
838            pol = vma->vm_ops->get_policy(vma, addr);
839        else
840            pol = vma->vm_policy;
841    } else if (addr)
842        return -EINVAL;
843
844    if (!pol)
845        pol = &default_policy; /* indicates default behavior */
846
847    if (flags & MPOL_F_NODE) {
848        if (flags & MPOL_F_ADDR) {
849            err = lookup_node(mm, addr);
850            if (err < 0)
851                goto out;
852            *policy = err;
853        } else if (pol == current->mempolicy &&
854                pol->mode == MPOL_INTERLEAVE) {
855            *policy = current->il_next;
856        } else {
857            err = -EINVAL;
858            goto out;
859        }
860    } else {
861        *policy = pol == &default_policy ? MPOL_DEFAULT :
862                        pol->mode;
863        /*
864         * Internal mempolicy flags must be masked off before exposing
865         * the policy to userspace.
866         */
867        *policy |= (pol->flags & MPOL_MODE_FLAGS);
868    }
869
870    if (vma) {
871        up_read(&current->mm->mmap_sem);
872        vma = NULL;
873    }
874
875    err = 0;
876    if (nmask) {
877        if (mpol_store_user_nodemask(pol)) {
878            *nmask = pol->w.user_nodemask;
879        } else {
880            task_lock(current);
881            get_policy_nodemask(pol, nmask);
882            task_unlock(current);
883        }
884    }
885
886 out:
887    mpol_cond_put(pol);
888    if (vma)
889        up_read(&current->mm->mmap_sem);
890    return err;
891}
892
893#ifdef CONFIG_MIGRATION
894/*
895 * page migration
896 */
897static void migrate_page_add(struct page *page, struct list_head *pagelist,
898                unsigned long flags)
899{
900    /*
901     * Avoid migrating a page that is shared with others.
902     */
903    if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
904        if (!isolate_lru_page(page)) {
905            list_add_tail(&page->lru, pagelist);
906            inc_zone_page_state(page, NR_ISOLATED_ANON +
907                        page_is_file_cache(page));
908        }
909    }
910}
911
912static struct page *new_node_page(struct page *page, unsigned long node, int **x)
913{
914    return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
915}
916
917/*
918 * Migrate pages from one node to a target node.
919 * Returns error or the number of pages not migrated.
920 */
921static int migrate_to_node(struct mm_struct *mm, int source, int dest,
922               int flags)
923{
924    nodemask_t nmask;
925    LIST_HEAD(pagelist);
926    int err = 0;
927    struct vm_area_struct *vma;
928
929    nodes_clear(nmask);
930    node_set(source, nmask);
931
932    vma = check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
933            flags | MPOL_MF_DISCONTIG_OK, &pagelist);
934    if (IS_ERR(vma))
935        return PTR_ERR(vma);
936
937    if (!list_empty(&pagelist)) {
938        err = migrate_pages(&pagelist, new_node_page, dest,
939                            false, MIGRATE_SYNC);
940        if (err)
941            putback_lru_pages(&pagelist);
942    }
943
944    return err;
945}
946
947/*
948 * Move pages between the two nodesets so as to preserve the physical
949 * layout as much as possible.
950 *
951 * Returns the number of page that could not be moved.
952 */
953int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
954             const nodemask_t *to, int flags)
955{
956    int busy = 0;
957    int err;
958    nodemask_t tmp;
959
960    err = migrate_prep();
961    if (err)
962        return err;
963
964    down_read(&mm->mmap_sem);
965
966    err = migrate_vmas(mm, from, to, flags);
967    if (err)
968        goto out;
969
970    /*
971     * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
972     * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
973     * bit in 'tmp', and return that <source, dest> pair for migration.
974     * The pair of nodemasks 'to' and 'from' define the map.
975     *
976     * If no pair of bits is found that way, fallback to picking some
977     * pair of 'source' and 'dest' bits that are not the same. If the
978     * 'source' and 'dest' bits are the same, this represents a node
979     * that will be migrating to itself, so no pages need move.
980     *
981     * If no bits are left in 'tmp', or if all remaining bits left
982     * in 'tmp' correspond to the same bit in 'to', return false
983     * (nothing left to migrate).
984     *
985     * This lets us pick a pair of nodes to migrate between, such that
986     * if possible the dest node is not already occupied by some other
987     * source node, minimizing the risk of overloading the memory on a
988     * node that would happen if we migrated incoming memory to a node
989     * before migrating outgoing memory source that same node.
990     *
991     * A single scan of tmp is sufficient. As we go, we remember the
992     * most recent <s, d> pair that moved (s != d). If we find a pair
993     * that not only moved, but what's better, moved to an empty slot
994     * (d is not set in tmp), then we break out then, with that pair.
995     * Otherwise when we finish scanning from_tmp, we at least have the
996     * most recent <s, d> pair that moved. If we get all the way through
997     * the scan of tmp without finding any node that moved, much less
998     * moved to an empty node, then there is nothing left worth migrating.
999     */
1000
1001    tmp = *from;
1002    while (!nodes_empty(tmp)) {
1003        int s,d;
1004        int source = -1;
1005        int dest = 0;
1006
1007        for_each_node_mask(s, tmp) {
1008
1009            /*
1010             * do_migrate_pages() tries to maintain the relative
1011             * node relationship of the pages established between
1012             * threads and memory areas.
1013                         *
1014             * However if the number of source nodes is not equal to
1015             * the number of destination nodes we can not preserve
1016             * this node relative relationship. In that case, skip
1017             * copying memory from a node that is in the destination
1018             * mask.
1019             *
1020             * Example: [2,3,4] -> [3,4,5] moves everything.
1021             * [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1022             */
1023
1024            if ((nodes_weight(*from) != nodes_weight(*to)) &&
1025                        (node_isset(s, *to)))
1026                continue;
1027
1028            d = node_remap(s, *from, *to);
1029            if (s == d)
1030                continue;
1031
1032            source = s; /* Node moved. Memorize */
1033            dest = d;
1034
1035            /* dest not in remaining from nodes? */
1036            if (!node_isset(dest, tmp))
1037                break;
1038        }
1039        if (source == -1)
1040            break;
1041
1042        node_clear(source, tmp);
1043        err = migrate_to_node(mm, source, dest, flags);
1044        if (err > 0)
1045            busy += err;
1046        if (err < 0)
1047            break;
1048    }
1049out:
1050    up_read(&mm->mmap_sem);
1051    if (err < 0)
1052        return err;
1053    return busy;
1054
1055}
1056
1057/*
1058 * Allocate a new page for page migration based on vma policy.
1059 * Start assuming that page is mapped by vma pointed to by @private.
1060 * Search forward from there, if not. N.B., this assumes that the
1061 * list of pages handed to migrate_pages()--which is how we get here--
1062 * is in virtual address order.
1063 */
1064static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
1065{
1066    struct vm_area_struct *vma = (struct vm_area_struct *)private;
1067    unsigned long uninitialized_var(address);
1068
1069    while (vma) {
1070        address = page_address_in_vma(page, vma);
1071        if (address != -EFAULT)
1072            break;
1073        vma = vma->vm_next;
1074    }
1075
1076    /*
1077     * if !vma, alloc_page_vma() will use task or system default policy
1078     */
1079    return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
1080}
1081#else
1082
1083static void migrate_page_add(struct page *page, struct list_head *pagelist,
1084                unsigned long flags)
1085{
1086}
1087
1088int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1089             const nodemask_t *to, int flags)
1090{
1091    return -ENOSYS;
1092}
1093
1094static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
1095{
1096    return NULL;
1097}
1098#endif
1099
1100static long do_mbind(unsigned long start, unsigned long len,
1101             unsigned short mode, unsigned short mode_flags,
1102             nodemask_t *nmask, unsigned long flags)
1103{
1104    struct vm_area_struct *vma;
1105    struct mm_struct *mm = current->mm;
1106    struct mempolicy *new;
1107    unsigned long end;
1108    int err;
1109    LIST_HEAD(pagelist);
1110
1111    if (flags & ~(unsigned long)(MPOL_MF_STRICT |
1112                     MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1113        return -EINVAL;
1114    if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1115        return -EPERM;
1116
1117    if (start & ~PAGE_MASK)
1118        return -EINVAL;
1119
1120    if (mode == MPOL_DEFAULT)
1121        flags &= ~MPOL_MF_STRICT;
1122
1123    len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1124    end = start + len;
1125
1126    if (end < start)
1127        return -EINVAL;
1128    if (end == start)
1129        return 0;
1130
1131    new = mpol_new(mode, mode_flags, nmask);
1132    if (IS_ERR(new))
1133        return PTR_ERR(new);
1134
1135    /*
1136     * If we are using the default policy then operation
1137     * on discontinuous address spaces is okay after all
1138     */
1139    if (!new)
1140        flags |= MPOL_MF_DISCONTIG_OK;
1141
1142    pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1143         start, start + len, mode, mode_flags,
1144         nmask ? nodes_addr(*nmask)[0] : -1);
1145
1146    if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1147
1148        err = migrate_prep();
1149        if (err)
1150            goto mpol_out;
1151    }
1152    {
1153        NODEMASK_SCRATCH(scratch);
1154        if (scratch) {
1155            down_write(&mm->mmap_sem);
1156            task_lock(current);
1157            err = mpol_set_nodemask(new, nmask, scratch);
1158            task_unlock(current);
1159            if (err)
1160                up_write(&mm->mmap_sem);
1161        } else
1162            err = -ENOMEM;
1163        NODEMASK_SCRATCH_FREE(scratch);
1164    }
1165    if (err)
1166        goto mpol_out;
1167
1168    vma = check_range(mm, start, end, nmask,
1169              flags | MPOL_MF_INVERT, &pagelist);
1170
1171    err = PTR_ERR(vma);
1172    if (!IS_ERR(vma)) {
1173        int nr_failed = 0;
1174
1175        err = mbind_range(mm, start, end, new);
1176
1177        if (!list_empty(&pagelist)) {
1178            nr_failed = migrate_pages(&pagelist, new_vma_page,
1179                        (unsigned long)vma,
1180                        false, MIGRATE_SYNC);
1181            if (nr_failed)
1182                putback_lru_pages(&pagelist);
1183        }
1184
1185        if (!err && nr_failed && (flags & MPOL_MF_STRICT))
1186            err = -EIO;
1187    } else
1188        putback_lru_pages(&pagelist);
1189
1190    up_write(&mm->mmap_sem);
1191 mpol_out:
1192    mpol_put(new);
1193    return err;
1194}
1195
1196/*
1197 * User space interface with variable sized bitmaps for nodelists.
1198 */
1199
1200/* Copy a node mask from user space. */
1201static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1202             unsigned long maxnode)
1203{
1204    unsigned long k;
1205    unsigned long nlongs;
1206    unsigned long endmask;
1207
1208    --maxnode;
1209    nodes_clear(*nodes);
1210    if (maxnode == 0 || !nmask)
1211        return 0;
1212    if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1213        return -EINVAL;
1214
1215    nlongs = BITS_TO_LONGS(maxnode);
1216    if ((maxnode % BITS_PER_LONG) == 0)
1217        endmask = ~0UL;
1218    else
1219        endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1220
1221    /* When the user specified more nodes than supported just check
1222       if the non supported part is all zero. */
1223    if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1224        if (nlongs > PAGE_SIZE/sizeof(long))
1225            return -EINVAL;
1226        for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1227            unsigned long t;
1228            if (get_user(t, nmask + k))
1229                return -EFAULT;
1230            if (k == nlongs - 1) {
1231                if (t & endmask)
1232                    return -EINVAL;
1233            } else if (t)
1234                return -EINVAL;
1235        }
1236        nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1237        endmask = ~0UL;
1238    }
1239
1240    if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1241        return -EFAULT;
1242    nodes_addr(*nodes)[nlongs-1] &= endmask;
1243    return 0;
1244}
1245
1246/* Copy a kernel node mask to user space */
1247static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1248                  nodemask_t *nodes)
1249{
1250    unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1251    const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1252
1253    if (copy > nbytes) {
1254        if (copy > PAGE_SIZE)
1255            return -EINVAL;
1256        if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1257            return -EFAULT;
1258        copy = nbytes;
1259    }
1260    return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1261}
1262
1263SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1264        unsigned long, mode, unsigned long __user *, nmask,
1265        unsigned long, maxnode, unsigned, flags)
1266{
1267    nodemask_t nodes;
1268    int err;
1269    unsigned short mode_flags;
1270
1271    mode_flags = mode & MPOL_MODE_FLAGS;
1272    mode &= ~MPOL_MODE_FLAGS;
1273    if (mode >= MPOL_MAX)
1274        return -EINVAL;
1275    if ((mode_flags & MPOL_F_STATIC_NODES) &&
1276        (mode_flags & MPOL_F_RELATIVE_NODES))
1277        return -EINVAL;
1278    err = get_nodes(&nodes, nmask, maxnode);
1279    if (err)
1280        return err;
1281    return do_mbind(start, len, mode, mode_flags, &nodes, flags);
1282}
1283
1284/* Set the process memory policy */
1285SYSCALL_DEFINE3(set_mempolicy, int, mode, unsigned long __user *, nmask,
1286        unsigned long, maxnode)
1287{
1288    int err;
1289    nodemask_t nodes;
1290    unsigned short flags;
1291
1292    flags = mode & MPOL_MODE_FLAGS;
1293    mode &= ~MPOL_MODE_FLAGS;
1294    if ((unsigned int)mode >= MPOL_MAX)
1295        return -EINVAL;
1296    if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1297        return -EINVAL;
1298    err = get_nodes(&nodes, nmask, maxnode);
1299    if (err)
1300        return err;
1301    return do_set_mempolicy(mode, flags, &nodes);
1302}
1303
1304SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1305        const unsigned long __user *, old_nodes,
1306        const unsigned long __user *, new_nodes)
1307{
1308    const struct cred *cred = current_cred(), *tcred;
1309    struct mm_struct *mm = NULL;
1310    struct task_struct *task;
1311    nodemask_t task_nodes;
1312    int err;
1313    nodemask_t *old;
1314    nodemask_t *new;
1315    NODEMASK_SCRATCH(scratch);
1316
1317    if (!scratch)
1318        return -ENOMEM;
1319
1320    old = &scratch->mask1;
1321    new = &scratch->mask2;
1322
1323    err = get_nodes(old, old_nodes, maxnode);
1324    if (err)
1325        goto out;
1326
1327    err = get_nodes(new, new_nodes, maxnode);
1328    if (err)
1329        goto out;
1330
1331    /* Find the mm_struct */
1332    rcu_read_lock();
1333    task = pid ? find_task_by_vpid(pid) : current;
1334    if (!task) {
1335        rcu_read_unlock();
1336        err = -ESRCH;
1337        goto out;
1338    }
1339    get_task_struct(task);
1340
1341    err = -EINVAL;
1342
1343    /*
1344     * Check if this process has the right to modify the specified
1345     * process. The right exists if the process has administrative
1346     * capabilities, superuser privileges or the same
1347     * userid as the target process.
1348     */
1349    tcred = __task_cred(task);
1350    if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1351        !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
1352        !capable(CAP_SYS_NICE)) {
1353        rcu_read_unlock();
1354        err = -EPERM;
1355        goto out_put;
1356    }
1357    rcu_read_unlock();
1358
1359    task_nodes = cpuset_mems_allowed(task);
1360    /* Is the user allowed to access the target nodes? */
1361    if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1362        err = -EPERM;
1363        goto out_put;
1364    }
1365
1366    if (!nodes_subset(*new, node_states[N_HIGH_MEMORY])) {
1367        err = -EINVAL;
1368        goto out_put;
1369    }
1370
1371    err = security_task_movememory(task);
1372    if (err)
1373        goto out_put;
1374
1375    mm = get_task_mm(task);
1376    put_task_struct(task);
1377
1378    if (!mm) {
1379        err = -EINVAL;
1380        goto out;
1381    }
1382
1383    err = do_migrate_pages(mm, old, new,
1384        capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1385
1386    mmput(mm);
1387out:
1388    NODEMASK_SCRATCH_FREE(scratch);
1389
1390    return err;
1391
1392out_put:
1393    put_task_struct(task);
1394    goto out;
1395
1396}
1397
1398
1399/* Retrieve NUMA policy */
1400SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1401        unsigned long __user *, nmask, unsigned long, maxnode,
1402        unsigned long, addr, unsigned long, flags)
1403{
1404    int err;
1405    int uninitialized_var(pval);
1406    nodemask_t nodes;
1407
1408    if (nmask != NULL && maxnode < MAX_NUMNODES)
1409        return -EINVAL;
1410
1411    err = do_get_mempolicy(&pval, &nodes, addr, flags);
1412
1413    if (err)
1414        return err;
1415
1416    if (policy && put_user(pval, policy))
1417        return -EFAULT;
1418
1419    if (nmask)
1420        err = copy_nodes_to_user(nmask, maxnode, &nodes);
1421
1422    return err;
1423}
1424
1425#ifdef CONFIG_COMPAT
1426
1427asmlinkage long compat_sys_get_mempolicy(int __user *policy,
1428                     compat_ulong_t __user *nmask,
1429                     compat_ulong_t maxnode,
1430                     compat_ulong_t addr, compat_ulong_t flags)
1431{
1432    long err;
1433    unsigned long __user *nm = NULL;
1434    unsigned long nr_bits, alloc_size;
1435    DECLARE_BITMAP(bm, MAX_NUMNODES);
1436
1437    nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1438    alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1439
1440    if (nmask)
1441        nm = compat_alloc_user_space(alloc_size);
1442
1443    err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1444
1445    if (!err && nmask) {
1446        unsigned long copy_size;
1447        copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
1448        err = copy_from_user(bm, nm, copy_size);
1449        /* ensure entire bitmap is zeroed */
1450        err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1451        err |= compat_put_bitmap(nmask, bm, nr_bits);
1452    }
1453
1454    return err;
1455}
1456
1457asmlinkage long compat_sys_set_mempolicy(int mode, compat_ulong_t __user *nmask,
1458                     compat_ulong_t maxnode)
1459{
1460    long err = 0;
1461    unsigned long __user *nm = NULL;
1462    unsigned long nr_bits, alloc_size;
1463    DECLARE_BITMAP(bm, MAX_NUMNODES);
1464
1465    nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1466    alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1467
1468    if (nmask) {
1469        err = compat_get_bitmap(bm, nmask, nr_bits);
1470        nm = compat_alloc_user_space(alloc_size);
1471        err |= copy_to_user(nm, bm, alloc_size);
1472    }
1473
1474    if (err)
1475        return -EFAULT;
1476
1477    return sys_set_mempolicy(mode, nm, nr_bits+1);
1478}
1479
1480asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
1481                 compat_ulong_t mode, compat_ulong_t __user *nmask,
1482                 compat_ulong_t maxnode, compat_ulong_t flags)
1483{
1484    long err = 0;
1485    unsigned long __user *nm = NULL;
1486    unsigned long nr_bits, alloc_size;
1487    nodemask_t bm;
1488
1489    nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1490    alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1491
1492    if (nmask) {
1493        err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
1494        nm = compat_alloc_user_space(alloc_size);
1495        err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
1496    }
1497
1498    if (err)
1499        return -EFAULT;
1500
1501    return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1502}
1503
1504#endif
1505
1506/*
1507 * get_vma_policy(@task, @vma, @addr)
1508 * @task - task for fallback if vma policy == default
1509 * @vma - virtual memory area whose policy is sought
1510 * @addr - address in @vma for shared policy lookup
1511 *
1512 * Returns effective policy for a VMA at specified address.
1513 * Falls back to @task or system default policy, as necessary.
1514 * Current or other task's task mempolicy and non-shared vma policies
1515 * are protected by the task's mmap_sem, which must be held for read by
1516 * the caller.
1517 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1518 * count--added by the get_policy() vm_op, as appropriate--to protect against
1519 * freeing by another task. It is the caller's responsibility to free the
1520 * extra reference for shared policies.
1521 */
1522struct mempolicy *get_vma_policy(struct task_struct *task,
1523        struct vm_area_struct *vma, unsigned long addr)
1524{
1525    struct mempolicy *pol = task->mempolicy;
1526
1527    if (vma) {
1528        if (vma->vm_ops && vma->vm_ops->get_policy) {
1529            struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
1530                                    addr);
1531            if (vpol)
1532                pol = vpol;
1533        } else if (vma->vm_policy)
1534            pol = vma->vm_policy;
1535    }
1536    if (!pol)
1537        pol = &default_policy;
1538    return pol;
1539}
1540
1541/*
1542 * Return a nodemask representing a mempolicy for filtering nodes for
1543 * page allocation
1544 */
1545static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
1546{
1547    /* Lower zones don't get a nodemask applied for MPOL_BIND */
1548    if (unlikely(policy->mode == MPOL_BIND) &&
1549            gfp_zone(gfp) >= policy_zone &&
1550            cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1551        return &policy->v.nodes;
1552
1553    return NULL;
1554}
1555
1556/* Return a zonelist indicated by gfp for node representing a mempolicy */
1557static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
1558    int nd)
1559{
1560    switch (policy->mode) {
1561    case MPOL_PREFERRED:
1562        if (!(policy->flags & MPOL_F_LOCAL))
1563            nd = policy->v.preferred_node;
1564        break;
1565    case MPOL_BIND:
1566        /*
1567         * Normally, MPOL_BIND allocations are node-local within the
1568         * allowed nodemask. However, if __GFP_THISNODE is set and the
1569         * current node isn't part of the mask, we use the zonelist for
1570         * the first node in the mask instead.
1571         */
1572        if (unlikely(gfp & __GFP_THISNODE) &&
1573                unlikely(!node_isset(nd, policy->v.nodes)))
1574            nd = first_node(policy->v.nodes);
1575        break;
1576    default:
1577        BUG();
1578    }
1579    return node_zonelist(nd, gfp);
1580}
1581
1582/* Do dynamic interleaving for a process */
1583static unsigned interleave_nodes(struct mempolicy *policy)
1584{
1585    unsigned nid, next;
1586    struct task_struct *me = current;
1587
1588    nid = me->il_next;
1589    next = next_node(nid, policy->v.nodes);
1590    if (next >= MAX_NUMNODES)
1591        next = first_node(policy->v.nodes);
1592    if (next < MAX_NUMNODES)
1593        me->il_next = next;
1594    return nid;
1595}
1596
1597/*
1598 * Depending on the memory policy provide a node from which to allocate the
1599 * next slab entry.
1600 * @policy must be protected by freeing by the caller. If @policy is
1601 * the current task's mempolicy, this protection is implicit, as only the
1602 * task can change it's policy. The system default policy requires no
1603 * such protection.
1604 */
1605unsigned slab_node(void)
1606{
1607    struct mempolicy *policy;
1608
1609    if (in_interrupt())
1610        return numa_node_id();
1611
1612    policy = current->mempolicy;
1613    if (!policy || policy->flags & MPOL_F_LOCAL)
1614        return numa_node_id();
1615
1616    switch (policy->mode) {
1617    case MPOL_PREFERRED:
1618        /*
1619         * handled MPOL_F_LOCAL above
1620         */
1621        return policy->v.preferred_node;
1622
1623    case MPOL_INTERLEAVE:
1624        return interleave_nodes(policy);
1625
1626    case MPOL_BIND: {
1627        /*
1628         * Follow bind policy behavior and start allocation at the
1629         * first node.
1630         */
1631        struct zonelist *zonelist;
1632        struct zone *zone;
1633        enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
1634        zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
1635        (void)first_zones_zonelist(zonelist, highest_zoneidx,
1636                            &policy->v.nodes,
1637                            &zone);
1638        return zone ? zone->node : numa_node_id();
1639    }
1640
1641    default:
1642        BUG();
1643    }
1644}
1645
1646/* Do static interleaving for a VMA with known offset. */
1647static unsigned offset_il_node(struct mempolicy *pol,
1648        struct vm_area_struct *vma, unsigned long off)
1649{
1650    unsigned nnodes = nodes_weight(pol->v.nodes);
1651    unsigned target;
1652    int c;
1653    int nid = -1;
1654
1655    if (!nnodes)
1656        return numa_node_id();
1657    target = (unsigned int)off % nnodes;
1658    c = 0;
1659    do {
1660        nid = next_node(nid, pol->v.nodes);
1661        c++;
1662    } while (c <= target);
1663    return nid;
1664}
1665
1666/* Determine a node number for interleave */
1667static inline unsigned interleave_nid(struct mempolicy *pol,
1668         struct vm_area_struct *vma, unsigned long addr, int shift)
1669{
1670    if (vma) {
1671        unsigned long off;
1672
1673        /*
1674         * for small pages, there is no difference between
1675         * shift and PAGE_SHIFT, so the bit-shift is safe.
1676         * for huge pages, since vm_pgoff is in units of small
1677         * pages, we need to shift off the always 0 bits to get
1678         * a useful offset.
1679         */
1680        BUG_ON(shift < PAGE_SHIFT);
1681        off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
1682        off += (addr - vma->vm_start) >> shift;
1683        return offset_il_node(pol, vma, off);
1684    } else
1685        return interleave_nodes(pol);
1686}
1687
1688/*
1689 * Return the bit number of a random bit set in the nodemask.
1690 * (returns -1 if nodemask is empty)
1691 */
1692int node_random(const nodemask_t *maskp)
1693{
1694    int w, bit = -1;
1695
1696    w = nodes_weight(*maskp);
1697    if (w)
1698        bit = bitmap_ord_to_pos(maskp->bits,
1699            get_random_int() % w, MAX_NUMNODES);
1700    return bit;
1701}
1702
1703#ifdef CONFIG_HUGETLBFS
1704/*
1705 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1706 * @vma = virtual memory area whose policy is sought
1707 * @addr = address in @vma for shared policy lookup and interleave policy
1708 * @gfp_flags = for requested zone
1709 * @mpol = pointer to mempolicy pointer for reference counted mempolicy
1710 * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
1711 *
1712 * Returns a zonelist suitable for a huge page allocation and a pointer
1713 * to the struct mempolicy for conditional unref after allocation.
1714 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1715 * @nodemask for filtering the zonelist.
1716 *
1717 * Must be protected by get_mems_allowed()
1718 */
1719struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
1720                gfp_t gfp_flags, struct mempolicy **mpol,
1721                nodemask_t **nodemask)
1722{
1723    struct zonelist *zl;
1724
1725    *mpol = get_vma_policy(current, vma, addr);
1726    *nodemask = NULL; /* assume !MPOL_BIND */
1727
1728    if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1729        zl = node_zonelist(interleave_nid(*mpol, vma, addr,
1730                huge_page_shift(hstate_vma(vma))), gfp_flags);
1731    } else {
1732        zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
1733        if ((*mpol)->mode == MPOL_BIND)
1734            *nodemask = &(*mpol)->v.nodes;
1735    }
1736    return zl;
1737}
1738
1739/*
1740 * init_nodemask_of_mempolicy
1741 *
1742 * If the current task's mempolicy is "default" [NULL], return 'false'
1743 * to indicate default policy. Otherwise, extract the policy nodemask
1744 * for 'bind' or 'interleave' policy into the argument nodemask, or
1745 * initialize the argument nodemask to contain the single node for
1746 * 'preferred' or 'local' policy and return 'true' to indicate presence
1747 * of non-default mempolicy.
1748 *
1749 * We don't bother with reference counting the mempolicy [mpol_get/put]
1750 * because the current task is examining it's own mempolicy and a task's
1751 * mempolicy is only ever changed by the task itself.
1752 *
1753 * N.B., it is the caller's responsibility to free a returned nodemask.
1754 */
1755bool init_nodemask_of_mempolicy(nodemask_t *mask)
1756{
1757    struct mempolicy *mempolicy;
1758    int nid;
1759
1760    if (!(mask && current->mempolicy))
1761        return false;
1762
1763    task_lock(current);
1764    mempolicy = current->mempolicy;
1765    switch (mempolicy->mode) {
1766    case MPOL_PREFERRED:
1767        if (mempolicy->flags & MPOL_F_LOCAL)
1768            nid = numa_node_id();
1769        else
1770            nid = mempolicy->v.preferred_node;
1771        init_nodemask_of_node(mask, nid);
1772        break;
1773
1774    case MPOL_BIND:
1775        /* Fall through */
1776    case MPOL_INTERLEAVE:
1777        *mask = mempolicy->v.nodes;
1778        break;
1779
1780    default:
1781        BUG();
1782    }
1783    task_unlock(current);
1784
1785    return true;
1786}
1787#endif
1788
1789/*
1790 * mempolicy_nodemask_intersects
1791 *
1792 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1793 * policy. Otherwise, check for intersection between mask and the policy
1794 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local'
1795 * policy, always return true since it may allocate elsewhere on fallback.
1796 *
1797 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
1798 */
1799bool mempolicy_nodemask_intersects(struct task_struct *tsk,
1800                    const nodemask_t *mask)
1801{
1802    struct mempolicy *mempolicy;
1803    bool ret = true;
1804
1805    if (!mask)
1806        return ret;
1807    task_lock(tsk);
1808    mempolicy = tsk->mempolicy;
1809    if (!mempolicy)
1810        goto out;
1811
1812    switch (mempolicy->mode) {
1813    case MPOL_PREFERRED:
1814        /*
1815         * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
1816         * allocate from, they may fallback to other nodes when oom.
1817         * Thus, it's possible for tsk to have allocated memory from
1818         * nodes in mask.
1819         */
1820        break;
1821    case MPOL_BIND:
1822    case MPOL_INTERLEAVE:
1823        ret = nodes_intersects(mempolicy->v.nodes, *mask);
1824        break;
1825    default:
1826        BUG();
1827    }
1828out:
1829    task_unlock(tsk);
1830    return ret;
1831}
1832
1833/* Allocate a page in interleaved policy.
1834   Own path because it needs to do special accounting. */
1835static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1836                    unsigned nid)
1837{
1838    struct zonelist *zl;
1839    struct page *page;
1840
1841    zl = node_zonelist(nid, gfp);
1842    page = __alloc_pages(gfp, order, zl);
1843    if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
1844        inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
1845    return page;
1846}
1847
1848/**
1849 * alloc_pages_vma - Allocate a page for a VMA.
1850 *
1851 * @gfp:
1852 * %GFP_USER user allocation.
1853 * %GFP_KERNEL kernel allocations,
1854 * %GFP_HIGHMEM highmem/user allocations,
1855 * %GFP_FS allocation should not call back into a file system.
1856 * %GFP_ATOMIC don't sleep.
1857 *
1858 * @order:Order of the GFP allocation.
1859 * @vma: Pointer to VMA or NULL if not available.
1860 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1861 *
1862 * This function allocates a page from the kernel page pool and applies
1863 * a NUMA policy associated with the VMA or the current process.
1864 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1865 * mm_struct of the VMA to prevent it from going away. Should be used for
1866 * all allocations for pages that will be mapped into
1867 * user space. Returns NULL when no page can be allocated.
1868 *
1869 * Should be called with the mm_sem of the vma hold.
1870 */
1871struct page *
1872alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
1873        unsigned long addr, int node)
1874{
1875    struct mempolicy *pol;
1876    struct zonelist *zl;
1877    struct page *page;
1878    unsigned int cpuset_mems_cookie;
1879
1880retry_cpuset:
1881    pol = get_vma_policy(current, vma, addr);
1882    cpuset_mems_cookie = get_mems_allowed();
1883
1884    if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
1885        unsigned nid;
1886
1887        nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
1888        mpol_cond_put(pol);
1889        page = alloc_page_interleave(gfp, order, nid);
1890        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
1891            goto retry_cpuset;
1892
1893        return page;
1894    }
1895    zl = policy_zonelist(gfp, pol, node);
1896    if (unlikely(mpol_needs_cond_ref(pol))) {
1897        /*
1898         * slow path: ref counted shared policy
1899         */
1900        struct page *page = __alloc_pages_nodemask(gfp, order,
1901                        zl, policy_nodemask(gfp, pol));
1902        __mpol_put(pol);
1903        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
1904            goto retry_cpuset;
1905        return page;
1906    }
1907    /*
1908     * fast path: default or task policy
1909     */
1910    page = __alloc_pages_nodemask(gfp, order, zl,
1911                      policy_nodemask(gfp, pol));
1912    if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
1913        goto retry_cpuset;
1914    return page;
1915}
1916
1917/**
1918 * alloc_pages_current - Allocate pages.
1919 *
1920 * @gfp:
1921 * %GFP_USER user allocation,
1922 * %GFP_KERNEL kernel allocation,
1923 * %GFP_HIGHMEM highmem allocation,
1924 * %GFP_FS don't call back into a file system.
1925 * %GFP_ATOMIC don't sleep.
1926 * @order: Power of two of allocation size in pages. 0 is a single page.
1927 *
1928 * Allocate a page from the kernel page pool. When not in
1929 * interrupt context and apply the current process NUMA policy.
1930 * Returns NULL when no page can be allocated.
1931 *
1932 * Don't call cpuset_update_task_memory_state() unless
1933 * 1) it's ok to take cpuset_sem (can WAIT), and
1934 * 2) allocating for current task (not interrupt).
1935 */
1936struct page *alloc_pages_current(gfp_t gfp, unsigned order)
1937{
1938    struct mempolicy *pol = current->mempolicy;
1939    struct page *page;
1940    unsigned int cpuset_mems_cookie;
1941
1942    if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
1943        pol = &default_policy;
1944
1945retry_cpuset:
1946    cpuset_mems_cookie = get_mems_allowed();
1947
1948    /*
1949     * No reference counting needed for current->mempolicy
1950     * nor system default_policy
1951     */
1952    if (pol->mode == MPOL_INTERLEAVE)
1953        page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
1954    else
1955        page = __alloc_pages_nodemask(gfp, order,
1956                policy_zonelist(gfp, pol, numa_node_id()),
1957                policy_nodemask(gfp, pol));
1958
1959    if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
1960        goto retry_cpuset;
1961
1962    return page;
1963}
1964EXPORT_SYMBOL(alloc_pages_current);
1965
1966/*
1967 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
1968 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1969 * with the mems_allowed returned by cpuset_mems_allowed(). This
1970 * keeps mempolicies cpuset relative after its cpuset moves. See
1971 * further kernel/cpuset.c update_nodemask().
1972 *
1973 * current's mempolicy may be rebinded by the other task(the task that changes
1974 * cpuset's mems), so we needn't do rebind work for current task.
1975 */
1976
1977/* Slow path of a mempolicy duplicate */
1978struct mempolicy *__mpol_dup(struct mempolicy *old)
1979{
1980    struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
1981
1982    if (!new)
1983        return ERR_PTR(-ENOMEM);
1984
1985    /* task's mempolicy is protected by alloc_lock */
1986    if (old == current->mempolicy) {
1987        task_lock(current);
1988        *new = *old;
1989        task_unlock(current);
1990    } else
1991        *new = *old;
1992
1993    rcu_read_lock();
1994    if (current_cpuset_is_being_rebound()) {
1995        nodemask_t mems = cpuset_mems_allowed(current);
1996        if (new->flags & MPOL_F_REBINDING)
1997            mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
1998        else
1999            mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
2000    }
2001    rcu_read_unlock();
2002    atomic_set(&new->refcnt, 1);
2003    return new;
2004}
2005
2006/*
2007 * If *frompol needs [has] an extra ref, copy *frompol to *tompol ,
2008 * eliminate the * MPOL_F_* flags that require conditional ref and
2009 * [NOTE!!!] drop the extra ref. Not safe to reference *frompol directly
2010 * after return. Use the returned value.
2011 *
2012 * Allows use of a mempolicy for, e.g., multiple allocations with a single
2013 * policy lookup, even if the policy needs/has extra ref on lookup.
2014 * shmem_readahead needs this.
2015 */
2016struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol,
2017                        struct mempolicy *frompol)
2018{
2019    if (!mpol_needs_cond_ref(frompol))
2020        return frompol;
2021
2022    *tompol = *frompol;
2023    tompol->flags &= ~MPOL_F_SHARED; /* copy doesn't need unref */
2024    __mpol_put(frompol);
2025    return tompol;
2026}
2027
2028/* Slow path of a mempolicy comparison */
2029bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2030{
2031    if (!a || !b)
2032        return false;
2033    if (a->mode != b->mode)
2034        return false;
2035    if (a->flags != b->flags)
2036        return false;
2037    if (mpol_store_user_nodemask(a))
2038        if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2039            return false;
2040
2041    switch (a->mode) {
2042    case MPOL_BIND:
2043        /* Fall through */
2044    case MPOL_INTERLEAVE:
2045        return !!nodes_equal(a->v.nodes, b->v.nodes);
2046    case MPOL_PREFERRED:
2047        return a->v.preferred_node == b->v.preferred_node;
2048    default:
2049        BUG();
2050        return false;
2051    }
2052}
2053
2054/*
2055 * Shared memory backing store policy support.
2056 *
2057 * Remember policies even when nobody has shared memory mapped.
2058 * The policies are kept in Red-Black tree linked from the inode.
2059 * They are protected by the sp->lock spinlock, which should be held
2060 * for any accesses to the tree.
2061 */
2062
2063/* lookup first element intersecting start-end */
2064/* Caller holds sp->lock */
2065static struct sp_node *
2066sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2067{
2068    struct rb_node *n = sp->root.rb_node;
2069
2070    while (n) {
2071        struct sp_node *p = rb_entry(n, struct sp_node, nd);
2072
2073        if (start >= p->end)
2074            n = n->rb_right;
2075        else if (end <= p->start)
2076            n = n->rb_left;
2077        else
2078            break;
2079    }
2080    if (!n)
2081        return NULL;
2082    for (;;) {
2083        struct sp_node *w = NULL;
2084        struct rb_node *prev = rb_prev(n);
2085        if (!prev)
2086            break;
2087        w = rb_entry(prev, struct sp_node, nd);
2088        if (w->end <= start)
2089            break;
2090        n = prev;
2091    }
2092    return rb_entry(n, struct sp_node, nd);
2093}
2094
2095/* Insert a new shared policy into the list. */
2096/* Caller holds sp->lock */
2097static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2098{
2099    struct rb_node **p = &sp->root.rb_node;
2100    struct rb_node *parent = NULL;
2101    struct sp_node *nd;
2102
2103    while (*p) {
2104        parent = *p;
2105        nd = rb_entry(parent, struct sp_node, nd);
2106        if (new->start < nd->start)
2107            p = &(*p)->rb_left;
2108        else if (new->end > nd->end)
2109            p = &(*p)->rb_right;
2110        else
2111            BUG();
2112    }
2113    rb_link_node(&new->nd, parent, p);
2114    rb_insert_color(&new->nd, &sp->root);
2115    pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
2116         new->policy ? new->policy->mode : 0);
2117}
2118
2119/* Find shared policy intersecting idx */
2120struct mempolicy *
2121mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2122{
2123    struct mempolicy *pol = NULL;
2124    struct sp_node *sn;
2125
2126    if (!sp->root.rb_node)
2127        return NULL;
2128    spin_lock(&sp->lock);
2129    sn = sp_lookup(sp, idx, idx+1);
2130    if (sn) {
2131        mpol_get(sn->policy);
2132        pol = sn->policy;
2133    }
2134    spin_unlock(&sp->lock);
2135    return pol;
2136}
2137
2138static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2139{
2140    pr_debug("deleting %lx-l%lx\n", n->start, n->end);
2141    rb_erase(&n->nd, &sp->root);
2142    mpol_put(n->policy);
2143    kmem_cache_free(sn_cache, n);
2144}
2145
2146static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2147                struct mempolicy *pol)
2148{
2149    struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2150
2151    if (!n)
2152        return NULL;
2153    n->start = start;
2154    n->end = end;
2155    mpol_get(pol);
2156    pol->flags |= MPOL_F_SHARED; /* for unref */
2157    n->policy = pol;
2158    return n;
2159}
2160
2161/* Replace a policy range. */
2162static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2163                 unsigned long end, struct sp_node *new)
2164{
2165    struct sp_node *n, *new2 = NULL;
2166
2167restart:
2168    spin_lock(&sp->lock);
2169    n = sp_lookup(sp, start, end);
2170    /* Take care of old policies in the same range. */
2171    while (n && n->start < end) {
2172        struct rb_node *next = rb_next(&n->nd);
2173        if (n->start >= start) {
2174            if (n->end <= end)
2175                sp_delete(sp, n);
2176            else
2177                n->start = end;
2178        } else {
2179            /* Old policy spanning whole new range. */
2180            if (n->end > end) {
2181                if (!new2) {
2182                    spin_unlock(&sp->lock);
2183                    new2 = sp_alloc(end, n->end, n->policy);
2184                    if (!new2)
2185                        return -ENOMEM;
2186                    goto restart;
2187                }
2188                n->end = start;
2189                sp_insert(sp, new2);
2190                new2 = NULL;
2191                break;
2192            } else
2193                n->end = start;
2194        }
2195        if (!next)
2196            break;
2197        n = rb_entry(next, struct sp_node, nd);
2198    }
2199    if (new)
2200        sp_insert(sp, new);
2201    spin_unlock(&sp->lock);
2202    if (new2) {
2203        mpol_put(new2->policy);
2204        kmem_cache_free(sn_cache, new2);
2205    }
2206    return 0;
2207}
2208
2209/**
2210 * mpol_shared_policy_init - initialize shared policy for inode
2211 * @sp: pointer to inode shared policy
2212 * @mpol: struct mempolicy to install
2213 *
2214 * Install non-NULL @mpol in inode's shared policy rb-tree.
2215 * On entry, the current task has a reference on a non-NULL @mpol.
2216 * This must be released on exit.
2217 * This is called at get_inode() calls and we can use GFP_KERNEL.
2218 */
2219void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2220{
2221    int ret;
2222
2223    sp->root = RB_ROOT; /* empty tree == default mempolicy */
2224    spin_lock_init(&sp->lock);
2225
2226    if (mpol) {
2227        struct vm_area_struct pvma;
2228        struct mempolicy *new;
2229        NODEMASK_SCRATCH(scratch);
2230
2231        if (!scratch)
2232            goto put_mpol;
2233        /* contextualize the tmpfs mount point mempolicy */
2234        new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
2235        if (IS_ERR(new))
2236            goto free_scratch; /* no valid nodemask intersection */
2237
2238        task_lock(current);
2239        ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
2240        task_unlock(current);
2241        if (ret)
2242            goto put_new;
2243
2244        /* Create pseudo-vma that contains just the policy */
2245        memset(&pvma, 0, sizeof(struct vm_area_struct));
2246        pvma.vm_end = TASK_SIZE; /* policy covers entire file */
2247        mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
2248
2249put_new:
2250        mpol_put(new); /* drop initial ref */
2251free_scratch:
2252        NODEMASK_SCRATCH_FREE(scratch);
2253put_mpol:
2254        mpol_put(mpol); /* drop our incoming ref on sb mpol */
2255    }
2256}
2257
2258int mpol_set_shared_policy(struct shared_policy *info,
2259            struct vm_area_struct *vma, struct mempolicy *npol)
2260{
2261    int err;
2262    struct sp_node *new = NULL;
2263    unsigned long sz = vma_pages(vma);
2264
2265    pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
2266         vma->vm_pgoff,
2267         sz, npol ? npol->mode : -1,
2268         npol ? npol->flags : -1,
2269         npol ? nodes_addr(npol->v.nodes)[0] : -1);
2270
2271    if (npol) {
2272        new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2273        if (!new)
2274            return -ENOMEM;
2275    }
2276    err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2277    if (err && new)
2278        kmem_cache_free(sn_cache, new);
2279    return err;
2280}
2281
2282/* Free a backing policy store on inode delete. */
2283void mpol_free_shared_policy(struct shared_policy *p)
2284{
2285    struct sp_node *n;
2286    struct rb_node *next;
2287
2288    if (!p->root.rb_node)
2289        return;
2290    spin_lock(&p->lock);
2291    next = rb_first(&p->root);
2292    while (next) {
2293        n = rb_entry(next, struct sp_node, nd);
2294        next = rb_next(&n->nd);
2295        rb_erase(&n->nd, &p->root);
2296        mpol_put(n->policy);
2297        kmem_cache_free(sn_cache, n);
2298    }
2299    spin_unlock(&p->lock);
2300}
2301
2302/* assumes fs == KERNEL_DS */
2303void __init numa_policy_init(void)
2304{
2305    nodemask_t interleave_nodes;
2306    unsigned long largest = 0;
2307    int nid, prefer = 0;
2308
2309    policy_cache = kmem_cache_create("numa_policy",
2310                     sizeof(struct mempolicy),
2311                     0, SLAB_PANIC, NULL);
2312
2313    sn_cache = kmem_cache_create("shared_policy_node",
2314                     sizeof(struct sp_node),
2315                     0, SLAB_PANIC, NULL);
2316
2317    /*
2318     * Set interleaving policy for system init. Interleaving is only
2319     * enabled across suitably sized nodes (default is >= 16MB), or
2320     * fall back to the largest node if they're all smaller.
2321     */
2322    nodes_clear(interleave_nodes);
2323    for_each_node_state(nid, N_HIGH_MEMORY) {
2324        unsigned long total_pages = node_present_pages(nid);
2325
2326        /* Preserve the largest node */
2327        if (largest < total_pages) {
2328            largest = total_pages;
2329            prefer = nid;
2330        }
2331
2332        /* Interleave this node? */
2333        if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2334            node_set(nid, interleave_nodes);
2335    }
2336
2337    /* All too small, use the largest */
2338    if (unlikely(nodes_empty(interleave_nodes)))
2339        node_set(prefer, interleave_nodes);
2340
2341    if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
2342        printk("numa_policy_init: interleaving failed\n");
2343}
2344
2345/* Reset policy of current process to default */
2346void numa_default_policy(void)
2347{
2348    do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
2349}
2350
2351/*
2352 * Parse and format mempolicy from/to strings
2353 */
2354
2355/*
2356 * "local" is pseudo-policy: MPOL_PREFERRED with MPOL_F_LOCAL flag
2357 * Used only for mpol_parse_str() and mpol_to_str()
2358 */
2359#define MPOL_LOCAL MPOL_MAX
2360static const char * const policy_modes[] =
2361{
2362    [MPOL_DEFAULT] = "default",
2363    [MPOL_PREFERRED] = "prefer",
2364    [MPOL_BIND] = "bind",
2365    [MPOL_INTERLEAVE] = "interleave",
2366    [MPOL_LOCAL] = "local"
2367};
2368
2369
2370#ifdef CONFIG_TMPFS
2371/**
2372 * mpol_parse_str - parse string to mempolicy
2373 * @str: string containing mempolicy to parse
2374 * @mpol: pointer to struct mempolicy pointer, returned on success.
2375 * @no_context: flag whether to "contextualize" the mempolicy
2376 *
2377 * Format of input:
2378 * <mode>[=<flags>][:<nodelist>]
2379 *
2380 * if @no_context is true, save the input nodemask in w.user_nodemask in
2381 * the returned mempolicy. This will be used to "clone" the mempolicy in
2382 * a specific context [cpuset] at a later time. Used to parse tmpfs mpol
2383 * mount option. Note that if 'static' or 'relative' mode flags were
2384 * specified, the input nodemask will already have been saved. Saving
2385 * it again is redundant, but safe.
2386 *
2387 * On success, returns 0, else 1
2388 */
2389int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
2390{
2391    struct mempolicy *new = NULL;
2392    unsigned short mode;
2393    unsigned short uninitialized_var(mode_flags);
2394    nodemask_t nodes;
2395    char *nodelist = strchr(str, ':');
2396    char *flags = strchr(str, '=');
2397    int err = 1;
2398
2399    if (nodelist) {
2400        /* NUL-terminate mode or flags string */
2401        *nodelist++ = '\0';
2402        if (nodelist_parse(nodelist, nodes))
2403            goto out;
2404        if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY]))
2405            goto out;
2406    } else
2407        nodes_clear(nodes);
2408
2409    if (flags)
2410        *flags++ = '\0'; /* terminate mode string */
2411
2412    for (mode = 0; mode <= MPOL_LOCAL; mode++) {
2413        if (!strcmp(str, policy_modes[mode])) {
2414            break;
2415        }
2416    }
2417    if (mode > MPOL_LOCAL)
2418        goto out;
2419
2420    switch (mode) {
2421    case MPOL_PREFERRED:
2422        /*
2423         * Insist on a nodelist of one node only
2424         */
2425        if (nodelist) {
2426            char *rest = nodelist;
2427            while (isdigit(*rest))
2428                rest++;
2429            if (*rest)
2430                goto out;
2431        }
2432        break;
2433    case MPOL_INTERLEAVE:
2434        /*
2435         * Default to online nodes with memory if no nodelist
2436         */
2437        if (!nodelist)
2438            nodes = node_states[N_HIGH_MEMORY];
2439        break;
2440    case MPOL_LOCAL:
2441        /*
2442         * Don't allow a nodelist; mpol_new() checks flags
2443         */
2444        if (nodelist)
2445            goto out;
2446        mode = MPOL_PREFERRED;
2447        break;
2448    case MPOL_DEFAULT:
2449        /*
2450         * Insist on a empty nodelist
2451         */
2452        if (!nodelist)
2453            err = 0;
2454        goto out;
2455    case MPOL_BIND:
2456        /*
2457         * Insist on a nodelist
2458         */
2459        if (!nodelist)
2460            goto out;
2461    }
2462
2463    mode_flags = 0;
2464    if (flags) {
2465        /*
2466         * Currently, we only support two mutually exclusive
2467         * mode flags.
2468         */
2469        if (!strcmp(flags, "static"))
2470            mode_flags |= MPOL_F_STATIC_NODES;
2471        else if (!strcmp(flags, "relative"))
2472            mode_flags |= MPOL_F_RELATIVE_NODES;
2473        else
2474            goto out;
2475    }
2476
2477    new = mpol_new(mode, mode_flags, &nodes);
2478    if (IS_ERR(new))
2479        goto out;
2480
2481    if (no_context) {
2482        /* save for contextualization */
2483        new->w.user_nodemask = nodes;
2484    } else {
2485        int ret;
2486        NODEMASK_SCRATCH(scratch);
2487        if (scratch) {
2488            task_lock(current);
2489            ret = mpol_set_nodemask(new, &nodes, scratch);
2490            task_unlock(current);
2491        } else
2492            ret = -ENOMEM;
2493        NODEMASK_SCRATCH_FREE(scratch);
2494        if (ret) {
2495            mpol_put(new);
2496            goto out;
2497        }
2498    }
2499    err = 0;
2500
2501out:
2502    /* Restore string for error message */
2503    if (nodelist)
2504        *--nodelist = ':';
2505    if (flags)
2506        *--flags = '=';
2507    if (!err)
2508        *mpol = new;
2509    return err;
2510}
2511#endif /* CONFIG_TMPFS */
2512
2513/**
2514 * mpol_to_str - format a mempolicy structure for printing
2515 * @buffer: to contain formatted mempolicy string
2516 * @maxlen: length of @buffer
2517 * @pol: pointer to mempolicy to be formatted
2518 * @no_context: "context free" mempolicy - use nodemask in w.user_nodemask
2519 *
2520 * Convert a mempolicy into a string.
2521 * Returns the number of characters in buffer (if positive)
2522 * or an error (negative)
2523 */
2524int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context)
2525{
2526    char *p = buffer;
2527    int l;
2528    nodemask_t nodes;
2529    unsigned short mode;
2530    unsigned short flags = pol ? pol->flags : 0;
2531
2532    /*
2533     * Sanity check: room for longest mode, flag and some nodes
2534     */
2535    VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
2536
2537    if (!pol || pol == &default_policy)
2538        mode = MPOL_DEFAULT;
2539    else
2540        mode = pol->mode;
2541
2542    switch (mode) {
2543    case MPOL_DEFAULT:
2544        nodes_clear(nodes);
2545        break;
2546
2547    case MPOL_PREFERRED:
2548        nodes_clear(nodes);
2549        if (flags & MPOL_F_LOCAL)
2550            mode = MPOL_LOCAL; /* pseudo-policy */
2551        else
2552            node_set(pol->v.preferred_node, nodes);
2553        break;
2554
2555    case MPOL_BIND:
2556        /* Fall through */
2557    case MPOL_INTERLEAVE:
2558        if (no_context)
2559            nodes = pol->w.user_nodemask;
2560        else
2561            nodes = pol->v.nodes;
2562        break;
2563
2564    default:
2565        BUG();
2566    }
2567
2568    l = strlen(policy_modes[mode]);
2569    if (buffer + maxlen < p + l + 1)
2570        return -ENOSPC;
2571
2572    strcpy(p, policy_modes[mode]);
2573    p += l;
2574
2575    if (flags & MPOL_MODE_FLAGS) {
2576        if (buffer + maxlen < p + 2)
2577            return -ENOSPC;
2578        *p++ = '=';
2579
2580        /*
2581         * Currently, the only defined flags are mutually exclusive
2582         */
2583        if (flags & MPOL_F_STATIC_NODES)
2584            p += snprintf(p, buffer + maxlen - p, "static");
2585        else if (flags & MPOL_F_RELATIVE_NODES)
2586            p += snprintf(p, buffer + maxlen - p, "relative");
2587    }
2588
2589    if (!nodes_empty(nodes)) {
2590        if (buffer + maxlen < p + 2)
2591            return -ENOSPC;
2592        *p++ = ':';
2593         p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
2594    }
2595    return p - buffer;
2596}
2597

Archive Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9



interactive