Root/mm/madvise.c

1/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
11#include <linux/mempolicy.h>
12#include <linux/page-isolation.h>
13#include <linux/hugetlb.h>
14#include <linux/sched.h>
15#include <linux/ksm.h>
16
17/*
18 * Any behaviour which results in changes to the vma->vm_flags needs to
19 * take mmap_sem for writing. Others, which simply traverse vmas, need
20 * to only take it for reading.
21 */
22static int madvise_need_mmap_write(int behavior)
23{
24    switch (behavior) {
25    case MADV_REMOVE:
26    case MADV_WILLNEED:
27    case MADV_DONTNEED:
28        return 0;
29    default:
30        /* be safe, default to 1. list exceptions explicitly */
31        return 1;
32    }
33}
34
35/*
36 * We can potentially split a vm area into separate
37 * areas, each area with its own behavior.
38 */
39static long madvise_behavior(struct vm_area_struct * vma,
40             struct vm_area_struct **prev,
41             unsigned long start, unsigned long end, int behavior)
42{
43    struct mm_struct * mm = vma->vm_mm;
44    int error = 0;
45    pgoff_t pgoff;
46    unsigned long new_flags = vma->vm_flags;
47
48    switch (behavior) {
49    case MADV_NORMAL:
50        new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
51        break;
52    case MADV_SEQUENTIAL:
53        new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
54        break;
55    case MADV_RANDOM:
56        new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
57        break;
58    case MADV_DONTFORK:
59        new_flags |= VM_DONTCOPY;
60        break;
61    case MADV_DOFORK:
62        if (vma->vm_flags & VM_IO) {
63            error = -EINVAL;
64            goto out;
65        }
66        new_flags &= ~VM_DONTCOPY;
67        break;
68    case MADV_MERGEABLE:
69    case MADV_UNMERGEABLE:
70        error = ksm_madvise(vma, start, end, behavior, &new_flags);
71        if (error)
72            goto out;
73        break;
74    }
75
76    if (new_flags == vma->vm_flags) {
77        *prev = vma;
78        goto out;
79    }
80
81    pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
82    *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
83                vma->vm_file, pgoff, vma_policy(vma));
84    if (*prev) {
85        vma = *prev;
86        goto success;
87    }
88
89    *prev = vma;
90
91    if (start != vma->vm_start) {
92        error = split_vma(mm, vma, start, 1);
93        if (error)
94            goto out;
95    }
96
97    if (end != vma->vm_end) {
98        error = split_vma(mm, vma, end, 0);
99        if (error)
100            goto out;
101    }
102
103success:
104    /*
105     * vm_flags is protected by the mmap_sem held in write mode.
106     */
107    vma->vm_flags = new_flags;
108
109out:
110    if (error == -ENOMEM)
111        error = -EAGAIN;
112    return error;
113}
114
115/*
116 * Schedule all required I/O operations. Do not wait for completion.
117 */
118static long madvise_willneed(struct vm_area_struct * vma,
119                 struct vm_area_struct ** prev,
120                 unsigned long start, unsigned long end)
121{
122    struct file *file = vma->vm_file;
123
124    if (!file)
125        return -EBADF;
126
127    if (file->f_mapping->a_ops->get_xip_mem) {
128        /* no bad return value, but ignore advice */
129        return 0;
130    }
131
132    *prev = vma;
133    start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
134    if (end > vma->vm_end)
135        end = vma->vm_end;
136    end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
137
138    force_page_cache_readahead(file->f_mapping, file, start, end - start);
139    return 0;
140}
141
142/*
143 * Application no longer needs these pages. If the pages are dirty,
144 * it's OK to just throw them away. The app will be more careful about
145 * data it wants to keep. Be sure to free swap resources too. The
146 * zap_page_range call sets things up for shrink_active_list to actually free
147 * these pages later if no one else has touched them in the meantime,
148 * although we could add these pages to a global reuse list for
149 * shrink_active_list to pick up before reclaiming other pages.
150 *
151 * NB: This interface discards data rather than pushes it out to swap,
152 * as some implementations do. This has performance implications for
153 * applications like large transactional databases which want to discard
154 * pages in anonymous maps after committing to backing store the data
155 * that was kept in them. There is no reason to write this data out to
156 * the swap area if the application is discarding it.
157 *
158 * An interface that causes the system to free clean pages and flush
159 * dirty pages is already available as msync(MS_INVALIDATE).
160 */
161static long madvise_dontneed(struct vm_area_struct * vma,
162                 struct vm_area_struct ** prev,
163                 unsigned long start, unsigned long end)
164{
165    *prev = vma;
166    if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
167        return -EINVAL;
168
169    if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
170        struct zap_details details = {
171            .nonlinear_vma = vma,
172            .last_index = ULONG_MAX,
173        };
174        zap_page_range(vma, start, end - start, &details);
175    } else
176        zap_page_range(vma, start, end - start, NULL);
177    return 0;
178}
179
180/*
181 * Application wants to free up the pages and associated backing store.
182 * This is effectively punching a hole into the middle of a file.
183 *
184 * NOTE: Currently, only shmfs/tmpfs is supported for this operation.
185 * Other filesystems return -ENOSYS.
186 */
187static long madvise_remove(struct vm_area_struct *vma,
188                struct vm_area_struct **prev,
189                unsigned long start, unsigned long end)
190{
191    struct address_space *mapping;
192    loff_t offset, endoff;
193    int error;
194
195    *prev = NULL; /* tell sys_madvise we drop mmap_sem */
196
197    if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
198        return -EINVAL;
199
200    if (!vma->vm_file || !vma->vm_file->f_mapping
201        || !vma->vm_file->f_mapping->host) {
202            return -EINVAL;
203    }
204
205    if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
206        return -EACCES;
207
208    mapping = vma->vm_file->f_mapping;
209
210    offset = (loff_t)(start - vma->vm_start)
211            + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
212    endoff = (loff_t)(end - vma->vm_start - 1)
213            + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
214
215    /* vmtruncate_range needs to take i_mutex and i_alloc_sem */
216    up_read(&current->mm->mmap_sem);
217    error = vmtruncate_range(mapping->host, offset, endoff);
218    down_read(&current->mm->mmap_sem);
219    return error;
220}
221
222#ifdef CONFIG_MEMORY_FAILURE
223/*
224 * Error injection support for memory error handling.
225 */
226static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
227{
228    int ret = 0;
229
230    if (!capable(CAP_SYS_ADMIN))
231        return -EPERM;
232    for (; start < end; start += PAGE_SIZE) {
233        struct page *p;
234        int ret = get_user_pages_fast(start, 1, 0, &p);
235        if (ret != 1)
236            return ret;
237        if (bhv == MADV_SOFT_OFFLINE) {
238            printk(KERN_INFO "Soft offlining page %lx at %lx\n",
239                page_to_pfn(p), start);
240            ret = soft_offline_page(p, MF_COUNT_INCREASED);
241            if (ret)
242                break;
243            continue;
244        }
245        printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n",
246               page_to_pfn(p), start);
247        /* Ignore return value for now */
248        __memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
249    }
250    return ret;
251}
252#endif
253
254static long
255madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
256        unsigned long start, unsigned long end, int behavior)
257{
258    switch (behavior) {
259    case MADV_REMOVE:
260        return madvise_remove(vma, prev, start, end);
261    case MADV_WILLNEED:
262        return madvise_willneed(vma, prev, start, end);
263    case MADV_DONTNEED:
264        return madvise_dontneed(vma, prev, start, end);
265    default:
266        return madvise_behavior(vma, prev, start, end, behavior);
267    }
268}
269
270static int
271madvise_behavior_valid(int behavior)
272{
273    switch (behavior) {
274    case MADV_DOFORK:
275    case MADV_DONTFORK:
276    case MADV_NORMAL:
277    case MADV_SEQUENTIAL:
278    case MADV_RANDOM:
279    case MADV_REMOVE:
280    case MADV_WILLNEED:
281    case MADV_DONTNEED:
282#ifdef CONFIG_KSM
283    case MADV_MERGEABLE:
284    case MADV_UNMERGEABLE:
285#endif
286        return 1;
287
288    default:
289        return 0;
290    }
291}
292
293/*
294 * The madvise(2) system call.
295 *
296 * Applications can use madvise() to advise the kernel how it should
297 * handle paging I/O in this VM area. The idea is to help the kernel
298 * use appropriate read-ahead and caching techniques. The information
299 * provided is advisory only, and can be safely disregarded by the
300 * kernel without affecting the correct operation of the application.
301 *
302 * behavior values:
303 * MADV_NORMAL - the default behavior is to read clusters. This
304 * results in some read-ahead and read-behind.
305 * MADV_RANDOM - the system should read the minimum amount of data
306 * on any access, since it is unlikely that the appli-
307 * cation will need more than what it asks for.
308 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
309 * once, so they can be aggressively read ahead, and
310 * can be freed soon after they are accessed.
311 * MADV_WILLNEED - the application is notifying the system to read
312 * some pages ahead.
313 * MADV_DONTNEED - the application is finished with the given range,
314 * so the kernel can free resources associated with it.
315 * MADV_REMOVE - the application wants to free up the given range of
316 * pages and associated backing store.
317 * MADV_DONTFORK - omit this area from child's address space when forking:
318 * typically, to avoid COWing pages pinned by get_user_pages().
319 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
320 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
321 * this area with pages of identical content from other such areas.
322 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
323 *
324 * return values:
325 * zero - success
326 * -EINVAL - start + len < 0, start is not page-aligned,
327 * "behavior" is not a valid value, or application
328 * is attempting to release locked or shared pages.
329 * -ENOMEM - addresses in the specified range are not currently
330 * mapped, or are outside the AS of the process.
331 * -EIO - an I/O error occurred while paging in data.
332 * -EBADF - map exists, but area maps something that isn't a file.
333 * -EAGAIN - a kernel resource was temporarily unavailable.
334 */
335SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
336{
337    unsigned long end, tmp;
338    struct vm_area_struct * vma, *prev;
339    int unmapped_error = 0;
340    int error = -EINVAL;
341    int write;
342    size_t len;
343
344#ifdef CONFIG_MEMORY_FAILURE
345    if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
346        return madvise_hwpoison(behavior, start, start+len_in);
347#endif
348    if (!madvise_behavior_valid(behavior))
349        return error;
350
351    write = madvise_need_mmap_write(behavior);
352    if (write)
353        down_write(&current->mm->mmap_sem);
354    else
355        down_read(&current->mm->mmap_sem);
356
357    if (start & ~PAGE_MASK)
358        goto out;
359    len = (len_in + ~PAGE_MASK) & PAGE_MASK;
360
361    /* Check to see whether len was rounded up from small -ve to zero */
362    if (len_in && !len)
363        goto out;
364
365    end = start + len;
366    if (end < start)
367        goto out;
368
369    error = 0;
370    if (end == start)
371        goto out;
372
373    /*
374     * If the interval [start,end) covers some unmapped address
375     * ranges, just ignore them, but return -ENOMEM at the end.
376     * - different from the way of handling in mlock etc.
377     */
378    vma = find_vma_prev(current->mm, start, &prev);
379    if (vma && start > vma->vm_start)
380        prev = vma;
381
382    for (;;) {
383        /* Still start < end. */
384        error = -ENOMEM;
385        if (!vma)
386            goto out;
387
388        /* Here start < (end|vma->vm_end). */
389        if (start < vma->vm_start) {
390            unmapped_error = -ENOMEM;
391            start = vma->vm_start;
392            if (start >= end)
393                goto out;
394        }
395
396        /* Here vma->vm_start <= start < (end|vma->vm_end) */
397        tmp = vma->vm_end;
398        if (end < tmp)
399            tmp = end;
400
401        /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
402        error = madvise_vma(vma, &prev, start, tmp, behavior);
403        if (error)
404            goto out;
405        start = tmp;
406        if (prev && start < prev->vm_end)
407            start = prev->vm_end;
408        error = unmapped_error;
409        if (start >= end)
410            goto out;
411        if (prev)
412            vma = prev->vm_next;
413        else /* madvise_remove dropped mmap_sem */
414            vma = find_vma(current->mm, start);
415    }
416out:
417    if (write)
418        up_write(&current->mm->mmap_sem);
419    else
420        up_read(&current->mm->mmap_sem);
421
422    return error;
423}
424

Archive Download this file



interactive