Root/mm/shmem.c

1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.
9 * Copyright (C) 2002-2005 Hugh Dickins.
10 * Copyright (C) 2002-2005 VERITAS Software Corporation.
11 * Copyright (C) 2004 Andi Kleen, SuSE Labs
12 *
13 * Extended attribute support for tmpfs:
14 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
16 *
17 * tiny-shmem:
18 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
19 *
20 * This file is released under the GPL.
21 */
22
23#include <linux/fs.h>
24#include <linux/init.h>
25#include <linux/vfs.h>
26#include <linux/mount.h>
27#include <linux/pagemap.h>
28#include <linux/file.h>
29#include <linux/mm.h>
30#include <linux/module.h>
31#include <linux/swap.h>
32
33static struct vfsmount *shm_mnt;
34
35#ifdef CONFIG_SHMEM
36/*
37 * This virtual memory filesystem is heavily based on the ramfs. It
38 * extends ramfs by the ability to use swap and honor resource limits
39 * which makes it a completely usable filesystem.
40 */
41
42#include <linux/xattr.h>
43#include <linux/exportfs.h>
44#include <linux/posix_acl.h>
45#include <linux/generic_acl.h>
46#include <linux/mman.h>
47#include <linux/string.h>
48#include <linux/slab.h>
49#include <linux/backing-dev.h>
50#include <linux/shmem_fs.h>
51#include <linux/writeback.h>
52#include <linux/blkdev.h>
53#include <linux/security.h>
54#include <linux/swapops.h>
55#include <linux/mempolicy.h>
56#include <linux/namei.h>
57#include <linux/ctype.h>
58#include <linux/migrate.h>
59#include <linux/highmem.h>
60#include <linux/seq_file.h>
61#include <linux/magic.h>
62
63#include <asm/uaccess.h>
64#include <asm/div64.h>
65#include <asm/pgtable.h>
66
67/*
68 * The maximum size of a shmem/tmpfs file is limited by the maximum size of
69 * its triple-indirect swap vector - see illustration at shmem_swp_entry().
70 *
71 * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel,
72 * but one eighth of that on a 64-bit kernel. With 8kB page size, maximum
73 * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel,
74 * MAX_LFS_FILESIZE being then more restrictive than swap vector layout.
75 *
76 * We use / and * instead of shifts in the definitions below, so that the swap
77 * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE.
78 */
79#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
80#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
81
82#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
83#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT)
84
85#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE)
86#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT))
87
88#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
89#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
90
91/* info->flags needs VM_flags to handle pagein/truncate races efficiently */
92#define SHMEM_PAGEIN VM_READ
93#define SHMEM_TRUNCATE VM_WRITE
94
95/* Definition to limit shmem_truncate's steps between cond_rescheds */
96#define LATENCY_LIMIT 64
97
98/* Pretend that each entry is of this size in directory's i_size */
99#define BOGO_DIRENT_SIZE 20
100
101/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
102enum sgp_type {
103    SGP_READ, /* don't exceed i_size, don't allocate page */
104    SGP_CACHE, /* don't exceed i_size, may allocate page */
105    SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
106    SGP_WRITE, /* may exceed i_size, may allocate page */
107};
108
109#ifdef CONFIG_TMPFS
110static unsigned long shmem_default_max_blocks(void)
111{
112    return totalram_pages / 2;
113}
114
115static unsigned long shmem_default_max_inodes(void)
116{
117    return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
118}
119#endif
120
121static int shmem_getpage(struct inode *inode, unsigned long idx,
122             struct page **pagep, enum sgp_type sgp, int *type);
123
124static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
125{
126    /*
127     * The above definition of ENTRIES_PER_PAGE, and the use of
128     * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
129     * might be reconsidered if it ever diverges from PAGE_SIZE.
130     *
131     * Mobility flags are masked out as swap vectors cannot move
132     */
133    return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
134                PAGE_CACHE_SHIFT-PAGE_SHIFT);
135}
136
137static inline void shmem_dir_free(struct page *page)
138{
139    __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
140}
141
142static struct page **shmem_dir_map(struct page *page)
143{
144    return (struct page **)kmap_atomic(page, KM_USER0);
145}
146
147static inline void shmem_dir_unmap(struct page **dir)
148{
149    kunmap_atomic(dir, KM_USER0);
150}
151
152static swp_entry_t *shmem_swp_map(struct page *page)
153{
154    return (swp_entry_t *)kmap_atomic(page, KM_USER1);
155}
156
157static inline void shmem_swp_balance_unmap(void)
158{
159    /*
160     * When passing a pointer to an i_direct entry, to code which
161     * also handles indirect entries and so will shmem_swp_unmap,
162     * we must arrange for the preempt count to remain in balance.
163     * What kmap_atomic of a lowmem page does depends on config
164     * and architecture, so pretend to kmap_atomic some lowmem page.
165     */
166    (void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
167}
168
169static inline void shmem_swp_unmap(swp_entry_t *entry)
170{
171    kunmap_atomic(entry, KM_USER1);
172}
173
174static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
175{
176    return sb->s_fs_info;
177}
178
179/*
180 * shmem_file_setup pre-accounts the whole fixed size of a VM object,
181 * for shared memory and for shared anonymous (/dev/zero) mappings
182 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
183 * consistent with the pre-accounting of private mappings ...
184 */
185static inline int shmem_acct_size(unsigned long flags, loff_t size)
186{
187    return (flags & VM_NORESERVE) ?
188        0 : security_vm_enough_memory_kern(VM_ACCT(size));
189}
190
191static inline void shmem_unacct_size(unsigned long flags, loff_t size)
192{
193    if (!(flags & VM_NORESERVE))
194        vm_unacct_memory(VM_ACCT(size));
195}
196
197/*
198 * ... whereas tmpfs objects are accounted incrementally as
199 * pages are allocated, in order to allow huge sparse files.
200 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
201 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
202 */
203static inline int shmem_acct_block(unsigned long flags)
204{
205    return (flags & VM_NORESERVE) ?
206        security_vm_enough_memory_kern(VM_ACCT(PAGE_CACHE_SIZE)) : 0;
207}
208
209static inline void shmem_unacct_blocks(unsigned long flags, long pages)
210{
211    if (flags & VM_NORESERVE)
212        vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
213}
214
215static const struct super_operations shmem_ops;
216static const struct address_space_operations shmem_aops;
217static const struct file_operations shmem_file_operations;
218static const struct inode_operations shmem_inode_operations;
219static const struct inode_operations shmem_dir_inode_operations;
220static const struct inode_operations shmem_special_inode_operations;
221static const struct vm_operations_struct shmem_vm_ops;
222
223static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
224    .ra_pages = 0, /* No readahead */
225    .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED,
226    .unplug_io_fn = default_unplug_io_fn,
227};
228
229static LIST_HEAD(shmem_swaplist);
230static DEFINE_MUTEX(shmem_swaplist_mutex);
231
232static void shmem_free_blocks(struct inode *inode, long pages)
233{
234    struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
235    if (sbinfo->max_blocks) {
236        spin_lock(&sbinfo->stat_lock);
237        sbinfo->free_blocks += pages;
238        inode->i_blocks -= pages*BLOCKS_PER_PAGE;
239        spin_unlock(&sbinfo->stat_lock);
240    }
241}
242
243static int shmem_reserve_inode(struct super_block *sb)
244{
245    struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
246    if (sbinfo->max_inodes) {
247        spin_lock(&sbinfo->stat_lock);
248        if (!sbinfo->free_inodes) {
249            spin_unlock(&sbinfo->stat_lock);
250            return -ENOSPC;
251        }
252        sbinfo->free_inodes--;
253        spin_unlock(&sbinfo->stat_lock);
254    }
255    return 0;
256}
257
258static void shmem_free_inode(struct super_block *sb)
259{
260    struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
261    if (sbinfo->max_inodes) {
262        spin_lock(&sbinfo->stat_lock);
263        sbinfo->free_inodes++;
264        spin_unlock(&sbinfo->stat_lock);
265    }
266}
267
268/**
269 * shmem_recalc_inode - recalculate the size of an inode
270 * @inode: inode to recalc
271 *
272 * We have to calculate the free blocks since the mm can drop
273 * undirtied hole pages behind our back.
274 *
275 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped
276 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
277 *
278 * It has to be called with the spinlock held.
279 */
280static void shmem_recalc_inode(struct inode *inode)
281{
282    struct shmem_inode_info *info = SHMEM_I(inode);
283    long freed;
284
285    freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
286    if (freed > 0) {
287        info->alloced -= freed;
288        shmem_unacct_blocks(info->flags, freed);
289        shmem_free_blocks(inode, freed);
290    }
291}
292
293/**
294 * shmem_swp_entry - find the swap vector position in the info structure
295 * @info: info structure for the inode
296 * @index: index of the page to find
297 * @page: optional page to add to the structure. Has to be preset to
298 * all zeros
299 *
300 * If there is no space allocated yet it will return NULL when
301 * page is NULL, else it will use the page for the needed block,
302 * setting it to NULL on return to indicate that it has been used.
303 *
304 * The swap vector is organized the following way:
305 *
306 * There are SHMEM_NR_DIRECT entries directly stored in the
307 * shmem_inode_info structure. So small files do not need an addional
308 * allocation.
309 *
310 * For pages with index > SHMEM_NR_DIRECT there is the pointer
311 * i_indirect which points to a page which holds in the first half
312 * doubly indirect blocks, in the second half triple indirect blocks:
313 *
314 * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
315 * following layout (for SHMEM_NR_DIRECT == 16):
316 *
317 * i_indirect -> dir --> 16-19
318 * | +-> 20-23
319 * |
320 * +-->dir2 --> 24-27
321 * | +-> 28-31
322 * | +-> 32-35
323 * | +-> 36-39
324 * |
325 * +-->dir3 --> 40-43
326 * +-> 44-47
327 * +-> 48-51
328 * +-> 52-55
329 */
330static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
331{
332    unsigned long offset;
333    struct page **dir;
334    struct page *subdir;
335
336    if (index < SHMEM_NR_DIRECT) {
337        shmem_swp_balance_unmap();
338        return info->i_direct+index;
339    }
340    if (!info->i_indirect) {
341        if (page) {
342            info->i_indirect = *page;
343            *page = NULL;
344        }
345        return NULL; /* need another page */
346    }
347
348    index -= SHMEM_NR_DIRECT;
349    offset = index % ENTRIES_PER_PAGE;
350    index /= ENTRIES_PER_PAGE;
351    dir = shmem_dir_map(info->i_indirect);
352
353    if (index >= ENTRIES_PER_PAGE/2) {
354        index -= ENTRIES_PER_PAGE/2;
355        dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
356        index %= ENTRIES_PER_PAGE;
357        subdir = *dir;
358        if (!subdir) {
359            if (page) {
360                *dir = *page;
361                *page = NULL;
362            }
363            shmem_dir_unmap(dir);
364            return NULL; /* need another page */
365        }
366        shmem_dir_unmap(dir);
367        dir = shmem_dir_map(subdir);
368    }
369
370    dir += index;
371    subdir = *dir;
372    if (!subdir) {
373        if (!page || !(subdir = *page)) {
374            shmem_dir_unmap(dir);
375            return NULL; /* need a page */
376        }
377        *dir = subdir;
378        *page = NULL;
379    }
380    shmem_dir_unmap(dir);
381    return shmem_swp_map(subdir) + offset;
382}
383
384static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
385{
386    long incdec = value? 1: -1;
387
388    entry->val = value;
389    info->swapped += incdec;
390    if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
391        struct page *page = kmap_atomic_to_page(entry);
392        set_page_private(page, page_private(page) + incdec);
393    }
394}
395
396/**
397 * shmem_swp_alloc - get the position of the swap entry for the page.
398 * @info: info structure for the inode
399 * @index: index of the page to find
400 * @sgp: check and recheck i_size? skip allocation?
401 *
402 * If the entry does not exist, allocate it.
403 */
404static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
405{
406    struct inode *inode = &info->vfs_inode;
407    struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
408    struct page *page = NULL;
409    swp_entry_t *entry;
410
411    if (sgp != SGP_WRITE &&
412        ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
413        return ERR_PTR(-EINVAL);
414
415    while (!(entry = shmem_swp_entry(info, index, &page))) {
416        if (sgp == SGP_READ)
417            return shmem_swp_map(ZERO_PAGE(0));
418        /*
419         * Test free_blocks against 1 not 0, since we have 1 data
420         * page (and perhaps indirect index pages) yet to allocate:
421         * a waste to allocate index if we cannot allocate data.
422         */
423        if (sbinfo->max_blocks) {
424            spin_lock(&sbinfo->stat_lock);
425            if (sbinfo->free_blocks <= 1) {
426                spin_unlock(&sbinfo->stat_lock);
427                return ERR_PTR(-ENOSPC);
428            }
429            sbinfo->free_blocks--;
430            inode->i_blocks += BLOCKS_PER_PAGE;
431            spin_unlock(&sbinfo->stat_lock);
432        }
433
434        spin_unlock(&info->lock);
435        page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
436        spin_lock(&info->lock);
437
438        if (!page) {
439            shmem_free_blocks(inode, 1);
440            return ERR_PTR(-ENOMEM);
441        }
442        if (sgp != SGP_WRITE &&
443            ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
444            entry = ERR_PTR(-EINVAL);
445            break;
446        }
447        if (info->next_index <= index)
448            info->next_index = index + 1;
449    }
450    if (page) {
451        /* another task gave its page, or truncated the file */
452        shmem_free_blocks(inode, 1);
453        shmem_dir_free(page);
454    }
455    if (info->next_index <= index && !IS_ERR(entry))
456        info->next_index = index + 1;
457    return entry;
458}
459
460/**
461 * shmem_free_swp - free some swap entries in a directory
462 * @dir: pointer to the directory
463 * @edir: pointer after last entry of the directory
464 * @punch_lock: pointer to spinlock when needed for the holepunch case
465 */
466static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
467                        spinlock_t *punch_lock)
468{
469    spinlock_t *punch_unlock = NULL;
470    swp_entry_t *ptr;
471    int freed = 0;
472
473    for (ptr = dir; ptr < edir; ptr++) {
474        if (ptr->val) {
475            if (unlikely(punch_lock)) {
476                punch_unlock = punch_lock;
477                punch_lock = NULL;
478                spin_lock(punch_unlock);
479                if (!ptr->val)
480                    continue;
481            }
482            free_swap_and_cache(*ptr);
483            *ptr = (swp_entry_t){0};
484            freed++;
485        }
486    }
487    if (punch_unlock)
488        spin_unlock(punch_unlock);
489    return freed;
490}
491
492static int shmem_map_and_free_swp(struct page *subdir, int offset,
493        int limit, struct page ***dir, spinlock_t *punch_lock)
494{
495    swp_entry_t *ptr;
496    int freed = 0;
497
498    ptr = shmem_swp_map(subdir);
499    for (; offset < limit; offset += LATENCY_LIMIT) {
500        int size = limit - offset;
501        if (size > LATENCY_LIMIT)
502            size = LATENCY_LIMIT;
503        freed += shmem_free_swp(ptr+offset, ptr+offset+size,
504                            punch_lock);
505        if (need_resched()) {
506            shmem_swp_unmap(ptr);
507            if (*dir) {
508                shmem_dir_unmap(*dir);
509                *dir = NULL;
510            }
511            cond_resched();
512            ptr = shmem_swp_map(subdir);
513        }
514    }
515    shmem_swp_unmap(ptr);
516    return freed;
517}
518
519static void shmem_free_pages(struct list_head *next)
520{
521    struct page *page;
522    int freed = 0;
523
524    do {
525        page = container_of(next, struct page, lru);
526        next = next->next;
527        shmem_dir_free(page);
528        freed++;
529        if (freed >= LATENCY_LIMIT) {
530            cond_resched();
531            freed = 0;
532        }
533    } while (next);
534}
535
536static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
537{
538    struct shmem_inode_info *info = SHMEM_I(inode);
539    unsigned long idx;
540    unsigned long size;
541    unsigned long limit;
542    unsigned long stage;
543    unsigned long diroff;
544    struct page **dir;
545    struct page *topdir;
546    struct page *middir;
547    struct page *subdir;
548    swp_entry_t *ptr;
549    LIST_HEAD(pages_to_free);
550    long nr_pages_to_free = 0;
551    long nr_swaps_freed = 0;
552    int offset;
553    int freed;
554    int punch_hole;
555    spinlock_t *needs_lock;
556    spinlock_t *punch_lock;
557    unsigned long upper_limit;
558
559    inode->i_ctime = inode->i_mtime = CURRENT_TIME;
560    idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
561    if (idx >= info->next_index)
562        return;
563
564    spin_lock(&info->lock);
565    info->flags |= SHMEM_TRUNCATE;
566    if (likely(end == (loff_t) -1)) {
567        limit = info->next_index;
568        upper_limit = SHMEM_MAX_INDEX;
569        info->next_index = idx;
570        needs_lock = NULL;
571        punch_hole = 0;
572    } else {
573        if (end + 1 >= inode->i_size) { /* we may free a little more */
574            limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
575                            PAGE_CACHE_SHIFT;
576            upper_limit = SHMEM_MAX_INDEX;
577        } else {
578            limit = (end + 1) >> PAGE_CACHE_SHIFT;
579            upper_limit = limit;
580        }
581        needs_lock = &info->lock;
582        punch_hole = 1;
583    }
584
585    topdir = info->i_indirect;
586    if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
587        info->i_indirect = NULL;
588        nr_pages_to_free++;
589        list_add(&topdir->lru, &pages_to_free);
590    }
591    spin_unlock(&info->lock);
592
593    if (info->swapped && idx < SHMEM_NR_DIRECT) {
594        ptr = info->i_direct;
595        size = limit;
596        if (size > SHMEM_NR_DIRECT)
597            size = SHMEM_NR_DIRECT;
598        nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
599    }
600
601    /*
602     * If there are no indirect blocks or we are punching a hole
603     * below indirect blocks, nothing to be done.
604     */
605    if (!topdir || limit <= SHMEM_NR_DIRECT)
606        goto done2;
607
608    /*
609     * The truncation case has already dropped info->lock, and we're safe
610     * because i_size and next_index have already been lowered, preventing
611     * access beyond. But in the punch_hole case, we still need to take
612     * the lock when updating the swap directory, because there might be
613     * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
614     * shmem_writepage. However, whenever we find we can remove a whole
615     * directory page (not at the misaligned start or end of the range),
616     * we first NULLify its pointer in the level above, and then have no
617     * need to take the lock when updating its contents: needs_lock and
618     * punch_lock (either pointing to info->lock or NULL) manage this.
619     */
620
621    upper_limit -= SHMEM_NR_DIRECT;
622    limit -= SHMEM_NR_DIRECT;
623    idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
624    offset = idx % ENTRIES_PER_PAGE;
625    idx -= offset;
626
627    dir = shmem_dir_map(topdir);
628    stage = ENTRIES_PER_PAGEPAGE/2;
629    if (idx < ENTRIES_PER_PAGEPAGE/2) {
630        middir = topdir;
631        diroff = idx/ENTRIES_PER_PAGE;
632    } else {
633        dir += ENTRIES_PER_PAGE/2;
634        dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
635        while (stage <= idx)
636            stage += ENTRIES_PER_PAGEPAGE;
637        middir = *dir;
638        if (*dir) {
639            diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
640                ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
641            if (!diroff && !offset && upper_limit >= stage) {
642                if (needs_lock) {
643                    spin_lock(needs_lock);
644                    *dir = NULL;
645                    spin_unlock(needs_lock);
646                    needs_lock = NULL;
647                } else
648                    *dir = NULL;
649                nr_pages_to_free++;
650                list_add(&middir->lru, &pages_to_free);
651            }
652            shmem_dir_unmap(dir);
653            dir = shmem_dir_map(middir);
654        } else {
655            diroff = 0;
656            offset = 0;
657            idx = stage;
658        }
659    }
660
661    for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
662        if (unlikely(idx == stage)) {
663            shmem_dir_unmap(dir);
664            dir = shmem_dir_map(topdir) +
665                ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
666            while (!*dir) {
667                dir++;
668                idx += ENTRIES_PER_PAGEPAGE;
669                if (idx >= limit)
670                    goto done1;
671            }
672            stage = idx + ENTRIES_PER_PAGEPAGE;
673            middir = *dir;
674            if (punch_hole)
675                needs_lock = &info->lock;
676            if (upper_limit >= stage) {
677                if (needs_lock) {
678                    spin_lock(needs_lock);
679                    *dir = NULL;
680                    spin_unlock(needs_lock);
681                    needs_lock = NULL;
682                } else
683                    *dir = NULL;
684                nr_pages_to_free++;
685                list_add(&middir->lru, &pages_to_free);
686            }
687            shmem_dir_unmap(dir);
688            cond_resched();
689            dir = shmem_dir_map(middir);
690            diroff = 0;
691        }
692        punch_lock = needs_lock;
693        subdir = dir[diroff];
694        if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
695            if (needs_lock) {
696                spin_lock(needs_lock);
697                dir[diroff] = NULL;
698                spin_unlock(needs_lock);
699                punch_lock = NULL;
700            } else
701                dir[diroff] = NULL;
702            nr_pages_to_free++;
703            list_add(&subdir->lru, &pages_to_free);
704        }
705        if (subdir && page_private(subdir) /* has swap entries */) {
706            size = limit - idx;
707            if (size > ENTRIES_PER_PAGE)
708                size = ENTRIES_PER_PAGE;
709            freed = shmem_map_and_free_swp(subdir,
710                    offset, size, &dir, punch_lock);
711            if (!dir)
712                dir = shmem_dir_map(middir);
713            nr_swaps_freed += freed;
714            if (offset || punch_lock) {
715                spin_lock(&info->lock);
716                set_page_private(subdir,
717                    page_private(subdir) - freed);
718                spin_unlock(&info->lock);
719            } else
720                BUG_ON(page_private(subdir) != freed);
721        }
722        offset = 0;
723    }
724done1:
725    shmem_dir_unmap(dir);
726done2:
727    if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
728        /*
729         * Call truncate_inode_pages again: racing shmem_unuse_inode
730         * may have swizzled a page in from swap since
731         * truncate_pagecache or generic_delete_inode did it, before we
732         * lowered next_index. Also, though shmem_getpage checks
733         * i_size before adding to cache, no recheck after: so fix the
734         * narrow window there too.
735         *
736         * Recalling truncate_inode_pages_range and unmap_mapping_range
737         * every time for punch_hole (which never got a chance to clear
738         * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
739         * yet hardly ever necessary: try to optimize them out later.
740         */
741        truncate_inode_pages_range(inode->i_mapping, start, end);
742        if (punch_hole)
743            unmap_mapping_range(inode->i_mapping, start,
744                            end - start, 1);
745    }
746
747    spin_lock(&info->lock);
748    info->flags &= ~SHMEM_TRUNCATE;
749    info->swapped -= nr_swaps_freed;
750    if (nr_pages_to_free)
751        shmem_free_blocks(inode, nr_pages_to_free);
752    shmem_recalc_inode(inode);
753    spin_unlock(&info->lock);
754
755    /*
756     * Empty swap vector directory pages to be freed?
757     */
758    if (!list_empty(&pages_to_free)) {
759        pages_to_free.prev->next = NULL;
760        shmem_free_pages(pages_to_free.next);
761    }
762}
763
764static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
765{
766    struct inode *inode = dentry->d_inode;
767    loff_t newsize = attr->ia_size;
768    int error;
769
770    if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)
771                    && newsize != inode->i_size) {
772        struct page *page = NULL;
773
774        if (newsize < inode->i_size) {
775            /*
776             * If truncating down to a partial page, then
777             * if that page is already allocated, hold it
778             * in memory until the truncation is over, so
779             * truncate_partial_page cannnot miss it were
780             * it assigned to swap.
781             */
782            if (newsize & (PAGE_CACHE_SIZE-1)) {
783                (void) shmem_getpage(inode,
784                    newsize >> PAGE_CACHE_SHIFT,
785                        &page, SGP_READ, NULL);
786                if (page)
787                    unlock_page(page);
788            }
789            /*
790             * Reset SHMEM_PAGEIN flag so that shmem_truncate can
791             * detect if any pages might have been added to cache
792             * after truncate_inode_pages. But we needn't bother
793             * if it's being fully truncated to zero-length: the
794             * nrpages check is efficient enough in that case.
795             */
796            if (newsize) {
797                struct shmem_inode_info *info = SHMEM_I(inode);
798                spin_lock(&info->lock);
799                info->flags &= ~SHMEM_PAGEIN;
800                spin_unlock(&info->lock);
801            }
802        }
803
804        error = simple_setsize(inode, newsize);
805        if (page)
806            page_cache_release(page);
807        if (error)
808            return error;
809        shmem_truncate_range(inode, newsize, (loff_t)-1);
810    }
811
812    error = inode_change_ok(inode, attr);
813    if (!error)
814        generic_setattr(inode, attr);
815#ifdef CONFIG_TMPFS_POSIX_ACL
816    if (!error && (attr->ia_valid & ATTR_MODE))
817        error = generic_acl_chmod(inode);
818#endif
819    return error;
820}
821
822static void shmem_delete_inode(struct inode *inode)
823{
824    struct shmem_inode_info *info = SHMEM_I(inode);
825
826    if (inode->i_mapping->a_ops == &shmem_aops) {
827        truncate_inode_pages(inode->i_mapping, 0);
828        shmem_unacct_size(info->flags, inode->i_size);
829        inode->i_size = 0;
830        shmem_truncate_range(inode, 0, (loff_t)-1);
831        if (!list_empty(&info->swaplist)) {
832            mutex_lock(&shmem_swaplist_mutex);
833            list_del_init(&info->swaplist);
834            mutex_unlock(&shmem_swaplist_mutex);
835        }
836    }
837    BUG_ON(inode->i_blocks);
838    shmem_free_inode(inode->i_sb);
839    clear_inode(inode);
840}
841
842static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
843{
844    swp_entry_t *ptr;
845
846    for (ptr = dir; ptr < edir; ptr++) {
847        if (ptr->val == entry.val)
848            return ptr - dir;
849    }
850    return -1;
851}
852
853static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
854{
855    struct inode *inode;
856    unsigned long idx;
857    unsigned long size;
858    unsigned long limit;
859    unsigned long stage;
860    struct page **dir;
861    struct page *subdir;
862    swp_entry_t *ptr;
863    int offset;
864    int error;
865
866    idx = 0;
867    ptr = info->i_direct;
868    spin_lock(&info->lock);
869    if (!info->swapped) {
870        list_del_init(&info->swaplist);
871        goto lost2;
872    }
873    limit = info->next_index;
874    size = limit;
875    if (size > SHMEM_NR_DIRECT)
876        size = SHMEM_NR_DIRECT;
877    offset = shmem_find_swp(entry, ptr, ptr+size);
878    if (offset >= 0)
879        goto found;
880    if (!info->i_indirect)
881        goto lost2;
882
883    dir = shmem_dir_map(info->i_indirect);
884    stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
885
886    for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
887        if (unlikely(idx == stage)) {
888            shmem_dir_unmap(dir-1);
889            if (cond_resched_lock(&info->lock)) {
890                /* check it has not been truncated */
891                if (limit > info->next_index) {
892                    limit = info->next_index;
893                    if (idx >= limit)
894                        goto lost2;
895                }
896            }
897            dir = shmem_dir_map(info->i_indirect) +
898                ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
899            while (!*dir) {
900                dir++;
901                idx += ENTRIES_PER_PAGEPAGE;
902                if (idx >= limit)
903                    goto lost1;
904            }
905            stage = idx + ENTRIES_PER_PAGEPAGE;
906            subdir = *dir;
907            shmem_dir_unmap(dir);
908            dir = shmem_dir_map(subdir);
909        }
910        subdir = *dir;
911        if (subdir && page_private(subdir)) {
912            ptr = shmem_swp_map(subdir);
913            size = limit - idx;
914            if (size > ENTRIES_PER_PAGE)
915                size = ENTRIES_PER_PAGE;
916            offset = shmem_find_swp(entry, ptr, ptr+size);
917            shmem_swp_unmap(ptr);
918            if (offset >= 0) {
919                shmem_dir_unmap(dir);
920                goto found;
921            }
922        }
923    }
924lost1:
925    shmem_dir_unmap(dir-1);
926lost2:
927    spin_unlock(&info->lock);
928    return 0;
929found:
930    idx += offset;
931    inode = igrab(&info->vfs_inode);
932    spin_unlock(&info->lock);
933
934    /*
935     * Move _head_ to start search for next from here.
936     * But be careful: shmem_delete_inode checks list_empty without taking
937     * mutex, and there's an instant in list_move_tail when info->swaplist
938     * would appear empty, if it were the only one on shmem_swaplist. We
939     * could avoid doing it if inode NULL; or use this minor optimization.
940     */
941    if (shmem_swaplist.next != &info->swaplist)
942        list_move_tail(&shmem_swaplist, &info->swaplist);
943    mutex_unlock(&shmem_swaplist_mutex);
944
945    error = 1;
946    if (!inode)
947        goto out;
948    /*
949     * Charge page using GFP_KERNEL while we can wait.
950     * Charged back to the user(not to caller) when swap account is used.
951     * add_to_page_cache() will be called with GFP_NOWAIT.
952     */
953    error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
954    if (error)
955        goto out;
956    error = radix_tree_preload(GFP_KERNEL);
957    if (error) {
958        mem_cgroup_uncharge_cache_page(page);
959        goto out;
960    }
961    error = 1;
962
963    spin_lock(&info->lock);
964    ptr = shmem_swp_entry(info, idx, NULL);
965    if (ptr && ptr->val == entry.val) {
966        error = add_to_page_cache_locked(page, inode->i_mapping,
967                        idx, GFP_NOWAIT);
968        /* does mem_cgroup_uncharge_cache_page on error */
969    } else /* we must compensate for our precharge above */
970        mem_cgroup_uncharge_cache_page(page);
971
972    if (error == -EEXIST) {
973        struct page *filepage = find_get_page(inode->i_mapping, idx);
974        error = 1;
975        if (filepage) {
976            /*
977             * There might be a more uptodate page coming down
978             * from a stacked writepage: forget our swappage if so.
979             */
980            if (PageUptodate(filepage))
981                error = 0;
982            page_cache_release(filepage);
983        }
984    }
985    if (!error) {
986        delete_from_swap_cache(page);
987        set_page_dirty(page);
988        info->flags |= SHMEM_PAGEIN;
989        shmem_swp_set(info, ptr, 0);
990        swap_free(entry);
991        error = 1; /* not an error, but entry was found */
992    }
993    if (ptr)
994        shmem_swp_unmap(ptr);
995    spin_unlock(&info->lock);
996    radix_tree_preload_end();
997out:
998    unlock_page(page);
999    page_cache_release(page);
1000    iput(inode); /* allows for NULL */
1001    return error;
1002}
1003
1004/*
1005 * shmem_unuse() search for an eventually swapped out shmem page.
1006 */
1007int shmem_unuse(swp_entry_t entry, struct page *page)
1008{
1009    struct list_head *p, *next;
1010    struct shmem_inode_info *info;
1011    int found = 0;
1012
1013    mutex_lock(&shmem_swaplist_mutex);
1014    list_for_each_safe(p, next, &shmem_swaplist) {
1015        info = list_entry(p, struct shmem_inode_info, swaplist);
1016        found = shmem_unuse_inode(info, entry, page);
1017        cond_resched();
1018        if (found)
1019            goto out;
1020    }
1021    mutex_unlock(&shmem_swaplist_mutex);
1022    /*
1023     * Can some race bring us here? We've been holding page lock,
1024     * so I think not; but would rather try again later than BUG()
1025     */
1026    unlock_page(page);
1027    page_cache_release(page);
1028out:
1029    return (found < 0) ? found : 0;
1030}
1031
1032/*
1033 * Move the page from the page cache to the swap cache.
1034 */
1035static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1036{
1037    struct shmem_inode_info *info;
1038    swp_entry_t *entry, swap;
1039    struct address_space *mapping;
1040    unsigned long index;
1041    struct inode *inode;
1042
1043    BUG_ON(!PageLocked(page));
1044    mapping = page->mapping;
1045    index = page->index;
1046    inode = mapping->host;
1047    info = SHMEM_I(inode);
1048    if (info->flags & VM_LOCKED)
1049        goto redirty;
1050    if (!total_swap_pages)
1051        goto redirty;
1052
1053    /*
1054     * shmem_backing_dev_info's capabilities prevent regular writeback or
1055     * sync from ever calling shmem_writepage; but a stacking filesystem
1056     * may use the ->writepage of its underlying filesystem, in which case
1057     * tmpfs should write out to swap only in response to memory pressure,
1058     * and not for the writeback threads or sync. However, in those cases,
1059     * we do still want to check if there's a redundant swappage to be
1060     * discarded.
1061     */
1062    if (wbc->for_reclaim)
1063        swap = get_swap_page();
1064    else
1065        swap.val = 0;
1066
1067    spin_lock(&info->lock);
1068    if (index >= info->next_index) {
1069        BUG_ON(!(info->flags & SHMEM_TRUNCATE));
1070        goto unlock;
1071    }
1072    entry = shmem_swp_entry(info, index, NULL);
1073    if (entry->val) {
1074        /*
1075         * The more uptodate page coming down from a stacked
1076         * writepage should replace our old swappage.
1077         */
1078        free_swap_and_cache(*entry);
1079        shmem_swp_set(info, entry, 0);
1080    }
1081    shmem_recalc_inode(inode);
1082
1083    if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1084        remove_from_page_cache(page);
1085        shmem_swp_set(info, entry, swap.val);
1086        shmem_swp_unmap(entry);
1087        if (list_empty(&info->swaplist))
1088            inode = igrab(inode);
1089        else
1090            inode = NULL;
1091        spin_unlock(&info->lock);
1092        swap_shmem_alloc(swap);
1093        BUG_ON(page_mapped(page));
1094        page_cache_release(page); /* pagecache ref */
1095        swap_writepage(page, wbc);
1096        if (inode) {
1097            mutex_lock(&shmem_swaplist_mutex);
1098            /* move instead of add in case we're racing */
1099            list_move_tail(&info->swaplist, &shmem_swaplist);
1100            mutex_unlock(&shmem_swaplist_mutex);
1101            iput(inode);
1102        }
1103        return 0;
1104    }
1105
1106    shmem_swp_unmap(entry);
1107unlock:
1108    spin_unlock(&info->lock);
1109    /*
1110     * add_to_swap_cache() doesn't return -EEXIST, so we can safely
1111     * clear SWAP_HAS_CACHE flag.
1112     */
1113    swapcache_free(swap, NULL);
1114redirty:
1115    set_page_dirty(page);
1116    if (wbc->for_reclaim)
1117        return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
1118    unlock_page(page);
1119    return 0;
1120}
1121
1122#ifdef CONFIG_NUMA
1123#ifdef CONFIG_TMPFS
1124static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1125{
1126    char buffer[64];
1127
1128    if (!mpol || mpol->mode == MPOL_DEFAULT)
1129        return; /* show nothing */
1130
1131    mpol_to_str(buffer, sizeof(buffer), mpol, 1);
1132
1133    seq_printf(seq, ",mpol=%s", buffer);
1134}
1135
1136static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1137{
1138    struct mempolicy *mpol = NULL;
1139    if (sbinfo->mpol) {
1140        spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
1141        mpol = sbinfo->mpol;
1142        mpol_get(mpol);
1143        spin_unlock(&sbinfo->stat_lock);
1144    }
1145    return mpol;
1146}
1147#endif /* CONFIG_TMPFS */
1148
1149static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1150            struct shmem_inode_info *info, unsigned long idx)
1151{
1152    struct mempolicy mpol, *spol;
1153    struct vm_area_struct pvma;
1154    struct page *page;
1155
1156    spol = mpol_cond_copy(&mpol,
1157                mpol_shared_policy_lookup(&info->policy, idx));
1158
1159    /* Create a pseudo vma that just contains the policy */
1160    pvma.vm_start = 0;
1161    pvma.vm_pgoff = idx;
1162    pvma.vm_ops = NULL;
1163    pvma.vm_policy = spol;
1164    page = swapin_readahead(entry, gfp, &pvma, 0);
1165    return page;
1166}
1167
1168static struct page *shmem_alloc_page(gfp_t gfp,
1169            struct shmem_inode_info *info, unsigned long idx)
1170{
1171    struct vm_area_struct pvma;
1172
1173    /* Create a pseudo vma that just contains the policy */
1174    pvma.vm_start = 0;
1175    pvma.vm_pgoff = idx;
1176    pvma.vm_ops = NULL;
1177    pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1178
1179    /*
1180     * alloc_page_vma() will drop the shared policy reference
1181     */
1182    return alloc_page_vma(gfp, &pvma, 0);
1183}
1184#else /* !CONFIG_NUMA */
1185#ifdef CONFIG_TMPFS
1186static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p)
1187{
1188}
1189#endif /* CONFIG_TMPFS */
1190
1191static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1192            struct shmem_inode_info *info, unsigned long idx)
1193{
1194    return swapin_readahead(entry, gfp, NULL, 0);
1195}
1196
1197static inline struct page *shmem_alloc_page(gfp_t gfp,
1198            struct shmem_inode_info *info, unsigned long idx)
1199{
1200    return alloc_page(gfp);
1201}
1202#endif /* CONFIG_NUMA */
1203
1204#if !defined(CONFIG_NUMA) || !defined(CONFIG_TMPFS)
1205static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1206{
1207    return NULL;
1208}
1209#endif
1210
1211/*
1212 * shmem_getpage - either get the page from swap or allocate a new one
1213 *
1214 * If we allocate a new one we do not mark it dirty. That's up to the
1215 * vm. If we swap it in we mark it dirty since we also free the swap
1216 * entry since a page cannot live in both the swap and page cache
1217 */
1218static int shmem_getpage(struct inode *inode, unsigned long idx,
1219            struct page **pagep, enum sgp_type sgp, int *type)
1220{
1221    struct address_space *mapping = inode->i_mapping;
1222    struct shmem_inode_info *info = SHMEM_I(inode);
1223    struct shmem_sb_info *sbinfo;
1224    struct page *filepage = *pagep;
1225    struct page *swappage;
1226    swp_entry_t *entry;
1227    swp_entry_t swap;
1228    gfp_t gfp;
1229    int error;
1230
1231    if (idx >= SHMEM_MAX_INDEX)
1232        return -EFBIG;
1233
1234    if (type)
1235        *type = 0;
1236
1237    /*
1238     * Normally, filepage is NULL on entry, and either found
1239     * uptodate immediately, or allocated and zeroed, or read
1240     * in under swappage, which is then assigned to filepage.
1241     * But shmem_readpage (required for splice) passes in a locked
1242     * filepage, which may be found not uptodate by other callers
1243     * too, and may need to be copied from the swappage read in.
1244     */
1245repeat:
1246    if (!filepage)
1247        filepage = find_lock_page(mapping, idx);
1248    if (filepage && PageUptodate(filepage))
1249        goto done;
1250    error = 0;
1251    gfp = mapping_gfp_mask(mapping);
1252    if (!filepage) {
1253        /*
1254         * Try to preload while we can wait, to not make a habit of
1255         * draining atomic reserves; but don't latch on to this cpu.
1256         */
1257        error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
1258        if (error)
1259            goto failed;
1260        radix_tree_preload_end();
1261    }
1262
1263    spin_lock(&info->lock);
1264    shmem_recalc_inode(inode);
1265    entry = shmem_swp_alloc(info, idx, sgp);
1266    if (IS_ERR(entry)) {
1267        spin_unlock(&info->lock);
1268        error = PTR_ERR(entry);
1269        goto failed;
1270    }
1271    swap = *entry;
1272
1273    if (swap.val) {
1274        /* Look it up and read it in.. */
1275        swappage = lookup_swap_cache(swap);
1276        if (!swappage) {
1277            shmem_swp_unmap(entry);
1278            /* here we actually do the io */
1279            if (type && !(*type & VM_FAULT_MAJOR)) {
1280                __count_vm_event(PGMAJFAULT);
1281                *type |= VM_FAULT_MAJOR;
1282            }
1283            spin_unlock(&info->lock);
1284            swappage = shmem_swapin(swap, gfp, info, idx);
1285            if (!swappage) {
1286                spin_lock(&info->lock);
1287                entry = shmem_swp_alloc(info, idx, sgp);
1288                if (IS_ERR(entry))
1289                    error = PTR_ERR(entry);
1290                else {
1291                    if (entry->val == swap.val)
1292                        error = -ENOMEM;
1293                    shmem_swp_unmap(entry);
1294                }
1295                spin_unlock(&info->lock);
1296                if (error)
1297                    goto failed;
1298                goto repeat;
1299            }
1300            wait_on_page_locked(swappage);
1301            page_cache_release(swappage);
1302            goto repeat;
1303        }
1304
1305        /* We have to do this with page locked to prevent races */
1306        if (!trylock_page(swappage)) {
1307            shmem_swp_unmap(entry);
1308            spin_unlock(&info->lock);
1309            wait_on_page_locked(swappage);
1310            page_cache_release(swappage);
1311            goto repeat;
1312        }
1313        if (PageWriteback(swappage)) {
1314            shmem_swp_unmap(entry);
1315            spin_unlock(&info->lock);
1316            wait_on_page_writeback(swappage);
1317            unlock_page(swappage);
1318            page_cache_release(swappage);
1319            goto repeat;
1320        }
1321        if (!PageUptodate(swappage)) {
1322            shmem_swp_unmap(entry);
1323            spin_unlock(&info->lock);
1324            unlock_page(swappage);
1325            page_cache_release(swappage);
1326            error = -EIO;
1327            goto failed;
1328        }
1329
1330        if (filepage) {
1331            shmem_swp_set(info, entry, 0);
1332            shmem_swp_unmap(entry);
1333            delete_from_swap_cache(swappage);
1334            spin_unlock(&info->lock);
1335            copy_highpage(filepage, swappage);
1336            unlock_page(swappage);
1337            page_cache_release(swappage);
1338            flush_dcache_page(filepage);
1339            SetPageUptodate(filepage);
1340            set_page_dirty(filepage);
1341            swap_free(swap);
1342        } else if (!(error = add_to_page_cache_locked(swappage, mapping,
1343                    idx, GFP_NOWAIT))) {
1344            info->flags |= SHMEM_PAGEIN;
1345            shmem_swp_set(info, entry, 0);
1346            shmem_swp_unmap(entry);
1347            delete_from_swap_cache(swappage);
1348            spin_unlock(&info->lock);
1349            filepage = swappage;
1350            set_page_dirty(filepage);
1351            swap_free(swap);
1352        } else {
1353            shmem_swp_unmap(entry);
1354            spin_unlock(&info->lock);
1355            if (error == -ENOMEM) {
1356                /*
1357                 * reclaim from proper memory cgroup and
1358                 * call memcg's OOM if needed.
1359                 */
1360                error = mem_cgroup_shmem_charge_fallback(
1361                                swappage,
1362                                current->mm,
1363                                gfp);
1364                if (error) {
1365                    unlock_page(swappage);
1366                    page_cache_release(swappage);
1367                    goto failed;
1368                }
1369            }
1370            unlock_page(swappage);
1371            page_cache_release(swappage);
1372            goto repeat;
1373        }
1374    } else if (sgp == SGP_READ && !filepage) {
1375        shmem_swp_unmap(entry);
1376        filepage = find_get_page(mapping, idx);
1377        if (filepage &&
1378            (!PageUptodate(filepage) || !trylock_page(filepage))) {
1379            spin_unlock(&info->lock);
1380            wait_on_page_locked(filepage);
1381            page_cache_release(filepage);
1382            filepage = NULL;
1383            goto repeat;
1384        }
1385        spin_unlock(&info->lock);
1386    } else {
1387        shmem_swp_unmap(entry);
1388        sbinfo = SHMEM_SB(inode->i_sb);
1389        if (sbinfo->max_blocks) {
1390            spin_lock(&sbinfo->stat_lock);
1391            if (sbinfo->free_blocks == 0 ||
1392                shmem_acct_block(info->flags)) {
1393                spin_unlock(&sbinfo->stat_lock);
1394                spin_unlock(&info->lock);
1395                error = -ENOSPC;
1396                goto failed;
1397            }
1398            sbinfo->free_blocks--;
1399            inode->i_blocks += BLOCKS_PER_PAGE;
1400            spin_unlock(&sbinfo->stat_lock);
1401        } else if (shmem_acct_block(info->flags)) {
1402            spin_unlock(&info->lock);
1403            error = -ENOSPC;
1404            goto failed;
1405        }
1406
1407        if (!filepage) {
1408            int ret;
1409
1410            spin_unlock(&info->lock);
1411            filepage = shmem_alloc_page(gfp, info, idx);
1412            if (!filepage) {
1413                shmem_unacct_blocks(info->flags, 1);
1414                shmem_free_blocks(inode, 1);
1415                error = -ENOMEM;
1416                goto failed;
1417            }
1418            SetPageSwapBacked(filepage);
1419
1420            /* Precharge page while we can wait, compensate after */
1421            error = mem_cgroup_cache_charge(filepage, current->mm,
1422                    GFP_KERNEL);
1423            if (error) {
1424                page_cache_release(filepage);
1425                shmem_unacct_blocks(info->flags, 1);
1426                shmem_free_blocks(inode, 1);
1427                filepage = NULL;
1428                goto failed;
1429            }
1430
1431            spin_lock(&info->lock);
1432            entry = shmem_swp_alloc(info, idx, sgp);
1433            if (IS_ERR(entry))
1434                error = PTR_ERR(entry);
1435            else {
1436                swap = *entry;
1437                shmem_swp_unmap(entry);
1438            }
1439            ret = error || swap.val;
1440            if (ret)
1441                mem_cgroup_uncharge_cache_page(filepage);
1442            else
1443                ret = add_to_page_cache_lru(filepage, mapping,
1444                        idx, GFP_NOWAIT);
1445            /*
1446             * At add_to_page_cache_lru() failure, uncharge will
1447             * be done automatically.
1448             */
1449            if (ret) {
1450                spin_unlock(&info->lock);
1451                page_cache_release(filepage);
1452                shmem_unacct_blocks(info->flags, 1);
1453                shmem_free_blocks(inode, 1);
1454                filepage = NULL;
1455                if (error)
1456                    goto failed;
1457                goto repeat;
1458            }
1459            info->flags |= SHMEM_PAGEIN;
1460        }
1461
1462        info->alloced++;
1463        spin_unlock(&info->lock);
1464        clear_highpage(filepage);
1465        flush_dcache_page(filepage);
1466        SetPageUptodate(filepage);
1467        if (sgp == SGP_DIRTY)
1468            set_page_dirty(filepage);
1469    }
1470done:
1471    *pagep = filepage;
1472    return 0;
1473
1474failed:
1475    if (*pagep != filepage) {
1476        unlock_page(filepage);
1477        page_cache_release(filepage);
1478    }
1479    return error;
1480}
1481
1482static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1483{
1484    struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1485    int error;
1486    int ret;
1487
1488    if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1489        return VM_FAULT_SIGBUS;
1490
1491    error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1492    if (error)
1493        return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1494
1495    return ret | VM_FAULT_LOCKED;
1496}
1497
1498#ifdef CONFIG_NUMA
1499static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1500{
1501    struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1502    return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1503}
1504
1505static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1506                      unsigned long addr)
1507{
1508    struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1509    unsigned long idx;
1510
1511    idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1512    return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1513}
1514#endif
1515
1516int shmem_lock(struct file *file, int lock, struct user_struct *user)
1517{
1518    struct inode *inode = file->f_path.dentry->d_inode;
1519    struct shmem_inode_info *info = SHMEM_I(inode);
1520    int retval = -ENOMEM;
1521
1522    spin_lock(&info->lock);
1523    if (lock && !(info->flags & VM_LOCKED)) {
1524        if (!user_shm_lock(inode->i_size, user))
1525            goto out_nomem;
1526        info->flags |= VM_LOCKED;
1527        mapping_set_unevictable(file->f_mapping);
1528    }
1529    if (!lock && (info->flags & VM_LOCKED) && user) {
1530        user_shm_unlock(inode->i_size, user);
1531        info->flags &= ~VM_LOCKED;
1532        mapping_clear_unevictable(file->f_mapping);
1533        scan_mapping_unevictable_pages(file->f_mapping);
1534    }
1535    retval = 0;
1536
1537out_nomem:
1538    spin_unlock(&info->lock);
1539    return retval;
1540}
1541
1542static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1543{
1544    file_accessed(file);
1545    vma->vm_ops = &shmem_vm_ops;
1546    vma->vm_flags |= VM_CAN_NONLINEAR;
1547    return 0;
1548}
1549
1550static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1551                     int mode, dev_t dev, unsigned long flags)
1552{
1553    struct inode *inode;
1554    struct shmem_inode_info *info;
1555    struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1556
1557    if (shmem_reserve_inode(sb))
1558        return NULL;
1559
1560    inode = new_inode(sb);
1561    if (inode) {
1562        inode_init_owner(inode, dir, mode);
1563        inode->i_blocks = 0;
1564        inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1565        inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1566        inode->i_generation = get_seconds();
1567        info = SHMEM_I(inode);
1568        memset(info, 0, (char *)inode - (char *)info);
1569        spin_lock_init(&info->lock);
1570        info->flags = flags & VM_NORESERVE;
1571        INIT_LIST_HEAD(&info->swaplist);
1572        cache_no_acl(inode);
1573
1574        switch (mode & S_IFMT) {
1575        default:
1576            inode->i_op = &shmem_special_inode_operations;
1577            init_special_inode(inode, mode, dev);
1578            break;
1579        case S_IFREG:
1580            inode->i_mapping->a_ops = &shmem_aops;
1581            inode->i_op = &shmem_inode_operations;
1582            inode->i_fop = &shmem_file_operations;
1583            mpol_shared_policy_init(&info->policy,
1584                         shmem_get_sbmpol(sbinfo));
1585            break;
1586        case S_IFDIR:
1587            inc_nlink(inode);
1588            /* Some things misbehave if size == 0 on a directory */
1589            inode->i_size = 2 * BOGO_DIRENT_SIZE;
1590            inode->i_op = &shmem_dir_inode_operations;
1591            inode->i_fop = &simple_dir_operations;
1592            break;
1593        case S_IFLNK:
1594            /*
1595             * Must not load anything in the rbtree,
1596             * mpol_free_shared_policy will not be called.
1597             */
1598            mpol_shared_policy_init(&info->policy, NULL);
1599            break;
1600        }
1601    } else
1602        shmem_free_inode(sb);
1603    return inode;
1604}
1605
1606#ifdef CONFIG_TMPFS
1607static const struct inode_operations shmem_symlink_inode_operations;
1608static const struct inode_operations shmem_symlink_inline_operations;
1609
1610/*
1611 * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin;
1612 * but providing them allows a tmpfs file to be used for splice, sendfile, and
1613 * below the loop driver, in the generic fashion that many filesystems support.
1614 */
1615static int shmem_readpage(struct file *file, struct page *page)
1616{
1617    struct inode *inode = page->mapping->host;
1618    int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
1619    unlock_page(page);
1620    return error;
1621}
1622
1623static int
1624shmem_write_begin(struct file *file, struct address_space *mapping,
1625            loff_t pos, unsigned len, unsigned flags,
1626            struct page **pagep, void **fsdata)
1627{
1628    struct inode *inode = mapping->host;
1629    pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1630    *pagep = NULL;
1631    return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1632}
1633
1634static int
1635shmem_write_end(struct file *file, struct address_space *mapping,
1636            loff_t pos, unsigned len, unsigned copied,
1637            struct page *page, void *fsdata)
1638{
1639    struct inode *inode = mapping->host;
1640
1641    if (pos + copied > inode->i_size)
1642        i_size_write(inode, pos + copied);
1643
1644    set_page_dirty(page);
1645    unlock_page(page);
1646    page_cache_release(page);
1647
1648    return copied;
1649}
1650
1651static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1652{
1653    struct inode *inode = filp->f_path.dentry->d_inode;
1654    struct address_space *mapping = inode->i_mapping;
1655    unsigned long index, offset;
1656    enum sgp_type sgp = SGP_READ;
1657
1658    /*
1659     * Might this read be for a stacking filesystem? Then when reading
1660     * holes of a sparse file, we actually need to allocate those pages,
1661     * and even mark them dirty, so it cannot exceed the max_blocks limit.
1662     */
1663    if (segment_eq(get_fs(), KERNEL_DS))
1664        sgp = SGP_DIRTY;
1665
1666    index = *ppos >> PAGE_CACHE_SHIFT;
1667    offset = *ppos & ~PAGE_CACHE_MASK;
1668
1669    for (;;) {
1670        struct page *page = NULL;
1671        unsigned long end_index, nr, ret;
1672        loff_t i_size = i_size_read(inode);
1673
1674        end_index = i_size >> PAGE_CACHE_SHIFT;
1675        if (index > end_index)
1676            break;
1677        if (index == end_index) {
1678            nr = i_size & ~PAGE_CACHE_MASK;
1679            if (nr <= offset)
1680                break;
1681        }
1682
1683        desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1684        if (desc->error) {
1685            if (desc->error == -EINVAL)
1686                desc->error = 0;
1687            break;
1688        }
1689        if (page)
1690            unlock_page(page);
1691
1692        /*
1693         * We must evaluate after, since reads (unlike writes)
1694         * are called without i_mutex protection against truncate
1695         */
1696        nr = PAGE_CACHE_SIZE;
1697        i_size = i_size_read(inode);
1698        end_index = i_size >> PAGE_CACHE_SHIFT;
1699        if (index == end_index) {
1700            nr = i_size & ~PAGE_CACHE_MASK;
1701            if (nr <= offset) {
1702                if (page)
1703                    page_cache_release(page);
1704                break;
1705            }
1706        }
1707        nr -= offset;
1708
1709        if (page) {
1710            /*
1711             * If users can be writing to this page using arbitrary
1712             * virtual addresses, take care about potential aliasing
1713             * before reading the page on the kernel side.
1714             */
1715            if (mapping_writably_mapped(mapping))
1716                flush_dcache_page(page);
1717            /*
1718             * Mark the page accessed if we read the beginning.
1719             */
1720            if (!offset)
1721                mark_page_accessed(page);
1722        } else {
1723            page = ZERO_PAGE(0);
1724            page_cache_get(page);
1725        }
1726
1727        /*
1728         * Ok, we have the page, and it's up-to-date, so
1729         * now we can copy it to user space...
1730         *
1731         * The actor routine returns how many bytes were actually used..
1732         * NOTE! This may not be the same as how much of a user buffer
1733         * we filled up (we may be padding etc), so we can only update
1734         * "pos" here (the actor routine has to update the user buffer
1735         * pointers and the remaining count).
1736         */
1737        ret = actor(desc, page, offset, nr);
1738        offset += ret;
1739        index += offset >> PAGE_CACHE_SHIFT;
1740        offset &= ~PAGE_CACHE_MASK;
1741
1742        page_cache_release(page);
1743        if (ret != nr || !desc->count)
1744            break;
1745
1746        cond_resched();
1747    }
1748
1749    *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1750    file_accessed(filp);
1751}
1752
1753static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1754        const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1755{
1756    struct file *filp = iocb->ki_filp;
1757    ssize_t retval;
1758    unsigned long seg;
1759    size_t count;
1760    loff_t *ppos = &iocb->ki_pos;
1761
1762    retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1763    if (retval)
1764        return retval;
1765
1766    for (seg = 0; seg < nr_segs; seg++) {
1767        read_descriptor_t desc;
1768
1769        desc.written = 0;
1770        desc.arg.buf = iov[seg].iov_base;
1771        desc.count = iov[seg].iov_len;
1772        if (desc.count == 0)
1773            continue;
1774        desc.error = 0;
1775        do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1776        retval += desc.written;
1777        if (desc.error) {
1778            retval = retval ?: desc.error;
1779            break;
1780        }
1781        if (desc.count > 0)
1782            break;
1783    }
1784    return retval;
1785}
1786
1787static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1788{
1789    struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1790
1791    buf->f_type = TMPFS_MAGIC;
1792    buf->f_bsize = PAGE_CACHE_SIZE;
1793    buf->f_namelen = NAME_MAX;
1794    spin_lock(&sbinfo->stat_lock);
1795    if (sbinfo->max_blocks) {
1796        buf->f_blocks = sbinfo->max_blocks;
1797        buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
1798    }
1799    if (sbinfo->max_inodes) {
1800        buf->f_files = sbinfo->max_inodes;
1801        buf->f_ffree = sbinfo->free_inodes;
1802    }
1803    /* else leave those fields 0 like simple_statfs */
1804    spin_unlock(&sbinfo->stat_lock);
1805    return 0;
1806}
1807
1808/*
1809 * File creation. Allocate an inode, and we're done..
1810 */
1811static int
1812shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1813{
1814    struct inode *inode;
1815    int error = -ENOSPC;
1816
1817    inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1818    if (inode) {
1819        error = security_inode_init_security(inode, dir, NULL, NULL,
1820                             NULL);
1821        if (error) {
1822            if (error != -EOPNOTSUPP) {
1823                iput(inode);
1824                return error;
1825            }
1826        }
1827#ifdef CONFIG_TMPFS_POSIX_ACL
1828        error = generic_acl_init(inode, dir);
1829        if (error) {
1830            iput(inode);
1831            return error;
1832        }
1833#else
1834        error = 0;
1835#endif
1836        dir->i_size += BOGO_DIRENT_SIZE;
1837        dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1838        d_instantiate(dentry, inode);
1839        dget(dentry); /* Extra count - pin the dentry in core */
1840    }
1841    return error;
1842}
1843
1844static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1845{
1846    int error;
1847
1848    if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1849        return error;
1850    inc_nlink(dir);
1851    return 0;
1852}
1853
1854static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1855        struct nameidata *nd)
1856{
1857    return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1858}
1859
1860/*
1861 * Link a file..
1862 */
1863static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1864{
1865    struct inode *inode = old_dentry->d_inode;
1866    int ret;
1867
1868    /*
1869     * No ordinary (disk based) filesystem counts links as inodes;
1870     * but each new link needs a new dentry, pinning lowmem, and
1871     * tmpfs dentries cannot be pruned until they are unlinked.
1872     */
1873    ret = shmem_reserve_inode(inode->i_sb);
1874    if (ret)
1875        goto out;
1876
1877    dir->i_size += BOGO_DIRENT_SIZE;
1878    inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1879    inc_nlink(inode);
1880    atomic_inc(&inode->i_count); /* New dentry reference */
1881    dget(dentry); /* Extra pinning count for the created dentry */
1882    d_instantiate(dentry, inode);
1883out:
1884    return ret;
1885}
1886
1887static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1888{
1889    struct inode *inode = dentry->d_inode;
1890
1891    if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1892        shmem_free_inode(inode->i_sb);
1893
1894    dir->i_size -= BOGO_DIRENT_SIZE;
1895    inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1896    drop_nlink(inode);
1897    dput(dentry); /* Undo the count from "create" - this does all the work */
1898    return 0;
1899}
1900
1901static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1902{
1903    if (!simple_empty(dentry))
1904        return -ENOTEMPTY;
1905
1906    drop_nlink(dentry->d_inode);
1907    drop_nlink(dir);
1908    return shmem_unlink(dir, dentry);
1909}
1910
1911/*
1912 * The VFS layer already does all the dentry stuff for rename,
1913 * we just have to decrement the usage count for the target if
1914 * it exists so that the VFS layer correctly free's it when it
1915 * gets overwritten.
1916 */
1917static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1918{
1919    struct inode *inode = old_dentry->d_inode;
1920    int they_are_dirs = S_ISDIR(inode->i_mode);
1921
1922    if (!simple_empty(new_dentry))
1923        return -ENOTEMPTY;
1924
1925    if (new_dentry->d_inode) {
1926        (void) shmem_unlink(new_dir, new_dentry);
1927        if (they_are_dirs)
1928            drop_nlink(old_dir);
1929    } else if (they_are_dirs) {
1930        drop_nlink(old_dir);
1931        inc_nlink(new_dir);
1932    }
1933
1934    old_dir->i_size -= BOGO_DIRENT_SIZE;
1935    new_dir->i_size += BOGO_DIRENT_SIZE;
1936    old_dir->i_ctime = old_dir->i_mtime =
1937    new_dir->i_ctime = new_dir->i_mtime =
1938    inode->i_ctime = CURRENT_TIME;
1939    return 0;
1940}
1941
1942static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1943{
1944    int error;
1945    int len;
1946    struct inode *inode;
1947    struct page *page = NULL;
1948    char *kaddr;
1949    struct shmem_inode_info *info;
1950
1951    len = strlen(symname) + 1;
1952    if (len > PAGE_CACHE_SIZE)
1953        return -ENAMETOOLONG;
1954
1955    inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
1956    if (!inode)
1957        return -ENOSPC;
1958
1959    error = security_inode_init_security(inode, dir, NULL, NULL,
1960                         NULL);
1961    if (error) {
1962        if (error != -EOPNOTSUPP) {
1963            iput(inode);
1964            return error;
1965        }
1966        error = 0;
1967    }
1968
1969    info = SHMEM_I(inode);
1970    inode->i_size = len-1;
1971    if (len <= (char *)inode - (char *)info) {
1972        /* do it inline */
1973        memcpy(info, symname, len);
1974        inode->i_op = &shmem_symlink_inline_operations;
1975    } else {
1976        error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1977        if (error) {
1978            iput(inode);
1979            return error;
1980        }
1981        inode->i_mapping->a_ops = &shmem_aops;
1982        inode->i_op = &shmem_symlink_inode_operations;
1983        kaddr = kmap_atomic(page, KM_USER0);
1984        memcpy(kaddr, symname, len);
1985        kunmap_atomic(kaddr, KM_USER0);
1986        set_page_dirty(page);
1987        unlock_page(page);
1988        page_cache_release(page);
1989    }
1990    dir->i_size += BOGO_DIRENT_SIZE;
1991    dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1992    d_instantiate(dentry, inode);
1993    dget(dentry);
1994    return 0;
1995}
1996
1997static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1998{
1999    nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
2000    return NULL;
2001}
2002
2003static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
2004{
2005    struct page *page = NULL;
2006    int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2007    nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
2008    if (page)
2009        unlock_page(page);
2010    return page;
2011}
2012
2013static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
2014{
2015    if (!IS_ERR(nd_get_link(nd))) {
2016        struct page *page = cookie;
2017        kunmap(page);
2018        mark_page_accessed(page);
2019        page_cache_release(page);
2020    }
2021}
2022
2023static const struct inode_operations shmem_symlink_inline_operations = {
2024    .readlink = generic_readlink,
2025    .follow_link = shmem_follow_link_inline,
2026};
2027
2028static const struct inode_operations shmem_symlink_inode_operations = {
2029    .readlink = generic_readlink,
2030    .follow_link = shmem_follow_link,
2031    .put_link = shmem_put_link,
2032};
2033
2034#ifdef CONFIG_TMPFS_POSIX_ACL
2035/*
2036 * Superblocks without xattr inode operations will get security.* xattr
2037 * support from the VFS "for free". As soon as we have any other xattrs
2038 * like ACLs, we also need to implement the security.* handlers at
2039 * filesystem level, though.
2040 */
2041
2042static size_t shmem_xattr_security_list(struct dentry *dentry, char *list,
2043                    size_t list_len, const char *name,
2044                    size_t name_len, int handler_flags)
2045{
2046    return security_inode_listsecurity(dentry->d_inode, list, list_len);
2047}
2048
2049static int shmem_xattr_security_get(struct dentry *dentry, const char *name,
2050        void *buffer, size_t size, int handler_flags)
2051{
2052    if (strcmp(name, "") == 0)
2053        return -EINVAL;
2054    return xattr_getsecurity(dentry->d_inode, name, buffer, size);
2055}
2056
2057static int shmem_xattr_security_set(struct dentry *dentry, const char *name,
2058        const void *value, size_t size, int flags, int handler_flags)
2059{
2060    if (strcmp(name, "") == 0)
2061        return -EINVAL;
2062    return security_inode_setsecurity(dentry->d_inode, name, value,
2063                      size, flags);
2064}
2065
2066static const struct xattr_handler shmem_xattr_security_handler = {
2067    .prefix = XATTR_SECURITY_PREFIX,
2068    .list = shmem_xattr_security_list,
2069    .get = shmem_xattr_security_get,
2070    .set = shmem_xattr_security_set,
2071};
2072
2073static const struct xattr_handler *shmem_xattr_handlers[] = {
2074    &generic_acl_access_handler,
2075    &generic_acl_default_handler,
2076    &shmem_xattr_security_handler,
2077    NULL
2078};
2079#endif
2080
2081static struct dentry *shmem_get_parent(struct dentry *child)
2082{
2083    return ERR_PTR(-ESTALE);
2084}
2085
2086static int shmem_match(struct inode *ino, void *vfh)
2087{
2088    __u32 *fh = vfh;
2089    __u64 inum = fh[2];
2090    inum = (inum << 32) | fh[1];
2091    return ino->i_ino == inum && fh[0] == ino->i_generation;
2092}
2093
2094static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2095        struct fid *fid, int fh_len, int fh_type)
2096{
2097    struct inode *inode;
2098    struct dentry *dentry = NULL;
2099    u64 inum = fid->raw[2];
2100    inum = (inum << 32) | fid->raw[1];
2101
2102    if (fh_len < 3)
2103        return NULL;
2104
2105    inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2106            shmem_match, fid->raw);
2107    if (inode) {
2108        dentry = d_find_alias(inode);
2109        iput(inode);
2110    }
2111
2112    return dentry;
2113}
2114
2115static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2116                int connectable)
2117{
2118    struct inode *inode = dentry->d_inode;
2119
2120    if (*len < 3)
2121        return 255;
2122
2123    if (hlist_unhashed(&inode->i_hash)) {
2124        /* Unfortunately insert_inode_hash is not idempotent,
2125         * so as we hash inodes here rather than at creation
2126         * time, we need a lock to ensure we only try
2127         * to do it once
2128         */
2129        static DEFINE_SPINLOCK(lock);
2130        spin_lock(&lock);
2131        if (hlist_unhashed(&inode->i_hash))
2132            __insert_inode_hash(inode,
2133                        inode->i_ino + inode->i_generation);
2134        spin_unlock(&lock);
2135    }
2136
2137    fh[0] = inode->i_generation;
2138    fh[1] = inode->i_ino;
2139    fh[2] = ((__u64)inode->i_ino) >> 32;
2140
2141    *len = 3;
2142    return 1;
2143}
2144
2145static const struct export_operations shmem_export_ops = {
2146    .get_parent = shmem_get_parent,
2147    .encode_fh = shmem_encode_fh,
2148    .fh_to_dentry = shmem_fh_to_dentry,
2149};
2150
2151static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2152                   bool remount)
2153{
2154    char *this_char, *value, *rest;
2155
2156    while (options != NULL) {
2157        this_char = options;
2158        for (;;) {
2159            /*
2160             * NUL-terminate this option: unfortunately,
2161             * mount options form a comma-separated list,
2162             * but mpol's nodelist may also contain commas.
2163             */
2164            options = strchr(options, ',');
2165            if (options == NULL)
2166                break;
2167            options++;
2168            if (!isdigit(*options)) {
2169                options[-1] = '\0';
2170                break;
2171            }
2172        }
2173        if (!*this_char)
2174            continue;
2175        if ((value = strchr(this_char,'=')) != NULL) {
2176            *value++ = 0;
2177        } else {
2178            printk(KERN_ERR
2179                "tmpfs: No value for mount option '%s'\n",
2180                this_char);
2181            return 1;
2182        }
2183
2184        if (!strcmp(this_char,"size")) {
2185            unsigned long long size;
2186            size = memparse(value,&rest);
2187            if (*rest == '%') {
2188                size <<= PAGE_SHIFT;
2189                size *= totalram_pages;
2190                do_div(size, 100);
2191                rest++;
2192            }
2193            if (*rest)
2194                goto bad_val;
2195            sbinfo->max_blocks =
2196                DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2197        } else if (!strcmp(this_char,"nr_blocks")) {
2198            sbinfo->max_blocks = memparse(value, &rest);
2199            if (*rest)
2200                goto bad_val;
2201        } else if (!strcmp(this_char,"nr_inodes")) {
2202            sbinfo->max_inodes = memparse(value, &rest);
2203            if (*rest)
2204                goto bad_val;
2205        } else if (!strcmp(this_char,"mode")) {
2206            if (remount)
2207                continue;
2208            sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2209            if (*rest)
2210                goto bad_val;
2211        } else if (!strcmp(this_char,"uid")) {
2212            if (remount)
2213                continue;
2214            sbinfo->uid = simple_strtoul(value, &rest, 0);
2215            if (*rest)
2216                goto bad_val;
2217        } else if (!strcmp(this_char,"gid")) {
2218            if (remount)
2219                continue;
2220            sbinfo->gid = simple_strtoul(value, &rest, 0);
2221            if (*rest)
2222                goto bad_val;
2223        } else if (!strcmp(this_char,"mpol")) {
2224            if (mpol_parse_str(value, &sbinfo->mpol, 1))
2225                goto bad_val;
2226        } else {
2227            printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2228                   this_char);
2229            return 1;
2230        }
2231    }
2232    return 0;
2233
2234bad_val:
2235    printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2236           value, this_char);
2237    return 1;
2238
2239}
2240
2241static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2242{
2243    struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2244    struct shmem_sb_info config = *sbinfo;
2245    unsigned long blocks;
2246    unsigned long inodes;
2247    int error = -EINVAL;
2248
2249    if (shmem_parse_options(data, &config, true))
2250        return error;
2251
2252    spin_lock(&sbinfo->stat_lock);
2253    blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2254    inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2255    if (config.max_blocks < blocks)
2256        goto out;
2257    if (config.max_inodes < inodes)
2258        goto out;
2259    /*
2260     * Those tests also disallow limited->unlimited while any are in
2261     * use, so i_blocks will always be zero when max_blocks is zero;
2262     * but we must separately disallow unlimited->limited, because
2263     * in that case we have no record of how much is already in use.
2264     */
2265    if (config.max_blocks && !sbinfo->max_blocks)
2266        goto out;
2267    if (config.max_inodes && !sbinfo->max_inodes)
2268        goto out;
2269
2270    error = 0;
2271    sbinfo->max_blocks = config.max_blocks;
2272    sbinfo->free_blocks = config.max_blocks - blocks;
2273    sbinfo->max_inodes = config.max_inodes;
2274    sbinfo->free_inodes = config.max_inodes - inodes;
2275
2276    mpol_put(sbinfo->mpol);
2277    sbinfo->mpol = config.mpol; /* transfers initial ref */
2278out:
2279    spin_unlock(&sbinfo->stat_lock);
2280    return error;
2281}
2282
2283static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2284{
2285    struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2286
2287    if (sbinfo->max_blocks != shmem_default_max_blocks())
2288        seq_printf(seq, ",size=%luk",
2289            sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2290    if (sbinfo->max_inodes != shmem_default_max_inodes())
2291        seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2292    if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2293        seq_printf(seq, ",mode=%03o", sbinfo->mode);
2294    if (sbinfo->uid != 0)
2295        seq_printf(seq, ",uid=%u", sbinfo->uid);
2296    if (sbinfo->gid != 0)
2297        seq_printf(seq, ",gid=%u", sbinfo->gid);
2298    shmem_show_mpol(seq, sbinfo->mpol);
2299    return 0;
2300}
2301#endif /* CONFIG_TMPFS */
2302
2303static void shmem_put_super(struct super_block *sb)
2304{
2305    kfree(sb->s_fs_info);
2306    sb->s_fs_info = NULL;
2307}
2308
2309int shmem_fill_super(struct super_block *sb, void *data, int silent)
2310{
2311    struct inode *inode;
2312    struct dentry *root;
2313    struct shmem_sb_info *sbinfo;
2314    int err = -ENOMEM;
2315
2316    /* Round up to L1_CACHE_BYTES to resist false sharing */
2317    sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2318                L1_CACHE_BYTES), GFP_KERNEL);
2319    if (!sbinfo)
2320        return -ENOMEM;
2321
2322    sbinfo->mode = S_IRWXUGO | S_ISVTX;
2323    sbinfo->uid = current_fsuid();
2324    sbinfo->gid = current_fsgid();
2325    sb->s_fs_info = sbinfo;
2326
2327#ifdef CONFIG_TMPFS
2328    /*
2329     * Per default we only allow half of the physical ram per
2330     * tmpfs instance, limiting inodes to one per page of lowmem;
2331     * but the internal instance is left unlimited.
2332     */
2333    if (!(sb->s_flags & MS_NOUSER)) {
2334        sbinfo->max_blocks = shmem_default_max_blocks();
2335        sbinfo->max_inodes = shmem_default_max_inodes();
2336        if (shmem_parse_options(data, sbinfo, false)) {
2337            err = -EINVAL;
2338            goto failed;
2339        }
2340    }
2341    sb->s_export_op = &shmem_export_ops;
2342#else
2343    sb->s_flags |= MS_NOUSER;
2344#endif
2345
2346    spin_lock_init(&sbinfo->stat_lock);
2347    sbinfo->free_blocks = sbinfo->max_blocks;
2348    sbinfo->free_inodes = sbinfo->max_inodes;
2349
2350    sb->s_maxbytes = SHMEM_MAX_BYTES;
2351    sb->s_blocksize = PAGE_CACHE_SIZE;
2352    sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2353    sb->s_magic = TMPFS_MAGIC;
2354    sb->s_op = &shmem_ops;
2355    sb->s_time_gran = 1;
2356#ifdef CONFIG_TMPFS_POSIX_ACL
2357    sb->s_xattr = shmem_xattr_handlers;
2358    sb->s_flags |= MS_POSIXACL;
2359#endif
2360
2361    inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2362    if (!inode)
2363        goto failed;
2364    inode->i_uid = sbinfo->uid;
2365    inode->i_gid = sbinfo->gid;
2366    root = d_alloc_root(inode);
2367    if (!root)
2368        goto failed_iput;
2369    sb->s_root = root;
2370    return 0;
2371
2372failed_iput:
2373    iput(inode);
2374failed:
2375    shmem_put_super(sb);
2376    return err;
2377}
2378
2379static struct kmem_cache *shmem_inode_cachep;
2380
2381static struct inode *shmem_alloc_inode(struct super_block *sb)
2382{
2383    struct shmem_inode_info *p;
2384    p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2385    if (!p)
2386        return NULL;
2387    return &p->vfs_inode;
2388}
2389
2390static void shmem_destroy_inode(struct inode *inode)
2391{
2392    if ((inode->i_mode & S_IFMT) == S_IFREG) {
2393        /* only struct inode is valid if it's an inline symlink */
2394        mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2395    }
2396    kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2397}
2398
2399static void init_once(void *foo)
2400{
2401    struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2402
2403    inode_init_once(&p->vfs_inode);
2404}
2405
2406static int init_inodecache(void)
2407{
2408    shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2409                sizeof(struct shmem_inode_info),
2410                0, SLAB_PANIC, init_once);
2411    return 0;
2412}
2413
2414static void destroy_inodecache(void)
2415{
2416    kmem_cache_destroy(shmem_inode_cachep);
2417}
2418
2419static const struct address_space_operations shmem_aops = {
2420    .writepage = shmem_writepage,
2421    .set_page_dirty = __set_page_dirty_no_writeback,
2422#ifdef CONFIG_TMPFS
2423    .readpage = shmem_readpage,
2424    .write_begin = shmem_write_begin,
2425    .write_end = shmem_write_end,
2426#endif
2427    .migratepage = migrate_page,
2428    .error_remove_page = generic_error_remove_page,
2429};
2430
2431static const struct file_operations shmem_file_operations = {
2432    .mmap = shmem_mmap,
2433#ifdef CONFIG_TMPFS
2434    .llseek = generic_file_llseek,
2435    .read = do_sync_read,
2436    .write = do_sync_write,
2437    .aio_read = shmem_file_aio_read,
2438    .aio_write = generic_file_aio_write,
2439    .fsync = noop_fsync,
2440    .splice_read = generic_file_splice_read,
2441    .splice_write = generic_file_splice_write,
2442#endif
2443};
2444
2445static const struct inode_operations shmem_inode_operations = {
2446    .setattr = shmem_notify_change,
2447    .truncate_range = shmem_truncate_range,
2448#ifdef CONFIG_TMPFS_POSIX_ACL
2449    .setxattr = generic_setxattr,
2450    .getxattr = generic_getxattr,
2451    .listxattr = generic_listxattr,
2452    .removexattr = generic_removexattr,
2453    .check_acl = generic_check_acl,
2454#endif
2455
2456};
2457
2458static const struct inode_operations shmem_dir_inode_operations = {
2459#ifdef CONFIG_TMPFS
2460    .create = shmem_create,
2461    .lookup = simple_lookup,
2462    .link = shmem_link,
2463    .unlink = shmem_unlink,
2464    .symlink = shmem_symlink,
2465    .mkdir = shmem_mkdir,
2466    .rmdir = shmem_rmdir,
2467    .mknod = shmem_mknod,
2468    .rename = shmem_rename,
2469#endif
2470#ifdef CONFIG_TMPFS_POSIX_ACL
2471    .setattr = shmem_notify_change,
2472    .setxattr = generic_setxattr,
2473    .getxattr = generic_getxattr,
2474    .listxattr = generic_listxattr,
2475    .removexattr = generic_removexattr,
2476    .check_acl = generic_check_acl,
2477#endif
2478};
2479
2480static const struct inode_operations shmem_special_inode_operations = {
2481#ifdef CONFIG_TMPFS_POSIX_ACL
2482    .setattr = shmem_notify_change,
2483    .setxattr = generic_setxattr,
2484    .getxattr = generic_getxattr,
2485    .listxattr = generic_listxattr,
2486    .removexattr = generic_removexattr,
2487    .check_acl = generic_check_acl,
2488#endif
2489};
2490
2491static const struct super_operations shmem_ops = {
2492    .alloc_inode = shmem_alloc_inode,
2493    .destroy_inode = shmem_destroy_inode,
2494#ifdef CONFIG_TMPFS
2495    .statfs = shmem_statfs,
2496    .remount_fs = shmem_remount_fs,
2497    .show_options = shmem_show_options,
2498#endif
2499    .delete_inode = shmem_delete_inode,
2500    .drop_inode = generic_delete_inode,
2501    .put_super = shmem_put_super,
2502};
2503
2504static const struct vm_operations_struct shmem_vm_ops = {
2505    .fault = shmem_fault,
2506#ifdef CONFIG_NUMA
2507    .set_policy = shmem_set_policy,
2508    .get_policy = shmem_get_policy,
2509#endif
2510};
2511
2512
2513static int shmem_get_sb(struct file_system_type *fs_type,
2514    int flags, const char *dev_name, void *data, struct vfsmount *mnt)
2515{
2516    return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
2517}
2518
2519static struct file_system_type tmpfs_fs_type = {
2520    .owner = THIS_MODULE,
2521    .name = "tmpfs",
2522    .get_sb = shmem_get_sb,
2523    .kill_sb = kill_litter_super,
2524};
2525
2526int __init init_tmpfs(void)
2527{
2528    int error;
2529
2530    error = bdi_init(&shmem_backing_dev_info);
2531    if (error)
2532        goto out4;
2533
2534    error = init_inodecache();
2535    if (error)
2536        goto out3;
2537
2538    error = register_filesystem(&tmpfs_fs_type);
2539    if (error) {
2540        printk(KERN_ERR "Could not register tmpfs\n");
2541        goto out2;
2542    }
2543
2544    shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2545                tmpfs_fs_type.name, NULL);
2546    if (IS_ERR(shm_mnt)) {
2547        error = PTR_ERR(shm_mnt);
2548        printk(KERN_ERR "Could not kern_mount tmpfs\n");
2549        goto out1;
2550    }
2551    return 0;
2552
2553out1:
2554    unregister_filesystem(&tmpfs_fs_type);
2555out2:
2556    destroy_inodecache();
2557out3:
2558    bdi_destroy(&shmem_backing_dev_info);
2559out4:
2560    shm_mnt = ERR_PTR(error);
2561    return error;
2562}
2563
2564#ifdef CONFIG_CGROUP_MEM_RES_CTLR
2565/**
2566 * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file
2567 * @inode: the inode to be searched
2568 * @pgoff: the offset to be searched
2569 * @pagep: the pointer for the found page to be stored
2570 * @ent: the pointer for the found swap entry to be stored
2571 *
2572 * If a page is found, refcount of it is incremented. Callers should handle
2573 * these refcount.
2574 */
2575void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff,
2576                    struct page **pagep, swp_entry_t *ent)
2577{
2578    swp_entry_t entry = { .val = 0 }, *ptr;
2579    struct page *page = NULL;
2580    struct shmem_inode_info *info = SHMEM_I(inode);
2581
2582    if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
2583        goto out;
2584
2585    spin_lock(&info->lock);
2586    ptr = shmem_swp_entry(info, pgoff, NULL);
2587#ifdef CONFIG_SWAP
2588    if (ptr && ptr->val) {
2589        entry.val = ptr->val;
2590        page = find_get_page(&swapper_space, entry.val);
2591    } else
2592#endif
2593        page = find_get_page(inode->i_mapping, pgoff);
2594    if (ptr)
2595        shmem_swp_unmap(ptr);
2596    spin_unlock(&info->lock);
2597out:
2598    *pagep = page;
2599    *ent = entry;
2600}
2601#endif
2602
2603#else /* !CONFIG_SHMEM */
2604
2605/*
2606 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
2607 *
2608 * This is intended for small system where the benefits of the full
2609 * shmem code (swap-backed and resource-limited) are outweighed by
2610 * their complexity. On systems without swap this code should be
2611 * effectively equivalent, but much lighter weight.
2612 */
2613
2614#include <linux/ramfs.h>
2615
2616static struct file_system_type tmpfs_fs_type = {
2617    .name = "tmpfs",
2618    .get_sb = ramfs_get_sb,
2619    .kill_sb = kill_litter_super,
2620};
2621
2622int __init init_tmpfs(void)
2623{
2624    BUG_ON(register_filesystem(&tmpfs_fs_type) != 0);
2625
2626    shm_mnt = kern_mount(&tmpfs_fs_type);
2627    BUG_ON(IS_ERR(shm_mnt));
2628
2629    return 0;
2630}
2631
2632int shmem_unuse(swp_entry_t entry, struct page *page)
2633{
2634    return 0;
2635}
2636
2637int shmem_lock(struct file *file, int lock, struct user_struct *user)
2638{
2639    return 0;
2640}
2641
2642#ifdef CONFIG_CGROUP_MEM_RES_CTLR
2643/**
2644 * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file
2645 * @inode: the inode to be searched
2646 * @pgoff: the offset to be searched
2647 * @pagep: the pointer for the found page to be stored
2648 * @ent: the pointer for the found swap entry to be stored
2649 *
2650 * If a page is found, refcount of it is incremented. Callers should handle
2651 * these refcount.
2652 */
2653void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff,
2654                    struct page **pagep, swp_entry_t *ent)
2655{
2656    struct page *page = NULL;
2657
2658    if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
2659        goto out;
2660    page = find_get_page(inode->i_mapping, pgoff);
2661out:
2662    *pagep = page;
2663    *ent = (swp_entry_t){ .val = 0 };
2664}
2665#endif
2666
2667#define shmem_vm_ops generic_file_vm_ops
2668#define shmem_file_operations ramfs_file_operations
2669#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
2670#define shmem_acct_size(flags, size) 0
2671#define shmem_unacct_size(flags, size) do {} while (0)
2672#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE
2673
2674#endif /* CONFIG_SHMEM */
2675
2676/* common code */
2677
2678/**
2679 * shmem_file_setup - get an unlinked file living in tmpfs
2680 * @name: name for dentry (to be seen in /proc/<pid>/maps
2681 * @size: size to be set for the file
2682 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2683 */
2684struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
2685{
2686    int error;
2687    struct file *file;
2688    struct inode *inode;
2689    struct path path;
2690    struct dentry *root;
2691    struct qstr this;
2692
2693    if (IS_ERR(shm_mnt))
2694        return (void *)shm_mnt;
2695
2696    if (size < 0 || size > SHMEM_MAX_BYTES)
2697        return ERR_PTR(-EINVAL);
2698
2699    if (shmem_acct_size(flags, size))
2700        return ERR_PTR(-ENOMEM);
2701
2702    error = -ENOMEM;
2703    this.name = name;
2704    this.len = strlen(name);
2705    this.hash = 0; /* will go */
2706    root = shm_mnt->mnt_root;
2707    path.dentry = d_alloc(root, &this);
2708    if (!path.dentry)
2709        goto put_memory;
2710    path.mnt = mntget(shm_mnt);
2711
2712    error = -ENOSPC;
2713    inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2714    if (!inode)
2715        goto put_dentry;
2716
2717    d_instantiate(path.dentry, inode);
2718    inode->i_size = size;
2719    inode->i_nlink = 0; /* It is unlinked */
2720#ifndef CONFIG_MMU
2721    error = ramfs_nommu_expand_for_mapping(inode, size);
2722    if (error)
2723        goto put_dentry;
2724#endif
2725
2726    error = -ENFILE;
2727    file = alloc_file(&path, FMODE_WRITE | FMODE_READ,
2728          &shmem_file_operations);
2729    if (!file)
2730        goto put_dentry;
2731
2732    return file;
2733
2734put_dentry:
2735    path_put(&path);
2736put_memory:
2737    shmem_unacct_size(flags, size);
2738    return ERR_PTR(error);
2739}
2740EXPORT_SYMBOL_GPL(shmem_file_setup);
2741
2742/**
2743 * shmem_zero_setup - setup a shared anonymous mapping
2744 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2745 */
2746int shmem_zero_setup(struct vm_area_struct *vma)
2747{
2748    struct file *file;
2749    loff_t size = vma->vm_end - vma->vm_start;
2750
2751    file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2752    if (IS_ERR(file))
2753        return PTR_ERR(file);
2754
2755    if (vma->vm_file)
2756        fput(vma->vm_file);
2757    vma->vm_file = file;
2758    vma->vm_ops = &shmem_vm_ops;
2759    return 0;
2760}
2761

Archive Download this file



interactive