Kernel

Kernel Git Source Tree

Root/mm/shmem.c

1	/*
2	* Resizable virtual memory filesystem for Linux.
3	*
4	* Copyright (C) 2000 Linus Torvalds.
5	* 2000 Transmeta Corp.
6	* 2000-2001 Christoph Rohland
7	* 2000-2001 SAP AG
8	* 2002 Red Hat Inc.
9	* Copyright (C) 2002-2011 Hugh Dickins.
10	* Copyright (C) 2011 Google Inc.
11	* Copyright (C) 2002-2005 VERITAS Software Corporation.
12	* Copyright (C) 2004 Andi Kleen, SuSE Labs
13	*
14	* Extended attribute support for tmpfs:
15	* Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16	* Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17	*
18	* tiny-shmem:
19	* Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20	*
21	* This file is released under the GPL.
22	*/
23
24	#include <linux/fs.h>
25	#include <linux/init.h>
26	#include <linux/vfs.h>
27	#include <linux/mount.h>
28	#include <linux/pagemap.h>
29	#include <linux/file.h>
30	#include <linux/mm.h>
31	#include <linux/export.h>
32	#include <linux/swap.h>
33
34	static struct vfsmount *shm_mnt;
35
36	#ifdef CONFIG_SHMEM
37	/*
38	* This virtual memory filesystem is heavily based on the ramfs. It
39	* extends ramfs by the ability to use swap and honor resource limits
40	* which makes it a completely usable filesystem.
41	*/
42
43	#include <linux/xattr.h>
44	#include <linux/exportfs.h>
45	#include <linux/posix_acl.h>
46	#include <linux/generic_acl.h>
47	#include <linux/mman.h>
48	#include <linux/string.h>
49	#include <linux/slab.h>
50	#include <linux/backing-dev.h>
51	#include <linux/shmem_fs.h>
52	#include <linux/writeback.h>
53	#include <linux/blkdev.h>
54	#include <linux/pagevec.h>
55	#include <linux/percpu_counter.h>
56	#include <linux/falloc.h>
57	#include <linux/splice.h>
58	#include <linux/security.h>
59	#include <linux/swapops.h>
60	#include <linux/mempolicy.h>
61	#include <linux/namei.h>
62	#include <linux/ctype.h>
63	#include <linux/migrate.h>
64	#include <linux/highmem.h>
65	#include <linux/seq_file.h>
66	#include <linux/magic.h>
67
68	#include <asm/uaccess.h>
69	#include <asm/pgtable.h>
70
71	#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
72	#define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
73
74	/* Pretend that each entry is of this size in directory's i_size */
75	#define BOGO_DIRENT_SIZE 20
76
77	/* Symlink up to this size is kmalloc'ed instead of using a swappable page */
78	#define SHORT_SYMLINK_LEN 128
79
80	struct shmem_xattr {
81	struct list_head list; /* anchored by shmem_inode_info->xattr_list */
82	char name; / xattr name */
83	size_t size;
84	char value[0];
85	};
86
87	/*
88	* shmem_fallocate and shmem_writepage communicate via inode->i_private
89	* (with i_mutex making sure that it has only one user at a time):
90	* we would prefer not to enlarge the shmem inode just for that.
91	*/
92	struct shmem_falloc {
93	pgoff_t start; /* start of range currently being fallocated */
94	pgoff_t next; /* the next page offset to be fallocated */
95	pgoff_t nr_falloced; /* how many new pages have been fallocated */
96	pgoff_t nr_unswapped; /* how often writepage refused to swap out */
97	};
98
99	/* Flag allocation requirements to shmem_getpage */
100	enum sgp_type {
101	SGP_READ, /* don't exceed i_size, don't allocate page */
102	SGP_CACHE, /* don't exceed i_size, may allocate page */
103	SGP_DIRTY, /* like SGP_CACHE, but set new page dirty */
104	SGP_WRITE, /* may exceed i_size, may allocate !Uptodate page */
105	SGP_FALLOC, /* like SGP_WRITE, but make existing page Uptodate */
106	};
107
108	#ifdef CONFIG_TMPFS
109	static unsigned long shmem_default_max_blocks(void)
110	{
111	return totalram_pages / 2;
112	}
113
114	static unsigned long shmem_default_max_inodes(void)
115	{
116	return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
117	}
118	#endif
119
120	static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
121	static int shmem_replace_page(struct page **pagep, gfp_t gfp,
122	struct shmem_inode_info *info, pgoff_t index);
123	static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
124	struct page *pagep, enum sgp_type sgp, gfp_t gfp, int fault_type);
125
126	static inline int shmem_getpage(struct inode *inode, pgoff_t index,
127	struct page *pagep, enum sgp_type sgp, int fault_type)
128	{
129	return shmem_getpage_gfp(inode, index, pagep, sgp,
130	mapping_gfp_mask(inode->i_mapping), fault_type);
131	}
132
133	static inline struct shmem_sb_info SHMEM_SB(struct super_block sb)
134	{
135	return sb->s_fs_info;
136	}
137
138	/*
139	* shmem_file_setup pre-accounts the whole fixed size of a VM object,
140	* for shared memory and for shared anonymous (/dev/zero) mappings
141	* (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
142	* consistent with the pre-accounting of private mappings ...
143	*/
144	static inline int shmem_acct_size(unsigned long flags, loff_t size)
145	{
146	return (flags & VM_NORESERVE) ?
147	0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
148	}
149
150	static inline void shmem_unacct_size(unsigned long flags, loff_t size)
151	{
152	if (!(flags & VM_NORESERVE))
153	vm_unacct_memory(VM_ACCT(size));
154	}
155
156	/*
157	* ... whereas tmpfs objects are accounted incrementally as
158	* pages are allocated, in order to allow huge sparse files.
159	* shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
160	* so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
161	*/
162	static inline int shmem_acct_block(unsigned long flags)
163	{
164	return (flags & VM_NORESERVE) ?
165	security_vm_enough_memory_mm(current->mm, VM_ACCT(PAGE_CACHE_SIZE)) : 0;
166	}
167
168	static inline void shmem_unacct_blocks(unsigned long flags, long pages)
169	{
170	if (flags & VM_NORESERVE)
171	vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
172	}
173
174	static const struct super_operations shmem_ops;
175	static const struct address_space_operations shmem_aops;
176	static const struct file_operations shmem_file_operations;
177	static const struct inode_operations shmem_inode_operations;
178	static const struct inode_operations shmem_dir_inode_operations;
179	static const struct inode_operations shmem_special_inode_operations;
180	static const struct vm_operations_struct shmem_vm_ops;
181
182	static struct backing_dev_info shmem_backing_dev_info __read_mostly = {
183	.ra_pages = 0, /* No readahead */
184	.capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK \| BDI_CAP_SWAP_BACKED,
185	};
186
187	static LIST_HEAD(shmem_swaplist);
188	static DEFINE_MUTEX(shmem_swaplist_mutex);
189
190	static int shmem_reserve_inode(struct super_block *sb)
191	{
192	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
193	if (sbinfo->max_inodes) {
194	spin_lock(&sbinfo->stat_lock);
195	if (!sbinfo->free_inodes) {
196	spin_unlock(&sbinfo->stat_lock);
197	return -ENOSPC;
198	}
199	sbinfo->free_inodes--;
200	spin_unlock(&sbinfo->stat_lock);
201	}
202	return 0;
203	}
204
205	static void shmem_free_inode(struct super_block *sb)
206	{
207	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
208	if (sbinfo->max_inodes) {
209	spin_lock(&sbinfo->stat_lock);
210	sbinfo->free_inodes++;
211	spin_unlock(&sbinfo->stat_lock);
212	}
213	}
214
215	/**
216	* shmem_recalc_inode - recalculate the block usage of an inode
217	* @inode: inode to recalc
218	*
219	* We have to calculate the free blocks since the mm can drop
220	* undirtied hole pages behind our back.
221	*
222	* But normally info->alloced == inode->i_mapping->nrpages + info->swapped
223	* So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
224	*
225	* It has to be called with the spinlock held.
226	*/
227	static void shmem_recalc_inode(struct inode *inode)
228	{
229	struct shmem_inode_info *info = SHMEM_I(inode);
230	long freed;
231
232	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
233	if (freed > 0) {
234	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
235	if (sbinfo->max_blocks)
236	percpu_counter_add(&sbinfo->used_blocks, -freed);
237	info->alloced -= freed;
238	inode->i_blocks -= freed * BLOCKS_PER_PAGE;
239	shmem_unacct_blocks(info->flags, freed);
240	}
241	}
242
243	/*
244	* Replace item expected in radix tree by a new item, while holding tree lock.
245	*/
246	static int shmem_radix_tree_replace(struct address_space *mapping,
247	pgoff_t index, void expected, void replacement)
248	{
249	void **pslot;
250	void *item = NULL;
251
252	VM_BUG_ON(!expected);
253	pslot = radix_tree_lookup_slot(&mapping->page_tree, index);
254	if (pslot)
255	item = radix_tree_deref_slot_protected(pslot,
256	&mapping->tree_lock);
257	if (item != expected)
258	return -ENOENT;
259	if (replacement)
260	radix_tree_replace_slot(pslot, replacement);
261	else
262	radix_tree_delete(&mapping->page_tree, index);
263	return 0;
264	}
265
266	/*
267	* Sometimes, before we decide whether to proceed or to fail, we must check
268	* that an entry was not already brought back from swap by a racing thread.
269	*
270	* Checking page is not enough: by the time a SwapCache page is locked, it
271	* might be reused, and again be SwapCache, using the same swap as before.
272	*/
273	static bool shmem_confirm_swap(struct address_space *mapping,
274	pgoff_t index, swp_entry_t swap)
275	{
276	void *item;
277
278	rcu_read_lock();
279	item = radix_tree_lookup(&mapping->page_tree, index);
280	rcu_read_unlock();
281	return item == swp_to_radix_entry(swap);
282	}
283
284	/*
285	* Like add_to_page_cache_locked, but error if expected item has gone.
286	*/
287	static int shmem_add_to_page_cache(struct page *page,
288	struct address_space *mapping,
289	pgoff_t index, gfp_t gfp, void *expected)
290	{
291	int error;
292
293	VM_BUG_ON(!PageLocked(page));
294	VM_BUG_ON(!PageSwapBacked(page));
295
296	page_cache_get(page);
297	page->mapping = mapping;
298	page->index = index;
299
300	spin_lock_irq(&mapping->tree_lock);
301	if (!expected)
302	error = radix_tree_insert(&mapping->page_tree, index, page);
303	else
304	error = shmem_radix_tree_replace(mapping, index, expected,
305	page);
306	if (!error) {
307	mapping->nrpages++;
308	__inc_zone_page_state(page, NR_FILE_PAGES);
309	__inc_zone_page_state(page, NR_SHMEM);
310	spin_unlock_irq(&mapping->tree_lock);
311	} else {
312	page->mapping = NULL;
313	spin_unlock_irq(&mapping->tree_lock);
314	page_cache_release(page);
315	}
316	return error;
317	}
318
319	/*
320	* Like delete_from_page_cache, but substitutes swap for page.
321	*/
322	static void shmem_delete_from_page_cache(struct page page, void radswap)
323	{
324	struct address_space *mapping = page->mapping;
325	int error;
326
327	spin_lock_irq(&mapping->tree_lock);
328	error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
329	page->mapping = NULL;
330	mapping->nrpages--;
331	__dec_zone_page_state(page, NR_FILE_PAGES);
332	__dec_zone_page_state(page, NR_SHMEM);
333	spin_unlock_irq(&mapping->tree_lock);
334	page_cache_release(page);
335	BUG_ON(error);
336	}
337
338	/*
339	* Like find_get_pages, but collecting swap entries as well as pages.
340	*/
341	static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping,
342	pgoff_t start, unsigned int nr_pages,
343	struct page *pages, pgoff_t indices)
344	{
345	unsigned int i;
346	unsigned int ret;
347	unsigned int nr_found;
348
349	rcu_read_lock();
350	restart:
351	nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
352	(void ***)pages, indices, start, nr_pages);
353	ret = 0;
354	for (i = 0; i < nr_found; i++) {
355	struct page *page;
356	repeat:
357	page = radix_tree_deref_slot((void **)pages[i]);
358	if (unlikely(!page))
359	continue;
360	if (radix_tree_exception(page)) {
361	if (radix_tree_deref_retry(page))
362	goto restart;
363	/*
364	* Otherwise, we must be storing a swap entry
365	* here as an exceptional entry: so return it
366	* without attempting to raise page count.
367	*/
368	goto export;
369	}
370	if (!page_cache_get_speculative(page))
371	goto repeat;
372
373	/* Has the page moved? */
374	if (unlikely(page != ((void *)pages[i]))) {
375	page_cache_release(page);
376	goto repeat;
377	}
378	export:
379	indices[ret] = indices[i];
380	pages[ret] = page;
381	ret++;
382	}
383	if (unlikely(!ret && nr_found))
384	goto restart;
385	rcu_read_unlock();
386	return ret;
387	}
388
389	/*
390	* Remove swap entry from radix tree, free the swap and its page cache.
391	*/
392	static int shmem_free_swap(struct address_space *mapping,
393	pgoff_t index, void *radswap)
394	{
395	int error;
396
397	spin_lock_irq(&mapping->tree_lock);
398	error = shmem_radix_tree_replace(mapping, index, radswap, NULL);
399	spin_unlock_irq(&mapping->tree_lock);
400	if (!error)
401	free_swap_and_cache(radix_to_swp_entry(radswap));
402	return error;
403	}
404
405	/*
406	* Pagevec may contain swap entries, so shuffle up pages before releasing.
407	*/
408	static void shmem_deswap_pagevec(struct pagevec *pvec)
409	{
410	int i, j;
411
412	for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
413	struct page *page = pvec->pages[i];
414	if (!radix_tree_exceptional_entry(page))
415	pvec->pages[j++] = page;
416	}
417	pvec->nr = j;
418	}
419
420	/*
421	* SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
422	*/
423	void shmem_unlock_mapping(struct address_space *mapping)
424	{
425	struct pagevec pvec;
426	pgoff_t indices[PAGEVEC_SIZE];
427	pgoff_t index = 0;
428
429	pagevec_init(&pvec, 0);
430	/*
431	* Minor point, but we might as well stop if someone else SHM_LOCKs it.
432	*/
433	while (!mapping_unevictable(mapping)) {
434	/*
435	* Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
436	* has finished, if it hits a row of PAGEVEC_SIZE swap entries.
437	*/
438	pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
439	PAGEVEC_SIZE, pvec.pages, indices);
440	if (!pvec.nr)
441	break;
442	index = indices[pvec.nr - 1] + 1;
443	shmem_deswap_pagevec(&pvec);
444	check_move_unevictable_pages(pvec.pages, pvec.nr);
445	pagevec_release(&pvec);
446	cond_resched();
447	}
448	}
449
450	/*
451	* Remove range of pages and swap entries from radix tree, and free them.
452	* If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
453	*/
454	static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
455	bool unfalloc)
456	{
457	struct address_space *mapping = inode->i_mapping;
458	struct shmem_inode_info *info = SHMEM_I(inode);
459	pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
460	pgoff_t end = (lend + 1) >> PAGE_CACHE_SHIFT;
461	unsigned int partial_start = lstart & (PAGE_CACHE_SIZE - 1);
462	unsigned int partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1);
463	struct pagevec pvec;
464	pgoff_t indices[PAGEVEC_SIZE];
465	long nr_swaps_freed = 0;
466	pgoff_t index;
467	int i;
468
469	if (lend == -1)
470	end = -1; /* unsigned, so actually very big */
471
472	pagevec_init(&pvec, 0);
473	index = start;
474	while (index < end) {
475	pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
476	min(end - index, (pgoff_t)PAGEVEC_SIZE),
477	pvec.pages, indices);
478	if (!pvec.nr)
479	break;
480	mem_cgroup_uncharge_start();
481	for (i = 0; i < pagevec_count(&pvec); i++) {
482	struct page *page = pvec.pages[i];
483
484	index = indices[i];
485	if (index >= end)
486	break;
487
488	if (radix_tree_exceptional_entry(page)) {
489	if (unfalloc)
490	continue;
491	nr_swaps_freed += !shmem_free_swap(mapping,
492	index, page);
493	continue;
494	}
495
496	if (!trylock_page(page))
497	continue;
498	if (!unfalloc \|\| !PageUptodate(page)) {
499	if (page->mapping == mapping) {
500	VM_BUG_ON(PageWriteback(page));
501	truncate_inode_page(mapping, page);
502	}
503	}
504	unlock_page(page);
505	}
506	shmem_deswap_pagevec(&pvec);
507	pagevec_release(&pvec);
508	mem_cgroup_uncharge_end();
509	cond_resched();
510	index++;
511	}
512
513	if (partial_start) {
514	struct page *page = NULL;
515	shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);
516	if (page) {
517	unsigned int top = PAGE_CACHE_SIZE;
518	if (start > end) {
519	top = partial_end;
520	partial_end = 0;
521	}
522	zero_user_segment(page, partial_start, top);
523	set_page_dirty(page);
524	unlock_page(page);
525	page_cache_release(page);
526	}
527	}
528	if (partial_end) {
529	struct page *page = NULL;
530	shmem_getpage(inode, end, &page, SGP_READ, NULL);
531	if (page) {
532	zero_user_segment(page, 0, partial_end);
533	set_page_dirty(page);
534	unlock_page(page);
535	page_cache_release(page);
536	}
537	}
538	if (start >= end)
539	return;
540
541	index = start;
542	for ( ; ; ) {
543	cond_resched();
544	pvec.nr = shmem_find_get_pages_and_swap(mapping, index,
545	min(end - index, (pgoff_t)PAGEVEC_SIZE),
546	pvec.pages, indices);
547	if (!pvec.nr) {
548	if (index == start \|\| unfalloc)
549	break;
550	index = start;
551	continue;
552	}
553	if ((index == start \|\| unfalloc) && indices[0] >= end) {
554	shmem_deswap_pagevec(&pvec);
555	pagevec_release(&pvec);
556	break;
557	}
558	mem_cgroup_uncharge_start();
559	for (i = 0; i < pagevec_count(&pvec); i++) {
560	struct page *page = pvec.pages[i];
561
562	index = indices[i];
563	if (index >= end)
564	break;
565
566	if (radix_tree_exceptional_entry(page)) {
567	if (unfalloc)
568	continue;
569	nr_swaps_freed += !shmem_free_swap(mapping,
570	index, page);
571	continue;
572	}
573
574	lock_page(page);
575	if (!unfalloc \|\| !PageUptodate(page)) {
576	if (page->mapping == mapping) {
577	VM_BUG_ON(PageWriteback(page));
578	truncate_inode_page(mapping, page);
579	}
580	}
581	unlock_page(page);
582	}
583	shmem_deswap_pagevec(&pvec);
584	pagevec_release(&pvec);
585	mem_cgroup_uncharge_end();
586	index++;
587	}
588
589	spin_lock(&info->lock);
590	info->swapped -= nr_swaps_freed;
591	shmem_recalc_inode(inode);
592	spin_unlock(&info->lock);
593	}
594
595	void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
596	{
597	shmem_undo_range(inode, lstart, lend, false);
598	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
599	}
600	EXPORT_SYMBOL_GPL(shmem_truncate_range);
601
602	static int shmem_setattr(struct dentry dentry, struct iattr attr)
603	{
604	struct inode *inode = dentry->d_inode;
605	int error;
606
607	error = inode_change_ok(inode, attr);
608	if (error)
609	return error;
610
611	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
612	loff_t oldsize = inode->i_size;
613	loff_t newsize = attr->ia_size;
614
615	if (newsize != oldsize) {
616	i_size_write(inode, newsize);
617	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
618	}
619	if (newsize < oldsize) {
620	loff_t holebegin = round_up(newsize, PAGE_SIZE);
621	unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
622	shmem_truncate_range(inode, newsize, (loff_t)-1);
623	/* unmap again to remove racily COWed private pages */
624	unmap_mapping_range(inode->i_mapping, holebegin, 0, 1);
625	}
626	}
627
628	setattr_copy(inode, attr);
629	#ifdef CONFIG_TMPFS_POSIX_ACL
630	if (attr->ia_valid & ATTR_MODE)
631	error = generic_acl_chmod(inode);
632	#endif
633	return error;
634	}
635
636	static void shmem_evict_inode(struct inode *inode)
637	{
638	struct shmem_inode_info *info = SHMEM_I(inode);
639	struct shmem_xattr xattr, nxattr;
640
641	if (inode->i_mapping->a_ops == &shmem_aops) {
642	shmem_unacct_size(info->flags, inode->i_size);
643	inode->i_size = 0;
644	shmem_truncate_range(inode, 0, (loff_t)-1);
645	if (!list_empty(&info->swaplist)) {
646	mutex_lock(&shmem_swaplist_mutex);
647	list_del_init(&info->swaplist);
648	mutex_unlock(&shmem_swaplist_mutex);
649	}
650	} else
651	kfree(info->symlink);
652
653	list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) {
654	kfree(xattr->name);
655	kfree(xattr);
656	}
657	BUG_ON(inode->i_blocks);
658	shmem_free_inode(inode->i_sb);
659	clear_inode(inode);
660	}
661
662	/*
663	* If swap found in inode, free it and move page from swapcache to filecache.
664	*/
665	static int shmem_unuse_inode(struct shmem_inode_info *info,
666	swp_entry_t swap, struct page **pagep)
667	{
668	struct address_space *mapping = info->vfs_inode.i_mapping;
669	void *radswap;
670	pgoff_t index;
671	gfp_t gfp;
672	int error = 0;
673
674	radswap = swp_to_radix_entry(swap);
675	index = radix_tree_locate_item(&mapping->page_tree, radswap);
676	if (index == -1)
677	return 0;
678
679	/*
680	* Move _head_ to start search for next from here.
681	* But be careful: shmem_evict_inode checks list_empty without taking
682	* mutex, and there's an instant in list_move_tail when info->swaplist
683	* would appear empty, if it were the only one on shmem_swaplist.
684	*/
685	if (shmem_swaplist.next != &info->swaplist)
686	list_move_tail(&shmem_swaplist, &info->swaplist);
687
688	gfp = mapping_gfp_mask(mapping);
689	if (shmem_should_replace_page(*pagep, gfp)) {
690	mutex_unlock(&shmem_swaplist_mutex);
691	error = shmem_replace_page(pagep, gfp, info, index);
692	mutex_lock(&shmem_swaplist_mutex);
693	/*
694	* We needed to drop mutex to make that restrictive page
695	* allocation, but the inode might have been freed while we
696	* dropped it: although a racing shmem_evict_inode() cannot
697	* complete without emptying the radix_tree, our page lock
698	* on this swapcache page is not enough to prevent that -
699	* free_swap_and_cache() of our swap entry will only
700	* trylock_page(), removing swap from radix_tree whatever.
701	*
702	* We must not proceed to shmem_add_to_page_cache() if the
703	* inode has been freed, but of course we cannot rely on
704	* inode or mapping or info to check that. However, we can
705	* safely check if our swap entry is still in use (and here
706	* it can't have got reused for another page): if it's still
707	* in use, then the inode cannot have been freed yet, and we
708	* can safely proceed (if it's no longer in use, that tells
709	* nothing about the inode, but we don't need to unuse swap).
710	*/
711	if (!page_swapcount(*pagep))
712	error = -ENOENT;
713	}
714
715	/*
716	* We rely on shmem_swaplist_mutex, not only to protect the swaplist,
717	* but also to hold up shmem_evict_inode(): so inode cannot be freed
718	* beneath us (pagelock doesn't help until the page is in pagecache).
719	*/
720	if (!error)
721	error = shmem_add_to_page_cache(*pagep, mapping, index,
722	GFP_NOWAIT, radswap);
723	if (error != -ENOMEM) {
724	/*
725	* Truncation and eviction use free_swap_and_cache(), which
726	* only does trylock page: if we raced, best clean up here.
727	*/
728	delete_from_swap_cache(*pagep);
729	set_page_dirty(*pagep);
730	if (!error) {
731	spin_lock(&info->lock);
732	info->swapped--;
733	spin_unlock(&info->lock);
734	swap_free(swap);
735	}
736	error = 1; /* not an error, but entry was found */
737	}
738	return error;
739	}
740
741	/*
742	* Search through swapped inodes to find and replace swap by page.
743	*/
744	int shmem_unuse(swp_entry_t swap, struct page *page)
745	{
746	struct list_head this, next;
747	struct shmem_inode_info *info;
748	int found = 0;
749	int error = 0;
750
751	/*
752	* There's a faint possibility that swap page was replaced before
753	* caller locked it: caller will come back later with the right page.
754	*/
755	if (unlikely(!PageSwapCache(page) \|\| page_private(page) != swap.val))
756	goto out;
757
758	/*
759	* Charge page using GFP_KERNEL while we can wait, before taking
760	* the shmem_swaplist_mutex which might hold up shmem_writepage().
761	* Charged back to the user (not to caller) when swap account is used.
762	*/
763	error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
764	if (error)
765	goto out;
766	/* No radix_tree_preload: swap entry keeps a place for page in tree */
767
768	mutex_lock(&shmem_swaplist_mutex);
769	list_for_each_safe(this, next, &shmem_swaplist) {
770	info = list_entry(this, struct shmem_inode_info, swaplist);
771	if (info->swapped)
772	found = shmem_unuse_inode(info, swap, &page);
773	else
774	list_del_init(&info->swaplist);
775	cond_resched();
776	if (found)
777	break;
778	}
779	mutex_unlock(&shmem_swaplist_mutex);
780
781	if (found < 0)
782	error = found;
783	out:
784	unlock_page(page);
785	page_cache_release(page);
786	return error;
787	}
788
789	/*
790	* Move the page from the page cache to the swap cache.
791	*/
792	static int shmem_writepage(struct page page, struct writeback_control wbc)
793	{
794	struct shmem_inode_info *info;
795	struct address_space *mapping;
796	struct inode *inode;
797	swp_entry_t swap;
798	pgoff_t index;
799
800	BUG_ON(!PageLocked(page));
801	mapping = page->mapping;
802	index = page->index;
803	inode = mapping->host;
804	info = SHMEM_I(inode);
805	if (info->flags & VM_LOCKED)
806	goto redirty;
807	if (!total_swap_pages)
808	goto redirty;
809
810	/*
811	* shmem_backing_dev_info's capabilities prevent regular writeback or
812	* sync from ever calling shmem_writepage; but a stacking filesystem
813	* might use ->writepage of its underlying filesystem, in which case
814	* tmpfs should write out to swap only in response to memory pressure,
815	* and not for the writeback threads or sync.
816	*/
817	if (!wbc->for_reclaim) {
818	WARN_ON_ONCE(1); /* Still happens? Tell us about it! */
819	goto redirty;
820	}
821
822	/*
823	* This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
824	* value into swapfile.c, the only way we can correctly account for a
825	* fallocated page arriving here is now to initialize it and write it.
826	*
827	* That's okay for a page already fallocated earlier, but if we have
828	* not yet completed the fallocation, then (a) we want to keep track
829	* of this page in case we have to undo it, and (b) it may not be a
830	* good idea to continue anyway, once we're pushing into swap. So
831	* reactivate the page, and let shmem_fallocate() quit when too many.
832	*/
833	if (!PageUptodate(page)) {
834	if (inode->i_private) {
835	struct shmem_falloc *shmem_falloc;
836	spin_lock(&inode->i_lock);
837	shmem_falloc = inode->i_private;
838	if (shmem_falloc &&
839	index >= shmem_falloc->start &&
840	index < shmem_falloc->next)
841	shmem_falloc->nr_unswapped++;
842	else
843	shmem_falloc = NULL;
844	spin_unlock(&inode->i_lock);
845	if (shmem_falloc)
846	goto redirty;
847	}
848	clear_highpage(page);
849	flush_dcache_page(page);
850	SetPageUptodate(page);
851	}
852
853	swap = get_swap_page();
854	if (!swap.val)
855	goto redirty;
856
857	/*
858	* Add inode to shmem_unuse()'s list of swapped-out inodes,
859	* if it's not already there. Do it now before the page is
860	* moved to swap cache, when its pagelock no longer protects
861	* the inode from eviction. But don't unlock the mutex until
862	* we've incremented swapped, because shmem_unuse_inode() will
863	* prune a !swapped inode from the swaplist under this mutex.
864	*/
865	mutex_lock(&shmem_swaplist_mutex);
866	if (list_empty(&info->swaplist))
867	list_add_tail(&info->swaplist, &shmem_swaplist);
868
869	if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
870	swap_shmem_alloc(swap);
871	shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
872
873	spin_lock(&info->lock);
874	info->swapped++;
875	shmem_recalc_inode(inode);
876	spin_unlock(&info->lock);
877
878	mutex_unlock(&shmem_swaplist_mutex);
879	BUG_ON(page_mapped(page));
880	swap_writepage(page, wbc);
881	return 0;
882	}
883
884	mutex_unlock(&shmem_swaplist_mutex);
885	swapcache_free(swap, NULL);
886	redirty:
887	set_page_dirty(page);
888	if (wbc->for_reclaim)
889	return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */
890	unlock_page(page);
891	return 0;
892	}
893
894	#ifdef CONFIG_NUMA
895	#ifdef CONFIG_TMPFS
896	static void shmem_show_mpol(struct seq_file seq, struct mempolicy mpol)
897	{
898	char buffer[64];
899
900	if (!mpol \|\| mpol->mode == MPOL_DEFAULT)
901	return; /* show nothing */
902
903	mpol_to_str(buffer, sizeof(buffer), mpol, 1);
904
905	seq_printf(seq, ",mpol=%s", buffer);
906	}
907
908	static struct mempolicy shmem_get_sbmpol(struct shmem_sb_info sbinfo)
909	{
910	struct mempolicy *mpol = NULL;
911	if (sbinfo->mpol) {
912	spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */
913	mpol = sbinfo->mpol;
914	mpol_get(mpol);
915	spin_unlock(&sbinfo->stat_lock);
916	}
917	return mpol;
918	}
919	#endif /* CONFIG_TMPFS */
920
921	static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
922	struct shmem_inode_info *info, pgoff_t index)
923	{
924	struct mempolicy mpol, *spol;
925	struct vm_area_struct pvma;
926
927	spol = mpol_cond_copy(&mpol,
928	mpol_shared_policy_lookup(&info->policy, index));
929
930	/* Create a pseudo vma that just contains the policy */
931	pvma.vm_start = 0;
932	/* Bias interleave by inode number to distribute better across nodes */
933	pvma.vm_pgoff = index + info->vfs_inode.i_ino;
934	pvma.vm_ops = NULL;
935	pvma.vm_policy = spol;
936	return swapin_readahead(swap, gfp, &pvma, 0);
937	}
938
939	static struct page *shmem_alloc_page(gfp_t gfp,
940	struct shmem_inode_info *info, pgoff_t index)
941	{
942	struct vm_area_struct pvma;
943
944	/* Create a pseudo vma that just contains the policy */
945	pvma.vm_start = 0;
946	/* Bias interleave by inode number to distribute better across nodes */
947	pvma.vm_pgoff = index + info->vfs_inode.i_ino;
948	pvma.vm_ops = NULL;
949	pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index);
950
951	/*
952	* alloc_page_vma() will drop the shared policy reference
953	*/
954	return alloc_page_vma(gfp, &pvma, 0);
955	}
956	#else /* !CONFIG_NUMA */
957	#ifdef CONFIG_TMPFS
958	static inline void shmem_show_mpol(struct seq_file seq, struct mempolicy mpol)
959	{
960	}
961	#endif /* CONFIG_TMPFS */
962
963	static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
964	struct shmem_inode_info *info, pgoff_t index)
965	{
966	return swapin_readahead(swap, gfp, NULL, 0);
967	}
968
969	static inline struct page *shmem_alloc_page(gfp_t gfp,
970	struct shmem_inode_info *info, pgoff_t index)
971	{
972	return alloc_page(gfp);
973	}
974	#endif /* CONFIG_NUMA */
975
976	#if !defined(CONFIG_NUMA) \|\| !defined(CONFIG_TMPFS)
977	static inline struct mempolicy shmem_get_sbmpol(struct shmem_sb_info sbinfo)
978	{
979	return NULL;
980	}
981	#endif
982
983	/*
984	* When a page is moved from swapcache to shmem filecache (either by the
985	* usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
986	* shmem_unuse_inode()), it may have been read in earlier from swap, in
987	* ignorance of the mapping it belongs to. If that mapping has special
988	* constraints (like the gma500 GEM driver, which requires RAM below 4GB),
989	* we may need to copy to a suitable page before moving to filecache.
990	*
991	* In a future release, this may well be extended to respect cpuset and
992	* NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
993	* but for now it is a simple matter of zone.
994	*/
995	static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
996	{
997	return page_zonenum(page) > gfp_zone(gfp);
998	}
999
1000	static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1001	struct shmem_inode_info *info, pgoff_t index)
1002	{
1003	struct page oldpage, newpage;
1004	struct address_space *swap_mapping;
1005	pgoff_t swap_index;
1006	int error;
1007
1008	oldpage = *pagep;
1009	swap_index = page_private(oldpage);
1010	swap_mapping = page_mapping(oldpage);
1011
1012	/*
1013	* We have arrived here because our zones are constrained, so don't
1014	* limit chance of success by further cpuset and node constraints.
1015	*/
1016	gfp &= ~GFP_CONSTRAINT_MASK;
1017	newpage = shmem_alloc_page(gfp, info, index);
1018	if (!newpage)
1019	return -ENOMEM;
1020
1021	page_cache_get(newpage);
1022	copy_highpage(newpage, oldpage);
1023	flush_dcache_page(newpage);
1024
1025	__set_page_locked(newpage);
1026	SetPageUptodate(newpage);
1027	SetPageSwapBacked(newpage);
1028	set_page_private(newpage, swap_index);
1029	SetPageSwapCache(newpage);
1030
1031	/*
1032	* Our caller will very soon move newpage out of swapcache, but it's
1033	* a nice clean interface for us to replace oldpage by newpage there.
1034	*/
1035	spin_lock_irq(&swap_mapping->tree_lock);
1036	error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1037	newpage);
1038	if (!error) {
1039	__inc_zone_page_state(newpage, NR_FILE_PAGES);
1040	__dec_zone_page_state(oldpage, NR_FILE_PAGES);
1041	}
1042	spin_unlock_irq(&swap_mapping->tree_lock);
1043
1044	if (unlikely(error)) {
1045	/*
1046	* Is this possible? I think not, now that our callers check
1047	* both PageSwapCache and page_private after getting page lock;
1048	* but be defensive. Reverse old to newpage for clear and free.
1049	*/
1050	oldpage = newpage;
1051	} else {
1052	mem_cgroup_replace_page_cache(oldpage, newpage);
1053	lru_cache_add_anon(newpage);
1054	*pagep = newpage;
1055	}
1056
1057	ClearPageSwapCache(oldpage);
1058	set_page_private(oldpage, 0);
1059
1060	unlock_page(oldpage);
1061	page_cache_release(oldpage);
1062	page_cache_release(oldpage);
1063	return error;
1064	}
1065
1066	/*
1067	* shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1068	*
1069	* If we allocate a new one we do not mark it dirty. That's up to the
1070	* vm. If we swap it in we mark it dirty since we also free the swap
1071	* entry since a page cannot live in both the swap and page cache
1072	*/
1073	static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1074	struct page *pagep, enum sgp_type sgp, gfp_t gfp, int fault_type)
1075	{
1076	struct address_space *mapping = inode->i_mapping;
1077	struct shmem_inode_info *info;
1078	struct shmem_sb_info *sbinfo;
1079	struct page *page;
1080	swp_entry_t swap;
1081	int error;
1082	int once = 0;
1083	int alloced = 0;
1084
1085	if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT))
1086	return -EFBIG;
1087	repeat:
1088	swap.val = 0;
1089	page = find_lock_page(mapping, index);
1090	if (radix_tree_exceptional_entry(page)) {
1091	swap = radix_to_swp_entry(page);
1092	page = NULL;
1093	}
1094
1095	if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1096	((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1097	error = -EINVAL;
1098	goto failed;
1099	}
1100
1101	/* fallocated page? */
1102	if (page && !PageUptodate(page)) {
1103	if (sgp != SGP_READ)
1104	goto clear;
1105	unlock_page(page);
1106	page_cache_release(page);
1107	page = NULL;
1108	}
1109	if (page \|\| (sgp == SGP_READ && !swap.val)) {
1110	*pagep = page;
1111	return 0;
1112	}
1113
1114	/*
1115	* Fast cache lookup did not find it:
1116	* bring it back from swap or allocate.
1117	*/
1118	info = SHMEM_I(inode);
1119	sbinfo = SHMEM_SB(inode->i_sb);
1120
1121	if (swap.val) {
1122	/* Look it up and read it in.. */
1123	page = lookup_swap_cache(swap);
1124	if (!page) {
1125	/* here we actually do the io */
1126	if (fault_type)
1127	*fault_type \|= VM_FAULT_MAJOR;
1128	page = shmem_swapin(swap, gfp, info, index);
1129	if (!page) {
1130	error = -ENOMEM;
1131	goto failed;
1132	}
1133	}
1134
1135	/* We have to do this with page locked to prevent races */
1136	lock_page(page);
1137	if (!PageSwapCache(page) \|\| page_private(page) != swap.val \|\|
1138	!shmem_confirm_swap(mapping, index, swap)) {
1139	error = -EEXIST; /* try again */
1140	goto unlock;
1141	}
1142	if (!PageUptodate(page)) {
1143	error = -EIO;
1144	goto failed;
1145	}
1146	wait_on_page_writeback(page);
1147
1148	if (shmem_should_replace_page(page, gfp)) {
1149	error = shmem_replace_page(&page, gfp, info, index);
1150	if (error)
1151	goto failed;
1152	}
1153
1154	error = mem_cgroup_cache_charge(page, current->mm,
1155	gfp & GFP_RECLAIM_MASK);
1156	if (!error) {
1157	error = shmem_add_to_page_cache(page, mapping, index,
1158	gfp, swp_to_radix_entry(swap));
1159	/* We already confirmed swap, and make no allocation */
1160	VM_BUG_ON(error);
1161	}
1162	if (error)
1163	goto failed;
1164
1165	spin_lock(&info->lock);
1166	info->swapped--;
1167	shmem_recalc_inode(inode);
1168	spin_unlock(&info->lock);
1169
1170	delete_from_swap_cache(page);
1171	set_page_dirty(page);
1172	swap_free(swap);
1173
1174	} else {
1175	if (shmem_acct_block(info->flags)) {
1176	error = -ENOSPC;
1177	goto failed;
1178	}
1179	if (sbinfo->max_blocks) {
1180	if (percpu_counter_compare(&sbinfo->used_blocks,
1181	sbinfo->max_blocks) >= 0) {
1182	error = -ENOSPC;
1183	goto unacct;
1184	}
1185	percpu_counter_inc(&sbinfo->used_blocks);
1186	}
1187
1188	page = shmem_alloc_page(gfp, info, index);
1189	if (!page) {
1190	error = -ENOMEM;
1191	goto decused;
1192	}
1193
1194	SetPageSwapBacked(page);
1195	__set_page_locked(page);
1196	error = mem_cgroup_cache_charge(page, current->mm,
1197	gfp & GFP_RECLAIM_MASK);
1198	if (error)
1199	goto decused;
1200	error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
1201	if (!error) {
1202	error = shmem_add_to_page_cache(page, mapping, index,
1203	gfp, NULL);
1204	radix_tree_preload_end();
1205	}
1206	if (error) {
1207	mem_cgroup_uncharge_cache_page(page);
1208	goto decused;
1209	}
1210	lru_cache_add_anon(page);
1211
1212	spin_lock(&info->lock);
1213	info->alloced++;
1214	inode->i_blocks += BLOCKS_PER_PAGE;
1215	shmem_recalc_inode(inode);
1216	spin_unlock(&info->lock);
1217	alloced = true;
1218
1219	/*
1220	* Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1221	*/
1222	if (sgp == SGP_FALLOC)
1223	sgp = SGP_WRITE;
1224	clear:
1225	/*
1226	* Let SGP_WRITE caller clear ends if write does not fill page;
1227	* but SGP_FALLOC on a page fallocated earlier must initialize
1228	* it now, lest undo on failure cancel our earlier guarantee.
1229	*/
1230	if (sgp != SGP_WRITE) {
1231	clear_highpage(page);
1232	flush_dcache_page(page);
1233	SetPageUptodate(page);
1234	}
1235	if (sgp == SGP_DIRTY)
1236	set_page_dirty(page);
1237	}
1238
1239	/* Perhaps the file has been truncated since we checked */
1240	if (sgp != SGP_WRITE && sgp != SGP_FALLOC &&
1241	((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
1242	error = -EINVAL;
1243	if (alloced)
1244	goto trunc;
1245	else
1246	goto failed;
1247	}
1248	*pagep = page;
1249	return 0;
1250
1251	/*
1252	* Error recovery.
1253	*/
1254	trunc:
1255	info = SHMEM_I(inode);
1256	ClearPageDirty(page);
1257	delete_from_page_cache(page);
1258	spin_lock(&info->lock);
1259	info->alloced--;
1260	inode->i_blocks -= BLOCKS_PER_PAGE;
1261	spin_unlock(&info->lock);
1262	decused:
1263	sbinfo = SHMEM_SB(inode->i_sb);
1264	if (sbinfo->max_blocks)
1265	percpu_counter_add(&sbinfo->used_blocks, -1);
1266	unacct:
1267	shmem_unacct_blocks(info->flags, 1);
1268	failed:
1269	if (swap.val && error != -EINVAL &&
1270	!shmem_confirm_swap(mapping, index, swap))
1271	error = -EEXIST;
1272	unlock:
1273	if (page) {
1274	unlock_page(page);
1275	page_cache_release(page);
1276	}
1277	if (error == -ENOSPC && !once++) {
1278	info = SHMEM_I(inode);
1279	spin_lock(&info->lock);
1280	shmem_recalc_inode(inode);
1281	spin_unlock(&info->lock);
1282	goto repeat;
1283	}
1284	if (error == -EEXIST) /* from above or from radix_tree_insert */
1285	goto repeat;
1286	return error;
1287	}
1288
1289	static int shmem_fault(struct vm_area_struct vma, struct vm_fault vmf)
1290	{
1291	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1292	int error;
1293	int ret = VM_FAULT_LOCKED;
1294
1295	error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1296	if (error)
1297	return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1298
1299	if (ret & VM_FAULT_MAJOR) {
1300	count_vm_event(PGMAJFAULT);
1301	mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1302	}
1303	return ret;
1304	}
1305
1306	#ifdef CONFIG_NUMA
1307	static int shmem_set_policy(struct vm_area_struct vma, struct mempolicy mpol)
1308	{
1309	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1310	return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
1311	}
1312
1313	static struct mempolicy shmem_get_policy(struct vm_area_struct vma,
1314	unsigned long addr)
1315	{
1316	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1317	pgoff_t index;
1318
1319	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1320	return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
1321	}
1322	#endif
1323
1324	int shmem_lock(struct file file, int lock, struct user_struct user)
1325	{
1326	struct inode *inode = file->f_path.dentry->d_inode;
1327	struct shmem_inode_info *info = SHMEM_I(inode);
1328	int retval = -ENOMEM;
1329
1330	spin_lock(&info->lock);
1331	if (lock && !(info->flags & VM_LOCKED)) {
1332	if (!user_shm_lock(inode->i_size, user))
1333	goto out_nomem;
1334	info->flags \|= VM_LOCKED;
1335	mapping_set_unevictable(file->f_mapping);
1336	}
1337	if (!lock && (info->flags & VM_LOCKED) && user) {
1338	user_shm_unlock(inode->i_size, user);
1339	info->flags &= ~VM_LOCKED;
1340	mapping_clear_unevictable(file->f_mapping);
1341	}
1342	retval = 0;
1343
1344	out_nomem:
1345	spin_unlock(&info->lock);
1346	return retval;
1347	}
1348
1349	static int shmem_mmap(struct file file, struct vm_area_struct vma)
1350	{
1351	file_accessed(file);
1352	vma->vm_ops = &shmem_vm_ops;
1353	vma->vm_flags \|= VM_CAN_NONLINEAR;
1354	return 0;
1355	}
1356
1357	static struct inode shmem_get_inode(struct super_block sb, const struct inode *dir,
1358	umode_t mode, dev_t dev, unsigned long flags)
1359	{
1360	struct inode *inode;
1361	struct shmem_inode_info *info;
1362	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1363
1364	if (shmem_reserve_inode(sb))
1365	return NULL;
1366
1367	inode = new_inode(sb);
1368	if (inode) {
1369	inode->i_ino = get_next_ino();
1370	inode_init_owner(inode, dir, mode);
1371	inode->i_blocks = 0;
1372	inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1373	inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1374	inode->i_generation = get_seconds();
1375	info = SHMEM_I(inode);
1376	memset(info, 0, (char )inode - (char )info);
1377	spin_lock_init(&info->lock);
1378	info->flags = flags & VM_NORESERVE;
1379	INIT_LIST_HEAD(&info->swaplist);
1380	INIT_LIST_HEAD(&info->xattr_list);
1381	cache_no_acl(inode);
1382
1383	switch (mode & S_IFMT) {
1384	default:
1385	inode->i_op = &shmem_special_inode_operations;
1386	init_special_inode(inode, mode, dev);
1387	break;
1388	case S_IFREG:
1389	inode->i_mapping->a_ops = &shmem_aops;
1390	inode->i_op = &shmem_inode_operations;
1391	inode->i_fop = &shmem_file_operations;
1392	mpol_shared_policy_init(&info->policy,
1393	shmem_get_sbmpol(sbinfo));
1394	break;
1395	case S_IFDIR:
1396	inc_nlink(inode);
1397	/* Some things misbehave if size == 0 on a directory */
1398	inode->i_size = 2 * BOGO_DIRENT_SIZE;
1399	inode->i_op = &shmem_dir_inode_operations;
1400	inode->i_fop = &simple_dir_operations;
1401	break;
1402	case S_IFLNK:
1403	/*
1404	* Must not load anything in the rbtree,
1405	* mpol_free_shared_policy will not be called.
1406	*/
1407	mpol_shared_policy_init(&info->policy, NULL);
1408	break;
1409	}
1410	} else
1411	shmem_free_inode(sb);
1412	return inode;
1413	}
1414
1415	#ifdef CONFIG_TMPFS
1416	static const struct inode_operations shmem_symlink_inode_operations;
1417	static const struct inode_operations shmem_short_symlink_operations;
1418
1419	#ifdef CONFIG_TMPFS_XATTR
1420	static int shmem_initxattrs(struct inode , const struct xattr , void *);
1421	#else
1422	#define shmem_initxattrs NULL
1423	#endif
1424
1425	static int
1426	shmem_write_begin(struct file file, struct address_space mapping,
1427	loff_t pos, unsigned len, unsigned flags,
1428	struct page pagep, void fsdata)
1429	{
1430	struct inode *inode = mapping->host;
1431	pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1432	return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1433	}
1434
1435	static int
1436	shmem_write_end(struct file file, struct address_space mapping,
1437	loff_t pos, unsigned len, unsigned copied,
1438	struct page page, void fsdata)
1439	{
1440	struct inode *inode = mapping->host;
1441
1442	if (pos + copied > inode->i_size)
1443	i_size_write(inode, pos + copied);
1444
1445	if (!PageUptodate(page)) {
1446	if (copied < PAGE_CACHE_SIZE) {
1447	unsigned from = pos & (PAGE_CACHE_SIZE - 1);
1448	zero_user_segments(page, 0, from,
1449	from + copied, PAGE_CACHE_SIZE);
1450	}
1451	SetPageUptodate(page);
1452	}
1453	set_page_dirty(page);
1454	unlock_page(page);
1455	page_cache_release(page);
1456
1457	return copied;
1458	}
1459
1460	static void do_shmem_file_read(struct file filp, loff_t ppos, read_descriptor_t *desc, read_actor_t actor)
1461	{
1462	struct inode *inode = filp->f_path.dentry->d_inode;
1463	struct address_space *mapping = inode->i_mapping;
1464	pgoff_t index;
1465	unsigned long offset;
1466	enum sgp_type sgp = SGP_READ;
1467
1468	/*
1469	* Might this read be for a stacking filesystem? Then when reading
1470	* holes of a sparse file, we actually need to allocate those pages,
1471	* and even mark them dirty, so it cannot exceed the max_blocks limit.
1472	*/
1473	if (segment_eq(get_fs(), KERNEL_DS))
1474	sgp = SGP_DIRTY;
1475
1476	index = *ppos >> PAGE_CACHE_SHIFT;
1477	offset = *ppos & ~PAGE_CACHE_MASK;
1478
1479	for (;;) {
1480	struct page *page = NULL;
1481	pgoff_t end_index;
1482	unsigned long nr, ret;
1483	loff_t i_size = i_size_read(inode);
1484
1485	end_index = i_size >> PAGE_CACHE_SHIFT;
1486	if (index > end_index)
1487	break;
1488	if (index == end_index) {
1489	nr = i_size & ~PAGE_CACHE_MASK;
1490	if (nr <= offset)
1491	break;
1492	}
1493
1494	desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1495	if (desc->error) {
1496	if (desc->error == -EINVAL)
1497	desc->error = 0;
1498	break;
1499	}
1500	if (page)
1501	unlock_page(page);
1502
1503	/*
1504	* We must evaluate after, since reads (unlike writes)
1505	* are called without i_mutex protection against truncate
1506	*/
1507	nr = PAGE_CACHE_SIZE;
1508	i_size = i_size_read(inode);
1509	end_index = i_size >> PAGE_CACHE_SHIFT;
1510	if (index == end_index) {
1511	nr = i_size & ~PAGE_CACHE_MASK;
1512	if (nr <= offset) {
1513	if (page)
1514	page_cache_release(page);
1515	break;
1516	}
1517	}
1518	nr -= offset;
1519
1520	if (page) {
1521	/*
1522	* If users can be writing to this page using arbitrary
1523	* virtual addresses, take care about potential aliasing
1524	* before reading the page on the kernel side.
1525	*/
1526	if (mapping_writably_mapped(mapping))
1527	flush_dcache_page(page);
1528	/*
1529	* Mark the page accessed if we read the beginning.
1530	*/
1531	if (!offset)
1532	mark_page_accessed(page);
1533	} else {
1534	page = ZERO_PAGE(0);
1535	page_cache_get(page);
1536	}
1537
1538	/*
1539	* Ok, we have the page, and it's up-to-date, so
1540	* now we can copy it to user space...
1541	*
1542	* The actor routine returns how many bytes were actually used..
1543	* NOTE! This may not be the same as how much of a user buffer
1544	* we filled up (we may be padding etc), so we can only update
1545	* "pos" here (the actor routine has to update the user buffer
1546	* pointers and the remaining count).
1547	*/
1548	ret = actor(desc, page, offset, nr);
1549	offset += ret;
1550	index += offset >> PAGE_CACHE_SHIFT;
1551	offset &= ~PAGE_CACHE_MASK;
1552
1553	page_cache_release(page);
1554	if (ret != nr \|\| !desc->count)
1555	break;
1556
1557	cond_resched();
1558	}
1559
1560	*ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1561	file_accessed(filp);
1562	}
1563
1564	static ssize_t shmem_file_aio_read(struct kiocb *iocb,
1565	const struct iovec *iov, unsigned long nr_segs, loff_t pos)
1566	{
1567	struct file *filp = iocb->ki_filp;
1568	ssize_t retval;
1569	unsigned long seg;
1570	size_t count;
1571	loff_t *ppos = &iocb->ki_pos;
1572
1573	retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
1574	if (retval)
1575	return retval;
1576
1577	for (seg = 0; seg < nr_segs; seg++) {
1578	read_descriptor_t desc;
1579
1580	desc.written = 0;
1581	desc.arg.buf = iov[seg].iov_base;
1582	desc.count = iov[seg].iov_len;
1583	if (desc.count == 0)
1584	continue;
1585	desc.error = 0;
1586	do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1587	retval += desc.written;
1588	if (desc.error) {
1589	retval = retval ?: desc.error;
1590	break;
1591	}
1592	if (desc.count > 0)
1593	break;
1594	}
1595	return retval;
1596	}
1597
1598	static ssize_t shmem_file_splice_read(struct file in, loff_t ppos,
1599	struct pipe_inode_info *pipe, size_t len,
1600	unsigned int flags)
1601	{
1602	struct address_space *mapping = in->f_mapping;
1603	struct inode *inode = mapping->host;
1604	unsigned int loff, nr_pages, req_pages;
1605	struct page *pages[PIPE_DEF_BUFFERS];
1606	struct partial_page partial[PIPE_DEF_BUFFERS];
1607	struct page *page;
1608	pgoff_t index, end_index;
1609	loff_t isize, left;
1610	int error, page_nr;
1611	struct splice_pipe_desc spd = {
1612	.pages = pages,
1613	.partial = partial,
1614	.nr_pages_max = PIPE_DEF_BUFFERS,
1615	.flags = flags,
1616	.ops = &page_cache_pipe_buf_ops,
1617	.spd_release = spd_release_page,
1618	};
1619
1620	isize = i_size_read(inode);
1621	if (unlikely(*ppos >= isize))
1622	return 0;
1623
1624	left = isize - *ppos;
1625	if (unlikely(left < len))
1626	len = left;
1627
1628	if (splice_grow_spd(pipe, &spd))
1629	return -ENOMEM;
1630
1631	index = *ppos >> PAGE_CACHE_SHIFT;
1632	loff = *ppos & ~PAGE_CACHE_MASK;
1633	req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1634	nr_pages = min(req_pages, pipe->buffers);
1635
1636	spd.nr_pages = find_get_pages_contig(mapping, index,
1637	nr_pages, spd.pages);
1638	index += spd.nr_pages;
1639	error = 0;
1640
1641	while (spd.nr_pages < nr_pages) {
1642	error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);
1643	if (error)
1644	break;
1645	unlock_page(page);
1646	spd.pages[spd.nr_pages++] = page;
1647	index++;
1648	}
1649
1650	index = *ppos >> PAGE_CACHE_SHIFT;
1651	nr_pages = spd.nr_pages;
1652	spd.nr_pages = 0;
1653
1654	for (page_nr = 0; page_nr < nr_pages; page_nr++) {
1655	unsigned int this_len;
1656
1657	if (!len)
1658	break;
1659
1660	this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
1661	page = spd.pages[page_nr];
1662
1663	if (!PageUptodate(page) \|\| page->mapping != mapping) {
1664	error = shmem_getpage(inode, index, &page,
1665	SGP_CACHE, NULL);
1666	if (error)
1667	break;
1668	unlock_page(page);
1669	page_cache_release(spd.pages[page_nr]);
1670	spd.pages[page_nr] = page;
1671	}
1672
1673	isize = i_size_read(inode);
1674	end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
1675	if (unlikely(!isize \|\| index > end_index))
1676	break;
1677
1678	if (end_index == index) {
1679	unsigned int plen;
1680
1681	plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
1682	if (plen <= loff)
1683	break;
1684
1685	this_len = min(this_len, plen - loff);
1686	len = this_len;
1687	}
1688
1689	spd.partial[page_nr].offset = loff;
1690	spd.partial[page_nr].len = this_len;
1691	len -= this_len;
1692	loff = 0;
1693	spd.nr_pages++;
1694	index++;
1695	}
1696
1697	while (page_nr < nr_pages)
1698	page_cache_release(spd.pages[page_nr++]);
1699
1700	if (spd.nr_pages)
1701	error = splice_to_pipe(pipe, &spd);
1702
1703	splice_shrink_spd(&spd);
1704
1705	if (error > 0) {
1706	*ppos += error;
1707	file_accessed(in);
1708	}
1709	return error;
1710	}
1711
1712	static long shmem_fallocate(struct file *file, int mode, loff_t offset,
1713	loff_t len)
1714	{
1715	struct inode *inode = file->f_path.dentry->d_inode;
1716	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1717	struct shmem_falloc shmem_falloc;
1718	pgoff_t start, index, end;
1719	int error;
1720
1721	mutex_lock(&inode->i_mutex);
1722
1723	if (mode & FALLOC_FL_PUNCH_HOLE) {
1724	struct address_space *mapping = file->f_mapping;
1725	loff_t unmap_start = round_up(offset, PAGE_SIZE);
1726	loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
1727
1728	if ((u64)unmap_end > (u64)unmap_start)
1729	unmap_mapping_range(mapping, unmap_start,
1730	1 + unmap_end - unmap_start, 0);
1731	shmem_truncate_range(inode, offset, offset + len - 1);
1732	/* No need to unmap again: hole-punching leaves COWed pages */
1733	error = 0;
1734	goto out;
1735	}
1736
1737	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
1738	error = inode_newsize_ok(inode, offset + len);
1739	if (error)
1740	goto out;
1741
1742	start = offset >> PAGE_CACHE_SHIFT;
1743	end = (offset + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1744	/* Try to avoid a swapstorm if len is impossible to satisfy */
1745	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
1746	error = -ENOSPC;
1747	goto out;
1748	}
1749
1750	shmem_falloc.start = start;
1751	shmem_falloc.next = start;
1752	shmem_falloc.nr_falloced = 0;
1753	shmem_falloc.nr_unswapped = 0;
1754	spin_lock(&inode->i_lock);
1755	inode->i_private = &shmem_falloc;
1756	spin_unlock(&inode->i_lock);
1757
1758	for (index = start; index < end; index++) {
1759	struct page *page;
1760
1761	/*
1762	* Good, the fallocate(2) manpage permits EINTR: we may have
1763	* been interrupted because we are using up too much memory.
1764	*/
1765	if (signal_pending(current))
1766	error = -EINTR;
1767	else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
1768	error = -ENOMEM;
1769	else
1770	error = shmem_getpage(inode, index, &page, SGP_FALLOC,
1771	NULL);
1772	if (error) {
1773	/* Remove the !PageUptodate pages we added */
1774	shmem_undo_range(inode,
1775	(loff_t)start << PAGE_CACHE_SHIFT,
1776	(loff_t)index << PAGE_CACHE_SHIFT, true);
1777	goto undone;
1778	}
1779
1780	/*
1781	* Inform shmem_writepage() how far we have reached.
1782	* No need for lock or barrier: we have the page lock.
1783	*/
1784	shmem_falloc.next++;
1785	if (!PageUptodate(page))
1786	shmem_falloc.nr_falloced++;
1787
1788	/*
1789	* If !PageUptodate, leave it that way so that freeable pages
1790	* can be recognized if we need to rollback on error later.
1791	* But set_page_dirty so that memory pressure will swap rather
1792	* than free the pages we are allocating (and SGP_CACHE pages
1793	* might still be clean: we now need to mark those dirty too).
1794	*/
1795	set_page_dirty(page);
1796	unlock_page(page);
1797	page_cache_release(page);
1798	cond_resched();
1799	}
1800
1801	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
1802	i_size_write(inode, offset + len);
1803	inode->i_ctime = CURRENT_TIME;
1804	undone:
1805	spin_lock(&inode->i_lock);
1806	inode->i_private = NULL;
1807	spin_unlock(&inode->i_lock);
1808	out:
1809	mutex_unlock(&inode->i_mutex);
1810	return error;
1811	}
1812
1813	static int shmem_statfs(struct dentry dentry, struct kstatfs buf)
1814	{
1815	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1816
1817	buf->f_type = TMPFS_MAGIC;
1818	buf->f_bsize = PAGE_CACHE_SIZE;
1819	buf->f_namelen = NAME_MAX;
1820	if (sbinfo->max_blocks) {
1821	buf->f_blocks = sbinfo->max_blocks;
1822	buf->f_bavail =
1823	buf->f_bfree = sbinfo->max_blocks -
1824	percpu_counter_sum(&sbinfo->used_blocks);
1825	}
1826	if (sbinfo->max_inodes) {
1827	buf->f_files = sbinfo->max_inodes;
1828	buf->f_ffree = sbinfo->free_inodes;
1829	}
1830	/* else leave those fields 0 like simple_statfs */
1831	return 0;
1832	}
1833
1834	/*
1835	* File creation. Allocate an inode, and we're done..
1836	*/
1837	static int
1838	shmem_mknod(struct inode dir, struct dentry dentry, umode_t mode, dev_t dev)
1839	{
1840	struct inode *inode;
1841	int error = -ENOSPC;
1842
1843	inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1844	if (inode) {
1845	error = security_inode_init_security(inode, dir,
1846	&dentry->d_name,
1847	shmem_initxattrs, NULL);
1848	if (error) {
1849	if (error != -EOPNOTSUPP) {
1850	iput(inode);
1851	return error;
1852	}
1853	}
1854	#ifdef CONFIG_TMPFS_POSIX_ACL
1855	error = generic_acl_init(inode, dir);
1856	if (error) {
1857	iput(inode);
1858	return error;
1859	}
1860	#else
1861	error = 0;
1862	#endif
1863	dir->i_size += BOGO_DIRENT_SIZE;
1864	dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1865	d_instantiate(dentry, inode);
1866	dget(dentry); /* Extra count - pin the dentry in core */
1867	}
1868	return error;
1869	}
1870
1871	static int shmem_mkdir(struct inode dir, struct dentry dentry, umode_t mode)
1872	{
1873	int error;
1874
1875	if ((error = shmem_mknod(dir, dentry, mode \| S_IFDIR, 0)))
1876	return error;
1877	inc_nlink(dir);
1878	return 0;
1879	}
1880
1881	static int shmem_create(struct inode dir, struct dentry dentry, umode_t mode,
1882	bool excl)
1883	{
1884	return shmem_mknod(dir, dentry, mode \| S_IFREG, 0);
1885	}
1886
1887	/*
1888	* Link a file..
1889	*/
1890	static int shmem_link(struct dentry old_dentry, struct inode dir, struct dentry *dentry)
1891	{
1892	struct inode *inode = old_dentry->d_inode;
1893	int ret;
1894
1895	/*
1896	* No ordinary (disk based) filesystem counts links as inodes;
1897	* but each new link needs a new dentry, pinning lowmem, and
1898	* tmpfs dentries cannot be pruned until they are unlinked.
1899	*/
1900	ret = shmem_reserve_inode(inode->i_sb);
1901	if (ret)
1902	goto out;
1903
1904	dir->i_size += BOGO_DIRENT_SIZE;
1905	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1906	inc_nlink(inode);
1907	ihold(inode); /* New dentry reference */
1908	dget(dentry); /* Extra pinning count for the created dentry */
1909	d_instantiate(dentry, inode);
1910	out:
1911	return ret;
1912	}
1913
1914	static int shmem_unlink(struct inode dir, struct dentry dentry)
1915	{
1916	struct inode *inode = dentry->d_inode;
1917
1918	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1919	shmem_free_inode(inode->i_sb);
1920
1921	dir->i_size -= BOGO_DIRENT_SIZE;
1922	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1923	drop_nlink(inode);
1924	dput(dentry); /* Undo the count from "create" - this does all the work */
1925	return 0;
1926	}
1927
1928	static int shmem_rmdir(struct inode dir, struct dentry dentry)
1929	{
1930	if (!simple_empty(dentry))
1931	return -ENOTEMPTY;
1932
1933	drop_nlink(dentry->d_inode);
1934	drop_nlink(dir);
1935	return shmem_unlink(dir, dentry);
1936	}
1937
1938	/*
1939	* The VFS layer already does all the dentry stuff for rename,
1940	* we just have to decrement the usage count for the target if
1941	* it exists so that the VFS layer correctly free's it when it
1942	* gets overwritten.
1943	*/
1944	static int shmem_rename(struct inode old_dir, struct dentry old_dentry, struct inode new_dir, struct dentry new_dentry)
1945	{
1946	struct inode *inode = old_dentry->d_inode;
1947	int they_are_dirs = S_ISDIR(inode->i_mode);
1948
1949	if (!simple_empty(new_dentry))
1950	return -ENOTEMPTY;
1951
1952	if (new_dentry->d_inode) {
1953	(void) shmem_unlink(new_dir, new_dentry);
1954	if (they_are_dirs)
1955	drop_nlink(old_dir);
1956	} else if (they_are_dirs) {
1957	drop_nlink(old_dir);
1958	inc_nlink(new_dir);
1959	}
1960
1961	old_dir->i_size -= BOGO_DIRENT_SIZE;
1962	new_dir->i_size += BOGO_DIRENT_SIZE;
1963	old_dir->i_ctime = old_dir->i_mtime =
1964	new_dir->i_ctime = new_dir->i_mtime =
1965	inode->i_ctime = CURRENT_TIME;
1966	return 0;
1967	}
1968
1969	static int shmem_symlink(struct inode dir, struct dentry dentry, const char *symname)
1970	{
1971	int error;
1972	int len;
1973	struct inode *inode;
1974	struct page *page;
1975	char *kaddr;
1976	struct shmem_inode_info *info;
1977
1978	len = strlen(symname) + 1;
1979	if (len > PAGE_CACHE_SIZE)
1980	return -ENAMETOOLONG;
1981
1982	inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK\|S_IRWXUGO, 0, VM_NORESERVE);
1983	if (!inode)
1984	return -ENOSPC;
1985
1986	error = security_inode_init_security(inode, dir, &dentry->d_name,
1987	shmem_initxattrs, NULL);
1988	if (error) {
1989	if (error != -EOPNOTSUPP) {
1990	iput(inode);
1991	return error;
1992	}
1993	error = 0;
1994	}
1995
1996	info = SHMEM_I(inode);
1997	inode->i_size = len-1;
1998	if (len <= SHORT_SYMLINK_LEN) {
1999	info->symlink = kmemdup(symname, len, GFP_KERNEL);
2000	if (!info->symlink) {
2001	iput(inode);
2002	return -ENOMEM;
2003	}
2004	inode->i_op = &shmem_short_symlink_operations;
2005	} else {
2006	error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
2007	if (error) {
2008	iput(inode);
2009	return error;
2010	}
2011	inode->i_mapping->a_ops = &shmem_aops;
2012	inode->i_op = &shmem_symlink_inode_operations;
2013	kaddr = kmap_atomic(page);
2014	memcpy(kaddr, symname, len);
2015	kunmap_atomic(kaddr);
2016	SetPageUptodate(page);
2017	set_page_dirty(page);
2018	unlock_page(page);
2019	page_cache_release(page);
2020	}
2021	dir->i_size += BOGO_DIRENT_SIZE;
2022	dir->i_ctime = dir->i_mtime = CURRENT_TIME;
2023	d_instantiate(dentry, inode);
2024	dget(dentry);
2025	return 0;
2026	}
2027
2028	static void shmem_follow_short_symlink(struct dentry dentry, struct nameidata *nd)
2029	{
2030	nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink);
2031	return NULL;
2032	}
2033
2034	static void shmem_follow_link(struct dentry dentry, struct nameidata *nd)
2035	{
2036	struct page *page = NULL;
2037	int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
2038	nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));
2039	if (page)
2040	unlock_page(page);
2041	return page;
2042	}
2043
2044	static void shmem_put_link(struct dentry dentry, struct nameidata nd, void *cookie)
2045	{
2046	if (!IS_ERR(nd_get_link(nd))) {
2047	struct page *page = cookie;
2048	kunmap(page);
2049	mark_page_accessed(page);
2050	page_cache_release(page);
2051	}
2052	}
2053
2054	#ifdef CONFIG_TMPFS_XATTR
2055	/*
2056	* Superblocks without xattr inode operations may get some security.* xattr
2057	* support from the LSM "for free". As soon as we have any other xattrs
2058	* like ACLs, we also need to implement the security.* handlers at
2059	* filesystem level, though.
2060	*/
2061
2062	/*
2063	* Allocate new xattr and copy in the value; but leave the name to callers.
2064	*/
2065	static struct shmem_xattr shmem_xattr_alloc(const void value, size_t size)
2066	{
2067	struct shmem_xattr *new_xattr;
2068	size_t len;
2069
2070	/* wrap around? */
2071	len = sizeof(*new_xattr) + size;
2072	if (len <= sizeof(*new_xattr))
2073	return NULL;
2074
2075	new_xattr = kmalloc(len, GFP_KERNEL);
2076	if (!new_xattr)
2077	return NULL;
2078
2079	new_xattr->size = size;
2080	memcpy(new_xattr->value, value, size);
2081	return new_xattr;
2082	}
2083
2084	/*
2085	* Callback for security_inode_init_security() for acquiring xattrs.
2086	*/
2087	static int shmem_initxattrs(struct inode *inode,
2088	const struct xattr *xattr_array,
2089	void *fs_info)
2090	{
2091	struct shmem_inode_info *info = SHMEM_I(inode);
2092	const struct xattr *xattr;
2093	struct shmem_xattr *new_xattr;
2094	size_t len;
2095
2096	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
2097	new_xattr = shmem_xattr_alloc(xattr->value, xattr->value_len);
2098	if (!new_xattr)
2099	return -ENOMEM;
2100
2101	len = strlen(xattr->name) + 1;
2102	new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
2103	GFP_KERNEL);
2104	if (!new_xattr->name) {
2105	kfree(new_xattr);
2106	return -ENOMEM;
2107	}
2108
2109	memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
2110	XATTR_SECURITY_PREFIX_LEN);
2111	memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
2112	xattr->name, len);
2113
2114	spin_lock(&info->lock);
2115	list_add(&new_xattr->list, &info->xattr_list);
2116	spin_unlock(&info->lock);
2117	}
2118
2119	return 0;
2120	}
2121
2122	static int shmem_xattr_get(struct dentry dentry, const char name,
2123	void *buffer, size_t size)
2124	{
2125	struct shmem_inode_info *info;
2126	struct shmem_xattr *xattr;
2127	int ret = -ENODATA;
2128
2129	info = SHMEM_I(dentry->d_inode);
2130
2131	spin_lock(&info->lock);
2132	list_for_each_entry(xattr, &info->xattr_list, list) {
2133	if (strcmp(name, xattr->name))
2134	continue;
2135
2136	ret = xattr->size;
2137	if (buffer) {
2138	if (size < xattr->size)
2139	ret = -ERANGE;
2140	else
2141	memcpy(buffer, xattr->value, xattr->size);
2142	}
2143	break;
2144	}
2145	spin_unlock(&info->lock);
2146	return ret;
2147	}
2148
2149	static int shmem_xattr_set(struct inode inode, const char name,
2150	const void *value, size_t size, int flags)
2151	{
2152	struct shmem_inode_info *info = SHMEM_I(inode);
2153	struct shmem_xattr *xattr;
2154	struct shmem_xattr *new_xattr = NULL;
2155	int err = 0;
2156
2157	/* value == NULL means remove */
2158	if (value) {
2159	new_xattr = shmem_xattr_alloc(value, size);
2160	if (!new_xattr)
2161	return -ENOMEM;
2162
2163	new_xattr->name = kstrdup(name, GFP_KERNEL);
2164	if (!new_xattr->name) {
2165	kfree(new_xattr);
2166	return -ENOMEM;
2167	}
2168	}
2169
2170	spin_lock(&info->lock);
2171	list_for_each_entry(xattr, &info->xattr_list, list) {
2172	if (!strcmp(name, xattr->name)) {
2173	if (flags & XATTR_CREATE) {
2174	xattr = new_xattr;
2175	err = -EEXIST;
2176	} else if (new_xattr) {
2177	list_replace(&xattr->list, &new_xattr->list);
2178	} else {
2179	list_del(&xattr->list);
2180	}
2181	goto out;
2182	}
2183	}
2184	if (flags & XATTR_REPLACE) {
2185	xattr = new_xattr;
2186	err = -ENODATA;
2187	} else {
2188	list_add(&new_xattr->list, &info->xattr_list);
2189	xattr = NULL;
2190	}
2191	out:
2192	spin_unlock(&info->lock);
2193	if (xattr)
2194	kfree(xattr->name);
2195	kfree(xattr);
2196	return err;
2197	}
2198
2199	static const struct xattr_handler *shmem_xattr_handlers[] = {
2200	#ifdef CONFIG_TMPFS_POSIX_ACL
2201	&generic_acl_access_handler,
2202	&generic_acl_default_handler,
2203	#endif
2204	NULL
2205	};
2206
2207	static int shmem_xattr_validate(const char *name)
2208	{
2209	struct { const char *prefix; size_t len; } arr[] = {
2210	{ XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN },
2211	{ XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN }
2212	};
2213	int i;
2214
2215	for (i = 0; i < ARRAY_SIZE(arr); i++) {
2216	size_t preflen = arr[i].len;
2217	if (strncmp(name, arr[i].prefix, preflen) == 0) {
2218	if (!name[preflen])
2219	return -EINVAL;
2220	return 0;
2221	}
2222	}
2223	return -EOPNOTSUPP;
2224	}
2225
2226	static ssize_t shmem_getxattr(struct dentry dentry, const char name,
2227	void *buffer, size_t size)
2228	{
2229	int err;
2230
2231	/*
2232	* If this is a request for a synthetic attribute in the system.*
2233	* namespace use the generic infrastructure to resolve a handler
2234	* for it via sb->s_xattr.
2235	*/
2236	if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2237	return generic_getxattr(dentry, name, buffer, size);
2238
2239	err = shmem_xattr_validate(name);
2240	if (err)
2241	return err;
2242
2243	return shmem_xattr_get(dentry, name, buffer, size);
2244	}
2245
2246	static int shmem_setxattr(struct dentry dentry, const char name,
2247	const void *value, size_t size, int flags)
2248	{
2249	int err;
2250
2251	/*
2252	* If this is a request for a synthetic attribute in the system.*
2253	* namespace use the generic infrastructure to resolve a handler
2254	* for it via sb->s_xattr.
2255	*/
2256	if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2257	return generic_setxattr(dentry, name, value, size, flags);
2258
2259	err = shmem_xattr_validate(name);
2260	if (err)
2261	return err;
2262
2263	if (size == 0)
2264	value = ""; /* empty EA, do not remove */
2265
2266	return shmem_xattr_set(dentry->d_inode, name, value, size, flags);
2267
2268	}
2269
2270	static int shmem_removexattr(struct dentry dentry, const char name)
2271	{
2272	int err;
2273
2274	/*
2275	* If this is a request for a synthetic attribute in the system.*
2276	* namespace use the generic infrastructure to resolve a handler
2277	* for it via sb->s_xattr.
2278	*/
2279	if (!strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN))
2280	return generic_removexattr(dentry, name);
2281
2282	err = shmem_xattr_validate(name);
2283	if (err)
2284	return err;
2285
2286	return shmem_xattr_set(dentry->d_inode, name, NULL, 0, XATTR_REPLACE);
2287	}
2288
2289	static bool xattr_is_trusted(const char *name)
2290	{
2291	return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN);
2292	}
2293
2294	static ssize_t shmem_listxattr(struct dentry dentry, char buffer, size_t size)
2295	{
2296	bool trusted = capable(CAP_SYS_ADMIN);
2297	struct shmem_xattr *xattr;
2298	struct shmem_inode_info *info;
2299	size_t used = 0;
2300
2301	info = SHMEM_I(dentry->d_inode);
2302
2303	spin_lock(&info->lock);
2304	list_for_each_entry(xattr, &info->xattr_list, list) {
2305	size_t len;
2306
2307	/* skip "trusted." attributes for unprivileged callers */
2308	if (!trusted && xattr_is_trusted(xattr->name))
2309	continue;
2310
2311	len = strlen(xattr->name) + 1;
2312	used += len;
2313	if (buffer) {
2314	if (size < used) {
2315	used = -ERANGE;
2316	break;
2317	}
2318	memcpy(buffer, xattr->name, len);
2319	buffer += len;
2320	}
2321	}
2322	spin_unlock(&info->lock);
2323
2324	return used;
2325	}
2326	#endif /* CONFIG_TMPFS_XATTR */
2327
2328	static const struct inode_operations shmem_short_symlink_operations = {
2329	.readlink = generic_readlink,
2330	.follow_link = shmem_follow_short_symlink,
2331	#ifdef CONFIG_TMPFS_XATTR
2332	.setxattr = shmem_setxattr,
2333	.getxattr = shmem_getxattr,
2334	.listxattr = shmem_listxattr,
2335	.removexattr = shmem_removexattr,
2336	#endif
2337	};
2338
2339	static const struct inode_operations shmem_symlink_inode_operations = {
2340	.readlink = generic_readlink,
2341	.follow_link = shmem_follow_link,
2342	.put_link = shmem_put_link,
2343	#ifdef CONFIG_TMPFS_XATTR
2344	.setxattr = shmem_setxattr,
2345	.getxattr = shmem_getxattr,
2346	.listxattr = shmem_listxattr,
2347	.removexattr = shmem_removexattr,
2348	#endif
2349	};
2350
2351	static struct dentry shmem_get_parent(struct dentry child)
2352	{
2353	return ERR_PTR(-ESTALE);
2354	}
2355
2356	static int shmem_match(struct inode ino, void vfh)
2357	{
2358	__u32 *fh = vfh;
2359	__u64 inum = fh[2];
2360	inum = (inum << 32) \| fh[1];
2361	return ino->i_ino == inum && fh[0] == ino->i_generation;
2362	}
2363
2364	static struct dentry shmem_fh_to_dentry(struct super_block sb,
2365	struct fid *fid, int fh_len, int fh_type)
2366	{
2367	struct inode *inode;
2368	struct dentry *dentry = NULL;
2369	u64 inum = fid->raw[2];
2370	inum = (inum << 32) \| fid->raw[1];
2371
2372	if (fh_len < 3)
2373	return NULL;
2374
2375	inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2376	shmem_match, fid->raw);
2377	if (inode) {
2378	dentry = d_find_alias(inode);
2379	iput(inode);
2380	}
2381
2382	return dentry;
2383	}
2384
2385	static int shmem_encode_fh(struct inode inode, __u32 fh, int *len,
2386	struct inode *parent)
2387	{
2388	if (*len < 3) {
2389	*len = 3;
2390	return 255;
2391	}
2392
2393	if (inode_unhashed(inode)) {
2394	/* Unfortunately insert_inode_hash is not idempotent,
2395	* so as we hash inodes here rather than at creation
2396	* time, we need a lock to ensure we only try
2397	* to do it once
2398	*/
2399	static DEFINE_SPINLOCK(lock);
2400	spin_lock(&lock);
2401	if (inode_unhashed(inode))
2402	__insert_inode_hash(inode,
2403	inode->i_ino + inode->i_generation);
2404	spin_unlock(&lock);
2405	}
2406
2407	fh[0] = inode->i_generation;
2408	fh[1] = inode->i_ino;
2409	fh[2] = ((__u64)inode->i_ino) >> 32;
2410
2411	*len = 3;
2412	return 1;
2413	}
2414
2415	static const struct export_operations shmem_export_ops = {
2416	.get_parent = shmem_get_parent,
2417	.encode_fh = shmem_encode_fh,
2418	.fh_to_dentry = shmem_fh_to_dentry,
2419	};
2420
2421	static int shmem_parse_options(char options, struct shmem_sb_info sbinfo,
2422	bool remount)
2423	{
2424	char this_char, value, *rest;
2425	uid_t uid;
2426	gid_t gid;
2427
2428	while (options != NULL) {
2429	this_char = options;
2430	for (;;) {
2431	/*
2432	* NUL-terminate this option: unfortunately,
2433	* mount options form a comma-separated list,
2434	* but mpol's nodelist may also contain commas.
2435	*/
2436	options = strchr(options, ',');
2437	if (options == NULL)
2438	break;
2439	options++;
2440	if (!isdigit(*options)) {
2441	options[-1] = '\0';
2442	break;
2443	}
2444	}
2445	if (!*this_char)
2446	continue;
2447	if ((value = strchr(this_char,'=')) != NULL) {
2448	*value++ = 0;
2449	} else {
2450	printk(KERN_ERR
2451	"tmpfs: No value for mount option '%s'\n",
2452	this_char);
2453	return 1;
2454	}
2455
2456	if (!strcmp(this_char,"size")) {
2457	unsigned long long size;
2458	size = memparse(value,&rest);
2459	if (*rest == '%') {
2460	size <<= PAGE_SHIFT;
2461	size *= totalram_pages;
2462	do_div(size, 100);
2463	rest++;
2464	}
2465	if (*rest)
2466	goto bad_val;
2467	sbinfo->max_blocks =
2468	DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2469	} else if (!strcmp(this_char,"nr_blocks")) {
2470	sbinfo->max_blocks = memparse(value, &rest);
2471	if (*rest)
2472	goto bad_val;
2473	} else if (!strcmp(this_char,"nr_inodes")) {
2474	sbinfo->max_inodes = memparse(value, &rest);
2475	if (*rest)
2476	goto bad_val;
2477	} else if (!strcmp(this_char,"mode")) {
2478	if (remount)
2479	continue;
2480	sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2481	if (*rest)
2482	goto bad_val;
2483	} else if (!strcmp(this_char,"uid")) {
2484	if (remount)
2485	continue;
2486	uid = simple_strtoul(value, &rest, 0);
2487	if (*rest)
2488	goto bad_val;
2489	sbinfo->uid = make_kuid(current_user_ns(), uid);
2490	if (!uid_valid(sbinfo->uid))
2491	goto bad_val;
2492	} else if (!strcmp(this_char,"gid")) {
2493	if (remount)
2494	continue;
2495	gid = simple_strtoul(value, &rest, 0);
2496	if (*rest)
2497	goto bad_val;
2498	sbinfo->gid = make_kgid(current_user_ns(), gid);
2499	if (!gid_valid(sbinfo->gid))
2500	goto bad_val;
2501	} else if (!strcmp(this_char,"mpol")) {
2502	if (mpol_parse_str(value, &sbinfo->mpol, 1))
2503	goto bad_val;
2504	} else {
2505	printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2506	this_char);
2507	return 1;
2508	}
2509	}
2510	return 0;
2511
2512	bad_val:
2513	printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2514	value, this_char);
2515	return 1;
2516
2517	}
2518
2519	static int shmem_remount_fs(struct super_block sb, int flags, char *data)
2520	{
2521	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2522	struct shmem_sb_info config = *sbinfo;
2523	unsigned long inodes;
2524	int error = -EINVAL;
2525
2526	if (shmem_parse_options(data, &config, true))
2527	return error;
2528
2529	spin_lock(&sbinfo->stat_lock);
2530	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2531	if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
2532	goto out;
2533	if (config.max_inodes < inodes)
2534	goto out;
2535	/*
2536	* Those tests disallow limited->unlimited while any are in use;
2537	* but we must separately disallow unlimited->limited, because
2538	* in that case we have no record of how much is already in use.
2539	*/
2540	if (config.max_blocks && !sbinfo->max_blocks)
2541	goto out;
2542	if (config.max_inodes && !sbinfo->max_inodes)
2543	goto out;
2544
2545	error = 0;
2546	sbinfo->max_blocks = config.max_blocks;
2547	sbinfo->max_inodes = config.max_inodes;
2548	sbinfo->free_inodes = config.max_inodes - inodes;
2549
2550	mpol_put(sbinfo->mpol);
2551	sbinfo->mpol = config.mpol; /* transfers initial ref */
2552	out:
2553	spin_unlock(&sbinfo->stat_lock);
2554	return error;
2555	}
2556
2557	static int shmem_show_options(struct seq_file seq, struct dentry root)
2558	{
2559	struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
2560
2561	if (sbinfo->max_blocks != shmem_default_max_blocks())
2562	seq_printf(seq, ",size=%luk",
2563	sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2564	if (sbinfo->max_inodes != shmem_default_max_inodes())
2565	seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2566	if (sbinfo->mode != (S_IRWXUGO \| S_ISVTX))
2567	seq_printf(seq, ",mode=%03ho", sbinfo->mode);
2568	if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
2569	seq_printf(seq, ",uid=%u",
2570	from_kuid_munged(&init_user_ns, sbinfo->uid));
2571	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
2572	seq_printf(seq, ",gid=%u",
2573	from_kgid_munged(&init_user_ns, sbinfo->gid));
2574	shmem_show_mpol(seq, sbinfo->mpol);
2575	return 0;
2576	}
2577	#endif /* CONFIG_TMPFS */
2578
2579	static void shmem_put_super(struct super_block *sb)
2580	{
2581	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2582
2583	percpu_counter_destroy(&sbinfo->used_blocks);
2584	kfree(sbinfo);
2585	sb->s_fs_info = NULL;
2586	}
2587
2588	int shmem_fill_super(struct super_block sb, void data, int silent)
2589	{
2590	struct inode *inode;
2591	struct shmem_sb_info *sbinfo;
2592	int err = -ENOMEM;
2593
2594	/* Round up to L1_CACHE_BYTES to resist false sharing */
2595	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
2596	L1_CACHE_BYTES), GFP_KERNEL);
2597	if (!sbinfo)
2598	return -ENOMEM;
2599
2600	sbinfo->mode = S_IRWXUGO \| S_ISVTX;
2601	sbinfo->uid = current_fsuid();
2602	sbinfo->gid = current_fsgid();
2603	sb->s_fs_info = sbinfo;
2604
2605	#ifdef CONFIG_TMPFS
2606	/*
2607	* Per default we only allow half of the physical ram per
2608	* tmpfs instance, limiting inodes to one per page of lowmem;
2609	* but the internal instance is left unlimited.
2610	*/
2611	if (!(sb->s_flags & MS_NOUSER)) {
2612	sbinfo->max_blocks = shmem_default_max_blocks();
2613	sbinfo->max_inodes = shmem_default_max_inodes();
2614	if (shmem_parse_options(data, sbinfo, false)) {
2615	err = -EINVAL;
2616	goto failed;
2617	}
2618	}
2619	sb->s_export_op = &shmem_export_ops;
2620	sb->s_flags \|= MS_NOSEC;
2621	#else
2622	sb->s_flags \|= MS_NOUSER;
2623	#endif
2624
2625	spin_lock_init(&sbinfo->stat_lock);
2626	if (percpu_counter_init(&sbinfo->used_blocks, 0))
2627	goto failed;
2628	sbinfo->free_inodes = sbinfo->max_inodes;
2629
2630	sb->s_maxbytes = MAX_LFS_FILESIZE;
2631	sb->s_blocksize = PAGE_CACHE_SIZE;
2632	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2633	sb->s_magic = TMPFS_MAGIC;
2634	sb->s_op = &shmem_ops;
2635	sb->s_time_gran = 1;
2636	#ifdef CONFIG_TMPFS_XATTR
2637	sb->s_xattr = shmem_xattr_handlers;
2638	#endif
2639	#ifdef CONFIG_TMPFS_POSIX_ACL
2640	sb->s_flags \|= MS_POSIXACL;
2641	#endif
2642
2643	inode = shmem_get_inode(sb, NULL, S_IFDIR \| sbinfo->mode, 0, VM_NORESERVE);
2644	if (!inode)
2645	goto failed;
2646	inode->i_uid = sbinfo->uid;
2647	inode->i_gid = sbinfo->gid;
2648	sb->s_root = d_make_root(inode);
2649	if (!sb->s_root)
2650	goto failed;
2651	return 0;
2652
2653	failed:
2654	shmem_put_super(sb);
2655	return err;
2656	}
2657
2658	static struct kmem_cache *shmem_inode_cachep;
2659
2660	static struct inode shmem_alloc_inode(struct super_block sb)
2661	{
2662	struct shmem_inode_info *info;
2663	info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2664	if (!info)
2665	return NULL;
2666	return &info->vfs_inode;
2667	}
2668
2669	static void shmem_destroy_callback(struct rcu_head *head)
2670	{
2671	struct inode *inode = container_of(head, struct inode, i_rcu);
2672	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2673	}
2674
2675	static void shmem_destroy_inode(struct inode *inode)
2676	{
2677	if (S_ISREG(inode->i_mode))
2678	mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2679	call_rcu(&inode->i_rcu, shmem_destroy_callback);
2680	}
2681
2682	static void shmem_init_inode(void *foo)
2683	{
2684	struct shmem_inode_info *info = foo;
2685	inode_init_once(&info->vfs_inode);
2686	}
2687
2688	static int shmem_init_inodecache(void)
2689	{
2690	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2691	sizeof(struct shmem_inode_info),
2692	0, SLAB_PANIC, shmem_init_inode);
2693	return 0;
2694	}
2695
2696	static void shmem_destroy_inodecache(void)
2697	{
2698	kmem_cache_destroy(shmem_inode_cachep);
2699	}
2700
2701	static const struct address_space_operations shmem_aops = {
2702	.writepage = shmem_writepage,
2703	.set_page_dirty = __set_page_dirty_no_writeback,
2704	#ifdef CONFIG_TMPFS
2705	.write_begin = shmem_write_begin,
2706	.write_end = shmem_write_end,
2707	#endif
2708	.migratepage = migrate_page,
2709	.error_remove_page = generic_error_remove_page,
2710	};
2711
2712	static const struct file_operations shmem_file_operations = {
2713	.mmap = shmem_mmap,
2714	#ifdef CONFIG_TMPFS
2715	.llseek = generic_file_llseek,
2716	.read = do_sync_read,
2717	.write = do_sync_write,
2718	.aio_read = shmem_file_aio_read,
2719	.aio_write = generic_file_aio_write,
2720	.fsync = noop_fsync,
2721	.splice_read = shmem_file_splice_read,
2722	.splice_write = generic_file_splice_write,
2723	.fallocate = shmem_fallocate,
2724	#endif
2725	};
2726
2727	static const struct inode_operations shmem_inode_operations = {
2728	.setattr = shmem_setattr,
2729	#ifdef CONFIG_TMPFS_XATTR
2730	.setxattr = shmem_setxattr,
2731	.getxattr = shmem_getxattr,
2732	.listxattr = shmem_listxattr,
2733	.removexattr = shmem_removexattr,
2734	#endif
2735	};
2736
2737	static const struct inode_operations shmem_dir_inode_operations = {
2738	#ifdef CONFIG_TMPFS
2739	.create = shmem_create,
2740	.lookup = simple_lookup,
2741	.link = shmem_link,
2742	.unlink = shmem_unlink,
2743	.symlink = shmem_symlink,
2744	.mkdir = shmem_mkdir,
2745	.rmdir = shmem_rmdir,
2746	.mknod = shmem_mknod,
2747	.rename = shmem_rename,
2748	#endif
2749	#ifdef CONFIG_TMPFS_XATTR
2750	.setxattr = shmem_setxattr,
2751	.getxattr = shmem_getxattr,
2752	.listxattr = shmem_listxattr,
2753	.removexattr = shmem_removexattr,
2754	#endif
2755	#ifdef CONFIG_TMPFS_POSIX_ACL
2756	.setattr = shmem_setattr,
2757	#endif
2758	};
2759
2760	static const struct inode_operations shmem_special_inode_operations = {
2761	#ifdef CONFIG_TMPFS_XATTR
2762	.setxattr = shmem_setxattr,
2763	.getxattr = shmem_getxattr,
2764	.listxattr = shmem_listxattr,
2765	.removexattr = shmem_removexattr,
2766	#endif
2767	#ifdef CONFIG_TMPFS_POSIX_ACL
2768	.setattr = shmem_setattr,
2769	#endif
2770	};
2771
2772	static const struct super_operations shmem_ops = {
2773	.alloc_inode = shmem_alloc_inode,
2774	.destroy_inode = shmem_destroy_inode,
2775	#ifdef CONFIG_TMPFS
2776	.statfs = shmem_statfs,
2777	.remount_fs = shmem_remount_fs,
2778	.show_options = shmem_show_options,
2779	#endif
2780	.evict_inode = shmem_evict_inode,
2781	.drop_inode = generic_delete_inode,
2782	.put_super = shmem_put_super,
2783	};
2784
2785	static const struct vm_operations_struct shmem_vm_ops = {
2786	.fault = shmem_fault,
2787	#ifdef CONFIG_NUMA
2788	.set_policy = shmem_set_policy,
2789	.get_policy = shmem_get_policy,
2790	#endif
2791	};
2792
2793	static struct dentry shmem_mount(struct file_system_type fs_type,
2794	int flags, const char dev_name, void data)
2795	{
2796	return mount_nodev(fs_type, flags, data, shmem_fill_super);
2797	}
2798
2799	static struct file_system_type shmem_fs_type = {
2800	.owner = THIS_MODULE,
2801	.name = "tmpfs",
2802	.mount = shmem_mount,
2803	.kill_sb = kill_litter_super,
2804	};
2805
2806	int __init shmem_init(void)
2807	{
2808	int error;
2809
2810	error = bdi_init(&shmem_backing_dev_info);
2811	if (error)
2812	goto out4;
2813
2814	error = shmem_init_inodecache();
2815	if (error)
2816	goto out3;
2817
2818	error = register_filesystem(&shmem_fs_type);
2819	if (error) {
2820	printk(KERN_ERR "Could not register tmpfs\n");
2821	goto out2;
2822	}
2823
2824	shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER,
2825	shmem_fs_type.name, NULL);
2826	if (IS_ERR(shm_mnt)) {
2827	error = PTR_ERR(shm_mnt);
2828	printk(KERN_ERR "Could not kern_mount tmpfs\n");
2829	goto out1;
2830	}
2831	return 0;
2832
2833	out1:
2834	unregister_filesystem(&shmem_fs_type);
2835	out2:
2836	shmem_destroy_inodecache();
2837	out3:
2838	bdi_destroy(&shmem_backing_dev_info);
2839	out4:
2840	shm_mnt = ERR_PTR(error);
2841	return error;
2842	}
2843
2844	#else /* !CONFIG_SHMEM */
2845
2846	/*
2847	* tiny-shmem: simple shmemfs and tmpfs using ramfs code
2848	*
2849	* This is intended for small system where the benefits of the full
2850	* shmem code (swap-backed and resource-limited) are outweighed by
2851	* their complexity. On systems without swap this code should be
2852	* effectively equivalent, but much lighter weight.
2853	*/
2854
2855	#include <linux/ramfs.h>
2856
2857	static struct file_system_type shmem_fs_type = {
2858	.name = "tmpfs",
2859	.mount = ramfs_mount,
2860	.kill_sb = kill_litter_super,
2861	};
2862
2863	int __init shmem_init(void)
2864	{
2865	BUG_ON(register_filesystem(&shmem_fs_type) != 0);
2866
2867	shm_mnt = kern_mount(&shmem_fs_type);
2868	BUG_ON(IS_ERR(shm_mnt));
2869
2870	return 0;
2871	}
2872
2873	int shmem_unuse(swp_entry_t swap, struct page *page)
2874	{
2875	return 0;
2876	}
2877
2878	int shmem_lock(struct file file, int lock, struct user_struct user)
2879	{
2880	return 0;
2881	}
2882
2883	void shmem_unlock_mapping(struct address_space *mapping)
2884	{
2885	}
2886
2887	void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
2888	{
2889	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
2890	}
2891	EXPORT_SYMBOL_GPL(shmem_truncate_range);
2892
2893	#define shmem_vm_ops generic_file_vm_ops
2894	#define shmem_file_operations ramfs_file_operations
2895	#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
2896	#define shmem_acct_size(flags, size) 0
2897	#define shmem_unacct_size(flags, size) do {} while (0)
2898
2899	#endif /* CONFIG_SHMEM */
2900
2901	/* common code */
2902
2903	/**
2904	* shmem_file_setup - get an unlinked file living in tmpfs
2905	* @name: name for dentry (to be seen in /proc/<pid>/maps
2906	* @size: size to be set for the file
2907	* @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
2908	*/
2909	struct file shmem_file_setup(const char name, loff_t size, unsigned long flags)
2910	{
2911	int error;
2912	struct file *file;
2913	struct inode *inode;
2914	struct path path;
2915	struct dentry *root;
2916	struct qstr this;
2917
2918	if (IS_ERR(shm_mnt))
2919	return (void *)shm_mnt;
2920
2921	if (size < 0 \|\| size > MAX_LFS_FILESIZE)
2922	return ERR_PTR(-EINVAL);
2923
2924	if (shmem_acct_size(flags, size))
2925	return ERR_PTR(-ENOMEM);
2926
2927	error = -ENOMEM;
2928	this.name = name;
2929	this.len = strlen(name);
2930	this.hash = 0; /* will go */
2931	root = shm_mnt->mnt_root;
2932	path.dentry = d_alloc(root, &this);
2933	if (!path.dentry)
2934	goto put_memory;
2935	path.mnt = mntget(shm_mnt);
2936
2937	error = -ENOSPC;
2938	inode = shmem_get_inode(root->d_sb, NULL, S_IFREG \| S_IRWXUGO, 0, flags);
2939	if (!inode)
2940	goto put_dentry;
2941
2942	d_instantiate(path.dentry, inode);
2943	inode->i_size = size;
2944	clear_nlink(inode); /* It is unlinked */
2945	#ifndef CONFIG_MMU
2946	error = ramfs_nommu_expand_for_mapping(inode, size);
2947	if (error)
2948	goto put_dentry;
2949	#endif
2950
2951	error = -ENFILE;
2952	file = alloc_file(&path, FMODE_WRITE \| FMODE_READ,
2953	&shmem_file_operations);
2954	if (!file)
2955	goto put_dentry;
2956
2957	return file;
2958
2959	put_dentry:
2960	path_put(&path);
2961	put_memory:
2962	shmem_unacct_size(flags, size);
2963	return ERR_PTR(error);
2964	}
2965	EXPORT_SYMBOL_GPL(shmem_file_setup);
2966
2967	/**
2968	* shmem_zero_setup - setup a shared anonymous mapping
2969	* @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2970	*/
2971	int shmem_zero_setup(struct vm_area_struct *vma)
2972	{
2973	struct file *file;
2974	loff_t size = vma->vm_end - vma->vm_start;
2975
2976	file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2977	if (IS_ERR(file))
2978	return PTR_ERR(file);
2979
2980	if (vma->vm_file)
2981	fput(vma->vm_file);
2982	vma->vm_file = file;
2983	vma->vm_ops = &shmem_vm_ops;
2984	vma->vm_flags \|= VM_CAN_NONLINEAR;
2985	return 0;
2986	}
2987
2988	/**
2989	* shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
2990	* @mapping: the page's address_space
2991	* @index: the page index
2992	* @gfp: the page allocator flags to use if allocating
2993	*
2994	* This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
2995	* with any new page allocations done using the specified allocation flags.
2996	* But read_cache_page_gfp() uses the ->readpage() method: which does not
2997	* suit tmpfs, since it may have pages in swapcache, and needs to find those
2998	* for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
2999	*
3000	* i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY \| __GFP_NOWARN in
3001	* with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
3002	*/
3003	struct page shmem_read_mapping_page_gfp(struct address_space mapping,
3004	pgoff_t index, gfp_t gfp)
3005	{
3006	#ifdef CONFIG_SHMEM
3007	struct inode *inode = mapping->host;
3008	struct page *page;
3009	int error;
3010
3011	BUG_ON(mapping->a_ops != &shmem_aops);
3012	error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);
3013	if (error)
3014	page = ERR_PTR(error);
3015	else
3016	unlock_page(page);
3017	return page;
3018	#else
3019	/*
3020	* The tiny !SHMEM case uses ramfs without swap
3021	*/
3022	return read_cache_page_gfp(mapping, index, gfp);
3023	#endif
3024	}
3025	EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
3026

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/shmem.c

interactive