Root/
1 | /* Cache page management and data I/O routines |
2 | * |
3 | * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved. |
4 | * Written by David Howells (dhowells@redhat.com) |
5 | * |
6 | * This program is free software; you can redistribute it and/or |
7 | * modify it under the terms of the GNU General Public License |
8 | * as published by the Free Software Foundation; either version |
9 | * 2 of the License, or (at your option) any later version. |
10 | */ |
11 | |
12 | #define FSCACHE_DEBUG_LEVEL PAGE |
13 | #include <linux/module.h> |
14 | #include <linux/fscache-cache.h> |
15 | #include <linux/buffer_head.h> |
16 | #include <linux/pagevec.h> |
17 | #include <linux/slab.h> |
18 | #include "internal.h" |
19 | |
20 | /* |
21 | * check to see if a page is being written to the cache |
22 | */ |
23 | bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page) |
24 | { |
25 | void *val; |
26 | |
27 | rcu_read_lock(); |
28 | val = radix_tree_lookup(&cookie->stores, page->index); |
29 | rcu_read_unlock(); |
30 | |
31 | return val != NULL; |
32 | } |
33 | EXPORT_SYMBOL(__fscache_check_page_write); |
34 | |
35 | /* |
36 | * wait for a page to finish being written to the cache |
37 | */ |
38 | void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page) |
39 | { |
40 | wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0); |
41 | |
42 | wait_event(*wq, !__fscache_check_page_write(cookie, page)); |
43 | } |
44 | EXPORT_SYMBOL(__fscache_wait_on_page_write); |
45 | |
46 | /* |
47 | * decide whether a page can be released, possibly by cancelling a store to it |
48 | * - we're allowed to sleep if __GFP_WAIT is flagged |
49 | */ |
50 | bool __fscache_maybe_release_page(struct fscache_cookie *cookie, |
51 | struct page *page, |
52 | gfp_t gfp) |
53 | { |
54 | struct page *xpage; |
55 | void *val; |
56 | |
57 | _enter("%p,%p,%x", cookie, page, gfp); |
58 | |
59 | rcu_read_lock(); |
60 | val = radix_tree_lookup(&cookie->stores, page->index); |
61 | if (!val) { |
62 | rcu_read_unlock(); |
63 | fscache_stat(&fscache_n_store_vmscan_not_storing); |
64 | __fscache_uncache_page(cookie, page); |
65 | return true; |
66 | } |
67 | |
68 | /* see if the page is actually undergoing storage - if so we can't get |
69 | * rid of it till the cache has finished with it */ |
70 | if (radix_tree_tag_get(&cookie->stores, page->index, |
71 | FSCACHE_COOKIE_STORING_TAG)) { |
72 | rcu_read_unlock(); |
73 | goto page_busy; |
74 | } |
75 | |
76 | /* the page is pending storage, so we attempt to cancel the store and |
77 | * discard the store request so that the page can be reclaimed */ |
78 | spin_lock(&cookie->stores_lock); |
79 | rcu_read_unlock(); |
80 | |
81 | if (radix_tree_tag_get(&cookie->stores, page->index, |
82 | FSCACHE_COOKIE_STORING_TAG)) { |
83 | /* the page started to undergo storage whilst we were looking, |
84 | * so now we can only wait or return */ |
85 | spin_unlock(&cookie->stores_lock); |
86 | goto page_busy; |
87 | } |
88 | |
89 | xpage = radix_tree_delete(&cookie->stores, page->index); |
90 | spin_unlock(&cookie->stores_lock); |
91 | |
92 | if (xpage) { |
93 | fscache_stat(&fscache_n_store_vmscan_cancelled); |
94 | fscache_stat(&fscache_n_store_radix_deletes); |
95 | ASSERTCMP(xpage, ==, page); |
96 | } else { |
97 | fscache_stat(&fscache_n_store_vmscan_gone); |
98 | } |
99 | |
100 | wake_up_bit(&cookie->flags, 0); |
101 | if (xpage) |
102 | page_cache_release(xpage); |
103 | __fscache_uncache_page(cookie, page); |
104 | return true; |
105 | |
106 | page_busy: |
107 | /* we might want to wait here, but that could deadlock the allocator as |
108 | * the work threads writing to the cache may all end up sleeping |
109 | * on memory allocation */ |
110 | fscache_stat(&fscache_n_store_vmscan_busy); |
111 | return false; |
112 | } |
113 | EXPORT_SYMBOL(__fscache_maybe_release_page); |
114 | |
115 | /* |
116 | * note that a page has finished being written to the cache |
117 | */ |
118 | static void fscache_end_page_write(struct fscache_object *object, |
119 | struct page *page) |
120 | { |
121 | struct fscache_cookie *cookie; |
122 | struct page *xpage = NULL; |
123 | |
124 | spin_lock(&object->lock); |
125 | cookie = object->cookie; |
126 | if (cookie) { |
127 | /* delete the page from the tree if it is now no longer |
128 | * pending */ |
129 | spin_lock(&cookie->stores_lock); |
130 | radix_tree_tag_clear(&cookie->stores, page->index, |
131 | FSCACHE_COOKIE_STORING_TAG); |
132 | if (!radix_tree_tag_get(&cookie->stores, page->index, |
133 | FSCACHE_COOKIE_PENDING_TAG)) { |
134 | fscache_stat(&fscache_n_store_radix_deletes); |
135 | xpage = radix_tree_delete(&cookie->stores, page->index); |
136 | } |
137 | spin_unlock(&cookie->stores_lock); |
138 | wake_up_bit(&cookie->flags, 0); |
139 | } |
140 | spin_unlock(&object->lock); |
141 | if (xpage) |
142 | page_cache_release(xpage); |
143 | } |
144 | |
145 | /* |
146 | * actually apply the changed attributes to a cache object |
147 | */ |
148 | static void fscache_attr_changed_op(struct fscache_operation *op) |
149 | { |
150 | struct fscache_object *object = op->object; |
151 | int ret; |
152 | |
153 | _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id); |
154 | |
155 | fscache_stat(&fscache_n_attr_changed_calls); |
156 | |
157 | if (fscache_object_is_active(object)) { |
158 | fscache_stat(&fscache_n_cop_attr_changed); |
159 | ret = object->cache->ops->attr_changed(object); |
160 | fscache_stat_d(&fscache_n_cop_attr_changed); |
161 | if (ret < 0) |
162 | fscache_abort_object(object); |
163 | } |
164 | |
165 | _leave(""); |
166 | } |
167 | |
168 | /* |
169 | * notification that the attributes on an object have changed |
170 | */ |
171 | int __fscache_attr_changed(struct fscache_cookie *cookie) |
172 | { |
173 | struct fscache_operation *op; |
174 | struct fscache_object *object; |
175 | |
176 | _enter("%p", cookie); |
177 | |
178 | ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); |
179 | |
180 | fscache_stat(&fscache_n_attr_changed); |
181 | |
182 | op = kzalloc(sizeof(*op), GFP_KERNEL); |
183 | if (!op) { |
184 | fscache_stat(&fscache_n_attr_changed_nomem); |
185 | _leave(" = -ENOMEM"); |
186 | return -ENOMEM; |
187 | } |
188 | |
189 | fscache_operation_init(op, fscache_attr_changed_op, NULL); |
190 | op->flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_EXCLUSIVE); |
191 | |
192 | spin_lock(&cookie->lock); |
193 | |
194 | if (hlist_empty(&cookie->backing_objects)) |
195 | goto nobufs; |
196 | object = hlist_entry(cookie->backing_objects.first, |
197 | struct fscache_object, cookie_link); |
198 | |
199 | if (fscache_submit_exclusive_op(object, op) < 0) |
200 | goto nobufs; |
201 | spin_unlock(&cookie->lock); |
202 | fscache_stat(&fscache_n_attr_changed_ok); |
203 | fscache_put_operation(op); |
204 | _leave(" = 0"); |
205 | return 0; |
206 | |
207 | nobufs: |
208 | spin_unlock(&cookie->lock); |
209 | kfree(op); |
210 | fscache_stat(&fscache_n_attr_changed_nobufs); |
211 | _leave(" = %d", -ENOBUFS); |
212 | return -ENOBUFS; |
213 | } |
214 | EXPORT_SYMBOL(__fscache_attr_changed); |
215 | |
216 | /* |
217 | * release a retrieval op reference |
218 | */ |
219 | static void fscache_release_retrieval_op(struct fscache_operation *_op) |
220 | { |
221 | struct fscache_retrieval *op = |
222 | container_of(_op, struct fscache_retrieval, op); |
223 | |
224 | _enter("{OP%x}", op->op.debug_id); |
225 | |
226 | fscache_hist(fscache_retrieval_histogram, op->start_time); |
227 | if (op->context) |
228 | fscache_put_context(op->op.object->cookie, op->context); |
229 | |
230 | _leave(""); |
231 | } |
232 | |
233 | /* |
234 | * allocate a retrieval op |
235 | */ |
236 | static struct fscache_retrieval *fscache_alloc_retrieval( |
237 | struct address_space *mapping, |
238 | fscache_rw_complete_t end_io_func, |
239 | void *context) |
240 | { |
241 | struct fscache_retrieval *op; |
242 | |
243 | /* allocate a retrieval operation and attempt to submit it */ |
244 | op = kzalloc(sizeof(*op), GFP_NOIO); |
245 | if (!op) { |
246 | fscache_stat(&fscache_n_retrievals_nomem); |
247 | return NULL; |
248 | } |
249 | |
250 | fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op); |
251 | op->op.flags = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING); |
252 | op->mapping = mapping; |
253 | op->end_io_func = end_io_func; |
254 | op->context = context; |
255 | op->start_time = jiffies; |
256 | INIT_LIST_HEAD(&op->to_do); |
257 | return op; |
258 | } |
259 | |
260 | /* |
261 | * wait for a deferred lookup to complete |
262 | */ |
263 | static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie) |
264 | { |
265 | unsigned long jif; |
266 | |
267 | _enter(""); |
268 | |
269 | if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) { |
270 | _leave(" = 0 [imm]"); |
271 | return 0; |
272 | } |
273 | |
274 | fscache_stat(&fscache_n_retrievals_wait); |
275 | |
276 | jif = jiffies; |
277 | if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP, |
278 | fscache_wait_bit_interruptible, |
279 | TASK_INTERRUPTIBLE) != 0) { |
280 | fscache_stat(&fscache_n_retrievals_intr); |
281 | _leave(" = -ERESTARTSYS"); |
282 | return -ERESTARTSYS; |
283 | } |
284 | |
285 | ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)); |
286 | |
287 | smp_rmb(); |
288 | fscache_hist(fscache_retrieval_delay_histogram, jif); |
289 | _leave(" = 0 [dly]"); |
290 | return 0; |
291 | } |
292 | |
293 | /* |
294 | * wait for an object to become active (or dead) |
295 | */ |
296 | static int fscache_wait_for_retrieval_activation(struct fscache_object *object, |
297 | struct fscache_retrieval *op, |
298 | atomic_t *stat_op_waits, |
299 | atomic_t *stat_object_dead) |
300 | { |
301 | int ret; |
302 | |
303 | if (!test_bit(FSCACHE_OP_WAITING, &op->op.flags)) |
304 | goto check_if_dead; |
305 | |
306 | _debug(">>> WT"); |
307 | fscache_stat(stat_op_waits); |
308 | if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING, |
309 | fscache_wait_bit_interruptible, |
310 | TASK_INTERRUPTIBLE) < 0) { |
311 | ret = fscache_cancel_op(&op->op); |
312 | if (ret == 0) |
313 | return -ERESTARTSYS; |
314 | |
315 | /* it's been removed from the pending queue by another party, |
316 | * so we should get to run shortly */ |
317 | wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING, |
318 | fscache_wait_bit, TASK_UNINTERRUPTIBLE); |
319 | } |
320 | _debug("<<< GO"); |
321 | |
322 | check_if_dead: |
323 | if (unlikely(fscache_object_is_dead(object))) { |
324 | fscache_stat(stat_object_dead); |
325 | return -ENOBUFS; |
326 | } |
327 | return 0; |
328 | } |
329 | |
330 | /* |
331 | * read a page from the cache or allocate a block in which to store it |
332 | * - we return: |
333 | * -ENOMEM - out of memory, nothing done |
334 | * -ERESTARTSYS - interrupted |
335 | * -ENOBUFS - no backing object available in which to cache the block |
336 | * -ENODATA - no data available in the backing object for this block |
337 | * 0 - dispatched a read - it'll call end_io_func() when finished |
338 | */ |
339 | int __fscache_read_or_alloc_page(struct fscache_cookie *cookie, |
340 | struct page *page, |
341 | fscache_rw_complete_t end_io_func, |
342 | void *context, |
343 | gfp_t gfp) |
344 | { |
345 | struct fscache_retrieval *op; |
346 | struct fscache_object *object; |
347 | int ret; |
348 | |
349 | _enter("%p,%p,,,", cookie, page); |
350 | |
351 | fscache_stat(&fscache_n_retrievals); |
352 | |
353 | if (hlist_empty(&cookie->backing_objects)) |
354 | goto nobufs; |
355 | |
356 | ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); |
357 | ASSERTCMP(page, !=, NULL); |
358 | |
359 | if (fscache_wait_for_deferred_lookup(cookie) < 0) |
360 | return -ERESTARTSYS; |
361 | |
362 | op = fscache_alloc_retrieval(page->mapping, end_io_func, context); |
363 | if (!op) { |
364 | _leave(" = -ENOMEM"); |
365 | return -ENOMEM; |
366 | } |
367 | |
368 | spin_lock(&cookie->lock); |
369 | |
370 | if (hlist_empty(&cookie->backing_objects)) |
371 | goto nobufs_unlock; |
372 | object = hlist_entry(cookie->backing_objects.first, |
373 | struct fscache_object, cookie_link); |
374 | |
375 | ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP); |
376 | |
377 | atomic_inc(&object->n_reads); |
378 | set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); |
379 | |
380 | if (fscache_submit_op(object, &op->op) < 0) |
381 | goto nobufs_unlock; |
382 | spin_unlock(&cookie->lock); |
383 | |
384 | fscache_stat(&fscache_n_retrieval_ops); |
385 | |
386 | /* pin the netfs read context in case we need to do the actual netfs |
387 | * read because we've encountered a cache read failure */ |
388 | fscache_get_context(object->cookie, op->context); |
389 | |
390 | /* we wait for the operation to become active, and then process it |
391 | * *here*, in this thread, and not in the thread pool */ |
392 | ret = fscache_wait_for_retrieval_activation( |
393 | object, op, |
394 | __fscache_stat(&fscache_n_retrieval_op_waits), |
395 | __fscache_stat(&fscache_n_retrievals_object_dead)); |
396 | if (ret < 0) |
397 | goto error; |
398 | |
399 | /* ask the cache to honour the operation */ |
400 | if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) { |
401 | fscache_stat(&fscache_n_cop_allocate_page); |
402 | ret = object->cache->ops->allocate_page(op, page, gfp); |
403 | fscache_stat_d(&fscache_n_cop_allocate_page); |
404 | if (ret == 0) |
405 | ret = -ENODATA; |
406 | } else { |
407 | fscache_stat(&fscache_n_cop_read_or_alloc_page); |
408 | ret = object->cache->ops->read_or_alloc_page(op, page, gfp); |
409 | fscache_stat_d(&fscache_n_cop_read_or_alloc_page); |
410 | } |
411 | |
412 | error: |
413 | if (ret == -ENOMEM) |
414 | fscache_stat(&fscache_n_retrievals_nomem); |
415 | else if (ret == -ERESTARTSYS) |
416 | fscache_stat(&fscache_n_retrievals_intr); |
417 | else if (ret == -ENODATA) |
418 | fscache_stat(&fscache_n_retrievals_nodata); |
419 | else if (ret < 0) |
420 | fscache_stat(&fscache_n_retrievals_nobufs); |
421 | else |
422 | fscache_stat(&fscache_n_retrievals_ok); |
423 | |
424 | fscache_put_retrieval(op); |
425 | _leave(" = %d", ret); |
426 | return ret; |
427 | |
428 | nobufs_unlock: |
429 | spin_unlock(&cookie->lock); |
430 | kfree(op); |
431 | nobufs: |
432 | fscache_stat(&fscache_n_retrievals_nobufs); |
433 | _leave(" = -ENOBUFS"); |
434 | return -ENOBUFS; |
435 | } |
436 | EXPORT_SYMBOL(__fscache_read_or_alloc_page); |
437 | |
438 | /* |
439 | * read a list of page from the cache or allocate a block in which to store |
440 | * them |
441 | * - we return: |
442 | * -ENOMEM - out of memory, some pages may be being read |
443 | * -ERESTARTSYS - interrupted, some pages may be being read |
444 | * -ENOBUFS - no backing object or space available in which to cache any |
445 | * pages not being read |
446 | * -ENODATA - no data available in the backing object for some or all of |
447 | * the pages |
448 | * 0 - dispatched a read on all pages |
449 | * |
450 | * end_io_func() will be called for each page read from the cache as it is |
451 | * finishes being read |
452 | * |
453 | * any pages for which a read is dispatched will be removed from pages and |
454 | * nr_pages |
455 | */ |
456 | int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie, |
457 | struct address_space *mapping, |
458 | struct list_head *pages, |
459 | unsigned *nr_pages, |
460 | fscache_rw_complete_t end_io_func, |
461 | void *context, |
462 | gfp_t gfp) |
463 | { |
464 | struct fscache_retrieval *op; |
465 | struct fscache_object *object; |
466 | int ret; |
467 | |
468 | _enter("%p,,%d,,,", cookie, *nr_pages); |
469 | |
470 | fscache_stat(&fscache_n_retrievals); |
471 | |
472 | if (hlist_empty(&cookie->backing_objects)) |
473 | goto nobufs; |
474 | |
475 | ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); |
476 | ASSERTCMP(*nr_pages, >, 0); |
477 | ASSERT(!list_empty(pages)); |
478 | |
479 | if (fscache_wait_for_deferred_lookup(cookie) < 0) |
480 | return -ERESTARTSYS; |
481 | |
482 | op = fscache_alloc_retrieval(mapping, end_io_func, context); |
483 | if (!op) |
484 | return -ENOMEM; |
485 | |
486 | spin_lock(&cookie->lock); |
487 | |
488 | if (hlist_empty(&cookie->backing_objects)) |
489 | goto nobufs_unlock; |
490 | object = hlist_entry(cookie->backing_objects.first, |
491 | struct fscache_object, cookie_link); |
492 | |
493 | atomic_inc(&object->n_reads); |
494 | set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags); |
495 | |
496 | if (fscache_submit_op(object, &op->op) < 0) |
497 | goto nobufs_unlock; |
498 | spin_unlock(&cookie->lock); |
499 | |
500 | fscache_stat(&fscache_n_retrieval_ops); |
501 | |
502 | /* pin the netfs read context in case we need to do the actual netfs |
503 | * read because we've encountered a cache read failure */ |
504 | fscache_get_context(object->cookie, op->context); |
505 | |
506 | /* we wait for the operation to become active, and then process it |
507 | * *here*, in this thread, and not in the thread pool */ |
508 | ret = fscache_wait_for_retrieval_activation( |
509 | object, op, |
510 | __fscache_stat(&fscache_n_retrieval_op_waits), |
511 | __fscache_stat(&fscache_n_retrievals_object_dead)); |
512 | if (ret < 0) |
513 | goto error; |
514 | |
515 | /* ask the cache to honour the operation */ |
516 | if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) { |
517 | fscache_stat(&fscache_n_cop_allocate_pages); |
518 | ret = object->cache->ops->allocate_pages( |
519 | op, pages, nr_pages, gfp); |
520 | fscache_stat_d(&fscache_n_cop_allocate_pages); |
521 | } else { |
522 | fscache_stat(&fscache_n_cop_read_or_alloc_pages); |
523 | ret = object->cache->ops->read_or_alloc_pages( |
524 | op, pages, nr_pages, gfp); |
525 | fscache_stat_d(&fscache_n_cop_read_or_alloc_pages); |
526 | } |
527 | |
528 | error: |
529 | if (ret == -ENOMEM) |
530 | fscache_stat(&fscache_n_retrievals_nomem); |
531 | else if (ret == -ERESTARTSYS) |
532 | fscache_stat(&fscache_n_retrievals_intr); |
533 | else if (ret == -ENODATA) |
534 | fscache_stat(&fscache_n_retrievals_nodata); |
535 | else if (ret < 0) |
536 | fscache_stat(&fscache_n_retrievals_nobufs); |
537 | else |
538 | fscache_stat(&fscache_n_retrievals_ok); |
539 | |
540 | fscache_put_retrieval(op); |
541 | _leave(" = %d", ret); |
542 | return ret; |
543 | |
544 | nobufs_unlock: |
545 | spin_unlock(&cookie->lock); |
546 | kfree(op); |
547 | nobufs: |
548 | fscache_stat(&fscache_n_retrievals_nobufs); |
549 | _leave(" = -ENOBUFS"); |
550 | return -ENOBUFS; |
551 | } |
552 | EXPORT_SYMBOL(__fscache_read_or_alloc_pages); |
553 | |
554 | /* |
555 | * allocate a block in the cache on which to store a page |
556 | * - we return: |
557 | * -ENOMEM - out of memory, nothing done |
558 | * -ERESTARTSYS - interrupted |
559 | * -ENOBUFS - no backing object available in which to cache the block |
560 | * 0 - block allocated |
561 | */ |
562 | int __fscache_alloc_page(struct fscache_cookie *cookie, |
563 | struct page *page, |
564 | gfp_t gfp) |
565 | { |
566 | struct fscache_retrieval *op; |
567 | struct fscache_object *object; |
568 | int ret; |
569 | |
570 | _enter("%p,%p,,,", cookie, page); |
571 | |
572 | fscache_stat(&fscache_n_allocs); |
573 | |
574 | if (hlist_empty(&cookie->backing_objects)) |
575 | goto nobufs; |
576 | |
577 | ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); |
578 | ASSERTCMP(page, !=, NULL); |
579 | |
580 | if (fscache_wait_for_deferred_lookup(cookie) < 0) |
581 | return -ERESTARTSYS; |
582 | |
583 | op = fscache_alloc_retrieval(page->mapping, NULL, NULL); |
584 | if (!op) |
585 | return -ENOMEM; |
586 | |
587 | spin_lock(&cookie->lock); |
588 | |
589 | if (hlist_empty(&cookie->backing_objects)) |
590 | goto nobufs_unlock; |
591 | object = hlist_entry(cookie->backing_objects.first, |
592 | struct fscache_object, cookie_link); |
593 | |
594 | if (fscache_submit_op(object, &op->op) < 0) |
595 | goto nobufs_unlock; |
596 | spin_unlock(&cookie->lock); |
597 | |
598 | fscache_stat(&fscache_n_alloc_ops); |
599 | |
600 | ret = fscache_wait_for_retrieval_activation( |
601 | object, op, |
602 | __fscache_stat(&fscache_n_alloc_op_waits), |
603 | __fscache_stat(&fscache_n_allocs_object_dead)); |
604 | if (ret < 0) |
605 | goto error; |
606 | |
607 | /* ask the cache to honour the operation */ |
608 | fscache_stat(&fscache_n_cop_allocate_page); |
609 | ret = object->cache->ops->allocate_page(op, page, gfp); |
610 | fscache_stat_d(&fscache_n_cop_allocate_page); |
611 | |
612 | error: |
613 | if (ret == -ERESTARTSYS) |
614 | fscache_stat(&fscache_n_allocs_intr); |
615 | else if (ret < 0) |
616 | fscache_stat(&fscache_n_allocs_nobufs); |
617 | else |
618 | fscache_stat(&fscache_n_allocs_ok); |
619 | |
620 | fscache_put_retrieval(op); |
621 | _leave(" = %d", ret); |
622 | return ret; |
623 | |
624 | nobufs_unlock: |
625 | spin_unlock(&cookie->lock); |
626 | kfree(op); |
627 | nobufs: |
628 | fscache_stat(&fscache_n_allocs_nobufs); |
629 | _leave(" = -ENOBUFS"); |
630 | return -ENOBUFS; |
631 | } |
632 | EXPORT_SYMBOL(__fscache_alloc_page); |
633 | |
634 | /* |
635 | * release a write op reference |
636 | */ |
637 | static void fscache_release_write_op(struct fscache_operation *_op) |
638 | { |
639 | _enter("{OP%x}", _op->debug_id); |
640 | } |
641 | |
642 | /* |
643 | * perform the background storage of a page into the cache |
644 | */ |
645 | static void fscache_write_op(struct fscache_operation *_op) |
646 | { |
647 | struct fscache_storage *op = |
648 | container_of(_op, struct fscache_storage, op); |
649 | struct fscache_object *object = op->op.object; |
650 | struct fscache_cookie *cookie; |
651 | struct page *page; |
652 | unsigned n; |
653 | void *results[1]; |
654 | int ret; |
655 | |
656 | _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage)); |
657 | |
658 | spin_lock(&object->lock); |
659 | cookie = object->cookie; |
660 | |
661 | if (!fscache_object_is_active(object) || !cookie) { |
662 | spin_unlock(&object->lock); |
663 | _leave(""); |
664 | return; |
665 | } |
666 | |
667 | spin_lock(&cookie->stores_lock); |
668 | |
669 | fscache_stat(&fscache_n_store_calls); |
670 | |
671 | /* find a page to store */ |
672 | page = NULL; |
673 | n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1, |
674 | FSCACHE_COOKIE_PENDING_TAG); |
675 | if (n != 1) |
676 | goto superseded; |
677 | page = results[0]; |
678 | _debug("gang %d [%lx]", n, page->index); |
679 | if (page->index > op->store_limit) { |
680 | fscache_stat(&fscache_n_store_pages_over_limit); |
681 | goto superseded; |
682 | } |
683 | |
684 | radix_tree_tag_set(&cookie->stores, page->index, |
685 | FSCACHE_COOKIE_STORING_TAG); |
686 | radix_tree_tag_clear(&cookie->stores, page->index, |
687 | FSCACHE_COOKIE_PENDING_TAG); |
688 | |
689 | spin_unlock(&cookie->stores_lock); |
690 | spin_unlock(&object->lock); |
691 | |
692 | fscache_stat(&fscache_n_store_pages); |
693 | fscache_stat(&fscache_n_cop_write_page); |
694 | ret = object->cache->ops->write_page(op, page); |
695 | fscache_stat_d(&fscache_n_cop_write_page); |
696 | fscache_end_page_write(object, page); |
697 | if (ret < 0) { |
698 | fscache_abort_object(object); |
699 | } else { |
700 | fscache_enqueue_operation(&op->op); |
701 | } |
702 | |
703 | _leave(""); |
704 | return; |
705 | |
706 | superseded: |
707 | /* this writer is going away and there aren't any more things to |
708 | * write */ |
709 | _debug("cease"); |
710 | spin_unlock(&cookie->stores_lock); |
711 | clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags); |
712 | spin_unlock(&object->lock); |
713 | _leave(""); |
714 | } |
715 | |
716 | /* |
717 | * request a page be stored in the cache |
718 | * - returns: |
719 | * -ENOMEM - out of memory, nothing done |
720 | * -ENOBUFS - no backing object available in which to cache the page |
721 | * 0 - dispatched a write - it'll call end_io_func() when finished |
722 | * |
723 | * if the cookie still has a backing object at this point, that object can be |
724 | * in one of a few states with respect to storage processing: |
725 | * |
726 | * (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is |
727 | * set) |
728 | * |
729 | * (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred |
730 | * fill op) |
731 | * |
732 | * (b) writes deferred till post-creation (mark page for writing and |
733 | * return immediately) |
734 | * |
735 | * (2) negative lookup, object created, initial fill being made from netfs |
736 | * (FSCACHE_COOKIE_INITIAL_FILL is set) |
737 | * |
738 | * (a) fill point not yet reached this page (mark page for writing and |
739 | * return) |
740 | * |
741 | * (b) fill point passed this page (queue op to store this page) |
742 | * |
743 | * (3) object extant (queue op to store this page) |
744 | * |
745 | * any other state is invalid |
746 | */ |
747 | int __fscache_write_page(struct fscache_cookie *cookie, |
748 | struct page *page, |
749 | gfp_t gfp) |
750 | { |
751 | struct fscache_storage *op; |
752 | struct fscache_object *object; |
753 | int ret; |
754 | |
755 | _enter("%p,%x,", cookie, (u32) page->flags); |
756 | |
757 | ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); |
758 | ASSERT(PageFsCache(page)); |
759 | |
760 | fscache_stat(&fscache_n_stores); |
761 | |
762 | op = kzalloc(sizeof(*op), GFP_NOIO); |
763 | if (!op) |
764 | goto nomem; |
765 | |
766 | fscache_operation_init(&op->op, fscache_write_op, |
767 | fscache_release_write_op); |
768 | op->op.flags = FSCACHE_OP_ASYNC | (1 << FSCACHE_OP_WAITING); |
769 | |
770 | ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM); |
771 | if (ret < 0) |
772 | goto nomem_free; |
773 | |
774 | ret = -ENOBUFS; |
775 | spin_lock(&cookie->lock); |
776 | |
777 | if (hlist_empty(&cookie->backing_objects)) |
778 | goto nobufs; |
779 | object = hlist_entry(cookie->backing_objects.first, |
780 | struct fscache_object, cookie_link); |
781 | if (test_bit(FSCACHE_IOERROR, &object->cache->flags)) |
782 | goto nobufs; |
783 | |
784 | /* add the page to the pending-storage radix tree on the backing |
785 | * object */ |
786 | spin_lock(&object->lock); |
787 | spin_lock(&cookie->stores_lock); |
788 | |
789 | _debug("store limit %llx", (unsigned long long) object->store_limit); |
790 | |
791 | ret = radix_tree_insert(&cookie->stores, page->index, page); |
792 | if (ret < 0) { |
793 | if (ret == -EEXIST) |
794 | goto already_queued; |
795 | _debug("insert failed %d", ret); |
796 | goto nobufs_unlock_obj; |
797 | } |
798 | |
799 | radix_tree_tag_set(&cookie->stores, page->index, |
800 | FSCACHE_COOKIE_PENDING_TAG); |
801 | page_cache_get(page); |
802 | |
803 | /* we only want one writer at a time, but we do need to queue new |
804 | * writers after exclusive ops */ |
805 | if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags)) |
806 | goto already_pending; |
807 | |
808 | spin_unlock(&cookie->stores_lock); |
809 | spin_unlock(&object->lock); |
810 | |
811 | op->op.debug_id = atomic_inc_return(&fscache_op_debug_id); |
812 | op->store_limit = object->store_limit; |
813 | |
814 | if (fscache_submit_op(object, &op->op) < 0) |
815 | goto submit_failed; |
816 | |
817 | spin_unlock(&cookie->lock); |
818 | radix_tree_preload_end(); |
819 | fscache_stat(&fscache_n_store_ops); |
820 | fscache_stat(&fscache_n_stores_ok); |
821 | |
822 | /* the work queue now carries its own ref on the object */ |
823 | fscache_put_operation(&op->op); |
824 | _leave(" = 0"); |
825 | return 0; |
826 | |
827 | already_queued: |
828 | fscache_stat(&fscache_n_stores_again); |
829 | already_pending: |
830 | spin_unlock(&cookie->stores_lock); |
831 | spin_unlock(&object->lock); |
832 | spin_unlock(&cookie->lock); |
833 | radix_tree_preload_end(); |
834 | kfree(op); |
835 | fscache_stat(&fscache_n_stores_ok); |
836 | _leave(" = 0"); |
837 | return 0; |
838 | |
839 | submit_failed: |
840 | spin_lock(&cookie->stores_lock); |
841 | radix_tree_delete(&cookie->stores, page->index); |
842 | spin_unlock(&cookie->stores_lock); |
843 | page_cache_release(page); |
844 | ret = -ENOBUFS; |
845 | goto nobufs; |
846 | |
847 | nobufs_unlock_obj: |
848 | spin_unlock(&cookie->stores_lock); |
849 | spin_unlock(&object->lock); |
850 | nobufs: |
851 | spin_unlock(&cookie->lock); |
852 | radix_tree_preload_end(); |
853 | kfree(op); |
854 | fscache_stat(&fscache_n_stores_nobufs); |
855 | _leave(" = -ENOBUFS"); |
856 | return -ENOBUFS; |
857 | |
858 | nomem_free: |
859 | kfree(op); |
860 | nomem: |
861 | fscache_stat(&fscache_n_stores_oom); |
862 | _leave(" = -ENOMEM"); |
863 | return -ENOMEM; |
864 | } |
865 | EXPORT_SYMBOL(__fscache_write_page); |
866 | |
867 | /* |
868 | * remove a page from the cache |
869 | */ |
870 | void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page) |
871 | { |
872 | struct fscache_object *object; |
873 | |
874 | _enter(",%p", page); |
875 | |
876 | ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX); |
877 | ASSERTCMP(page, !=, NULL); |
878 | |
879 | fscache_stat(&fscache_n_uncaches); |
880 | |
881 | /* cache withdrawal may beat us to it */ |
882 | if (!PageFsCache(page)) |
883 | goto done; |
884 | |
885 | /* get the object */ |
886 | spin_lock(&cookie->lock); |
887 | |
888 | if (hlist_empty(&cookie->backing_objects)) { |
889 | ClearPageFsCache(page); |
890 | goto done_unlock; |
891 | } |
892 | |
893 | object = hlist_entry(cookie->backing_objects.first, |
894 | struct fscache_object, cookie_link); |
895 | |
896 | /* there might now be stuff on disk we could read */ |
897 | clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags); |
898 | |
899 | /* only invoke the cache backend if we managed to mark the page |
900 | * uncached here; this deals with synchronisation vs withdrawal */ |
901 | if (TestClearPageFsCache(page) && |
902 | object->cache->ops->uncache_page) { |
903 | /* the cache backend releases the cookie lock */ |
904 | fscache_stat(&fscache_n_cop_uncache_page); |
905 | object->cache->ops->uncache_page(object, page); |
906 | fscache_stat_d(&fscache_n_cop_uncache_page); |
907 | goto done; |
908 | } |
909 | |
910 | done_unlock: |
911 | spin_unlock(&cookie->lock); |
912 | done: |
913 | _leave(""); |
914 | } |
915 | EXPORT_SYMBOL(__fscache_uncache_page); |
916 | |
917 | /** |
918 | * fscache_mark_pages_cached - Mark pages as being cached |
919 | * @op: The retrieval op pages are being marked for |
920 | * @pagevec: The pages to be marked |
921 | * |
922 | * Mark a bunch of netfs pages as being cached. After this is called, |
923 | * the netfs must call fscache_uncache_page() to remove the mark. |
924 | */ |
925 | void fscache_mark_pages_cached(struct fscache_retrieval *op, |
926 | struct pagevec *pagevec) |
927 | { |
928 | struct fscache_cookie *cookie = op->op.object->cookie; |
929 | unsigned long loop; |
930 | |
931 | #ifdef CONFIG_FSCACHE_STATS |
932 | atomic_add(pagevec->nr, &fscache_n_marks); |
933 | #endif |
934 | |
935 | for (loop = 0; loop < pagevec->nr; loop++) { |
936 | struct page *page = pagevec->pages[loop]; |
937 | |
938 | _debug("- mark %p{%lx}", page, page->index); |
939 | if (TestSetPageFsCache(page)) { |
940 | static bool once_only; |
941 | if (!once_only) { |
942 | once_only = true; |
943 | printk(KERN_WARNING "FS-Cache:" |
944 | " Cookie type %s marked page %lx" |
945 | " multiple times\n", |
946 | cookie->def->name, page->index); |
947 | } |
948 | } |
949 | } |
950 | |
951 | if (cookie->def->mark_pages_cached) |
952 | cookie->def->mark_pages_cached(cookie->netfs_data, |
953 | op->mapping, pagevec); |
954 | pagevec_reinit(pagevec); |
955 | } |
956 | EXPORT_SYMBOL(fscache_mark_pages_cached); |
957 | |
958 | /* |
959 | * Uncache all the pages in an inode that are marked PG_fscache, assuming them |
960 | * to be associated with the given cookie. |
961 | */ |
962 | void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie, |
963 | struct inode *inode) |
964 | { |
965 | struct address_space *mapping = inode->i_mapping; |
966 | struct pagevec pvec; |
967 | pgoff_t next; |
968 | int i; |
969 | |
970 | _enter("%p,%p", cookie, inode); |
971 | |
972 | if (!mapping || mapping->nrpages == 0) { |
973 | _leave(" [no pages]"); |
974 | return; |
975 | } |
976 | |
977 | pagevec_init(&pvec, 0); |
978 | next = 0; |
979 | do { |
980 | if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) |
981 | break; |
982 | for (i = 0; i < pagevec_count(&pvec); i++) { |
983 | struct page *page = pvec.pages[i]; |
984 | next = page->index; |
985 | if (PageFsCache(page)) { |
986 | __fscache_wait_on_page_write(cookie, page); |
987 | __fscache_uncache_page(cookie, page); |
988 | } |
989 | } |
990 | pagevec_release(&pvec); |
991 | cond_resched(); |
992 | } while (++next); |
993 | |
994 | _leave(""); |
995 | } |
996 | EXPORT_SYMBOL(__fscache_uncache_all_inode_pages); |
997 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9