Root/
1 | /* |
2 | * fs/fs-writeback.c |
3 | * |
4 | * Copyright (C) 2002, Linus Torvalds. |
5 | * |
6 | * Contains all the functions related to writing back and waiting |
7 | * upon dirty inodes against superblocks, and writing back dirty |
8 | * pages against inodes. ie: data writeback. Writeout of the |
9 | * inode itself is not handled here. |
10 | * |
11 | * 10Apr2002 Andrew Morton |
12 | * Split out of fs/inode.c |
13 | * Additions for address_space-based writeback |
14 | */ |
15 | |
16 | #include <linux/kernel.h> |
17 | #include <linux/module.h> |
18 | #include <linux/spinlock.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/sched.h> |
21 | #include <linux/fs.h> |
22 | #include <linux/mm.h> |
23 | #include <linux/kthread.h> |
24 | #include <linux/freezer.h> |
25 | #include <linux/writeback.h> |
26 | #include <linux/blkdev.h> |
27 | #include <linux/backing-dev.h> |
28 | #include <linux/buffer_head.h> |
29 | #include <linux/tracepoint.h> |
30 | #include "internal.h" |
31 | |
32 | /* |
33 | * Passed into wb_writeback(), essentially a subset of writeback_control |
34 | */ |
35 | struct wb_writeback_work { |
36 | long nr_pages; |
37 | struct super_block *sb; |
38 | enum writeback_sync_modes sync_mode; |
39 | unsigned int for_kupdate:1; |
40 | unsigned int range_cyclic:1; |
41 | unsigned int for_background:1; |
42 | |
43 | struct list_head list; /* pending work list */ |
44 | struct completion *done; /* set if the caller waits */ |
45 | }; |
46 | |
47 | /* |
48 | * Include the creation of the trace points after defining the |
49 | * wb_writeback_work structure so that the definition remains local to this |
50 | * file. |
51 | */ |
52 | #define CREATE_TRACE_POINTS |
53 | #include <trace/events/writeback.h> |
54 | |
55 | /* |
56 | * We don't actually have pdflush, but this one is exported though /proc... |
57 | */ |
58 | int nr_pdflush_threads; |
59 | |
60 | /** |
61 | * writeback_in_progress - determine whether there is writeback in progress |
62 | * @bdi: the device's backing_dev_info structure. |
63 | * |
64 | * Determine whether there is writeback waiting to be handled against a |
65 | * backing device. |
66 | */ |
67 | int writeback_in_progress(struct backing_dev_info *bdi) |
68 | { |
69 | return test_bit(BDI_writeback_running, &bdi->state); |
70 | } |
71 | |
72 | static inline struct backing_dev_info *inode_to_bdi(struct inode *inode) |
73 | { |
74 | struct super_block *sb = inode->i_sb; |
75 | |
76 | if (strcmp(sb->s_type->name, "bdev") == 0) |
77 | return inode->i_mapping->backing_dev_info; |
78 | |
79 | return sb->s_bdi; |
80 | } |
81 | |
82 | static void bdi_queue_work(struct backing_dev_info *bdi, |
83 | struct wb_writeback_work *work) |
84 | { |
85 | trace_writeback_queue(bdi, work); |
86 | |
87 | spin_lock_bh(&bdi->wb_lock); |
88 | list_add_tail(&work->list, &bdi->work_list); |
89 | if (bdi->wb.task) { |
90 | wake_up_process(bdi->wb.task); |
91 | } else { |
92 | /* |
93 | * The bdi thread isn't there, wake up the forker thread which |
94 | * will create and run it. |
95 | */ |
96 | trace_writeback_nothread(bdi, work); |
97 | wake_up_process(default_backing_dev_info.wb.task); |
98 | } |
99 | spin_unlock_bh(&bdi->wb_lock); |
100 | } |
101 | |
102 | static void |
103 | __bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages, |
104 | bool range_cyclic, bool for_background) |
105 | { |
106 | struct wb_writeback_work *work; |
107 | |
108 | /* |
109 | * This is WB_SYNC_NONE writeback, so if allocation fails just |
110 | * wakeup the thread for old dirty data writeback |
111 | */ |
112 | work = kzalloc(sizeof(*work), GFP_ATOMIC); |
113 | if (!work) { |
114 | if (bdi->wb.task) { |
115 | trace_writeback_nowork(bdi); |
116 | wake_up_process(bdi->wb.task); |
117 | } |
118 | return; |
119 | } |
120 | |
121 | work->sync_mode = WB_SYNC_NONE; |
122 | work->nr_pages = nr_pages; |
123 | work->range_cyclic = range_cyclic; |
124 | work->for_background = for_background; |
125 | |
126 | bdi_queue_work(bdi, work); |
127 | } |
128 | |
129 | /** |
130 | * bdi_start_writeback - start writeback |
131 | * @bdi: the backing device to write from |
132 | * @nr_pages: the number of pages to write |
133 | * |
134 | * Description: |
135 | * This does WB_SYNC_NONE opportunistic writeback. The IO is only |
136 | * started when this function returns, we make no guarentees on |
137 | * completion. Caller need not hold sb s_umount semaphore. |
138 | * |
139 | */ |
140 | void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages) |
141 | { |
142 | __bdi_start_writeback(bdi, nr_pages, true, false); |
143 | } |
144 | |
145 | /** |
146 | * bdi_start_background_writeback - start background writeback |
147 | * @bdi: the backing device to write from |
148 | * |
149 | * Description: |
150 | * This does WB_SYNC_NONE background writeback. The IO is only |
151 | * started when this function returns, we make no guarentees on |
152 | * completion. Caller need not hold sb s_umount semaphore. |
153 | */ |
154 | void bdi_start_background_writeback(struct backing_dev_info *bdi) |
155 | { |
156 | __bdi_start_writeback(bdi, LONG_MAX, true, true); |
157 | } |
158 | |
159 | /* |
160 | * Redirty an inode: set its when-it-was dirtied timestamp and move it to the |
161 | * furthest end of its superblock's dirty-inode list. |
162 | * |
163 | * Before stamping the inode's ->dirtied_when, we check to see whether it is |
164 | * already the most-recently-dirtied inode on the b_dirty list. If that is |
165 | * the case then the inode must have been redirtied while it was being written |
166 | * out and we don't reset its dirtied_when. |
167 | */ |
168 | static void redirty_tail(struct inode *inode) |
169 | { |
170 | struct bdi_writeback *wb = &inode_to_bdi(inode)->wb; |
171 | |
172 | if (!list_empty(&wb->b_dirty)) { |
173 | struct inode *tail; |
174 | |
175 | tail = list_entry(wb->b_dirty.next, struct inode, i_list); |
176 | if (time_before(inode->dirtied_when, tail->dirtied_when)) |
177 | inode->dirtied_when = jiffies; |
178 | } |
179 | list_move(&inode->i_list, &wb->b_dirty); |
180 | } |
181 | |
182 | /* |
183 | * requeue inode for re-scanning after bdi->b_io list is exhausted. |
184 | */ |
185 | static void requeue_io(struct inode *inode) |
186 | { |
187 | struct bdi_writeback *wb = &inode_to_bdi(inode)->wb; |
188 | |
189 | list_move(&inode->i_list, &wb->b_more_io); |
190 | } |
191 | |
192 | static void inode_sync_complete(struct inode *inode) |
193 | { |
194 | /* |
195 | * Prevent speculative execution through spin_unlock(&inode_lock); |
196 | */ |
197 | smp_mb(); |
198 | wake_up_bit(&inode->i_state, __I_SYNC); |
199 | } |
200 | |
201 | static bool inode_dirtied_after(struct inode *inode, unsigned long t) |
202 | { |
203 | bool ret = time_after(inode->dirtied_when, t); |
204 | #ifndef CONFIG_64BIT |
205 | /* |
206 | * For inodes being constantly redirtied, dirtied_when can get stuck. |
207 | * It _appears_ to be in the future, but is actually in distant past. |
208 | * This test is necessary to prevent such wrapped-around relative times |
209 | * from permanently stopping the whole bdi writeback. |
210 | */ |
211 | ret = ret && time_before_eq(inode->dirtied_when, jiffies); |
212 | #endif |
213 | return ret; |
214 | } |
215 | |
216 | /* |
217 | * Move expired dirty inodes from @delaying_queue to @dispatch_queue. |
218 | */ |
219 | static void move_expired_inodes(struct list_head *delaying_queue, |
220 | struct list_head *dispatch_queue, |
221 | unsigned long *older_than_this) |
222 | { |
223 | LIST_HEAD(tmp); |
224 | struct list_head *pos, *node; |
225 | struct super_block *sb = NULL; |
226 | struct inode *inode; |
227 | int do_sb_sort = 0; |
228 | |
229 | while (!list_empty(delaying_queue)) { |
230 | inode = list_entry(delaying_queue->prev, struct inode, i_list); |
231 | if (older_than_this && |
232 | inode_dirtied_after(inode, *older_than_this)) |
233 | break; |
234 | if (sb && sb != inode->i_sb) |
235 | do_sb_sort = 1; |
236 | sb = inode->i_sb; |
237 | list_move(&inode->i_list, &tmp); |
238 | } |
239 | |
240 | /* just one sb in list, splice to dispatch_queue and we're done */ |
241 | if (!do_sb_sort) { |
242 | list_splice(&tmp, dispatch_queue); |
243 | return; |
244 | } |
245 | |
246 | /* Move inodes from one superblock together */ |
247 | while (!list_empty(&tmp)) { |
248 | inode = list_entry(tmp.prev, struct inode, i_list); |
249 | sb = inode->i_sb; |
250 | list_for_each_prev_safe(pos, node, &tmp) { |
251 | inode = list_entry(pos, struct inode, i_list); |
252 | if (inode->i_sb == sb) |
253 | list_move(&inode->i_list, dispatch_queue); |
254 | } |
255 | } |
256 | } |
257 | |
258 | /* |
259 | * Queue all expired dirty inodes for io, eldest first. |
260 | * Before |
261 | * newly dirtied b_dirty b_io b_more_io |
262 | * =============> gf edc BA |
263 | * After |
264 | * newly dirtied b_dirty b_io b_more_io |
265 | * =============> g fBAedc |
266 | * | |
267 | * +--> dequeue for IO |
268 | */ |
269 | static void queue_io(struct bdi_writeback *wb, unsigned long *older_than_this) |
270 | { |
271 | list_splice_init(&wb->b_more_io, &wb->b_io); |
272 | move_expired_inodes(&wb->b_dirty, &wb->b_io, older_than_this); |
273 | } |
274 | |
275 | static int write_inode(struct inode *inode, struct writeback_control *wbc) |
276 | { |
277 | if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) |
278 | return inode->i_sb->s_op->write_inode(inode, wbc); |
279 | return 0; |
280 | } |
281 | |
282 | /* |
283 | * Wait for writeback on an inode to complete. |
284 | */ |
285 | static void inode_wait_for_writeback(struct inode *inode) |
286 | { |
287 | DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC); |
288 | wait_queue_head_t *wqh; |
289 | |
290 | wqh = bit_waitqueue(&inode->i_state, __I_SYNC); |
291 | while (inode->i_state & I_SYNC) { |
292 | spin_unlock(&inode_lock); |
293 | __wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE); |
294 | spin_lock(&inode_lock); |
295 | } |
296 | } |
297 | |
298 | /* |
299 | * Write out an inode's dirty pages. Called under inode_lock. Either the |
300 | * caller has ref on the inode (either via __iget or via syscall against an fd) |
301 | * or the inode has I_WILL_FREE set (via generic_forget_inode) |
302 | * |
303 | * If `wait' is set, wait on the writeout. |
304 | * |
305 | * The whole writeout design is quite complex and fragile. We want to avoid |
306 | * starvation of particular inodes when others are being redirtied, prevent |
307 | * livelocks, etc. |
308 | * |
309 | * Called under inode_lock. |
310 | */ |
311 | static int |
312 | writeback_single_inode(struct inode *inode, struct writeback_control *wbc) |
313 | { |
314 | struct address_space *mapping = inode->i_mapping; |
315 | unsigned dirty; |
316 | int ret; |
317 | |
318 | if (!atomic_read(&inode->i_count)) |
319 | WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING))); |
320 | else |
321 | WARN_ON(inode->i_state & I_WILL_FREE); |
322 | |
323 | if (inode->i_state & I_SYNC) { |
324 | /* |
325 | * If this inode is locked for writeback and we are not doing |
326 | * writeback-for-data-integrity, move it to b_more_io so that |
327 | * writeback can proceed with the other inodes on s_io. |
328 | * |
329 | * We'll have another go at writing back this inode when we |
330 | * completed a full scan of b_io. |
331 | */ |
332 | if (wbc->sync_mode != WB_SYNC_ALL) { |
333 | requeue_io(inode); |
334 | return 0; |
335 | } |
336 | |
337 | /* |
338 | * It's a data-integrity sync. We must wait. |
339 | */ |
340 | inode_wait_for_writeback(inode); |
341 | } |
342 | |
343 | BUG_ON(inode->i_state & I_SYNC); |
344 | |
345 | /* Set I_SYNC, reset I_DIRTY_PAGES */ |
346 | inode->i_state |= I_SYNC; |
347 | inode->i_state &= ~I_DIRTY_PAGES; |
348 | spin_unlock(&inode_lock); |
349 | |
350 | ret = do_writepages(mapping, wbc); |
351 | |
352 | /* |
353 | * Make sure to wait on the data before writing out the metadata. |
354 | * This is important for filesystems that modify metadata on data |
355 | * I/O completion. |
356 | */ |
357 | if (wbc->sync_mode == WB_SYNC_ALL) { |
358 | int err = filemap_fdatawait(mapping); |
359 | if (ret == 0) |
360 | ret = err; |
361 | } |
362 | |
363 | /* |
364 | * Some filesystems may redirty the inode during the writeback |
365 | * due to delalloc, clear dirty metadata flags right before |
366 | * write_inode() |
367 | */ |
368 | spin_lock(&inode_lock); |
369 | dirty = inode->i_state & I_DIRTY; |
370 | inode->i_state &= ~(I_DIRTY_SYNC | I_DIRTY_DATASYNC); |
371 | spin_unlock(&inode_lock); |
372 | /* Don't write the inode if only I_DIRTY_PAGES was set */ |
373 | if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) { |
374 | int err = write_inode(inode, wbc); |
375 | if (ret == 0) |
376 | ret = err; |
377 | } |
378 | |
379 | spin_lock(&inode_lock); |
380 | inode->i_state &= ~I_SYNC; |
381 | if (!(inode->i_state & I_FREEING)) { |
382 | if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) { |
383 | /* |
384 | * We didn't write back all the pages. nfs_writepages() |
385 | * sometimes bales out without doing anything. |
386 | */ |
387 | inode->i_state |= I_DIRTY_PAGES; |
388 | if (wbc->nr_to_write <= 0) { |
389 | /* |
390 | * slice used up: queue for next turn |
391 | */ |
392 | requeue_io(inode); |
393 | } else { |
394 | /* |
395 | * Writeback blocked by something other than |
396 | * congestion. Delay the inode for some time to |
397 | * avoid spinning on the CPU (100% iowait) |
398 | * retrying writeback of the dirty page/inode |
399 | * that cannot be performed immediately. |
400 | */ |
401 | redirty_tail(inode); |
402 | } |
403 | } else if (inode->i_state & I_DIRTY) { |
404 | /* |
405 | * Filesystems can dirty the inode during writeback |
406 | * operations, such as delayed allocation during |
407 | * submission or metadata updates after data IO |
408 | * completion. |
409 | */ |
410 | redirty_tail(inode); |
411 | } else if (atomic_read(&inode->i_count)) { |
412 | /* |
413 | * The inode is clean, inuse |
414 | */ |
415 | list_move(&inode->i_list, &inode_in_use); |
416 | } else { |
417 | /* |
418 | * The inode is clean, unused |
419 | */ |
420 | list_move(&inode->i_list, &inode_unused); |
421 | } |
422 | } |
423 | inode_sync_complete(inode); |
424 | return ret; |
425 | } |
426 | |
427 | /* |
428 | * For background writeback the caller does not have the sb pinned |
429 | * before calling writeback. So make sure that we do pin it, so it doesn't |
430 | * go away while we are writing inodes from it. |
431 | */ |
432 | static bool pin_sb_for_writeback(struct super_block *sb) |
433 | { |
434 | spin_lock(&sb_lock); |
435 | if (list_empty(&sb->s_instances)) { |
436 | spin_unlock(&sb_lock); |
437 | return false; |
438 | } |
439 | |
440 | sb->s_count++; |
441 | spin_unlock(&sb_lock); |
442 | |
443 | if (down_read_trylock(&sb->s_umount)) { |
444 | if (sb->s_root) |
445 | return true; |
446 | up_read(&sb->s_umount); |
447 | } |
448 | |
449 | put_super(sb); |
450 | return false; |
451 | } |
452 | |
453 | /* |
454 | * Write a portion of b_io inodes which belong to @sb. |
455 | * |
456 | * If @only_this_sb is true, then find and write all such |
457 | * inodes. Otherwise write only ones which go sequentially |
458 | * in reverse order. |
459 | * |
460 | * Return 1, if the caller writeback routine should be |
461 | * interrupted. Otherwise return 0. |
462 | */ |
463 | static int writeback_sb_inodes(struct super_block *sb, struct bdi_writeback *wb, |
464 | struct writeback_control *wbc, bool only_this_sb) |
465 | { |
466 | while (!list_empty(&wb->b_io)) { |
467 | long pages_skipped; |
468 | struct inode *inode = list_entry(wb->b_io.prev, |
469 | struct inode, i_list); |
470 | |
471 | if (inode->i_sb != sb) { |
472 | if (only_this_sb) { |
473 | /* |
474 | * We only want to write back data for this |
475 | * superblock, move all inodes not belonging |
476 | * to it back onto the dirty list. |
477 | */ |
478 | redirty_tail(inode); |
479 | continue; |
480 | } |
481 | |
482 | /* |
483 | * The inode belongs to a different superblock. |
484 | * Bounce back to the caller to unpin this and |
485 | * pin the next superblock. |
486 | */ |
487 | return 0; |
488 | } |
489 | |
490 | if (inode->i_state & (I_NEW | I_WILL_FREE)) { |
491 | requeue_io(inode); |
492 | continue; |
493 | } |
494 | /* |
495 | * Was this inode dirtied after sync_sb_inodes was called? |
496 | * This keeps sync from extra jobs and livelock. |
497 | */ |
498 | if (inode_dirtied_after(inode, wbc->wb_start)) |
499 | return 1; |
500 | |
501 | BUG_ON(inode->i_state & I_FREEING); |
502 | __iget(inode); |
503 | pages_skipped = wbc->pages_skipped; |
504 | writeback_single_inode(inode, wbc); |
505 | if (wbc->pages_skipped != pages_skipped) { |
506 | /* |
507 | * writeback is not making progress due to locked |
508 | * buffers. Skip this inode for now. |
509 | */ |
510 | redirty_tail(inode); |
511 | } |
512 | spin_unlock(&inode_lock); |
513 | iput(inode); |
514 | cond_resched(); |
515 | spin_lock(&inode_lock); |
516 | if (wbc->nr_to_write <= 0) { |
517 | wbc->more_io = 1; |
518 | return 1; |
519 | } |
520 | if (!list_empty(&wb->b_more_io)) |
521 | wbc->more_io = 1; |
522 | } |
523 | /* b_io is empty */ |
524 | return 1; |
525 | } |
526 | |
527 | void writeback_inodes_wb(struct bdi_writeback *wb, |
528 | struct writeback_control *wbc) |
529 | { |
530 | int ret = 0; |
531 | |
532 | if (!wbc->wb_start) |
533 | wbc->wb_start = jiffies; /* livelock avoidance */ |
534 | spin_lock(&inode_lock); |
535 | if (!wbc->for_kupdate || list_empty(&wb->b_io)) |
536 | queue_io(wb, wbc->older_than_this); |
537 | |
538 | while (!list_empty(&wb->b_io)) { |
539 | struct inode *inode = list_entry(wb->b_io.prev, |
540 | struct inode, i_list); |
541 | struct super_block *sb = inode->i_sb; |
542 | |
543 | if (!pin_sb_for_writeback(sb)) { |
544 | requeue_io(inode); |
545 | continue; |
546 | } |
547 | ret = writeback_sb_inodes(sb, wb, wbc, false); |
548 | drop_super(sb); |
549 | |
550 | if (ret) |
551 | break; |
552 | } |
553 | spin_unlock(&inode_lock); |
554 | /* Leave any unwritten inodes on b_io */ |
555 | } |
556 | |
557 | static void __writeback_inodes_sb(struct super_block *sb, |
558 | struct bdi_writeback *wb, struct writeback_control *wbc) |
559 | { |
560 | WARN_ON(!rwsem_is_locked(&sb->s_umount)); |
561 | |
562 | spin_lock(&inode_lock); |
563 | if (!wbc->for_kupdate || list_empty(&wb->b_io)) |
564 | queue_io(wb, wbc->older_than_this); |
565 | writeback_sb_inodes(sb, wb, wbc, true); |
566 | spin_unlock(&inode_lock); |
567 | } |
568 | |
569 | /* |
570 | * The maximum number of pages to writeout in a single bdi flush/kupdate |
571 | * operation. We do this so we don't hold I_SYNC against an inode for |
572 | * enormous amounts of time, which would block a userspace task which has |
573 | * been forced to throttle against that inode. Also, the code reevaluates |
574 | * the dirty each time it has written this many pages. |
575 | */ |
576 | #define MAX_WRITEBACK_PAGES 1024 |
577 | |
578 | static inline bool over_bground_thresh(void) |
579 | { |
580 | unsigned long background_thresh, dirty_thresh; |
581 | |
582 | global_dirty_limits(&background_thresh, &dirty_thresh); |
583 | |
584 | return (global_page_state(NR_FILE_DIRTY) + |
585 | global_page_state(NR_UNSTABLE_NFS) >= background_thresh); |
586 | } |
587 | |
588 | /* |
589 | * Explicit flushing or periodic writeback of "old" data. |
590 | * |
591 | * Define "old": the first time one of an inode's pages is dirtied, we mark the |
592 | * dirtying-time in the inode's address_space. So this periodic writeback code |
593 | * just walks the superblock inode list, writing back any inodes which are |
594 | * older than a specific point in time. |
595 | * |
596 | * Try to run once per dirty_writeback_interval. But if a writeback event |
597 | * takes longer than a dirty_writeback_interval interval, then leave a |
598 | * one-second gap. |
599 | * |
600 | * older_than_this takes precedence over nr_to_write. So we'll only write back |
601 | * all dirty pages if they are all attached to "old" mappings. |
602 | */ |
603 | static long wb_writeback(struct bdi_writeback *wb, |
604 | struct wb_writeback_work *work) |
605 | { |
606 | struct writeback_control wbc = { |
607 | .sync_mode = work->sync_mode, |
608 | .older_than_this = NULL, |
609 | .for_kupdate = work->for_kupdate, |
610 | .for_background = work->for_background, |
611 | .range_cyclic = work->range_cyclic, |
612 | }; |
613 | unsigned long oldest_jif; |
614 | long wrote = 0; |
615 | struct inode *inode; |
616 | |
617 | if (wbc.for_kupdate) { |
618 | wbc.older_than_this = &oldest_jif; |
619 | oldest_jif = jiffies - |
620 | msecs_to_jiffies(dirty_expire_interval * 10); |
621 | } |
622 | if (!wbc.range_cyclic) { |
623 | wbc.range_start = 0; |
624 | wbc.range_end = LLONG_MAX; |
625 | } |
626 | |
627 | wbc.wb_start = jiffies; /* livelock avoidance */ |
628 | for (;;) { |
629 | /* |
630 | * Stop writeback when nr_pages has been consumed |
631 | */ |
632 | if (work->nr_pages <= 0) |
633 | break; |
634 | |
635 | /* |
636 | * For background writeout, stop when we are below the |
637 | * background dirty threshold |
638 | */ |
639 | if (work->for_background && !over_bground_thresh()) |
640 | break; |
641 | |
642 | wbc.more_io = 0; |
643 | wbc.nr_to_write = MAX_WRITEBACK_PAGES; |
644 | wbc.pages_skipped = 0; |
645 | |
646 | trace_wbc_writeback_start(&wbc, wb->bdi); |
647 | if (work->sb) |
648 | __writeback_inodes_sb(work->sb, wb, &wbc); |
649 | else |
650 | writeback_inodes_wb(wb, &wbc); |
651 | trace_wbc_writeback_written(&wbc, wb->bdi); |
652 | |
653 | work->nr_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write; |
654 | wrote += MAX_WRITEBACK_PAGES - wbc.nr_to_write; |
655 | |
656 | /* |
657 | * If we consumed everything, see if we have more |
658 | */ |
659 | if (wbc.nr_to_write <= 0) |
660 | continue; |
661 | /* |
662 | * Didn't write everything and we don't have more IO, bail |
663 | */ |
664 | if (!wbc.more_io) |
665 | break; |
666 | /* |
667 | * Did we write something? Try for more |
668 | */ |
669 | if (wbc.nr_to_write < MAX_WRITEBACK_PAGES) |
670 | continue; |
671 | /* |
672 | * Nothing written. Wait for some inode to |
673 | * become available for writeback. Otherwise |
674 | * we'll just busyloop. |
675 | */ |
676 | spin_lock(&inode_lock); |
677 | if (!list_empty(&wb->b_more_io)) { |
678 | inode = list_entry(wb->b_more_io.prev, |
679 | struct inode, i_list); |
680 | trace_wbc_writeback_wait(&wbc, wb->bdi); |
681 | inode_wait_for_writeback(inode); |
682 | } |
683 | spin_unlock(&inode_lock); |
684 | } |
685 | |
686 | return wrote; |
687 | } |
688 | |
689 | /* |
690 | * Return the next wb_writeback_work struct that hasn't been processed yet. |
691 | */ |
692 | static struct wb_writeback_work * |
693 | get_next_work_item(struct backing_dev_info *bdi) |
694 | { |
695 | struct wb_writeback_work *work = NULL; |
696 | |
697 | spin_lock_bh(&bdi->wb_lock); |
698 | if (!list_empty(&bdi->work_list)) { |
699 | work = list_entry(bdi->work_list.next, |
700 | struct wb_writeback_work, list); |
701 | list_del_init(&work->list); |
702 | } |
703 | spin_unlock_bh(&bdi->wb_lock); |
704 | return work; |
705 | } |
706 | |
707 | static long wb_check_old_data_flush(struct bdi_writeback *wb) |
708 | { |
709 | unsigned long expired; |
710 | long nr_pages; |
711 | |
712 | /* |
713 | * When set to zero, disable periodic writeback |
714 | */ |
715 | if (!dirty_writeback_interval) |
716 | return 0; |
717 | |
718 | expired = wb->last_old_flush + |
719 | msecs_to_jiffies(dirty_writeback_interval * 10); |
720 | if (time_before(jiffies, expired)) |
721 | return 0; |
722 | |
723 | wb->last_old_flush = jiffies; |
724 | nr_pages = global_page_state(NR_FILE_DIRTY) + |
725 | global_page_state(NR_UNSTABLE_NFS) + |
726 | (inodes_stat.nr_inodes - inodes_stat.nr_unused); |
727 | |
728 | if (nr_pages) { |
729 | struct wb_writeback_work work = { |
730 | .nr_pages = nr_pages, |
731 | .sync_mode = WB_SYNC_NONE, |
732 | .for_kupdate = 1, |
733 | .range_cyclic = 1, |
734 | }; |
735 | |
736 | return wb_writeback(wb, &work); |
737 | } |
738 | |
739 | return 0; |
740 | } |
741 | |
742 | /* |
743 | * Retrieve work items and do the writeback they describe |
744 | */ |
745 | long wb_do_writeback(struct bdi_writeback *wb, int force_wait) |
746 | { |
747 | struct backing_dev_info *bdi = wb->bdi; |
748 | struct wb_writeback_work *work; |
749 | long wrote = 0; |
750 | |
751 | set_bit(BDI_writeback_running, &wb->bdi->state); |
752 | while ((work = get_next_work_item(bdi)) != NULL) { |
753 | /* |
754 | * Override sync mode, in case we must wait for completion |
755 | * because this thread is exiting now. |
756 | */ |
757 | if (force_wait) |
758 | work->sync_mode = WB_SYNC_ALL; |
759 | |
760 | trace_writeback_exec(bdi, work); |
761 | |
762 | wrote += wb_writeback(wb, work); |
763 | |
764 | /* |
765 | * Notify the caller of completion if this is a synchronous |
766 | * work item, otherwise just free it. |
767 | */ |
768 | if (work->done) |
769 | complete(work->done); |
770 | else |
771 | kfree(work); |
772 | } |
773 | |
774 | /* |
775 | * Check for periodic writeback, kupdated() style |
776 | */ |
777 | wrote += wb_check_old_data_flush(wb); |
778 | clear_bit(BDI_writeback_running, &wb->bdi->state); |
779 | |
780 | return wrote; |
781 | } |
782 | |
783 | /* |
784 | * Handle writeback of dirty data for the device backed by this bdi. Also |
785 | * wakes up periodically and does kupdated style flushing. |
786 | */ |
787 | int bdi_writeback_thread(void *data) |
788 | { |
789 | struct bdi_writeback *wb = data; |
790 | struct backing_dev_info *bdi = wb->bdi; |
791 | long pages_written; |
792 | |
793 | current->flags |= PF_FLUSHER | PF_SWAPWRITE; |
794 | set_freezable(); |
795 | wb->last_active = jiffies; |
796 | |
797 | /* |
798 | * Our parent may run at a different priority, just set us to normal |
799 | */ |
800 | set_user_nice(current, 0); |
801 | |
802 | trace_writeback_thread_start(bdi); |
803 | |
804 | while (!kthread_should_stop()) { |
805 | /* |
806 | * Remove own delayed wake-up timer, since we are already awake |
807 | * and we'll take care of the preriodic write-back. |
808 | */ |
809 | del_timer(&wb->wakeup_timer); |
810 | |
811 | pages_written = wb_do_writeback(wb, 0); |
812 | |
813 | trace_writeback_pages_written(pages_written); |
814 | |
815 | if (pages_written) |
816 | wb->last_active = jiffies; |
817 | |
818 | set_current_state(TASK_INTERRUPTIBLE); |
819 | if (!list_empty(&bdi->work_list) || kthread_should_stop()) { |
820 | __set_current_state(TASK_RUNNING); |
821 | continue; |
822 | } |
823 | |
824 | if (wb_has_dirty_io(wb) && dirty_writeback_interval) |
825 | schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10)); |
826 | else { |
827 | /* |
828 | * We have nothing to do, so can go sleep without any |
829 | * timeout and save power. When a work is queued or |
830 | * something is made dirty - we will be woken up. |
831 | */ |
832 | schedule(); |
833 | } |
834 | |
835 | try_to_freeze(); |
836 | } |
837 | |
838 | /* Flush any work that raced with us exiting */ |
839 | if (!list_empty(&bdi->work_list)) |
840 | wb_do_writeback(wb, 1); |
841 | |
842 | trace_writeback_thread_stop(bdi); |
843 | return 0; |
844 | } |
845 | |
846 | |
847 | /* |
848 | * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back |
849 | * the whole world. |
850 | */ |
851 | void wakeup_flusher_threads(long nr_pages) |
852 | { |
853 | struct backing_dev_info *bdi; |
854 | |
855 | if (!nr_pages) { |
856 | nr_pages = global_page_state(NR_FILE_DIRTY) + |
857 | global_page_state(NR_UNSTABLE_NFS); |
858 | } |
859 | |
860 | rcu_read_lock(); |
861 | list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) { |
862 | if (!bdi_has_dirty_io(bdi)) |
863 | continue; |
864 | __bdi_start_writeback(bdi, nr_pages, false, false); |
865 | } |
866 | rcu_read_unlock(); |
867 | } |
868 | |
869 | static noinline void block_dump___mark_inode_dirty(struct inode *inode) |
870 | { |
871 | if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) { |
872 | struct dentry *dentry; |
873 | const char *name = "?"; |
874 | |
875 | dentry = d_find_alias(inode); |
876 | if (dentry) { |
877 | spin_lock(&dentry->d_lock); |
878 | name = (const char *) dentry->d_name.name; |
879 | } |
880 | printk(KERN_DEBUG |
881 | "%s(%d): dirtied inode %lu (%s) on %s\n", |
882 | current->comm, task_pid_nr(current), inode->i_ino, |
883 | name, inode->i_sb->s_id); |
884 | if (dentry) { |
885 | spin_unlock(&dentry->d_lock); |
886 | dput(dentry); |
887 | } |
888 | } |
889 | } |
890 | |
891 | /** |
892 | * __mark_inode_dirty - internal function |
893 | * @inode: inode to mark |
894 | * @flags: what kind of dirty (i.e. I_DIRTY_SYNC) |
895 | * Mark an inode as dirty. Callers should use mark_inode_dirty or |
896 | * mark_inode_dirty_sync. |
897 | * |
898 | * Put the inode on the super block's dirty list. |
899 | * |
900 | * CAREFUL! We mark it dirty unconditionally, but move it onto the |
901 | * dirty list only if it is hashed or if it refers to a blockdev. |
902 | * If it was not hashed, it will never be added to the dirty list |
903 | * even if it is later hashed, as it will have been marked dirty already. |
904 | * |
905 | * In short, make sure you hash any inodes _before_ you start marking |
906 | * them dirty. |
907 | * |
908 | * This function *must* be atomic for the I_DIRTY_PAGES case - |
909 | * set_page_dirty() is called under spinlock in several places. |
910 | * |
911 | * Note that for blockdevs, inode->dirtied_when represents the dirtying time of |
912 | * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of |
913 | * the kernel-internal blockdev inode represents the dirtying time of the |
914 | * blockdev's pages. This is why for I_DIRTY_PAGES we always use |
915 | * page->mapping->host, so the page-dirtying time is recorded in the internal |
916 | * blockdev inode. |
917 | */ |
918 | void __mark_inode_dirty(struct inode *inode, int flags) |
919 | { |
920 | struct super_block *sb = inode->i_sb; |
921 | struct backing_dev_info *bdi = NULL; |
922 | bool wakeup_bdi = false; |
923 | |
924 | /* |
925 | * Don't do this for I_DIRTY_PAGES - that doesn't actually |
926 | * dirty the inode itself |
927 | */ |
928 | if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) { |
929 | if (sb->s_op->dirty_inode) |
930 | sb->s_op->dirty_inode(inode); |
931 | } |
932 | |
933 | /* |
934 | * make sure that changes are seen by all cpus before we test i_state |
935 | * -- mikulas |
936 | */ |
937 | smp_mb(); |
938 | |
939 | /* avoid the locking if we can */ |
940 | if ((inode->i_state & flags) == flags) |
941 | return; |
942 | |
943 | if (unlikely(block_dump)) |
944 | block_dump___mark_inode_dirty(inode); |
945 | |
946 | spin_lock(&inode_lock); |
947 | if ((inode->i_state & flags) != flags) { |
948 | const int was_dirty = inode->i_state & I_DIRTY; |
949 | |
950 | inode->i_state |= flags; |
951 | |
952 | /* |
953 | * If the inode is being synced, just update its dirty state. |
954 | * The unlocker will place the inode on the appropriate |
955 | * superblock list, based upon its state. |
956 | */ |
957 | if (inode->i_state & I_SYNC) |
958 | goto out; |
959 | |
960 | /* |
961 | * Only add valid (hashed) inodes to the superblock's |
962 | * dirty list. Add blockdev inodes as well. |
963 | */ |
964 | if (!S_ISBLK(inode->i_mode)) { |
965 | if (hlist_unhashed(&inode->i_hash)) |
966 | goto out; |
967 | } |
968 | if (inode->i_state & I_FREEING) |
969 | goto out; |
970 | |
971 | /* |
972 | * If the inode was already on b_dirty/b_io/b_more_io, don't |
973 | * reposition it (that would break b_dirty time-ordering). |
974 | */ |
975 | if (!was_dirty) { |
976 | bdi = inode_to_bdi(inode); |
977 | |
978 | if (bdi_cap_writeback_dirty(bdi)) { |
979 | WARN(!test_bit(BDI_registered, &bdi->state), |
980 | "bdi-%s not registered\n", bdi->name); |
981 | |
982 | /* |
983 | * If this is the first dirty inode for this |
984 | * bdi, we have to wake-up the corresponding |
985 | * bdi thread to make sure background |
986 | * write-back happens later. |
987 | */ |
988 | if (!wb_has_dirty_io(&bdi->wb)) |
989 | wakeup_bdi = true; |
990 | } |
991 | |
992 | inode->dirtied_when = jiffies; |
993 | list_move(&inode->i_list, &bdi->wb.b_dirty); |
994 | } |
995 | } |
996 | out: |
997 | spin_unlock(&inode_lock); |
998 | |
999 | if (wakeup_bdi) |
1000 | bdi_wakeup_thread_delayed(bdi); |
1001 | } |
1002 | EXPORT_SYMBOL(__mark_inode_dirty); |
1003 | |
1004 | /* |
1005 | * Write out a superblock's list of dirty inodes. A wait will be performed |
1006 | * upon no inodes, all inodes or the final one, depending upon sync_mode. |
1007 | * |
1008 | * If older_than_this is non-NULL, then only write out inodes which |
1009 | * had their first dirtying at a time earlier than *older_than_this. |
1010 | * |
1011 | * If `bdi' is non-zero then we're being asked to writeback a specific queue. |
1012 | * This function assumes that the blockdev superblock's inodes are backed by |
1013 | * a variety of queues, so all inodes are searched. For other superblocks, |
1014 | * assume that all inodes are backed by the same queue. |
1015 | * |
1016 | * The inodes to be written are parked on bdi->b_io. They are moved back onto |
1017 | * bdi->b_dirty as they are selected for writing. This way, none can be missed |
1018 | * on the writer throttling path, and we get decent balancing between many |
1019 | * throttled threads: we don't want them all piling up on inode_sync_wait. |
1020 | */ |
1021 | static void wait_sb_inodes(struct super_block *sb) |
1022 | { |
1023 | struct inode *inode, *old_inode = NULL; |
1024 | |
1025 | /* |
1026 | * We need to be protected against the filesystem going from |
1027 | * r/o to r/w or vice versa. |
1028 | */ |
1029 | WARN_ON(!rwsem_is_locked(&sb->s_umount)); |
1030 | |
1031 | spin_lock(&inode_lock); |
1032 | |
1033 | /* |
1034 | * Data integrity sync. Must wait for all pages under writeback, |
1035 | * because there may have been pages dirtied before our sync |
1036 | * call, but which had writeout started before we write it out. |
1037 | * In which case, the inode may not be on the dirty list, but |
1038 | * we still have to wait for that writeout. |
1039 | */ |
1040 | list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { |
1041 | struct address_space *mapping; |
1042 | |
1043 | if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) |
1044 | continue; |
1045 | mapping = inode->i_mapping; |
1046 | if (mapping->nrpages == 0) |
1047 | continue; |
1048 | __iget(inode); |
1049 | spin_unlock(&inode_lock); |
1050 | /* |
1051 | * We hold a reference to 'inode' so it couldn't have |
1052 | * been removed from s_inodes list while we dropped the |
1053 | * inode_lock. We cannot iput the inode now as we can |
1054 | * be holding the last reference and we cannot iput it |
1055 | * under inode_lock. So we keep the reference and iput |
1056 | * it later. |
1057 | */ |
1058 | iput(old_inode); |
1059 | old_inode = inode; |
1060 | |
1061 | filemap_fdatawait(mapping); |
1062 | |
1063 | cond_resched(); |
1064 | |
1065 | spin_lock(&inode_lock); |
1066 | } |
1067 | spin_unlock(&inode_lock); |
1068 | iput(old_inode); |
1069 | } |
1070 | |
1071 | /** |
1072 | * writeback_inodes_sb - writeback dirty inodes from given super_block |
1073 | * @sb: the superblock |
1074 | * |
1075 | * Start writeback on some inodes on this super_block. No guarantees are made |
1076 | * on how many (if any) will be written, and this function does not wait |
1077 | * for IO completion of submitted IO. The number of pages submitted is |
1078 | * returned. |
1079 | */ |
1080 | void writeback_inodes_sb(struct super_block *sb) |
1081 | { |
1082 | unsigned long nr_dirty = global_page_state(NR_FILE_DIRTY); |
1083 | unsigned long nr_unstable = global_page_state(NR_UNSTABLE_NFS); |
1084 | DECLARE_COMPLETION_ONSTACK(done); |
1085 | struct wb_writeback_work work = { |
1086 | .sb = sb, |
1087 | .sync_mode = WB_SYNC_NONE, |
1088 | .done = &done, |
1089 | }; |
1090 | |
1091 | WARN_ON(!rwsem_is_locked(&sb->s_umount)); |
1092 | |
1093 | work.nr_pages = nr_dirty + nr_unstable + |
1094 | (inodes_stat.nr_inodes - inodes_stat.nr_unused); |
1095 | |
1096 | bdi_queue_work(sb->s_bdi, &work); |
1097 | wait_for_completion(&done); |
1098 | } |
1099 | EXPORT_SYMBOL(writeback_inodes_sb); |
1100 | |
1101 | /** |
1102 | * writeback_inodes_sb_if_idle - start writeback if none underway |
1103 | * @sb: the superblock |
1104 | * |
1105 | * Invoke writeback_inodes_sb if no writeback is currently underway. |
1106 | * Returns 1 if writeback was started, 0 if not. |
1107 | */ |
1108 | int writeback_inodes_sb_if_idle(struct super_block *sb) |
1109 | { |
1110 | if (!writeback_in_progress(sb->s_bdi)) { |
1111 | down_read(&sb->s_umount); |
1112 | writeback_inodes_sb(sb); |
1113 | up_read(&sb->s_umount); |
1114 | return 1; |
1115 | } else |
1116 | return 0; |
1117 | } |
1118 | EXPORT_SYMBOL(writeback_inodes_sb_if_idle); |
1119 | |
1120 | /** |
1121 | * sync_inodes_sb - sync sb inode pages |
1122 | * @sb: the superblock |
1123 | * |
1124 | * This function writes and waits on any dirty inode belonging to this |
1125 | * super_block. The number of pages synced is returned. |
1126 | */ |
1127 | void sync_inodes_sb(struct super_block *sb) |
1128 | { |
1129 | DECLARE_COMPLETION_ONSTACK(done); |
1130 | struct wb_writeback_work work = { |
1131 | .sb = sb, |
1132 | .sync_mode = WB_SYNC_ALL, |
1133 | .nr_pages = LONG_MAX, |
1134 | .range_cyclic = 0, |
1135 | .done = &done, |
1136 | }; |
1137 | |
1138 | WARN_ON(!rwsem_is_locked(&sb->s_umount)); |
1139 | |
1140 | bdi_queue_work(sb->s_bdi, &work); |
1141 | wait_for_completion(&done); |
1142 | |
1143 | wait_sb_inodes(sb); |
1144 | } |
1145 | EXPORT_SYMBOL(sync_inodes_sb); |
1146 | |
1147 | /** |
1148 | * write_inode_now - write an inode to disk |
1149 | * @inode: inode to write to disk |
1150 | * @sync: whether the write should be synchronous or not |
1151 | * |
1152 | * This function commits an inode to disk immediately if it is dirty. This is |
1153 | * primarily needed by knfsd. |
1154 | * |
1155 | * The caller must either have a ref on the inode or must have set I_WILL_FREE. |
1156 | */ |
1157 | int write_inode_now(struct inode *inode, int sync) |
1158 | { |
1159 | int ret; |
1160 | struct writeback_control wbc = { |
1161 | .nr_to_write = LONG_MAX, |
1162 | .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE, |
1163 | .range_start = 0, |
1164 | .range_end = LLONG_MAX, |
1165 | }; |
1166 | |
1167 | if (!mapping_cap_writeback_dirty(inode->i_mapping)) |
1168 | wbc.nr_to_write = 0; |
1169 | |
1170 | might_sleep(); |
1171 | spin_lock(&inode_lock); |
1172 | ret = writeback_single_inode(inode, &wbc); |
1173 | spin_unlock(&inode_lock); |
1174 | if (sync) |
1175 | inode_sync_wait(inode); |
1176 | return ret; |
1177 | } |
1178 | EXPORT_SYMBOL(write_inode_now); |
1179 | |
1180 | /** |
1181 | * sync_inode - write an inode and its pages to disk. |
1182 | * @inode: the inode to sync |
1183 | * @wbc: controls the writeback mode |
1184 | * |
1185 | * sync_inode() will write an inode and its pages to disk. It will also |
1186 | * correctly update the inode on its superblock's dirty inode lists and will |
1187 | * update inode->i_state. |
1188 | * |
1189 | * The caller must have a ref on the inode. |
1190 | */ |
1191 | int sync_inode(struct inode *inode, struct writeback_control *wbc) |
1192 | { |
1193 | int ret; |
1194 | |
1195 | spin_lock(&inode_lock); |
1196 | ret = writeback_single_inode(inode, wbc); |
1197 | spin_unlock(&inode_lock); |
1198 | return ret; |
1199 | } |
1200 | EXPORT_SYMBOL(sync_inode); |
1201 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9