Root/mm/page-writeback.c

1/*
2 * mm/page-writeback.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
6 *
7 * Contains functions related to writing back dirty pages at the
8 * address_space level.
9 *
10 * 10Apr2002 Andrew Morton
11 * Initial version
12 */
13
14#include <linux/kernel.h>
15#include <linux/export.h>
16#include <linux/spinlock.h>
17#include <linux/fs.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/slab.h>
21#include <linux/pagemap.h>
22#include <linux/writeback.h>
23#include <linux/init.h>
24#include <linux/backing-dev.h>
25#include <linux/task_io_accounting_ops.h>
26#include <linux/blkdev.h>
27#include <linux/mpage.h>
28#include <linux/rmap.h>
29#include <linux/percpu.h>
30#include <linux/notifier.h>
31#include <linux/smp.h>
32#include <linux/sysctl.h>
33#include <linux/cpu.h>
34#include <linux/syscalls.h>
35#include <linux/buffer_head.h> /* __set_page_dirty_buffers */
36#include <linux/pagevec.h>
37#include <linux/timer.h>
38#include <trace/events/writeback.h>
39
40/*
41 * Sleep at most 200ms at a time in balance_dirty_pages().
42 */
43#define MAX_PAUSE max(HZ/5, 1)
44
45/*
46 * Try to keep balance_dirty_pages() call intervals higher than this many pages
47 * by raising pause time to max_pause when falls below it.
48 */
49#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10))
50
51/*
52 * Estimate write bandwidth at 200ms intervals.
53 */
54#define BANDWIDTH_INTERVAL max(HZ/5, 1)
55
56#define RATELIMIT_CALC_SHIFT 10
57
58/*
59 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
60 * will look to see if it needs to force writeback or throttling.
61 */
62static long ratelimit_pages = 32;
63
64/* The following parameters are exported via /proc/sys/vm */
65
66/*
67 * Start background writeback (via writeback threads) at this percentage
68 */
69int dirty_background_ratio = 10;
70
71/*
72 * dirty_background_bytes starts at 0 (disabled) so that it is a function of
73 * dirty_background_ratio * the amount of dirtyable memory
74 */
75unsigned long dirty_background_bytes;
76
77/*
78 * free highmem will not be subtracted from the total free memory
79 * for calculating free ratios if vm_highmem_is_dirtyable is true
80 */
81int vm_highmem_is_dirtyable;
82
83/*
84 * The generator of dirty data starts writeback at this percentage
85 */
86int vm_dirty_ratio = 20;
87
88/*
89 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
90 * vm_dirty_ratio * the amount of dirtyable memory
91 */
92unsigned long vm_dirty_bytes;
93
94/*
95 * The interval between `kupdate'-style writebacks
96 */
97unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
98
99EXPORT_SYMBOL_GPL(dirty_writeback_interval);
100
101/*
102 * The longest time for which data is allowed to remain dirty
103 */
104unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
105
106/*
107 * Flag that makes the machine dump writes/reads and block dirtyings.
108 */
109int block_dump;
110
111/*
112 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
113 * a full sync is triggered after this time elapses without any disk activity.
114 */
115int laptop_mode;
116
117EXPORT_SYMBOL(laptop_mode);
118
119/* End of sysctl-exported parameters */
120
121unsigned long global_dirty_limit;
122
123/*
124 * Scale the writeback cache size proportional to the relative writeout speeds.
125 *
126 * We do this by keeping a floating proportion between BDIs, based on page
127 * writeback completions [end_page_writeback()]. Those devices that write out
128 * pages fastest will get the larger share, while the slower will get a smaller
129 * share.
130 *
131 * We use page writeout completions because we are interested in getting rid of
132 * dirty pages. Having them written out is the primary goal.
133 *
134 * We introduce a concept of time, a period over which we measure these events,
135 * because demand can/will vary over time. The length of this period itself is
136 * measured in page writeback completions.
137 *
138 */
139static struct fprop_global writeout_completions;
140
141static void writeout_period(unsigned long t);
142/* Timer for aging of writeout_completions */
143static struct timer_list writeout_period_timer =
144        TIMER_DEFERRED_INITIALIZER(writeout_period, 0, 0);
145static unsigned long writeout_period_time = 0;
146
147/*
148 * Length of period for aging writeout fractions of bdis. This is an
149 * arbitrarily chosen number. The longer the period, the slower fractions will
150 * reflect changes in current writeout rate.
151 */
152#define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
153
154/*
155 * Work out the current dirty-memory clamping and background writeout
156 * thresholds.
157 *
158 * The main aim here is to lower them aggressively if there is a lot of mapped
159 * memory around. To avoid stressing page reclaim with lots of unreclaimable
160 * pages. It is better to clamp down on writers than to start swapping, and
161 * performing lots of scanning.
162 *
163 * We only allow 1/2 of the currently-unmapped memory to be dirtied.
164 *
165 * We don't permit the clamping level to fall below 5% - that is getting rather
166 * excessive.
167 *
168 * We make sure that the background writeout level is below the adjusted
169 * clamping level.
170 */
171
172/*
173 * In a memory zone, there is a certain amount of pages we consider
174 * available for the page cache, which is essentially the number of
175 * free and reclaimable pages, minus some zone reserves to protect
176 * lowmem and the ability to uphold the zone's watermarks without
177 * requiring writeback.
178 *
179 * This number of dirtyable pages is the base value of which the
180 * user-configurable dirty ratio is the effictive number of pages that
181 * are allowed to be actually dirtied. Per individual zone, or
182 * globally by using the sum of dirtyable pages over all zones.
183 *
184 * Because the user is allowed to specify the dirty limit globally as
185 * absolute number of bytes, calculating the per-zone dirty limit can
186 * require translating the configured limit into a percentage of
187 * global dirtyable memory first.
188 */
189
190static unsigned long highmem_dirtyable_memory(unsigned long total)
191{
192#ifdef CONFIG_HIGHMEM
193    int node;
194    unsigned long x = 0;
195
196    for_each_node_state(node, N_HIGH_MEMORY) {
197        struct zone *z =
198            &NODE_DATA(node)->node_zones[ZONE_HIGHMEM];
199
200        x += zone_page_state(z, NR_FREE_PAGES) +
201             zone_reclaimable_pages(z) - z->dirty_balance_reserve;
202    }
203    /*
204     * Make sure that the number of highmem pages is never larger
205     * than the number of the total dirtyable memory. This can only
206     * occur in very strange VM situations but we want to make sure
207     * that this does not occur.
208     */
209    return min(x, total);
210#else
211    return 0;
212#endif
213}
214
215/**
216 * global_dirtyable_memory - number of globally dirtyable pages
217 *
218 * Returns the global number of pages potentially available for dirty
219 * page cache. This is the base value for the global dirty limits.
220 */
221static unsigned long global_dirtyable_memory(void)
222{
223    unsigned long x;
224
225    x = global_page_state(NR_FREE_PAGES) + global_reclaimable_pages() -
226        dirty_balance_reserve;
227
228    if (!vm_highmem_is_dirtyable)
229        x -= highmem_dirtyable_memory(x);
230
231    return x + 1; /* Ensure that we never return 0 */
232}
233
234/*
235 * global_dirty_limits - background-writeback and dirty-throttling thresholds
236 *
237 * Calculate the dirty thresholds based on sysctl parameters
238 * - vm.dirty_background_ratio or vm.dirty_background_bytes
239 * - vm.dirty_ratio or vm.dirty_bytes
240 * The dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and
241 * real-time tasks.
242 */
243void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
244{
245    unsigned long background;
246    unsigned long dirty;
247    unsigned long uninitialized_var(available_memory);
248    struct task_struct *tsk;
249
250    if (!vm_dirty_bytes || !dirty_background_bytes)
251        available_memory = global_dirtyable_memory();
252
253    if (vm_dirty_bytes)
254        dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE);
255    else
256        dirty = (vm_dirty_ratio * available_memory) / 100;
257
258    if (dirty_background_bytes)
259        background = DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE);
260    else
261        background = (dirty_background_ratio * available_memory) / 100;
262
263    if (background >= dirty)
264        background = dirty / 2;
265    tsk = current;
266    if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
267        background += background / 4;
268        dirty += dirty / 4;
269    }
270    *pbackground = background;
271    *pdirty = dirty;
272    trace_global_dirty_state(background, dirty);
273}
274
275/**
276 * zone_dirtyable_memory - number of dirtyable pages in a zone
277 * @zone: the zone
278 *
279 * Returns the zone's number of pages potentially available for dirty
280 * page cache. This is the base value for the per-zone dirty limits.
281 */
282static unsigned long zone_dirtyable_memory(struct zone *zone)
283{
284    /*
285     * The effective global number of dirtyable pages may exclude
286     * highmem as a big-picture measure to keep the ratio between
287     * dirty memory and lowmem reasonable.
288     *
289     * But this function is purely about the individual zone and a
290     * highmem zone can hold its share of dirty pages, so we don't
291     * care about vm_highmem_is_dirtyable here.
292     */
293    return zone_page_state(zone, NR_FREE_PAGES) +
294           zone_reclaimable_pages(zone) -
295           zone->dirty_balance_reserve;
296}
297
298/**
299 * zone_dirty_limit - maximum number of dirty pages allowed in a zone
300 * @zone: the zone
301 *
302 * Returns the maximum number of dirty pages allowed in a zone, based
303 * on the zone's dirtyable memory.
304 */
305static unsigned long zone_dirty_limit(struct zone *zone)
306{
307    unsigned long zone_memory = zone_dirtyable_memory(zone);
308    struct task_struct *tsk = current;
309    unsigned long dirty;
310
311    if (vm_dirty_bytes)
312        dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
313            zone_memory / global_dirtyable_memory();
314    else
315        dirty = vm_dirty_ratio * zone_memory / 100;
316
317    if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk))
318        dirty += dirty / 4;
319
320    return dirty;
321}
322
323/**
324 * zone_dirty_ok - tells whether a zone is within its dirty limits
325 * @zone: the zone to check
326 *
327 * Returns %true when the dirty pages in @zone are within the zone's
328 * dirty limit, %false if the limit is exceeded.
329 */
330bool zone_dirty_ok(struct zone *zone)
331{
332    unsigned long limit = zone_dirty_limit(zone);
333
334    return zone_page_state(zone, NR_FILE_DIRTY) +
335           zone_page_state(zone, NR_UNSTABLE_NFS) +
336           zone_page_state(zone, NR_WRITEBACK) <= limit;
337}
338
339int dirty_background_ratio_handler(struct ctl_table *table, int write,
340        void __user *buffer, size_t *lenp,
341        loff_t *ppos)
342{
343    int ret;
344
345    ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
346    if (ret == 0 && write)
347        dirty_background_bytes = 0;
348    return ret;
349}
350
351int dirty_background_bytes_handler(struct ctl_table *table, int write,
352        void __user *buffer, size_t *lenp,
353        loff_t *ppos)
354{
355    int ret;
356
357    ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
358    if (ret == 0 && write)
359        dirty_background_ratio = 0;
360    return ret;
361}
362
363int dirty_ratio_handler(struct ctl_table *table, int write,
364        void __user *buffer, size_t *lenp,
365        loff_t *ppos)
366{
367    int old_ratio = vm_dirty_ratio;
368    int ret;
369
370    ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
371    if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
372        writeback_set_ratelimit();
373        vm_dirty_bytes = 0;
374    }
375    return ret;
376}
377
378int dirty_bytes_handler(struct ctl_table *table, int write,
379        void __user *buffer, size_t *lenp,
380        loff_t *ppos)
381{
382    unsigned long old_bytes = vm_dirty_bytes;
383    int ret;
384
385    ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
386    if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
387        writeback_set_ratelimit();
388        vm_dirty_ratio = 0;
389    }
390    return ret;
391}
392
393static unsigned long wp_next_time(unsigned long cur_time)
394{
395    cur_time += VM_COMPLETIONS_PERIOD_LEN;
396    /* 0 has a special meaning... */
397    if (!cur_time)
398        return 1;
399    return cur_time;
400}
401
402/*
403 * Increment the BDI's writeout completion count and the global writeout
404 * completion count. Called from test_clear_page_writeback().
405 */
406static inline void __bdi_writeout_inc(struct backing_dev_info *bdi)
407{
408    __inc_bdi_stat(bdi, BDI_WRITTEN);
409    __fprop_inc_percpu_max(&writeout_completions, &bdi->completions,
410                   bdi->max_prop_frac);
411    /* First event after period switching was turned off? */
412    if (!unlikely(writeout_period_time)) {
413        /*
414         * We can race with other __bdi_writeout_inc calls here but
415         * it does not cause any harm since the resulting time when
416         * timer will fire and what is in writeout_period_time will be
417         * roughly the same.
418         */
419        writeout_period_time = wp_next_time(jiffies);
420        mod_timer(&writeout_period_timer, writeout_period_time);
421    }
422}
423
424void bdi_writeout_inc(struct backing_dev_info *bdi)
425{
426    unsigned long flags;
427
428    local_irq_save(flags);
429    __bdi_writeout_inc(bdi);
430    local_irq_restore(flags);
431}
432EXPORT_SYMBOL_GPL(bdi_writeout_inc);
433
434/*
435 * Obtain an accurate fraction of the BDI's portion.
436 */
437static void bdi_writeout_fraction(struct backing_dev_info *bdi,
438        long *numerator, long *denominator)
439{
440    fprop_fraction_percpu(&writeout_completions, &bdi->completions,
441                numerator, denominator);
442}
443
444/*
445 * On idle system, we can be called long after we scheduled because we use
446 * deferred timers so count with missed periods.
447 */
448static void writeout_period(unsigned long t)
449{
450    int miss_periods = (jiffies - writeout_period_time) /
451                         VM_COMPLETIONS_PERIOD_LEN;
452
453    if (fprop_new_period(&writeout_completions, miss_periods + 1)) {
454        writeout_period_time = wp_next_time(writeout_period_time +
455                miss_periods * VM_COMPLETIONS_PERIOD_LEN);
456        mod_timer(&writeout_period_timer, writeout_period_time);
457    } else {
458        /*
459         * Aging has zeroed all fractions. Stop wasting CPU on period
460         * updates.
461         */
462        writeout_period_time = 0;
463    }
464}
465
466/*
467 * bdi_min_ratio keeps the sum of the minimum dirty shares of all
468 * registered backing devices, which, for obvious reasons, can not
469 * exceed 100%.
470 */
471static unsigned int bdi_min_ratio;
472
473int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
474{
475    int ret = 0;
476
477    spin_lock_bh(&bdi_lock);
478    if (min_ratio > bdi->max_ratio) {
479        ret = -EINVAL;
480    } else {
481        min_ratio -= bdi->min_ratio;
482        if (bdi_min_ratio + min_ratio < 100) {
483            bdi_min_ratio += min_ratio;
484            bdi->min_ratio += min_ratio;
485        } else {
486            ret = -EINVAL;
487        }
488    }
489    spin_unlock_bh(&bdi_lock);
490
491    return ret;
492}
493
494int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
495{
496    int ret = 0;
497
498    if (max_ratio > 100)
499        return -EINVAL;
500
501    spin_lock_bh(&bdi_lock);
502    if (bdi->min_ratio > max_ratio) {
503        ret = -EINVAL;
504    } else {
505        bdi->max_ratio = max_ratio;
506        bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100;
507    }
508    spin_unlock_bh(&bdi_lock);
509
510    return ret;
511}
512EXPORT_SYMBOL(bdi_set_max_ratio);
513
514static unsigned long dirty_freerun_ceiling(unsigned long thresh,
515                       unsigned long bg_thresh)
516{
517    return (thresh + bg_thresh) / 2;
518}
519
520static unsigned long hard_dirty_limit(unsigned long thresh)
521{
522    return max(thresh, global_dirty_limit);
523}
524
525/**
526 * bdi_dirty_limit - @bdi's share of dirty throttling threshold
527 * @bdi: the backing_dev_info to query
528 * @dirty: global dirty limit in pages
529 *
530 * Returns @bdi's dirty limit in pages. The term "dirty" in the context of
531 * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages.
532 *
533 * Note that balance_dirty_pages() will only seriously take it as a hard limit
534 * when sleeping max_pause per page is not enough to keep the dirty pages under
535 * control. For example, when the device is completely stalled due to some error
536 * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
537 * In the other normal situations, it acts more gently by throttling the tasks
538 * more (rather than completely block them) when the bdi dirty pages go high.
539 *
540 * It allocates high/low dirty limits to fast/slow devices, in order to prevent
541 * - starving fast devices
542 * - piling up dirty pages (that will take long time to sync) on slow devices
543 *
544 * The bdi's share of dirty limit will be adapting to its throughput and
545 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
546 */
547unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty)
548{
549    u64 bdi_dirty;
550    long numerator, denominator;
551
552    /*
553     * Calculate this BDI's share of the dirty ratio.
554     */
555    bdi_writeout_fraction(bdi, &numerator, &denominator);
556
557    bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;
558    bdi_dirty *= numerator;
559    do_div(bdi_dirty, denominator);
560
561    bdi_dirty += (dirty * bdi->min_ratio) / 100;
562    if (bdi_dirty > (dirty * bdi->max_ratio) / 100)
563        bdi_dirty = dirty * bdi->max_ratio / 100;
564
565    return bdi_dirty;
566}
567
568/*
569 * Dirty position control.
570 *
571 * (o) global/bdi setpoints
572 *
573 * We want the dirty pages be balanced around the global/bdi setpoints.
574 * When the number of dirty pages is higher/lower than the setpoint, the
575 * dirty position control ratio (and hence task dirty ratelimit) will be
576 * decreased/increased to bring the dirty pages back to the setpoint.
577 *
578 * pos_ratio = 1 << RATELIMIT_CALC_SHIFT
579 *
580 * if (dirty < setpoint) scale up pos_ratio
581 * if (dirty > setpoint) scale down pos_ratio
582 *
583 * if (bdi_dirty < bdi_setpoint) scale up pos_ratio
584 * if (bdi_dirty > bdi_setpoint) scale down pos_ratio
585 *
586 * task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
587 *
588 * (o) global control line
589 *
590 * ^ pos_ratio
591 * |
592 * | |<===== global dirty control scope ======>|
593 * 2.0 .............*
594 * | .*
595 * | . *
596 * | . *
597 * | . *
598 * | . *
599 * | . *
600 * 1.0 ................................*
601 * | . . *
602 * | . . *
603 * | . . *
604 * | . . *
605 * | . . *
606 * 0 +------------.------------------.----------------------*------------->
607 * freerun^ setpoint^ limit^ dirty pages
608 *
609 * (o) bdi control line
610 *
611 * ^ pos_ratio
612 * |
613 * | *
614 * | *
615 * | *
616 * | *
617 * | * |<=========== span ============>|
618 * 1.0 .......................*
619 * | . *
620 * | . *
621 * | . *
622 * | . *
623 * | . *
624 * | . *
625 * | . *
626 * | . *
627 * | . *
628 * | . *
629 * | . *
630 * 1/4 ...............................................* * * * * * * * * * * *
631 * | . .
632 * | . .
633 * | . .
634 * 0 +----------------------.-------------------------------.------------->
635 * bdi_setpoint^ x_intercept^
636 *
637 * The bdi control line won't drop below pos_ratio=1/4, so that bdi_dirty can
638 * be smoothly throttled down to normal if it starts high in situations like
639 * - start writing to a slow SD card and a fast disk at the same time. The SD
640 * card's bdi_dirty may rush to many times higher than bdi_setpoint.
641 * - the bdi dirty thresh drops quickly due to change of JBOD workload
642 */
643static unsigned long bdi_position_ratio(struct backing_dev_info *bdi,
644                    unsigned long thresh,
645                    unsigned long bg_thresh,
646                    unsigned long dirty,
647                    unsigned long bdi_thresh,
648                    unsigned long bdi_dirty)
649{
650    unsigned long write_bw = bdi->avg_write_bandwidth;
651    unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
652    unsigned long limit = hard_dirty_limit(thresh);
653    unsigned long x_intercept;
654    unsigned long setpoint; /* dirty pages' target balance point */
655    unsigned long bdi_setpoint;
656    unsigned long span;
657    long long pos_ratio; /* for scaling up/down the rate limit */
658    long x;
659
660    if (unlikely(dirty >= limit))
661        return 0;
662
663    /*
664     * global setpoint
665     *
666     * setpoint - dirty 3
667     * f(dirty) := 1.0 + (----------------)
668     * limit - setpoint
669     *
670     * it's a 3rd order polynomial that subjects to
671     *
672     * (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast
673     * (2) f(setpoint) = 1.0 => the balance point
674     * (3) f(limit) = 0 => the hard limit
675     * (4) df/dx <= 0 => negative feedback control
676     * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
677     * => fast response on large errors; small oscillation near setpoint
678     */
679    setpoint = (freerun + limit) / 2;
680    x = div_s64((setpoint - dirty) << RATELIMIT_CALC_SHIFT,
681            limit - setpoint + 1);
682    pos_ratio = x;
683    pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
684    pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
685    pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
686
687    /*
688     * We have computed basic pos_ratio above based on global situation. If
689     * the bdi is over/under its share of dirty pages, we want to scale
690     * pos_ratio further down/up. That is done by the following mechanism.
691     */
692
693    /*
694     * bdi setpoint
695     *
696     * f(bdi_dirty) := 1.0 + k * (bdi_dirty - bdi_setpoint)
697     *
698     * x_intercept - bdi_dirty
699     * := --------------------------
700     * x_intercept - bdi_setpoint
701     *
702     * The main bdi control line is a linear function that subjects to
703     *
704     * (1) f(bdi_setpoint) = 1.0
705     * (2) k = - 1 / (8 * write_bw) (in single bdi case)
706     * or equally: x_intercept = bdi_setpoint + 8 * write_bw
707     *
708     * For single bdi case, the dirty pages are observed to fluctuate
709     * regularly within range
710     * [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2]
711     * for various filesystems, where (2) can yield in a reasonable 12.5%
712     * fluctuation range for pos_ratio.
713     *
714     * For JBOD case, bdi_thresh (not bdi_dirty!) could fluctuate up to its
715     * own size, so move the slope over accordingly and choose a slope that
716     * yields 100% pos_ratio fluctuation on suddenly doubled bdi_thresh.
717     */
718    if (unlikely(bdi_thresh > thresh))
719        bdi_thresh = thresh;
720    /*
721     * It's very possible that bdi_thresh is close to 0 not because the
722     * device is slow, but that it has remained inactive for long time.
723     * Honour such devices a reasonable good (hopefully IO efficient)
724     * threshold, so that the occasional writes won't be blocked and active
725     * writes can rampup the threshold quickly.
726     */
727    bdi_thresh = max(bdi_thresh, (limit - dirty) / 8);
728    /*
729     * scale global setpoint to bdi's:
730     * bdi_setpoint = setpoint * bdi_thresh / thresh
731     */
732    x = div_u64((u64)bdi_thresh << 16, thresh + 1);
733    bdi_setpoint = setpoint * (u64)x >> 16;
734    /*
735     * Use span=(8*write_bw) in single bdi case as indicated by
736     * (thresh - bdi_thresh ~= 0) and transit to bdi_thresh in JBOD case.
737     *
738     * bdi_thresh thresh - bdi_thresh
739     * span = ---------- * (8 * write_bw) + ------------------- * bdi_thresh
740     * thresh thresh
741     */
742    span = (thresh - bdi_thresh + 8 * write_bw) * (u64)x >> 16;
743    x_intercept = bdi_setpoint + span;
744
745    if (bdi_dirty < x_intercept - span / 4) {
746        pos_ratio = div_u64(pos_ratio * (x_intercept - bdi_dirty),
747                    x_intercept - bdi_setpoint + 1);
748    } else
749        pos_ratio /= 4;
750
751    /*
752     * bdi reserve area, safeguard against dirty pool underrun and disk idle
753     * It may push the desired control point of global dirty pages higher
754     * than setpoint.
755     */
756    x_intercept = bdi_thresh / 2;
757    if (bdi_dirty < x_intercept) {
758        if (bdi_dirty > x_intercept / 8)
759            pos_ratio = div_u64(pos_ratio * x_intercept, bdi_dirty);
760        else
761            pos_ratio *= 8;
762    }
763
764    return pos_ratio;
765}
766
767static void bdi_update_write_bandwidth(struct backing_dev_info *bdi,
768                       unsigned long elapsed,
769                       unsigned long written)
770{
771    const unsigned long period = roundup_pow_of_two(3 * HZ);
772    unsigned long avg = bdi->avg_write_bandwidth;
773    unsigned long old = bdi->write_bandwidth;
774    u64 bw;
775
776    /*
777     * bw = written * HZ / elapsed
778     *
779     * bw * elapsed + write_bandwidth * (period - elapsed)
780     * write_bandwidth = ---------------------------------------------------
781     * period
782     */
783    bw = written - bdi->written_stamp;
784    bw *= HZ;
785    if (unlikely(elapsed > period)) {
786        do_div(bw, elapsed);
787        avg = bw;
788        goto out;
789    }
790    bw += (u64)bdi->write_bandwidth * (period - elapsed);
791    bw >>= ilog2(period);
792
793    /*
794     * one more level of smoothing, for filtering out sudden spikes
795     */
796    if (avg > old && old >= (unsigned long)bw)
797        avg -= (avg - old) >> 3;
798
799    if (avg < old && old <= (unsigned long)bw)
800        avg += (old - avg) >> 3;
801
802out:
803    bdi->write_bandwidth = bw;
804    bdi->avg_write_bandwidth = avg;
805}
806
807/*
808 * The global dirtyable memory and dirty threshold could be suddenly knocked
809 * down by a large amount (eg. on the startup of KVM in a swapless system).
810 * This may throw the system into deep dirty exceeded state and throttle
811 * heavy/light dirtiers alike. To retain good responsiveness, maintain
812 * global_dirty_limit for tracking slowly down to the knocked down dirty
813 * threshold.
814 */
815static void update_dirty_limit(unsigned long thresh, unsigned long dirty)
816{
817    unsigned long limit = global_dirty_limit;
818
819    /*
820     * Follow up in one step.
821     */
822    if (limit < thresh) {
823        limit = thresh;
824        goto update;
825    }
826
827    /*
828     * Follow down slowly. Use the higher one as the target, because thresh
829     * may drop below dirty. This is exactly the reason to introduce
830     * global_dirty_limit which is guaranteed to lie above the dirty pages.
831     */
832    thresh = max(thresh, dirty);
833    if (limit > thresh) {
834        limit -= (limit - thresh) >> 5;
835        goto update;
836    }
837    return;
838update:
839    global_dirty_limit = limit;
840}
841
842static void global_update_bandwidth(unsigned long thresh,
843                    unsigned long dirty,
844                    unsigned long now)
845{
846    static DEFINE_SPINLOCK(dirty_lock);
847    static unsigned long update_time;
848
849    /*
850     * check locklessly first to optimize away locking for the most time
851     */
852    if (time_before(now, update_time + BANDWIDTH_INTERVAL))
853        return;
854
855    spin_lock(&dirty_lock);
856    if (time_after_eq(now, update_time + BANDWIDTH_INTERVAL)) {
857        update_dirty_limit(thresh, dirty);
858        update_time = now;
859    }
860    spin_unlock(&dirty_lock);
861}
862
863/*
864 * Maintain bdi->dirty_ratelimit, the base dirty throttle rate.
865 *
866 * Normal bdi tasks will be curbed at or below it in long term.
867 * Obviously it should be around (write_bw / N) when there are N dd tasks.
868 */
869static void bdi_update_dirty_ratelimit(struct backing_dev_info *bdi,
870                       unsigned long thresh,
871                       unsigned long bg_thresh,
872                       unsigned long dirty,
873                       unsigned long bdi_thresh,
874                       unsigned long bdi_dirty,
875                       unsigned long dirtied,
876                       unsigned long elapsed)
877{
878    unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh);
879    unsigned long limit = hard_dirty_limit(thresh);
880    unsigned long setpoint = (freerun + limit) / 2;
881    unsigned long write_bw = bdi->avg_write_bandwidth;
882    unsigned long dirty_ratelimit = bdi->dirty_ratelimit;
883    unsigned long dirty_rate;
884    unsigned long task_ratelimit;
885    unsigned long balanced_dirty_ratelimit;
886    unsigned long pos_ratio;
887    unsigned long step;
888    unsigned long x;
889
890    /*
891     * The dirty rate will match the writeout rate in long term, except
892     * when dirty pages are truncated by userspace or re-dirtied by FS.
893     */
894    dirty_rate = (dirtied - bdi->dirtied_stamp) * HZ / elapsed;
895
896    pos_ratio = bdi_position_ratio(bdi, thresh, bg_thresh, dirty,
897                       bdi_thresh, bdi_dirty);
898    /*
899     * task_ratelimit reflects each dd's dirty rate for the past 200ms.
900     */
901    task_ratelimit = (u64)dirty_ratelimit *
902                    pos_ratio >> RATELIMIT_CALC_SHIFT;
903    task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */
904
905    /*
906     * A linear estimation of the "balanced" throttle rate. The theory is,
907     * if there are N dd tasks, each throttled at task_ratelimit, the bdi's
908     * dirty_rate will be measured to be (N * task_ratelimit). So the below
909     * formula will yield the balanced rate limit (write_bw / N).
910     *
911     * Note that the expanded form is not a pure rate feedback:
912     * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1)
913     * but also takes pos_ratio into account:
914     * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2)
915     *
916     * (1) is not realistic because pos_ratio also takes part in balancing
917     * the dirty rate. Consider the state
918     * pos_ratio = 0.5 (3)
919     * rate = 2 * (write_bw / N) (4)
920     * If (1) is used, it will stuck in that state! Because each dd will
921     * be throttled at
922     * task_ratelimit = pos_ratio * rate = (write_bw / N) (5)
923     * yielding
924     * dirty_rate = N * task_ratelimit = write_bw (6)
925     * put (6) into (1) we get
926     * rate_(i+1) = rate_(i) (7)
927     *
928     * So we end up using (2) to always keep
929     * rate_(i+1) ~= (write_bw / N) (8)
930     * regardless of the value of pos_ratio. As long as (8) is satisfied,
931     * pos_ratio is able to drive itself to 1.0, which is not only where
932     * the dirty count meet the setpoint, but also where the slope of
933     * pos_ratio is most flat and hence task_ratelimit is least fluctuated.
934     */
935    balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
936                       dirty_rate | 1);
937    /*
938     * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
939     */
940    if (unlikely(balanced_dirty_ratelimit > write_bw))
941        balanced_dirty_ratelimit = write_bw;
942
943    /*
944     * We could safely do this and return immediately:
945     *
946     * bdi->dirty_ratelimit = balanced_dirty_ratelimit;
947     *
948     * However to get a more stable dirty_ratelimit, the below elaborated
949     * code makes use of task_ratelimit to filter out singular points and
950     * limit the step size.
951     *
952     * The below code essentially only uses the relative value of
953     *
954     * task_ratelimit - dirty_ratelimit
955     * = (pos_ratio - 1) * dirty_ratelimit
956     *
957     * which reflects the direction and size of dirty position error.
958     */
959
960    /*
961     * dirty_ratelimit will follow balanced_dirty_ratelimit iff
962     * task_ratelimit is on the same side of dirty_ratelimit, too.
963     * For example, when
964     * - dirty_ratelimit > balanced_dirty_ratelimit
965     * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
966     * lowering dirty_ratelimit will help meet both the position and rate
967     * control targets. Otherwise, don't update dirty_ratelimit if it will
968     * only help meet the rate target. After all, what the users ultimately
969     * feel and care are stable dirty rate and small position error.
970     *
971     * |task_ratelimit - dirty_ratelimit| is used to limit the step size
972     * and filter out the singular points of balanced_dirty_ratelimit. Which
973     * keeps jumping around randomly and can even leap far away at times
974     * due to the small 200ms estimation period of dirty_rate (we want to
975     * keep that period small to reduce time lags).
976     */
977    step = 0;
978    if (dirty < setpoint) {
979        x = min(bdi->balanced_dirty_ratelimit,
980             min(balanced_dirty_ratelimit, task_ratelimit));
981        if (dirty_ratelimit < x)
982            step = x - dirty_ratelimit;
983    } else {
984        x = max(bdi->balanced_dirty_ratelimit,
985             max(balanced_dirty_ratelimit, task_ratelimit));
986        if (dirty_ratelimit > x)
987            step = dirty_ratelimit - x;
988    }
989
990    /*
991     * Don't pursue 100% rate matching. It's impossible since the balanced
992     * rate itself is constantly fluctuating. So decrease the track speed
993     * when it gets close to the target. Helps eliminate pointless tremors.
994     */
995    step >>= dirty_ratelimit / (2 * step + 1);
996    /*
997     * Limit the tracking speed to avoid overshooting.
998     */
999    step = (step + 7) / 8;
1000
1001    if (dirty_ratelimit < balanced_dirty_ratelimit)
1002        dirty_ratelimit += step;
1003    else
1004        dirty_ratelimit -= step;
1005
1006    bdi->dirty_ratelimit = max(dirty_ratelimit, 1UL);
1007    bdi->balanced_dirty_ratelimit = balanced_dirty_ratelimit;
1008
1009    trace_bdi_dirty_ratelimit(bdi, dirty_rate, task_ratelimit);
1010}
1011
1012void __bdi_update_bandwidth(struct backing_dev_info *bdi,
1013                unsigned long thresh,
1014                unsigned long bg_thresh,
1015                unsigned long dirty,
1016                unsigned long bdi_thresh,
1017                unsigned long bdi_dirty,
1018                unsigned long start_time)
1019{
1020    unsigned long now = jiffies;
1021    unsigned long elapsed = now - bdi->bw_time_stamp;
1022    unsigned long dirtied;
1023    unsigned long written;
1024
1025    /*
1026     * rate-limit, only update once every 200ms.
1027     */
1028    if (elapsed < BANDWIDTH_INTERVAL)
1029        return;
1030
1031    dirtied = percpu_counter_read(&bdi->bdi_stat[BDI_DIRTIED]);
1032    written = percpu_counter_read(&bdi->bdi_stat[BDI_WRITTEN]);
1033
1034    /*
1035     * Skip quiet periods when disk bandwidth is under-utilized.
1036     * (at least 1s idle time between two flusher runs)
1037     */
1038    if (elapsed > HZ && time_before(bdi->bw_time_stamp, start_time))
1039        goto snapshot;
1040
1041    if (thresh) {
1042        global_update_bandwidth(thresh, dirty, now);
1043        bdi_update_dirty_ratelimit(bdi, thresh, bg_thresh, dirty,
1044                       bdi_thresh, bdi_dirty,
1045                       dirtied, elapsed);
1046    }
1047    bdi_update_write_bandwidth(bdi, elapsed, written);
1048
1049snapshot:
1050    bdi->dirtied_stamp = dirtied;
1051    bdi->written_stamp = written;
1052    bdi->bw_time_stamp = now;
1053}
1054
1055static void bdi_update_bandwidth(struct backing_dev_info *bdi,
1056                 unsigned long thresh,
1057                 unsigned long bg_thresh,
1058                 unsigned long dirty,
1059                 unsigned long bdi_thresh,
1060                 unsigned long bdi_dirty,
1061                 unsigned long start_time)
1062{
1063    if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL))
1064        return;
1065    spin_lock(&bdi->wb.list_lock);
1066    __bdi_update_bandwidth(bdi, thresh, bg_thresh, dirty,
1067                   bdi_thresh, bdi_dirty, start_time);
1068    spin_unlock(&bdi->wb.list_lock);
1069}
1070
1071/*
1072 * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr()
1073 * will look to see if it needs to start dirty throttling.
1074 *
1075 * If dirty_poll_interval is too low, big NUMA machines will call the expensive
1076 * global_page_state() too often. So scale it near-sqrt to the safety margin
1077 * (the number of pages we may dirty without exceeding the dirty limits).
1078 */
1079static unsigned long dirty_poll_interval(unsigned long dirty,
1080                     unsigned long thresh)
1081{
1082    if (thresh > dirty)
1083        return 1UL << (ilog2(thresh - dirty) >> 1);
1084
1085    return 1;
1086}
1087
1088static long bdi_max_pause(struct backing_dev_info *bdi,
1089              unsigned long bdi_dirty)
1090{
1091    long bw = bdi->avg_write_bandwidth;
1092    long t;
1093
1094    /*
1095     * Limit pause time for small memory systems. If sleeping for too long
1096     * time, a small pool of dirty/writeback pages may go empty and disk go
1097     * idle.
1098     *
1099     * 8 serves as the safety ratio.
1100     */
1101    t = bdi_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
1102    t++;
1103
1104    return min_t(long, t, MAX_PAUSE);
1105}
1106
1107static long bdi_min_pause(struct backing_dev_info *bdi,
1108              long max_pause,
1109              unsigned long task_ratelimit,
1110              unsigned long dirty_ratelimit,
1111              int *nr_dirtied_pause)
1112{
1113    long hi = ilog2(bdi->avg_write_bandwidth);
1114    long lo = ilog2(bdi->dirty_ratelimit);
1115    long t; /* target pause */
1116    long pause; /* estimated next pause */
1117    int pages; /* target nr_dirtied_pause */
1118
1119    /* target for 10ms pause on 1-dd case */
1120    t = max(1, HZ / 100);
1121
1122    /*
1123     * Scale up pause time for concurrent dirtiers in order to reduce CPU
1124     * overheads.
1125     *
1126     * (N * 10ms) on 2^N concurrent tasks.
1127     */
1128    if (hi > lo)
1129        t += (hi - lo) * (10 * HZ) / 1024;
1130
1131    /*
1132     * This is a bit convoluted. We try to base the next nr_dirtied_pause
1133     * on the much more stable dirty_ratelimit. However the next pause time
1134     * will be computed based on task_ratelimit and the two rate limits may
1135     * depart considerably at some time. Especially if task_ratelimit goes
1136     * below dirty_ratelimit/2 and the target pause is max_pause, the next
1137     * pause time will be max_pause*2 _trimmed down_ to max_pause. As a
1138     * result task_ratelimit won't be executed faithfully, which could
1139     * eventually bring down dirty_ratelimit.
1140     *
1141     * We apply two rules to fix it up:
1142     * 1) try to estimate the next pause time and if necessary, use a lower
1143     * nr_dirtied_pause so as not to exceed max_pause. When this happens,
1144     * nr_dirtied_pause will be "dancing" with task_ratelimit.
1145     * 2) limit the target pause time to max_pause/2, so that the normal
1146     * small fluctuations of task_ratelimit won't trigger rule (1) and
1147     * nr_dirtied_pause will remain as stable as dirty_ratelimit.
1148     */
1149    t = min(t, 1 + max_pause / 2);
1150    pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1151
1152    /*
1153     * Tiny nr_dirtied_pause is found to hurt I/O performance in the test
1154     * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
1155     * When the 16 consecutive reads are often interrupted by some dirty
1156     * throttling pause during the async writes, cfq will go into idles
1157     * (deadline is fine). So push nr_dirtied_pause as high as possible
1158     * until reaches DIRTY_POLL_THRESH=32 pages.
1159     */
1160    if (pages < DIRTY_POLL_THRESH) {
1161        t = max_pause;
1162        pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
1163        if (pages > DIRTY_POLL_THRESH) {
1164            pages = DIRTY_POLL_THRESH;
1165            t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
1166        }
1167    }
1168
1169    pause = HZ * pages / (task_ratelimit + 1);
1170    if (pause > max_pause) {
1171        t = max_pause;
1172        pages = task_ratelimit * t / roundup_pow_of_two(HZ);
1173    }
1174
1175    *nr_dirtied_pause = pages;
1176    /*
1177     * The minimal pause time will normally be half the target pause time.
1178     */
1179    return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
1180}
1181
1182/*
1183 * balance_dirty_pages() must be called by processes which are generating dirty
1184 * data. It looks at the number of dirty pages in the machine and will force
1185 * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2.
1186 * If we're over `background_thresh' then the writeback threads are woken to
1187 * perform some writeout.
1188 */
1189static void balance_dirty_pages(struct address_space *mapping,
1190                unsigned long pages_dirtied)
1191{
1192    unsigned long nr_reclaimable; /* = file_dirty + unstable_nfs */
1193    unsigned long bdi_reclaimable;
1194    unsigned long nr_dirty; /* = file_dirty + writeback + unstable_nfs */
1195    unsigned long bdi_dirty;
1196    unsigned long freerun;
1197    unsigned long background_thresh;
1198    unsigned long dirty_thresh;
1199    unsigned long bdi_thresh;
1200    long period;
1201    long pause;
1202    long max_pause;
1203    long min_pause;
1204    int nr_dirtied_pause;
1205    bool dirty_exceeded = false;
1206    unsigned long task_ratelimit;
1207    unsigned long dirty_ratelimit;
1208    unsigned long pos_ratio;
1209    struct backing_dev_info *bdi = mapping->backing_dev_info;
1210    unsigned long start_time = jiffies;
1211
1212    for (;;) {
1213        unsigned long now = jiffies;
1214
1215        /*
1216         * Unstable writes are a feature of certain networked
1217         * filesystems (i.e. NFS) in which data may have been
1218         * written to the server's write cache, but has not yet
1219         * been flushed to permanent storage.
1220         */
1221        nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
1222                    global_page_state(NR_UNSTABLE_NFS);
1223        nr_dirty = nr_reclaimable + global_page_state(NR_WRITEBACK);
1224
1225        global_dirty_limits(&background_thresh, &dirty_thresh);
1226
1227        /*
1228         * Throttle it only when the background writeback cannot
1229         * catch-up. This avoids (excessively) small writeouts
1230         * when the bdi limits are ramping up.
1231         */
1232        freerun = dirty_freerun_ceiling(dirty_thresh,
1233                        background_thresh);
1234        if (nr_dirty <= freerun) {
1235            current->dirty_paused_when = now;
1236            current->nr_dirtied = 0;
1237            current->nr_dirtied_pause =
1238                dirty_poll_interval(nr_dirty, dirty_thresh);
1239            break;
1240        }
1241
1242        if (unlikely(!writeback_in_progress(bdi)))
1243            bdi_start_background_writeback(bdi);
1244
1245        /*
1246         * bdi_thresh is not treated as some limiting factor as
1247         * dirty_thresh, due to reasons
1248         * - in JBOD setup, bdi_thresh can fluctuate a lot
1249         * - in a system with HDD and USB key, the USB key may somehow
1250         * go into state (bdi_dirty >> bdi_thresh) either because
1251         * bdi_dirty starts high, or because bdi_thresh drops low.
1252         * In this case we don't want to hard throttle the USB key
1253         * dirtiers for 100 seconds until bdi_dirty drops under
1254         * bdi_thresh. Instead the auxiliary bdi control line in
1255         * bdi_position_ratio() will let the dirtier task progress
1256         * at some rate <= (write_bw / 2) for bringing down bdi_dirty.
1257         */
1258        bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);
1259
1260        /*
1261         * In order to avoid the stacked BDI deadlock we need
1262         * to ensure we accurately count the 'dirty' pages when
1263         * the threshold is low.
1264         *
1265         * Otherwise it would be possible to get thresh+n pages
1266         * reported dirty, even though there are thresh-m pages
1267         * actually dirty; with m+n sitting in the percpu
1268         * deltas.
1269         */
1270        if (bdi_thresh < 2 * bdi_stat_error(bdi)) {
1271            bdi_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE);
1272            bdi_dirty = bdi_reclaimable +
1273                    bdi_stat_sum(bdi, BDI_WRITEBACK);
1274        } else {
1275            bdi_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE);
1276            bdi_dirty = bdi_reclaimable +
1277                    bdi_stat(bdi, BDI_WRITEBACK);
1278        }
1279
1280        dirty_exceeded = (bdi_dirty > bdi_thresh) &&
1281                  (nr_dirty > dirty_thresh);
1282        if (dirty_exceeded && !bdi->dirty_exceeded)
1283            bdi->dirty_exceeded = 1;
1284
1285        bdi_update_bandwidth(bdi, dirty_thresh, background_thresh,
1286                     nr_dirty, bdi_thresh, bdi_dirty,
1287                     start_time);
1288
1289        dirty_ratelimit = bdi->dirty_ratelimit;
1290        pos_ratio = bdi_position_ratio(bdi, dirty_thresh,
1291                           background_thresh, nr_dirty,
1292                           bdi_thresh, bdi_dirty);
1293        task_ratelimit = ((u64)dirty_ratelimit * pos_ratio) >>
1294                            RATELIMIT_CALC_SHIFT;
1295        max_pause = bdi_max_pause(bdi, bdi_dirty);
1296        min_pause = bdi_min_pause(bdi, max_pause,
1297                      task_ratelimit, dirty_ratelimit,
1298                      &nr_dirtied_pause);
1299
1300        if (unlikely(task_ratelimit == 0)) {
1301            period = max_pause;
1302            pause = max_pause;
1303            goto pause;
1304        }
1305        period = HZ * pages_dirtied / task_ratelimit;
1306        pause = period;
1307        if (current->dirty_paused_when)
1308            pause -= now - current->dirty_paused_when;
1309        /*
1310         * For less than 1s think time (ext3/4 may block the dirtier
1311         * for up to 800ms from time to time on 1-HDD; so does xfs,
1312         * however at much less frequency), try to compensate it in
1313         * future periods by updating the virtual time; otherwise just
1314         * do a reset, as it may be a light dirtier.
1315         */
1316        if (pause < min_pause) {
1317            trace_balance_dirty_pages(bdi,
1318                          dirty_thresh,
1319                          background_thresh,
1320                          nr_dirty,
1321                          bdi_thresh,
1322                          bdi_dirty,
1323                          dirty_ratelimit,
1324                          task_ratelimit,
1325                          pages_dirtied,
1326                          period,
1327                          min(pause, 0L),
1328                          start_time);
1329            if (pause < -HZ) {
1330                current->dirty_paused_when = now;
1331                current->nr_dirtied = 0;
1332            } else if (period) {
1333                current->dirty_paused_when += period;
1334                current->nr_dirtied = 0;
1335            } else if (current->nr_dirtied_pause <= pages_dirtied)
1336                current->nr_dirtied_pause += pages_dirtied;
1337            break;
1338        }
1339        if (unlikely(pause > max_pause)) {
1340            /* for occasional dropped task_ratelimit */
1341            now += min(pause - max_pause, max_pause);
1342            pause = max_pause;
1343        }
1344
1345pause:
1346        trace_balance_dirty_pages(bdi,
1347                      dirty_thresh,
1348                      background_thresh,
1349                      nr_dirty,
1350                      bdi_thresh,
1351                      bdi_dirty,
1352                      dirty_ratelimit,
1353                      task_ratelimit,
1354                      pages_dirtied,
1355                      period,
1356                      pause,
1357                      start_time);
1358        __set_current_state(TASK_KILLABLE);
1359        io_schedule_timeout(pause);
1360
1361        current->dirty_paused_when = now + pause;
1362        current->nr_dirtied = 0;
1363        current->nr_dirtied_pause = nr_dirtied_pause;
1364
1365        /*
1366         * This is typically equal to (nr_dirty < dirty_thresh) and can
1367         * also keep "1000+ dd on a slow USB stick" under control.
1368         */
1369        if (task_ratelimit)
1370            break;
1371
1372        /*
1373         * In the case of an unresponding NFS server and the NFS dirty
1374         * pages exceeds dirty_thresh, give the other good bdi's a pipe
1375         * to go through, so that tasks on them still remain responsive.
1376         *
1377         * In theory 1 page is enough to keep the comsumer-producer
1378         * pipe going: the flusher cleans 1 page => the task dirties 1
1379         * more page. However bdi_dirty has accounting errors. So use
1380         * the larger and more IO friendly bdi_stat_error.
1381         */
1382        if (bdi_dirty <= bdi_stat_error(bdi))
1383            break;
1384
1385        if (fatal_signal_pending(current))
1386            break;
1387    }
1388
1389    if (!dirty_exceeded && bdi->dirty_exceeded)
1390        bdi->dirty_exceeded = 0;
1391
1392    if (writeback_in_progress(bdi))
1393        return;
1394
1395    /*
1396     * In laptop mode, we wait until hitting the higher threshold before
1397     * starting background writeout, and then write out all the way down
1398     * to the lower threshold. So slow writers cause minimal disk activity.
1399     *
1400     * In normal mode, we start background writeout at the lower
1401     * background_thresh, to keep the amount of dirty memory low.
1402     */
1403    if (laptop_mode)
1404        return;
1405
1406    if (nr_reclaimable > background_thresh)
1407        bdi_start_background_writeback(bdi);
1408}
1409
1410void set_page_dirty_balance(struct page *page, int page_mkwrite)
1411{
1412    if (set_page_dirty(page) || page_mkwrite) {
1413        struct address_space *mapping = page_mapping(page);
1414
1415        if (mapping)
1416            balance_dirty_pages_ratelimited(mapping);
1417    }
1418}
1419
1420static DEFINE_PER_CPU(int, bdp_ratelimits);
1421
1422/*
1423 * Normal tasks are throttled by
1424 * loop {
1425 * dirty tsk->nr_dirtied_pause pages;
1426 * take a snap in balance_dirty_pages();
1427 * }
1428 * However there is a worst case. If every task exit immediately when dirtied
1429 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
1430 * called to throttle the page dirties. The solution is to save the not yet
1431 * throttled page dirties in dirty_throttle_leaks on task exit and charge them
1432 * randomly into the running tasks. This works well for the above worst case,
1433 * as the new task will pick up and accumulate the old task's leaked dirty
1434 * count and eventually get throttled.
1435 */
1436DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
1437
1438/**
1439 * balance_dirty_pages_ratelimited_nr - balance dirty memory state
1440 * @mapping: address_space which was dirtied
1441 * @nr_pages_dirtied: number of pages which the caller has just dirtied
1442 *
1443 * Processes which are dirtying memory should call in here once for each page
1444 * which was newly dirtied. The function will periodically check the system's
1445 * dirty state and will initiate writeback if needed.
1446 *
1447 * On really big machines, get_writeback_state is expensive, so try to avoid
1448 * calling it too often (ratelimiting). But once we're over the dirty memory
1449 * limit we decrease the ratelimiting by a lot, to prevent individual processes
1450 * from overshooting the limit by (ratelimit_pages) each.
1451 */
1452void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
1453                    unsigned long nr_pages_dirtied)
1454{
1455    struct backing_dev_info *bdi = mapping->backing_dev_info;
1456    int ratelimit;
1457    int *p;
1458
1459    if (!bdi_cap_account_dirty(bdi))
1460        return;
1461
1462    ratelimit = current->nr_dirtied_pause;
1463    if (bdi->dirty_exceeded)
1464        ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10));
1465
1466    preempt_disable();
1467    /*
1468     * This prevents one CPU to accumulate too many dirtied pages without
1469     * calling into balance_dirty_pages(), which can happen when there are
1470     * 1000+ tasks, all of them start dirtying pages at exactly the same
1471     * time, hence all honoured too large initial task->nr_dirtied_pause.
1472     */
1473    p = &__get_cpu_var(bdp_ratelimits);
1474    if (unlikely(current->nr_dirtied >= ratelimit))
1475        *p = 0;
1476    else if (unlikely(*p >= ratelimit_pages)) {
1477        *p = 0;
1478        ratelimit = 0;
1479    }
1480    /*
1481     * Pick up the dirtied pages by the exited tasks. This avoids lots of
1482     * short-lived tasks (eg. gcc invocations in a kernel build) escaping
1483     * the dirty throttling and livelock other long-run dirtiers.
1484     */
1485    p = &__get_cpu_var(dirty_throttle_leaks);
1486    if (*p > 0 && current->nr_dirtied < ratelimit) {
1487        nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
1488        *p -= nr_pages_dirtied;
1489        current->nr_dirtied += nr_pages_dirtied;
1490    }
1491    preempt_enable();
1492
1493    if (unlikely(current->nr_dirtied >= ratelimit))
1494        balance_dirty_pages(mapping, current->nr_dirtied);
1495}
1496EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr);
1497
1498void throttle_vm_writeout(gfp_t gfp_mask)
1499{
1500    unsigned long background_thresh;
1501    unsigned long dirty_thresh;
1502
1503        for ( ; ; ) {
1504        global_dirty_limits(&background_thresh, &dirty_thresh);
1505        dirty_thresh = hard_dirty_limit(dirty_thresh);
1506
1507                /*
1508                 * Boost the allowable dirty threshold a bit for page
1509                 * allocators so they don't get DoS'ed by heavy writers
1510                 */
1511                dirty_thresh += dirty_thresh / 10; /* wheeee... */
1512
1513                if (global_page_state(NR_UNSTABLE_NFS) +
1514            global_page_state(NR_WRITEBACK) <= dirty_thresh)
1515                            break;
1516                congestion_wait(BLK_RW_ASYNC, HZ/10);
1517
1518        /*
1519         * The caller might hold locks which can prevent IO completion
1520         * or progress in the filesystem. So we cannot just sit here
1521         * waiting for IO to complete.
1522         */
1523        if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO))
1524            break;
1525        }
1526}
1527
1528/*
1529 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
1530 */
1531int dirty_writeback_centisecs_handler(ctl_table *table, int write,
1532    void __user *buffer, size_t *length, loff_t *ppos)
1533{
1534    proc_dointvec(table, write, buffer, length, ppos);
1535    return 0;
1536}
1537
1538#ifdef CONFIG_BLOCK
1539void laptop_mode_timer_fn(unsigned long data)
1540{
1541    struct request_queue *q = (struct request_queue *)data;
1542    int nr_pages = global_page_state(NR_FILE_DIRTY) +
1543        global_page_state(NR_UNSTABLE_NFS);
1544
1545    /*
1546     * We want to write everything out, not just down to the dirty
1547     * threshold
1548     */
1549    if (bdi_has_dirty_io(&q->backing_dev_info))
1550        bdi_start_writeback(&q->backing_dev_info, nr_pages,
1551                    WB_REASON_LAPTOP_TIMER);
1552}
1553
1554/*
1555 * We've spun up the disk and we're in laptop mode: schedule writeback
1556 * of all dirty data a few seconds from now. If the flush is already scheduled
1557 * then push it back - the user is still using the disk.
1558 */
1559void laptop_io_completion(struct backing_dev_info *info)
1560{
1561    mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
1562}
1563
1564/*
1565 * We're in laptop mode and we've just synced. The sync's writes will have
1566 * caused another writeback to be scheduled by laptop_io_completion.
1567 * Nothing needs to be written back anymore, so we unschedule the writeback.
1568 */
1569void laptop_sync_completion(void)
1570{
1571    struct backing_dev_info *bdi;
1572
1573    rcu_read_lock();
1574
1575    list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
1576        del_timer(&bdi->laptop_mode_wb_timer);
1577
1578    rcu_read_unlock();
1579}
1580#endif
1581
1582/*
1583 * If ratelimit_pages is too high then we can get into dirty-data overload
1584 * if a large number of processes all perform writes at the same time.
1585 * If it is too low then SMP machines will call the (expensive)
1586 * get_writeback_state too often.
1587 *
1588 * Here we set ratelimit_pages to a level which ensures that when all CPUs are
1589 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
1590 * thresholds.
1591 */
1592
1593void writeback_set_ratelimit(void)
1594{
1595    unsigned long background_thresh;
1596    unsigned long dirty_thresh;
1597    global_dirty_limits(&background_thresh, &dirty_thresh);
1598    global_dirty_limit = dirty_thresh;
1599    ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
1600    if (ratelimit_pages < 16)
1601        ratelimit_pages = 16;
1602}
1603
1604static int __cpuinit
1605ratelimit_handler(struct notifier_block *self, unsigned long u, void *v)
1606{
1607    writeback_set_ratelimit();
1608    return NOTIFY_DONE;
1609}
1610
1611static struct notifier_block __cpuinitdata ratelimit_nb = {
1612    .notifier_call = ratelimit_handler,
1613    .next = NULL,
1614};
1615
1616/*
1617 * Called early on to tune the page writeback dirty limits.
1618 *
1619 * We used to scale dirty pages according to how total memory
1620 * related to pages that could be allocated for buffers (by
1621 * comparing nr_free_buffer_pages() to vm_total_pages.
1622 *
1623 * However, that was when we used "dirty_ratio" to scale with
1624 * all memory, and we don't do that any more. "dirty_ratio"
1625 * is now applied to total non-HIGHPAGE memory (by subtracting
1626 * totalhigh_pages from vm_total_pages), and as such we can't
1627 * get into the old insane situation any more where we had
1628 * large amounts of dirty pages compared to a small amount of
1629 * non-HIGHMEM memory.
1630 *
1631 * But we might still want to scale the dirty_ratio by how
1632 * much memory the box has..
1633 */
1634void __init page_writeback_init(void)
1635{
1636    writeback_set_ratelimit();
1637    register_cpu_notifier(&ratelimit_nb);
1638
1639    fprop_global_init(&writeout_completions);
1640}
1641
1642/**
1643 * tag_pages_for_writeback - tag pages to be written by write_cache_pages
1644 * @mapping: address space structure to write
1645 * @start: starting page index
1646 * @end: ending page index (inclusive)
1647 *
1648 * This function scans the page range from @start to @end (inclusive) and tags
1649 * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is
1650 * that write_cache_pages (or whoever calls this function) will then use
1651 * TOWRITE tag to identify pages eligible for writeback. This mechanism is
1652 * used to avoid livelocking of writeback by a process steadily creating new
1653 * dirty pages in the file (thus it is important for this function to be quick
1654 * so that it can tag pages faster than a dirtying process can create them).
1655 */
1656/*
1657 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency.
1658 */
1659void tag_pages_for_writeback(struct address_space *mapping,
1660                 pgoff_t start, pgoff_t end)
1661{
1662#define WRITEBACK_TAG_BATCH 4096
1663    unsigned long tagged;
1664
1665    do {
1666        spin_lock_irq(&mapping->tree_lock);
1667        tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree,
1668                &start, end, WRITEBACK_TAG_BATCH,
1669                PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
1670        spin_unlock_irq(&mapping->tree_lock);
1671        WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH);
1672        cond_resched();
1673        /* We check 'start' to handle wrapping when end == ~0UL */
1674    } while (tagged >= WRITEBACK_TAG_BATCH && start);
1675}
1676EXPORT_SYMBOL(tag_pages_for_writeback);
1677
1678/**
1679 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
1680 * @mapping: address space structure to write
1681 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
1682 * @writepage: function called for each page
1683 * @data: data passed to writepage function
1684 *
1685 * If a page is already under I/O, write_cache_pages() skips it, even
1686 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
1687 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
1688 * and msync() need to guarantee that all the data which was dirty at the time
1689 * the call was made get new I/O started against them. If wbc->sync_mode is
1690 * WB_SYNC_ALL then we were called for data integrity and we must wait for
1691 * existing IO to complete.
1692 *
1693 * To avoid livelocks (when other process dirties new pages), we first tag
1694 * pages which should be written back with TOWRITE tag and only then start
1695 * writing them. For data-integrity sync we have to be careful so that we do
1696 * not miss some pages (e.g., because some other process has cleared TOWRITE
1697 * tag we set). The rule we follow is that TOWRITE tag can be cleared only
1698 * by the process clearing the DIRTY tag (and submitting the page for IO).
1699 */
1700int write_cache_pages(struct address_space *mapping,
1701              struct writeback_control *wbc, writepage_t writepage,
1702              void *data)
1703{
1704    int ret = 0;
1705    int done = 0;
1706    struct pagevec pvec;
1707    int nr_pages;
1708    pgoff_t uninitialized_var(writeback_index);
1709    pgoff_t index;
1710    pgoff_t end; /* Inclusive */
1711    pgoff_t done_index;
1712    int cycled;
1713    int range_whole = 0;
1714    int tag;
1715
1716    pagevec_init(&pvec, 0);
1717    if (wbc->range_cyclic) {
1718        writeback_index = mapping->writeback_index; /* prev offset */
1719        index = writeback_index;
1720        if (index == 0)
1721            cycled = 1;
1722        else
1723            cycled = 0;
1724        end = -1;
1725    } else {
1726        index = wbc->range_start >> PAGE_CACHE_SHIFT;
1727        end = wbc->range_end >> PAGE_CACHE_SHIFT;
1728        if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1729            range_whole = 1;
1730        cycled = 1; /* ignore range_cyclic tests */
1731    }
1732    if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1733        tag = PAGECACHE_TAG_TOWRITE;
1734    else
1735        tag = PAGECACHE_TAG_DIRTY;
1736retry:
1737    if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1738        tag_pages_for_writeback(mapping, index, end);
1739    done_index = index;
1740    while (!done && (index <= end)) {
1741        int i;
1742
1743        nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1744                  min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
1745        if (nr_pages == 0)
1746            break;
1747
1748        for (i = 0; i < nr_pages; i++) {
1749            struct page *page = pvec.pages[i];
1750
1751            /*
1752             * At this point, the page may be truncated or
1753             * invalidated (changing page->mapping to NULL), or
1754             * even swizzled back from swapper_space to tmpfs file
1755             * mapping. However, page->index will not change
1756             * because we have a reference on the page.
1757             */
1758            if (page->index > end) {
1759                /*
1760                 * can't be range_cyclic (1st pass) because
1761                 * end == -1 in that case.
1762                 */
1763                done = 1;
1764                break;
1765            }
1766
1767            done_index = page->index;
1768
1769            lock_page(page);
1770
1771            /*
1772             * Page truncated or invalidated. We can freely skip it
1773             * then, even for data integrity operations: the page
1774             * has disappeared concurrently, so there could be no
1775             * real expectation of this data interity operation
1776             * even if there is now a new, dirty page at the same
1777             * pagecache address.
1778             */
1779            if (unlikely(page->mapping != mapping)) {
1780continue_unlock:
1781                unlock_page(page);
1782                continue;
1783            }
1784
1785            if (!PageDirty(page)) {
1786                /* someone wrote it for us */
1787                goto continue_unlock;
1788            }
1789
1790            if (PageWriteback(page)) {
1791                if (wbc->sync_mode != WB_SYNC_NONE)
1792                    wait_on_page_writeback(page);
1793                else
1794                    goto continue_unlock;
1795            }
1796
1797            BUG_ON(PageWriteback(page));
1798            if (!clear_page_dirty_for_io(page))
1799                goto continue_unlock;
1800
1801            trace_wbc_writepage(wbc, mapping->backing_dev_info);
1802            ret = (*writepage)(page, wbc, data);
1803            if (unlikely(ret)) {
1804                if (ret == AOP_WRITEPAGE_ACTIVATE) {
1805                    unlock_page(page);
1806                    ret = 0;
1807                } else {
1808                    /*
1809                     * done_index is set past this page,
1810                     * so media errors will not choke
1811                     * background writeout for the entire
1812                     * file. This has consequences for
1813                     * range_cyclic semantics (ie. it may
1814                     * not be suitable for data integrity
1815                     * writeout).
1816                     */
1817                    done_index = page->index + 1;
1818                    done = 1;
1819                    break;
1820                }
1821            }
1822
1823            /*
1824             * We stop writing back only if we are not doing
1825             * integrity sync. In case of integrity sync we have to
1826             * keep going until we have written all the pages
1827             * we tagged for writeback prior to entering this loop.
1828             */
1829            if (--wbc->nr_to_write <= 0 &&
1830                wbc->sync_mode == WB_SYNC_NONE) {
1831                done = 1;
1832                break;
1833            }
1834        }
1835        pagevec_release(&pvec);
1836        cond_resched();
1837    }
1838    if (!cycled && !done) {
1839        /*
1840         * range_cyclic:
1841         * We hit the last page and there is more work to be done: wrap
1842         * back to the start of the file
1843         */
1844        cycled = 1;
1845        index = 0;
1846        end = writeback_index - 1;
1847        goto retry;
1848    }
1849    if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1850        mapping->writeback_index = done_index;
1851
1852    return ret;
1853}
1854EXPORT_SYMBOL(write_cache_pages);
1855
1856/*
1857 * Function used by generic_writepages to call the real writepage
1858 * function and set the mapping flags on error
1859 */
1860static int __writepage(struct page *page, struct writeback_control *wbc,
1861               void *data)
1862{
1863    struct address_space *mapping = data;
1864    int ret = mapping->a_ops->writepage(page, wbc);
1865    mapping_set_error(mapping, ret);
1866    return ret;
1867}
1868
1869/**
1870 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them.
1871 * @mapping: address space structure to write
1872 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
1873 *
1874 * This is a library function, which implements the writepages()
1875 * address_space_operation.
1876 */
1877int generic_writepages(struct address_space *mapping,
1878               struct writeback_control *wbc)
1879{
1880    struct blk_plug plug;
1881    int ret;
1882
1883    /* deal with chardevs and other special file */
1884    if (!mapping->a_ops->writepage)
1885        return 0;
1886
1887    blk_start_plug(&plug);
1888    ret = write_cache_pages(mapping, wbc, __writepage, mapping);
1889    blk_finish_plug(&plug);
1890    return ret;
1891}
1892
1893EXPORT_SYMBOL(generic_writepages);
1894
1895int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
1896{
1897    int ret;
1898
1899    if (wbc->nr_to_write <= 0)
1900        return 0;
1901    if (mapping->a_ops->writepages)
1902        ret = mapping->a_ops->writepages(mapping, wbc);
1903    else
1904        ret = generic_writepages(mapping, wbc);
1905    return ret;
1906}
1907
1908/**
1909 * write_one_page - write out a single page and optionally wait on I/O
1910 * @page: the page to write
1911 * @wait: if true, wait on writeout
1912 *
1913 * The page must be locked by the caller and will be unlocked upon return.
1914 *
1915 * write_one_page() returns a negative error code if I/O failed.
1916 */
1917int write_one_page(struct page *page, int wait)
1918{
1919    struct address_space *mapping = page->mapping;
1920    int ret = 0;
1921    struct writeback_control wbc = {
1922        .sync_mode = WB_SYNC_ALL,
1923        .nr_to_write = 1,
1924    };
1925
1926    BUG_ON(!PageLocked(page));
1927
1928    if (wait)
1929        wait_on_page_writeback(page);
1930
1931    if (clear_page_dirty_for_io(page)) {
1932        page_cache_get(page);
1933        ret = mapping->a_ops->writepage(page, &wbc);
1934        if (ret == 0 && wait) {
1935            wait_on_page_writeback(page);
1936            if (PageError(page))
1937                ret = -EIO;
1938        }
1939        page_cache_release(page);
1940    } else {
1941        unlock_page(page);
1942    }
1943    return ret;
1944}
1945EXPORT_SYMBOL(write_one_page);
1946
1947/*
1948 * For address_spaces which do not use buffers nor write back.
1949 */
1950int __set_page_dirty_no_writeback(struct page *page)
1951{
1952    if (!PageDirty(page))
1953        return !TestSetPageDirty(page);
1954    return 0;
1955}
1956
1957/*
1958 * Helper function for set_page_dirty family.
1959 * NOTE: This relies on being atomic wrt interrupts.
1960 */
1961void account_page_dirtied(struct page *page, struct address_space *mapping)
1962{
1963    if (mapping_cap_account_dirty(mapping)) {
1964        __inc_zone_page_state(page, NR_FILE_DIRTY);
1965        __inc_zone_page_state(page, NR_DIRTIED);
1966        __inc_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
1967        __inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTIED);
1968        task_io_account_write(PAGE_CACHE_SIZE);
1969        current->nr_dirtied++;
1970        this_cpu_inc(bdp_ratelimits);
1971    }
1972}
1973EXPORT_SYMBOL(account_page_dirtied);
1974
1975/*
1976 * Helper function for set_page_writeback family.
1977 * NOTE: Unlike account_page_dirtied this does not rely on being atomic
1978 * wrt interrupts.
1979 */
1980void account_page_writeback(struct page *page)
1981{
1982    inc_zone_page_state(page, NR_WRITEBACK);
1983}
1984EXPORT_SYMBOL(account_page_writeback);
1985
1986/*
1987 * For address_spaces which do not use buffers. Just tag the page as dirty in
1988 * its radix tree.
1989 *
1990 * This is also used when a single buffer is being dirtied: we want to set the
1991 * page dirty in that case, but not all the buffers. This is a "bottom-up"
1992 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying.
1993 *
1994 * Most callers have locked the page, which pins the address_space in memory.
1995 * But zap_pte_range() does not lock the page, however in that case the
1996 * mapping is pinned by the vma's ->vm_file reference.
1997 *
1998 * We take care to handle the case where the page was truncated from the
1999 * mapping by re-checking page_mapping() inside tree_lock.
2000 */
2001int __set_page_dirty_nobuffers(struct page *page)
2002{
2003    if (!TestSetPageDirty(page)) {
2004        struct address_space *mapping = page_mapping(page);
2005        struct address_space *mapping2;
2006
2007        if (!mapping)
2008            return 1;
2009
2010        spin_lock_irq(&mapping->tree_lock);
2011        mapping2 = page_mapping(page);
2012        if (mapping2) { /* Race with truncate? */
2013            BUG_ON(mapping2 != mapping);
2014            WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
2015            account_page_dirtied(page, mapping);
2016            radix_tree_tag_set(&mapping->page_tree,
2017                page_index(page), PAGECACHE_TAG_DIRTY);
2018        }
2019        spin_unlock_irq(&mapping->tree_lock);
2020        if (mapping->host) {
2021            /* !PageAnon && !swapper_space */
2022            __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2023        }
2024        return 1;
2025    }
2026    return 0;
2027}
2028EXPORT_SYMBOL(__set_page_dirty_nobuffers);
2029
2030/*
2031 * Call this whenever redirtying a page, to de-account the dirty counters
2032 * (NR_DIRTIED, BDI_DIRTIED, tsk->nr_dirtied), so that they match the written
2033 * counters (NR_WRITTEN, BDI_WRITTEN) in long term. The mismatches will lead to
2034 * systematic errors in balanced_dirty_ratelimit and the dirty pages position
2035 * control.
2036 */
2037void account_page_redirty(struct page *page)
2038{
2039    struct address_space *mapping = page->mapping;
2040    if (mapping && mapping_cap_account_dirty(mapping)) {
2041        current->nr_dirtied--;
2042        dec_zone_page_state(page, NR_DIRTIED);
2043        dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTIED);
2044    }
2045}
2046EXPORT_SYMBOL(account_page_redirty);
2047
2048/*
2049 * When a writepage implementation decides that it doesn't want to write this
2050 * page for some reason, it should redirty the locked page via
2051 * redirty_page_for_writepage() and it should then unlock the page and return 0
2052 */
2053int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
2054{
2055    wbc->pages_skipped++;
2056    account_page_redirty(page);
2057    return __set_page_dirty_nobuffers(page);
2058}
2059EXPORT_SYMBOL(redirty_page_for_writepage);
2060
2061/*
2062 * Dirty a page.
2063 *
2064 * For pages with a mapping this should be done under the page lock
2065 * for the benefit of asynchronous memory errors who prefer a consistent
2066 * dirty state. This rule can be broken in some special cases,
2067 * but should be better not to.
2068 *
2069 * If the mapping doesn't provide a set_page_dirty a_op, then
2070 * just fall through and assume that it wants buffer_heads.
2071 */
2072int set_page_dirty(struct page *page)
2073{
2074    struct address_space *mapping = page_mapping(page);
2075
2076    if (likely(mapping)) {
2077        int (*spd)(struct page *) = mapping->a_ops->set_page_dirty;
2078        /*
2079         * readahead/lru_deactivate_page could remain
2080         * PG_readahead/PG_reclaim due to race with end_page_writeback
2081         * About readahead, if the page is written, the flags would be
2082         * reset. So no problem.
2083         * About lru_deactivate_page, if the page is redirty, the flag
2084         * will be reset. So no problem. but if the page is used by readahead
2085         * it will confuse readahead and make it restart the size rampup
2086         * process. But it's a trivial problem.
2087         */
2088        ClearPageReclaim(page);
2089#ifdef CONFIG_BLOCK
2090        if (!spd)
2091            spd = __set_page_dirty_buffers;
2092#endif
2093        return (*spd)(page);
2094    }
2095    if (!PageDirty(page)) {
2096        if (!TestSetPageDirty(page))
2097            return 1;
2098    }
2099    return 0;
2100}
2101EXPORT_SYMBOL(set_page_dirty);
2102
2103/*
2104 * set_page_dirty() is racy if the caller has no reference against
2105 * page->mapping->host, and if the page is unlocked. This is because another
2106 * CPU could truncate the page off the mapping and then free the mapping.
2107 *
2108 * Usually, the page _is_ locked, or the caller is a user-space process which
2109 * holds a reference on the inode by having an open file.
2110 *
2111 * In other cases, the page should be locked before running set_page_dirty().
2112 */
2113int set_page_dirty_lock(struct page *page)
2114{
2115    int ret;
2116
2117    lock_page(page);
2118    ret = set_page_dirty(page);
2119    unlock_page(page);
2120    return ret;
2121}
2122EXPORT_SYMBOL(set_page_dirty_lock);
2123
2124/*
2125 * Clear a page's dirty flag, while caring for dirty memory accounting.
2126 * Returns true if the page was previously dirty.
2127 *
2128 * This is for preparing to put the page under writeout. We leave the page
2129 * tagged as dirty in the radix tree so that a concurrent write-for-sync
2130 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage
2131 * implementation will run either set_page_writeback() or set_page_dirty(),
2132 * at which stage we bring the page's dirty flag and radix-tree dirty tag
2133 * back into sync.
2134 *
2135 * This incoherency between the page's dirty flag and radix-tree tag is
2136 * unfortunate, but it only exists while the page is locked.
2137 */
2138int clear_page_dirty_for_io(struct page *page)
2139{
2140    struct address_space *mapping = page_mapping(page);
2141
2142    BUG_ON(!PageLocked(page));
2143
2144    if (mapping && mapping_cap_account_dirty(mapping)) {
2145        /*
2146         * Yes, Virginia, this is indeed insane.
2147         *
2148         * We use this sequence to make sure that
2149         * (a) we account for dirty stats properly
2150         * (b) we tell the low-level filesystem to
2151         * mark the whole page dirty if it was
2152         * dirty in a pagetable. Only to then
2153         * (c) clean the page again and return 1 to
2154         * cause the writeback.
2155         *
2156         * This way we avoid all nasty races with the
2157         * dirty bit in multiple places and clearing
2158         * them concurrently from different threads.
2159         *
2160         * Note! Normally the "set_page_dirty(page)"
2161         * has no effect on the actual dirty bit - since
2162         * that will already usually be set. But we
2163         * need the side effects, and it can help us
2164         * avoid races.
2165         *
2166         * We basically use the page "master dirty bit"
2167         * as a serialization point for all the different
2168         * threads doing their things.
2169         */
2170        if (page_mkclean(page))
2171            set_page_dirty(page);
2172        /*
2173         * We carefully synchronise fault handlers against
2174         * installing a dirty pte and marking the page dirty
2175         * at this point. We do this by having them hold the
2176         * page lock at some point after installing their
2177         * pte, but before marking the page dirty.
2178         * Pages are always locked coming in here, so we get
2179         * the desired exclusion. See mm/memory.c:do_wp_page()
2180         * for more comments.
2181         */
2182        if (TestClearPageDirty(page)) {
2183            dec_zone_page_state(page, NR_FILE_DIRTY);
2184            dec_bdi_stat(mapping->backing_dev_info,
2185                    BDI_RECLAIMABLE);
2186            return 1;
2187        }
2188        return 0;
2189    }
2190    return TestClearPageDirty(page);
2191}
2192EXPORT_SYMBOL(clear_page_dirty_for_io);
2193
2194int test_clear_page_writeback(struct page *page)
2195{
2196    struct address_space *mapping = page_mapping(page);
2197    int ret;
2198
2199    if (mapping) {
2200        struct backing_dev_info *bdi = mapping->backing_dev_info;
2201        unsigned long flags;
2202
2203        spin_lock_irqsave(&mapping->tree_lock, flags);
2204        ret = TestClearPageWriteback(page);
2205        if (ret) {
2206            radix_tree_tag_clear(&mapping->page_tree,
2207                        page_index(page),
2208                        PAGECACHE_TAG_WRITEBACK);
2209            if (bdi_cap_account_writeback(bdi)) {
2210                __dec_bdi_stat(bdi, BDI_WRITEBACK);
2211                __bdi_writeout_inc(bdi);
2212            }
2213        }
2214        spin_unlock_irqrestore(&mapping->tree_lock, flags);
2215    } else {
2216        ret = TestClearPageWriteback(page);
2217    }
2218    if (ret) {
2219        dec_zone_page_state(page, NR_WRITEBACK);
2220        inc_zone_page_state(page, NR_WRITTEN);
2221    }
2222    return ret;
2223}
2224
2225int test_set_page_writeback(struct page *page)
2226{
2227    struct address_space *mapping = page_mapping(page);
2228    int ret;
2229
2230    if (mapping) {
2231        struct backing_dev_info *bdi = mapping->backing_dev_info;
2232        unsigned long flags;
2233
2234        spin_lock_irqsave(&mapping->tree_lock, flags);
2235        ret = TestSetPageWriteback(page);
2236        if (!ret) {
2237            radix_tree_tag_set(&mapping->page_tree,
2238                        page_index(page),
2239                        PAGECACHE_TAG_WRITEBACK);
2240            if (bdi_cap_account_writeback(bdi))
2241                __inc_bdi_stat(bdi, BDI_WRITEBACK);
2242        }
2243        if (!PageDirty(page))
2244            radix_tree_tag_clear(&mapping->page_tree,
2245                        page_index(page),
2246                        PAGECACHE_TAG_DIRTY);
2247        radix_tree_tag_clear(&mapping->page_tree,
2248                     page_index(page),
2249                     PAGECACHE_TAG_TOWRITE);
2250        spin_unlock_irqrestore(&mapping->tree_lock, flags);
2251    } else {
2252        ret = TestSetPageWriteback(page);
2253    }
2254    if (!ret)
2255        account_page_writeback(page);
2256    return ret;
2257
2258}
2259EXPORT_SYMBOL(test_set_page_writeback);
2260
2261/*
2262 * Return true if any of the pages in the mapping are marked with the
2263 * passed tag.
2264 */
2265int mapping_tagged(struct address_space *mapping, int tag)
2266{
2267    return radix_tree_tagged(&mapping->page_tree, tag);
2268}
2269EXPORT_SYMBOL(mapping_tagged);
2270

Archive Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9



interactive