Root/
1 | /* |
2 | * Linux VM pressure |
3 | * |
4 | * Copyright 2012 Linaro Ltd. |
5 | * Anton Vorontsov <anton.vorontsov@linaro.org> |
6 | * |
7 | * Based on ideas from Andrew Morton, David Rientjes, KOSAKI Motohiro, |
8 | * Leonid Moiseichuk, Mel Gorman, Minchan Kim and Pekka Enberg. |
9 | * |
10 | * This program is free software; you can redistribute it and/or modify it |
11 | * under the terms of the GNU General Public License version 2 as published |
12 | * by the Free Software Foundation. |
13 | */ |
14 | |
15 | #include <linux/cgroup.h> |
16 | #include <linux/fs.h> |
17 | #include <linux/log2.h> |
18 | #include <linux/sched.h> |
19 | #include <linux/mm.h> |
20 | #include <linux/vmstat.h> |
21 | #include <linux/eventfd.h> |
22 | #include <linux/slab.h> |
23 | #include <linux/swap.h> |
24 | #include <linux/printk.h> |
25 | #include <linux/vmpressure.h> |
26 | |
27 | /* |
28 | * The window size (vmpressure_win) is the number of scanned pages before |
29 | * we try to analyze scanned/reclaimed ratio. So the window is used as a |
30 | * rate-limit tunable for the "low" level notification, and also for |
31 | * averaging the ratio for medium/critical levels. Using small window |
32 | * sizes can cause lot of false positives, but too big window size will |
33 | * delay the notifications. |
34 | * |
35 | * As the vmscan reclaimer logic works with chunks which are multiple of |
36 | * SWAP_CLUSTER_MAX, it makes sense to use it for the window size as well. |
37 | * |
38 | * TODO: Make the window size depend on machine size, as we do for vmstat |
39 | * thresholds. Currently we set it to 512 pages (2MB for 4KB pages). |
40 | */ |
41 | static const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16; |
42 | |
43 | /* |
44 | * These thresholds are used when we account memory pressure through |
45 | * scanned/reclaimed ratio. The current values were chosen empirically. In |
46 | * essence, they are percents: the higher the value, the more number |
47 | * unsuccessful reclaims there were. |
48 | */ |
49 | static const unsigned int vmpressure_level_med = 60; |
50 | static const unsigned int vmpressure_level_critical = 95; |
51 | |
52 | /* |
53 | * When there are too little pages left to scan, vmpressure() may miss the |
54 | * critical pressure as number of pages will be less than "window size". |
55 | * However, in that case the vmscan priority will raise fast as the |
56 | * reclaimer will try to scan LRUs more deeply. |
57 | * |
58 | * The vmscan logic considers these special priorities: |
59 | * |
60 | * prio == DEF_PRIORITY (12): reclaimer starts with that value |
61 | * prio <= DEF_PRIORITY - 2 : kswapd becomes somewhat overwhelmed |
62 | * prio == 0 : close to OOM, kernel scans every page in an lru |
63 | * |
64 | * Any value in this range is acceptable for this tunable (i.e. from 12 to |
65 | * 0). Current value for the vmpressure_level_critical_prio is chosen |
66 | * empirically, but the number, in essence, means that we consider |
67 | * critical level when scanning depth is ~10% of the lru size (vmscan |
68 | * scans 'lru_size >> prio' pages, so it is actually 12.5%, or one |
69 | * eights). |
70 | */ |
71 | static const unsigned int vmpressure_level_critical_prio = ilog2(100 / 10); |
72 | |
73 | static struct vmpressure *work_to_vmpressure(struct work_struct *work) |
74 | { |
75 | return container_of(work, struct vmpressure, work); |
76 | } |
77 | |
78 | static struct vmpressure *vmpressure_parent(struct vmpressure *vmpr) |
79 | { |
80 | struct cgroup_subsys_state *css = vmpressure_to_css(vmpr); |
81 | struct mem_cgroup *memcg = mem_cgroup_from_css(css); |
82 | |
83 | memcg = parent_mem_cgroup(memcg); |
84 | if (!memcg) |
85 | return NULL; |
86 | return memcg_to_vmpressure(memcg); |
87 | } |
88 | |
89 | enum vmpressure_levels { |
90 | VMPRESSURE_LOW = 0, |
91 | VMPRESSURE_MEDIUM, |
92 | VMPRESSURE_CRITICAL, |
93 | VMPRESSURE_NUM_LEVELS, |
94 | }; |
95 | |
96 | static const char * const vmpressure_str_levels[] = { |
97 | [VMPRESSURE_LOW] = "low", |
98 | [VMPRESSURE_MEDIUM] = "medium", |
99 | [VMPRESSURE_CRITICAL] = "critical", |
100 | }; |
101 | |
102 | static enum vmpressure_levels vmpressure_level(unsigned long pressure) |
103 | { |
104 | if (pressure >= vmpressure_level_critical) |
105 | return VMPRESSURE_CRITICAL; |
106 | else if (pressure >= vmpressure_level_med) |
107 | return VMPRESSURE_MEDIUM; |
108 | return VMPRESSURE_LOW; |
109 | } |
110 | |
111 | static enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, |
112 | unsigned long reclaimed) |
113 | { |
114 | unsigned long scale = scanned + reclaimed; |
115 | unsigned long pressure; |
116 | |
117 | /* |
118 | * We calculate the ratio (in percents) of how many pages were |
119 | * scanned vs. reclaimed in a given time frame (window). Note that |
120 | * time is in VM reclaimer's "ticks", i.e. number of pages |
121 | * scanned. This makes it possible to set desired reaction time |
122 | * and serves as a ratelimit. |
123 | */ |
124 | pressure = scale - (reclaimed * scale / scanned); |
125 | pressure = pressure * 100 / scale; |
126 | |
127 | pr_debug("%s: %3lu (s: %lu r: %lu)\n", __func__, pressure, |
128 | scanned, reclaimed); |
129 | |
130 | return vmpressure_level(pressure); |
131 | } |
132 | |
133 | struct vmpressure_event { |
134 | struct eventfd_ctx *efd; |
135 | enum vmpressure_levels level; |
136 | struct list_head node; |
137 | }; |
138 | |
139 | static bool vmpressure_event(struct vmpressure *vmpr, |
140 | unsigned long scanned, unsigned long reclaimed) |
141 | { |
142 | struct vmpressure_event *ev; |
143 | enum vmpressure_levels level; |
144 | bool signalled = false; |
145 | |
146 | level = vmpressure_calc_level(scanned, reclaimed); |
147 | |
148 | mutex_lock(&vmpr->events_lock); |
149 | |
150 | list_for_each_entry(ev, &vmpr->events, node) { |
151 | if (level >= ev->level) { |
152 | eventfd_signal(ev->efd, 1); |
153 | signalled = true; |
154 | } |
155 | } |
156 | |
157 | mutex_unlock(&vmpr->events_lock); |
158 | |
159 | return signalled; |
160 | } |
161 | |
162 | static void vmpressure_work_fn(struct work_struct *work) |
163 | { |
164 | struct vmpressure *vmpr = work_to_vmpressure(work); |
165 | unsigned long scanned; |
166 | unsigned long reclaimed; |
167 | |
168 | /* |
169 | * Several contexts might be calling vmpressure(), so it is |
170 | * possible that the work was rescheduled again before the old |
171 | * work context cleared the counters. In that case we will run |
172 | * just after the old work returns, but then scanned might be zero |
173 | * here. No need for any locks here since we don't care if |
174 | * vmpr->reclaimed is in sync. |
175 | */ |
176 | if (!vmpr->scanned) |
177 | return; |
178 | |
179 | spin_lock(&vmpr->sr_lock); |
180 | scanned = vmpr->scanned; |
181 | reclaimed = vmpr->reclaimed; |
182 | vmpr->scanned = 0; |
183 | vmpr->reclaimed = 0; |
184 | spin_unlock(&vmpr->sr_lock); |
185 | |
186 | do { |
187 | if (vmpressure_event(vmpr, scanned, reclaimed)) |
188 | break; |
189 | /* |
190 | * If not handled, propagate the event upward into the |
191 | * hierarchy. |
192 | */ |
193 | } while ((vmpr = vmpressure_parent(vmpr))); |
194 | } |
195 | |
196 | /** |
197 | * vmpressure() - Account memory pressure through scanned/reclaimed ratio |
198 | * @gfp: reclaimer's gfp mask |
199 | * @memcg: cgroup memory controller handle |
200 | * @scanned: number of pages scanned |
201 | * @reclaimed: number of pages reclaimed |
202 | * |
203 | * This function should be called from the vmscan reclaim path to account |
204 | * "instantaneous" memory pressure (scanned/reclaimed ratio). The raw |
205 | * pressure index is then further refined and averaged over time. |
206 | * |
207 | * This function does not return any value. |
208 | */ |
209 | void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, |
210 | unsigned long scanned, unsigned long reclaimed) |
211 | { |
212 | struct vmpressure *vmpr = memcg_to_vmpressure(memcg); |
213 | |
214 | /* |
215 | * Here we only want to account pressure that userland is able to |
216 | * help us with. For example, suppose that DMA zone is under |
217 | * pressure; if we notify userland about that kind of pressure, |
218 | * then it will be mostly a waste as it will trigger unnecessary |
219 | * freeing of memory by userland (since userland is more likely to |
220 | * have HIGHMEM/MOVABLE pages instead of the DMA fallback). That |
221 | * is why we include only movable, highmem and FS/IO pages. |
222 | * Indirect reclaim (kswapd) sets sc->gfp_mask to GFP_KERNEL, so |
223 | * we account it too. |
224 | */ |
225 | if (!(gfp & (__GFP_HIGHMEM | __GFP_MOVABLE | __GFP_IO | __GFP_FS))) |
226 | return; |
227 | |
228 | /* |
229 | * If we got here with no pages scanned, then that is an indicator |
230 | * that reclaimer was unable to find any shrinkable LRUs at the |
231 | * current scanning depth. But it does not mean that we should |
232 | * report the critical pressure, yet. If the scanning priority |
233 | * (scanning depth) goes too high (deep), we will be notified |
234 | * through vmpressure_prio(). But so far, keep calm. |
235 | */ |
236 | if (!scanned) |
237 | return; |
238 | |
239 | spin_lock(&vmpr->sr_lock); |
240 | vmpr->scanned += scanned; |
241 | vmpr->reclaimed += reclaimed; |
242 | scanned = vmpr->scanned; |
243 | spin_unlock(&vmpr->sr_lock); |
244 | |
245 | if (scanned < vmpressure_win) |
246 | return; |
247 | schedule_work(&vmpr->work); |
248 | } |
249 | |
250 | /** |
251 | * vmpressure_prio() - Account memory pressure through reclaimer priority level |
252 | * @gfp: reclaimer's gfp mask |
253 | * @memcg: cgroup memory controller handle |
254 | * @prio: reclaimer's priority |
255 | * |
256 | * This function should be called from the reclaim path every time when |
257 | * the vmscan's reclaiming priority (scanning depth) changes. |
258 | * |
259 | * This function does not return any value. |
260 | */ |
261 | void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) |
262 | { |
263 | /* |
264 | * We only use prio for accounting critical level. For more info |
265 | * see comment for vmpressure_level_critical_prio variable above. |
266 | */ |
267 | if (prio > vmpressure_level_critical_prio) |
268 | return; |
269 | |
270 | /* |
271 | * OK, the prio is below the threshold, updating vmpressure |
272 | * information before shrinker dives into long shrinking of long |
273 | * range vmscan. Passing scanned = vmpressure_win, reclaimed = 0 |
274 | * to the vmpressure() basically means that we signal 'critical' |
275 | * level. |
276 | */ |
277 | vmpressure(gfp, memcg, vmpressure_win, 0); |
278 | } |
279 | |
280 | /** |
281 | * vmpressure_register_event() - Bind vmpressure notifications to an eventfd |
282 | * @memcg: memcg that is interested in vmpressure notifications |
283 | * @eventfd: eventfd context to link notifications with |
284 | * @args: event arguments (used to set up a pressure level threshold) |
285 | * |
286 | * This function associates eventfd context with the vmpressure |
287 | * infrastructure, so that the notifications will be delivered to the |
288 | * @eventfd. The @args parameter is a string that denotes pressure level |
289 | * threshold (one of vmpressure_str_levels, i.e. "low", "medium", or |
290 | * "critical"). |
291 | * |
292 | * To be used as memcg event method. |
293 | */ |
294 | int vmpressure_register_event(struct mem_cgroup *memcg, |
295 | struct eventfd_ctx *eventfd, const char *args) |
296 | { |
297 | struct vmpressure *vmpr = memcg_to_vmpressure(memcg); |
298 | struct vmpressure_event *ev; |
299 | int level; |
300 | |
301 | for (level = 0; level < VMPRESSURE_NUM_LEVELS; level++) { |
302 | if (!strcmp(vmpressure_str_levels[level], args)) |
303 | break; |
304 | } |
305 | |
306 | if (level >= VMPRESSURE_NUM_LEVELS) |
307 | return -EINVAL; |
308 | |
309 | ev = kzalloc(sizeof(*ev), GFP_KERNEL); |
310 | if (!ev) |
311 | return -ENOMEM; |
312 | |
313 | ev->efd = eventfd; |
314 | ev->level = level; |
315 | |
316 | mutex_lock(&vmpr->events_lock); |
317 | list_add(&ev->node, &vmpr->events); |
318 | mutex_unlock(&vmpr->events_lock); |
319 | |
320 | return 0; |
321 | } |
322 | |
323 | /** |
324 | * vmpressure_unregister_event() - Unbind eventfd from vmpressure |
325 | * @memcg: memcg handle |
326 | * @eventfd: eventfd context that was used to link vmpressure with the @cg |
327 | * |
328 | * This function does internal manipulations to detach the @eventfd from |
329 | * the vmpressure notifications, and then frees internal resources |
330 | * associated with the @eventfd (but the @eventfd itself is not freed). |
331 | * |
332 | * To be used as memcg event method. |
333 | */ |
334 | void vmpressure_unregister_event(struct mem_cgroup *memcg, |
335 | struct eventfd_ctx *eventfd) |
336 | { |
337 | struct vmpressure *vmpr = memcg_to_vmpressure(memcg); |
338 | struct vmpressure_event *ev; |
339 | |
340 | mutex_lock(&vmpr->events_lock); |
341 | list_for_each_entry(ev, &vmpr->events, node) { |
342 | if (ev->efd != eventfd) |
343 | continue; |
344 | list_del(&ev->node); |
345 | kfree(ev); |
346 | break; |
347 | } |
348 | mutex_unlock(&vmpr->events_lock); |
349 | } |
350 | |
351 | /** |
352 | * vmpressure_init() - Initialize vmpressure control structure |
353 | * @vmpr: Structure to be initialized |
354 | * |
355 | * This function should be called on every allocated vmpressure structure |
356 | * before any usage. |
357 | */ |
358 | void vmpressure_init(struct vmpressure *vmpr) |
359 | { |
360 | spin_lock_init(&vmpr->sr_lock); |
361 | mutex_init(&vmpr->events_lock); |
362 | INIT_LIST_HEAD(&vmpr->events); |
363 | INIT_WORK(&vmpr->work, vmpressure_work_fn); |
364 | } |
365 | |
366 | /** |
367 | * vmpressure_cleanup() - shuts down vmpressure control structure |
368 | * @vmpr: Structure to be cleaned up |
369 | * |
370 | * This function should be called before the structure in which it is |
371 | * embedded is cleaned up. |
372 | */ |
373 | void vmpressure_cleanup(struct vmpressure *vmpr) |
374 | { |
375 | /* |
376 | * Make sure there is no pending work before eventfd infrastructure |
377 | * goes away. |
378 | */ |
379 | flush_work(&vmpr->work); |
380 | } |
381 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9