Root/
1 | /* |
2 | * cpu_rmap.c: CPU affinity reverse-map support |
3 | * Copyright 2011 Solarflare Communications Inc. |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify it |
6 | * under the terms of the GNU General Public License version 2 as published |
7 | * by the Free Software Foundation, incorporated herein by reference. |
8 | */ |
9 | |
10 | #include <linux/cpu_rmap.h> |
11 | #include <linux/interrupt.h> |
12 | #include <linux/export.h> |
13 | |
14 | /* |
15 | * These functions maintain a mapping from CPUs to some ordered set of |
16 | * objects with CPU affinities. This can be seen as a reverse-map of |
17 | * CPU affinity. However, we do not assume that the object affinities |
18 | * cover all CPUs in the system. For those CPUs not directly covered |
19 | * by object affinities, we attempt to find a nearest object based on |
20 | * CPU topology. |
21 | */ |
22 | |
23 | /** |
24 | * alloc_cpu_rmap - allocate CPU affinity reverse-map |
25 | * @size: Number of objects to be mapped |
26 | * @flags: Allocation flags e.g. %GFP_KERNEL |
27 | */ |
28 | struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags) |
29 | { |
30 | struct cpu_rmap *rmap; |
31 | unsigned int cpu; |
32 | size_t obj_offset; |
33 | |
34 | /* This is a silly number of objects, and we use u16 indices. */ |
35 | if (size > 0xffff) |
36 | return NULL; |
37 | |
38 | /* Offset of object pointer array from base structure */ |
39 | obj_offset = ALIGN(offsetof(struct cpu_rmap, near[nr_cpu_ids]), |
40 | sizeof(void *)); |
41 | |
42 | rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags); |
43 | if (!rmap) |
44 | return NULL; |
45 | |
46 | kref_init(&rmap->refcount); |
47 | rmap->obj = (void **)((char *)rmap + obj_offset); |
48 | |
49 | /* Initially assign CPUs to objects on a rota, since we have |
50 | * no idea where the objects are. Use infinite distance, so |
51 | * any object with known distance is preferable. Include the |
52 | * CPUs that are not present/online, since we definitely want |
53 | * any newly-hotplugged CPUs to have some object assigned. |
54 | */ |
55 | for_each_possible_cpu(cpu) { |
56 | rmap->near[cpu].index = cpu % size; |
57 | rmap->near[cpu].dist = CPU_RMAP_DIST_INF; |
58 | } |
59 | |
60 | rmap->size = size; |
61 | return rmap; |
62 | } |
63 | EXPORT_SYMBOL(alloc_cpu_rmap); |
64 | |
65 | /** |
66 | * cpu_rmap_release - internal reclaiming helper called from kref_put |
67 | * @ref: kref to struct cpu_rmap |
68 | */ |
69 | static void cpu_rmap_release(struct kref *ref) |
70 | { |
71 | struct cpu_rmap *rmap = container_of(ref, struct cpu_rmap, refcount); |
72 | kfree(rmap); |
73 | } |
74 | |
75 | /** |
76 | * cpu_rmap_get - internal helper to get new ref on a cpu_rmap |
77 | * @rmap: reverse-map allocated with alloc_cpu_rmap() |
78 | */ |
79 | static inline void cpu_rmap_get(struct cpu_rmap *rmap) |
80 | { |
81 | kref_get(&rmap->refcount); |
82 | } |
83 | |
84 | /** |
85 | * cpu_rmap_put - release ref on a cpu_rmap |
86 | * @rmap: reverse-map allocated with alloc_cpu_rmap() |
87 | */ |
88 | int cpu_rmap_put(struct cpu_rmap *rmap) |
89 | { |
90 | return kref_put(&rmap->refcount, cpu_rmap_release); |
91 | } |
92 | EXPORT_SYMBOL(cpu_rmap_put); |
93 | |
94 | /* Reevaluate nearest object for given CPU, comparing with the given |
95 | * neighbours at the given distance. |
96 | */ |
97 | static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu, |
98 | const struct cpumask *mask, u16 dist) |
99 | { |
100 | int neigh; |
101 | |
102 | for_each_cpu(neigh, mask) { |
103 | if (rmap->near[cpu].dist > dist && |
104 | rmap->near[neigh].dist <= dist) { |
105 | rmap->near[cpu].index = rmap->near[neigh].index; |
106 | rmap->near[cpu].dist = dist; |
107 | return true; |
108 | } |
109 | } |
110 | return false; |
111 | } |
112 | |
113 | #ifdef DEBUG |
114 | static void debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix) |
115 | { |
116 | unsigned index; |
117 | unsigned int cpu; |
118 | |
119 | pr_info("cpu_rmap %p, %s:\n", rmap, prefix); |
120 | |
121 | for_each_possible_cpu(cpu) { |
122 | index = rmap->near[cpu].index; |
123 | pr_info("cpu %d -> obj %u (distance %u)\n", |
124 | cpu, index, rmap->near[cpu].dist); |
125 | } |
126 | } |
127 | #else |
128 | static inline void |
129 | debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix) |
130 | { |
131 | } |
132 | #endif |
133 | |
134 | /** |
135 | * cpu_rmap_add - add object to a rmap |
136 | * @rmap: CPU rmap allocated with alloc_cpu_rmap() |
137 | * @obj: Object to add to rmap |
138 | * |
139 | * Return index of object. |
140 | */ |
141 | int cpu_rmap_add(struct cpu_rmap *rmap, void *obj) |
142 | { |
143 | u16 index; |
144 | |
145 | BUG_ON(rmap->used >= rmap->size); |
146 | index = rmap->used++; |
147 | rmap->obj[index] = obj; |
148 | return index; |
149 | } |
150 | EXPORT_SYMBOL(cpu_rmap_add); |
151 | |
152 | /** |
153 | * cpu_rmap_update - update CPU rmap following a change of object affinity |
154 | * @rmap: CPU rmap to update |
155 | * @index: Index of object whose affinity changed |
156 | * @affinity: New CPU affinity of object |
157 | */ |
158 | int cpu_rmap_update(struct cpu_rmap *rmap, u16 index, |
159 | const struct cpumask *affinity) |
160 | { |
161 | cpumask_var_t update_mask; |
162 | unsigned int cpu; |
163 | |
164 | if (unlikely(!zalloc_cpumask_var(&update_mask, GFP_KERNEL))) |
165 | return -ENOMEM; |
166 | |
167 | /* Invalidate distance for all CPUs for which this used to be |
168 | * the nearest object. Mark those CPUs for update. |
169 | */ |
170 | for_each_online_cpu(cpu) { |
171 | if (rmap->near[cpu].index == index) { |
172 | rmap->near[cpu].dist = CPU_RMAP_DIST_INF; |
173 | cpumask_set_cpu(cpu, update_mask); |
174 | } |
175 | } |
176 | |
177 | debug_print_rmap(rmap, "after invalidating old distances"); |
178 | |
179 | /* Set distance to 0 for all CPUs in the new affinity mask. |
180 | * Mark all CPUs within their NUMA nodes for update. |
181 | */ |
182 | for_each_cpu(cpu, affinity) { |
183 | rmap->near[cpu].index = index; |
184 | rmap->near[cpu].dist = 0; |
185 | cpumask_or(update_mask, update_mask, |
186 | cpumask_of_node(cpu_to_node(cpu))); |
187 | } |
188 | |
189 | debug_print_rmap(rmap, "after updating neighbours"); |
190 | |
191 | /* Update distances based on topology */ |
192 | for_each_cpu(cpu, update_mask) { |
193 | if (cpu_rmap_copy_neigh(rmap, cpu, |
194 | topology_thread_cpumask(cpu), 1)) |
195 | continue; |
196 | if (cpu_rmap_copy_neigh(rmap, cpu, |
197 | topology_core_cpumask(cpu), 2)) |
198 | continue; |
199 | if (cpu_rmap_copy_neigh(rmap, cpu, |
200 | cpumask_of_node(cpu_to_node(cpu)), 3)) |
201 | continue; |
202 | /* We could continue into NUMA node distances, but for now |
203 | * we give up. |
204 | */ |
205 | } |
206 | |
207 | debug_print_rmap(rmap, "after copying neighbours"); |
208 | |
209 | free_cpumask_var(update_mask); |
210 | return 0; |
211 | } |
212 | EXPORT_SYMBOL(cpu_rmap_update); |
213 | |
214 | /* Glue between IRQ affinity notifiers and CPU rmaps */ |
215 | |
216 | struct irq_glue { |
217 | struct irq_affinity_notify notify; |
218 | struct cpu_rmap *rmap; |
219 | u16 index; |
220 | }; |
221 | |
222 | /** |
223 | * free_irq_cpu_rmap - free a CPU affinity reverse-map used for IRQs |
224 | * @rmap: Reverse-map allocated with alloc_irq_cpu_map(), or %NULL |
225 | * |
226 | * Must be called in process context, before freeing the IRQs. |
227 | */ |
228 | void free_irq_cpu_rmap(struct cpu_rmap *rmap) |
229 | { |
230 | struct irq_glue *glue; |
231 | u16 index; |
232 | |
233 | if (!rmap) |
234 | return; |
235 | |
236 | for (index = 0; index < rmap->used; index++) { |
237 | glue = rmap->obj[index]; |
238 | irq_set_affinity_notifier(glue->notify.irq, NULL); |
239 | } |
240 | |
241 | cpu_rmap_put(rmap); |
242 | } |
243 | EXPORT_SYMBOL(free_irq_cpu_rmap); |
244 | |
245 | /** |
246 | * irq_cpu_rmap_notify - callback for IRQ subsystem when IRQ affinity updated |
247 | * @notify: struct irq_affinity_notify passed by irq/manage.c |
248 | * @mask: cpu mask for new SMP affinity |
249 | * |
250 | * This is executed in workqueue context. |
251 | */ |
252 | static void |
253 | irq_cpu_rmap_notify(struct irq_affinity_notify *notify, const cpumask_t *mask) |
254 | { |
255 | struct irq_glue *glue = |
256 | container_of(notify, struct irq_glue, notify); |
257 | int rc; |
258 | |
259 | rc = cpu_rmap_update(glue->rmap, glue->index, mask); |
260 | if (rc) |
261 | pr_warning("irq_cpu_rmap_notify: update failed: %d\n", rc); |
262 | } |
263 | |
264 | /** |
265 | * irq_cpu_rmap_release - reclaiming callback for IRQ subsystem |
266 | * @ref: kref to struct irq_affinity_notify passed by irq/manage.c |
267 | */ |
268 | static void irq_cpu_rmap_release(struct kref *ref) |
269 | { |
270 | struct irq_glue *glue = |
271 | container_of(ref, struct irq_glue, notify.kref); |
272 | |
273 | cpu_rmap_put(glue->rmap); |
274 | kfree(glue); |
275 | } |
276 | |
277 | /** |
278 | * irq_cpu_rmap_add - add an IRQ to a CPU affinity reverse-map |
279 | * @rmap: The reverse-map |
280 | * @irq: The IRQ number |
281 | * |
282 | * This adds an IRQ affinity notifier that will update the reverse-map |
283 | * automatically. |
284 | * |
285 | * Must be called in process context, after the IRQ is allocated but |
286 | * before it is bound with request_irq(). |
287 | */ |
288 | int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq) |
289 | { |
290 | struct irq_glue *glue = kzalloc(sizeof(*glue), GFP_KERNEL); |
291 | int rc; |
292 | |
293 | if (!glue) |
294 | return -ENOMEM; |
295 | glue->notify.notify = irq_cpu_rmap_notify; |
296 | glue->notify.release = irq_cpu_rmap_release; |
297 | glue->rmap = rmap; |
298 | cpu_rmap_get(rmap); |
299 | glue->index = cpu_rmap_add(rmap, glue); |
300 | rc = irq_set_affinity_notifier(irq, &glue->notify); |
301 | if (rc) { |
302 | cpu_rmap_put(glue->rmap); |
303 | kfree(glue); |
304 | } |
305 | return rc; |
306 | } |
307 | EXPORT_SYMBOL(irq_cpu_rmap_add); |
308 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9