Root/
1 | /* |
2 | * xor offload engine api |
3 | * |
4 | * Copyright © 2006, Intel Corporation. |
5 | * |
6 | * Dan Williams <dan.j.williams@intel.com> |
7 | * |
8 | * with architecture considerations by: |
9 | * Neil Brown <neilb@suse.de> |
10 | * Jeff Garzik <jeff@garzik.org> |
11 | * |
12 | * This program is free software; you can redistribute it and/or modify it |
13 | * under the terms and conditions of the GNU General Public License, |
14 | * version 2, as published by the Free Software Foundation. |
15 | * |
16 | * This program is distributed in the hope it will be useful, but WITHOUT |
17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
19 | * more details. |
20 | * |
21 | * You should have received a copy of the GNU General Public License along with |
22 | * this program; if not, write to the Free Software Foundation, Inc., |
23 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
24 | * |
25 | */ |
26 | #include <linux/kernel.h> |
27 | #include <linux/interrupt.h> |
28 | #include <linux/mm.h> |
29 | #include <linux/dma-mapping.h> |
30 | #include <linux/raid/xor.h> |
31 | #include <linux/async_tx.h> |
32 | |
33 | /* do_async_xor - dma map the pages and perform the xor with an engine */ |
34 | static __async_inline struct dma_async_tx_descriptor * |
35 | do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, |
36 | unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src, |
37 | struct async_submit_ctl *submit) |
38 | { |
39 | struct dma_device *dma = chan->device; |
40 | struct dma_async_tx_descriptor *tx = NULL; |
41 | int src_off = 0; |
42 | int i; |
43 | dma_async_tx_callback cb_fn_orig = submit->cb_fn; |
44 | void *cb_param_orig = submit->cb_param; |
45 | enum async_tx_flags flags_orig = submit->flags; |
46 | enum dma_ctrl_flags dma_flags; |
47 | int xor_src_cnt = 0; |
48 | dma_addr_t dma_dest; |
49 | |
50 | /* map the dest bidrectional in case it is re-used as a source */ |
51 | dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL); |
52 | for (i = 0; i < src_cnt; i++) { |
53 | /* only map the dest once */ |
54 | if (!src_list[i]) |
55 | continue; |
56 | if (unlikely(src_list[i] == dest)) { |
57 | dma_src[xor_src_cnt++] = dma_dest; |
58 | continue; |
59 | } |
60 | dma_src[xor_src_cnt++] = dma_map_page(dma->dev, src_list[i], offset, |
61 | len, DMA_TO_DEVICE); |
62 | } |
63 | src_cnt = xor_src_cnt; |
64 | |
65 | while (src_cnt) { |
66 | submit->flags = flags_orig; |
67 | dma_flags = 0; |
68 | xor_src_cnt = min(src_cnt, (int)dma->max_xor); |
69 | /* if we are submitting additional xors, leave the chain open, |
70 | * clear the callback parameters, and leave the destination |
71 | * buffer mapped |
72 | */ |
73 | if (src_cnt > xor_src_cnt) { |
74 | submit->flags &= ~ASYNC_TX_ACK; |
75 | submit->flags |= ASYNC_TX_FENCE; |
76 | dma_flags = DMA_COMPL_SKIP_DEST_UNMAP; |
77 | submit->cb_fn = NULL; |
78 | submit->cb_param = NULL; |
79 | } else { |
80 | submit->cb_fn = cb_fn_orig; |
81 | submit->cb_param = cb_param_orig; |
82 | } |
83 | if (submit->cb_fn) |
84 | dma_flags |= DMA_PREP_INTERRUPT; |
85 | if (submit->flags & ASYNC_TX_FENCE) |
86 | dma_flags |= DMA_PREP_FENCE; |
87 | /* Since we have clobbered the src_list we are committed |
88 | * to doing this asynchronously. Drivers force forward progress |
89 | * in case they can not provide a descriptor |
90 | */ |
91 | tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off], |
92 | xor_src_cnt, len, dma_flags); |
93 | |
94 | if (unlikely(!tx)) |
95 | async_tx_quiesce(&submit->depend_tx); |
96 | |
97 | /* spin wait for the preceeding transactions to complete */ |
98 | while (unlikely(!tx)) { |
99 | dma_async_issue_pending(chan); |
100 | tx = dma->device_prep_dma_xor(chan, dma_dest, |
101 | &dma_src[src_off], |
102 | xor_src_cnt, len, |
103 | dma_flags); |
104 | } |
105 | |
106 | async_tx_submit(chan, tx, submit); |
107 | submit->depend_tx = tx; |
108 | |
109 | if (src_cnt > xor_src_cnt) { |
110 | /* drop completed sources */ |
111 | src_cnt -= xor_src_cnt; |
112 | src_off += xor_src_cnt; |
113 | |
114 | /* use the intermediate result a source */ |
115 | dma_src[--src_off] = dma_dest; |
116 | src_cnt++; |
117 | } else |
118 | break; |
119 | } |
120 | |
121 | return tx; |
122 | } |
123 | |
124 | static void |
125 | do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, |
126 | int src_cnt, size_t len, struct async_submit_ctl *submit) |
127 | { |
128 | int i; |
129 | int xor_src_cnt = 0; |
130 | int src_off = 0; |
131 | void *dest_buf; |
132 | void **srcs; |
133 | |
134 | if (submit->scribble) |
135 | srcs = submit->scribble; |
136 | else |
137 | srcs = (void **) src_list; |
138 | |
139 | /* convert to buffer pointers */ |
140 | for (i = 0; i < src_cnt; i++) |
141 | if (src_list[i]) |
142 | srcs[xor_src_cnt++] = page_address(src_list[i]) + offset; |
143 | src_cnt = xor_src_cnt; |
144 | /* set destination address */ |
145 | dest_buf = page_address(dest) + offset; |
146 | |
147 | if (submit->flags & ASYNC_TX_XOR_ZERO_DST) |
148 | memset(dest_buf, 0, len); |
149 | |
150 | while (src_cnt > 0) { |
151 | /* process up to 'MAX_XOR_BLOCKS' sources */ |
152 | xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS); |
153 | xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]); |
154 | |
155 | /* drop completed sources */ |
156 | src_cnt -= xor_src_cnt; |
157 | src_off += xor_src_cnt; |
158 | } |
159 | |
160 | async_tx_sync_epilog(submit); |
161 | } |
162 | |
163 | /** |
164 | * async_xor - attempt to xor a set of blocks with a dma engine. |
165 | * @dest: destination page |
166 | * @src_list: array of source pages |
167 | * @offset: common src/dst offset to start transaction |
168 | * @src_cnt: number of source pages |
169 | * @len: length in bytes |
170 | * @submit: submission / completion modifiers |
171 | * |
172 | * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST |
173 | * |
174 | * xor_blocks always uses the dest as a source so the |
175 | * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in |
176 | * the calculation. The assumption with dma eninges is that they only |
177 | * use the destination buffer as a source when it is explicity specified |
178 | * in the source list. |
179 | * |
180 | * src_list note: if the dest is also a source it must be at index zero. |
181 | * The contents of this array will be overwritten if a scribble region |
182 | * is not specified. |
183 | */ |
184 | struct dma_async_tx_descriptor * |
185 | async_xor(struct page *dest, struct page **src_list, unsigned int offset, |
186 | int src_cnt, size_t len, struct async_submit_ctl *submit) |
187 | { |
188 | struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, |
189 | &dest, 1, src_list, |
190 | src_cnt, len); |
191 | dma_addr_t *dma_src = NULL; |
192 | |
193 | BUG_ON(src_cnt <= 1); |
194 | |
195 | if (submit->scribble) |
196 | dma_src = submit->scribble; |
197 | else if (sizeof(dma_addr_t) <= sizeof(struct page *)) |
198 | dma_src = (dma_addr_t *) src_list; |
199 | |
200 | if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) { |
201 | /* run the xor asynchronously */ |
202 | pr_debug("%s (async): len: %zu\n", __func__, len); |
203 | |
204 | return do_async_xor(chan, dest, src_list, offset, src_cnt, len, |
205 | dma_src, submit); |
206 | } else { |
207 | /* run the xor synchronously */ |
208 | pr_debug("%s (sync): len: %zu\n", __func__, len); |
209 | WARN_ONCE(chan, "%s: no space for dma address conversion\n", |
210 | __func__); |
211 | |
212 | /* in the sync case the dest is an implied source |
213 | * (assumes the dest is the first source) |
214 | */ |
215 | if (submit->flags & ASYNC_TX_XOR_DROP_DST) { |
216 | src_cnt--; |
217 | src_list++; |
218 | } |
219 | |
220 | /* wait for any prerequisite operations */ |
221 | async_tx_quiesce(&submit->depend_tx); |
222 | |
223 | do_sync_xor(dest, src_list, offset, src_cnt, len, submit); |
224 | |
225 | return NULL; |
226 | } |
227 | } |
228 | EXPORT_SYMBOL_GPL(async_xor); |
229 | |
230 | static int page_is_zero(struct page *p, unsigned int offset, size_t len) |
231 | { |
232 | char *a = page_address(p) + offset; |
233 | return ((*(u32 *) a) == 0 && |
234 | memcmp(a, a + 4, len - 4) == 0); |
235 | } |
236 | |
237 | static inline struct dma_chan * |
238 | xor_val_chan(struct async_submit_ctl *submit, struct page *dest, |
239 | struct page **src_list, int src_cnt, size_t len) |
240 | { |
241 | #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA |
242 | return NULL; |
243 | #endif |
244 | return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list, |
245 | src_cnt, len); |
246 | } |
247 | |
248 | /** |
249 | * async_xor_val - attempt a xor parity check with a dma engine. |
250 | * @dest: destination page used if the xor is performed synchronously |
251 | * @src_list: array of source pages |
252 | * @offset: offset in pages to start transaction |
253 | * @src_cnt: number of source pages |
254 | * @len: length in bytes |
255 | * @result: 0 if sum == 0 else non-zero |
256 | * @submit: submission / completion modifiers |
257 | * |
258 | * honored flags: ASYNC_TX_ACK |
259 | * |
260 | * src_list note: if the dest is also a source it must be at index zero. |
261 | * The contents of this array will be overwritten if a scribble region |
262 | * is not specified. |
263 | */ |
264 | struct dma_async_tx_descriptor * |
265 | async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, |
266 | int src_cnt, size_t len, enum sum_check_flags *result, |
267 | struct async_submit_ctl *submit) |
268 | { |
269 | struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); |
270 | struct dma_device *device = chan ? chan->device : NULL; |
271 | struct dma_async_tx_descriptor *tx = NULL; |
272 | dma_addr_t *dma_src = NULL; |
273 | |
274 | BUG_ON(src_cnt <= 1); |
275 | |
276 | if (submit->scribble) |
277 | dma_src = submit->scribble; |
278 | else if (sizeof(dma_addr_t) <= sizeof(struct page *)) |
279 | dma_src = (dma_addr_t *) src_list; |
280 | |
281 | if (dma_src && device && src_cnt <= device->max_xor && |
282 | is_dma_xor_aligned(device, offset, 0, len)) { |
283 | unsigned long dma_prep_flags = 0; |
284 | int i; |
285 | |
286 | pr_debug("%s: (async) len: %zu\n", __func__, len); |
287 | |
288 | if (submit->cb_fn) |
289 | dma_prep_flags |= DMA_PREP_INTERRUPT; |
290 | if (submit->flags & ASYNC_TX_FENCE) |
291 | dma_prep_flags |= DMA_PREP_FENCE; |
292 | for (i = 0; i < src_cnt; i++) |
293 | dma_src[i] = dma_map_page(device->dev, src_list[i], |
294 | offset, len, DMA_TO_DEVICE); |
295 | |
296 | tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, |
297 | len, result, |
298 | dma_prep_flags); |
299 | if (unlikely(!tx)) { |
300 | async_tx_quiesce(&submit->depend_tx); |
301 | |
302 | while (!tx) { |
303 | dma_async_issue_pending(chan); |
304 | tx = device->device_prep_dma_xor_val(chan, |
305 | dma_src, src_cnt, len, result, |
306 | dma_prep_flags); |
307 | } |
308 | } |
309 | |
310 | async_tx_submit(chan, tx, submit); |
311 | } else { |
312 | enum async_tx_flags flags_orig = submit->flags; |
313 | |
314 | pr_debug("%s: (sync) len: %zu\n", __func__, len); |
315 | WARN_ONCE(device && src_cnt <= device->max_xor, |
316 | "%s: no space for dma address conversion\n", |
317 | __func__); |
318 | |
319 | submit->flags |= ASYNC_TX_XOR_DROP_DST; |
320 | submit->flags &= ~ASYNC_TX_ACK; |
321 | |
322 | tx = async_xor(dest, src_list, offset, src_cnt, len, submit); |
323 | |
324 | async_tx_quiesce(&tx); |
325 | |
326 | *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; |
327 | |
328 | async_tx_sync_epilog(submit); |
329 | submit->flags = flags_orig; |
330 | } |
331 | |
332 | return tx; |
333 | } |
334 | EXPORT_SYMBOL_GPL(async_xor_val); |
335 | |
336 | MODULE_AUTHOR("Intel Corporation"); |
337 | MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); |
338 | MODULE_LICENSE("GPL"); |
339 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9