Root/
| 1 | /* |
| 2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. |
| 3 | * Portions based on net/core/datagram.c and copyrighted by their authors. |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License as published by the Free |
| 7 | * Software Foundation; either version 2 of the License, or (at your option) |
| 8 | * any later version. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 13 | * more details. |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License along with |
| 16 | * this program; if not, write to the Free Software Foundation, Inc., 59 |
| 17 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 18 | * |
| 19 | * The full GNU General Public License is included in this distribution in the |
| 20 | * file called COPYING. |
| 21 | */ |
| 22 | |
| 23 | /* |
| 24 | * This code allows the net stack to make use of a DMA engine for |
| 25 | * skb to iovec copies. |
| 26 | */ |
| 27 | |
| 28 | #include <linux/dmaengine.h> |
| 29 | #include <linux/pagemap.h> |
| 30 | #include <linux/slab.h> |
| 31 | #include <net/tcp.h> /* for memcpy_toiovec */ |
| 32 | #include <asm/io.h> |
| 33 | #include <asm/uaccess.h> |
| 34 | |
| 35 | static int num_pages_spanned(struct iovec *iov) |
| 36 | { |
| 37 | return |
| 38 | ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - |
| 39 | ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); |
| 40 | } |
| 41 | |
| 42 | /* |
| 43 | * Pin down all the iovec pages needed for len bytes. |
| 44 | * Return a struct dma_pinned_list to keep track of pages pinned down. |
| 45 | * |
| 46 | * We are allocating a single chunk of memory, and then carving it up into |
| 47 | * 3 sections, the latter 2 whose size depends on the number of iovecs and the |
| 48 | * total number of pages, respectively. |
| 49 | */ |
| 50 | struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) |
| 51 | { |
| 52 | struct dma_pinned_list *local_list; |
| 53 | struct page **pages; |
| 54 | int i; |
| 55 | int ret; |
| 56 | int nr_iovecs = 0; |
| 57 | int iovec_len_used = 0; |
| 58 | int iovec_pages_used = 0; |
| 59 | |
| 60 | /* don't pin down non-user-based iovecs */ |
| 61 | if (segment_eq(get_fs(), KERNEL_DS)) |
| 62 | return NULL; |
| 63 | |
| 64 | /* determine how many iovecs/pages there are, up front */ |
| 65 | do { |
| 66 | iovec_len_used += iov[nr_iovecs].iov_len; |
| 67 | iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); |
| 68 | nr_iovecs++; |
| 69 | } while (iovec_len_used < len); |
| 70 | |
| 71 | /* single kmalloc for pinned list, page_list[], and the page arrays */ |
| 72 | local_list = kmalloc(sizeof(*local_list) |
| 73 | + (nr_iovecs * sizeof (struct dma_page_list)) |
| 74 | + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); |
| 75 | if (!local_list) |
| 76 | goto out; |
| 77 | |
| 78 | /* list of pages starts right after the page list array */ |
| 79 | pages = (struct page **) &local_list->page_list[nr_iovecs]; |
| 80 | |
| 81 | local_list->nr_iovecs = 0; |
| 82 | |
| 83 | for (i = 0; i < nr_iovecs; i++) { |
| 84 | struct dma_page_list *page_list = &local_list->page_list[i]; |
| 85 | |
| 86 | len -= iov[i].iov_len; |
| 87 | |
| 88 | if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) |
| 89 | goto unpin; |
| 90 | |
| 91 | page_list->nr_pages = num_pages_spanned(&iov[i]); |
| 92 | page_list->base_address = iov[i].iov_base; |
| 93 | |
| 94 | page_list->pages = pages; |
| 95 | pages += page_list->nr_pages; |
| 96 | |
| 97 | /* pin pages down */ |
| 98 | down_read(¤t->mm->mmap_sem); |
| 99 | ret = get_user_pages( |
| 100 | current, |
| 101 | current->mm, |
| 102 | (unsigned long) iov[i].iov_base, |
| 103 | page_list->nr_pages, |
| 104 | 1, /* write */ |
| 105 | 0, /* force */ |
| 106 | page_list->pages, |
| 107 | NULL); |
| 108 | up_read(¤t->mm->mmap_sem); |
| 109 | |
| 110 | if (ret != page_list->nr_pages) |
| 111 | goto unpin; |
| 112 | |
| 113 | local_list->nr_iovecs = i + 1; |
| 114 | } |
| 115 | |
| 116 | return local_list; |
| 117 | |
| 118 | unpin: |
| 119 | dma_unpin_iovec_pages(local_list); |
| 120 | out: |
| 121 | return NULL; |
| 122 | } |
| 123 | |
| 124 | void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) |
| 125 | { |
| 126 | int i, j; |
| 127 | |
| 128 | if (!pinned_list) |
| 129 | return; |
| 130 | |
| 131 | for (i = 0; i < pinned_list->nr_iovecs; i++) { |
| 132 | struct dma_page_list *page_list = &pinned_list->page_list[i]; |
| 133 | for (j = 0; j < page_list->nr_pages; j++) { |
| 134 | set_page_dirty_lock(page_list->pages[j]); |
| 135 | page_cache_release(page_list->pages[j]); |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | kfree(pinned_list); |
| 140 | } |
| 141 | |
| 142 | |
| 143 | /* |
| 144 | * We have already pinned down the pages we will be using in the iovecs. |
| 145 | * Each entry in iov array has corresponding entry in pinned_list->page_list. |
| 146 | * Using array indexing to keep iov[] and page_list[] in sync. |
| 147 | * Initial elements in iov array's iov->iov_len will be 0 if already copied into |
| 148 | * by another call. |
| 149 | * iov array length remaining guaranteed to be bigger than len. |
| 150 | */ |
| 151 | dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, |
| 152 | struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) |
| 153 | { |
| 154 | int iov_byte_offset; |
| 155 | int copy; |
| 156 | dma_cookie_t dma_cookie = 0; |
| 157 | int iovec_idx; |
| 158 | int page_idx; |
| 159 | |
| 160 | if (!chan) |
| 161 | return memcpy_toiovec(iov, kdata, len); |
| 162 | |
| 163 | iovec_idx = 0; |
| 164 | while (iovec_idx < pinned_list->nr_iovecs) { |
| 165 | struct dma_page_list *page_list; |
| 166 | |
| 167 | /* skip already used-up iovecs */ |
| 168 | while (!iov[iovec_idx].iov_len) |
| 169 | iovec_idx++; |
| 170 | |
| 171 | page_list = &pinned_list->page_list[iovec_idx]; |
| 172 | |
| 173 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); |
| 174 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) |
| 175 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; |
| 176 | |
| 177 | /* break up copies to not cross page boundary */ |
| 178 | while (iov[iovec_idx].iov_len) { |
| 179 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); |
| 180 | copy = min_t(int, copy, iov[iovec_idx].iov_len); |
| 181 | |
| 182 | dma_cookie = dma_async_memcpy_buf_to_pg(chan, |
| 183 | page_list->pages[page_idx], |
| 184 | iov_byte_offset, |
| 185 | kdata, |
| 186 | copy); |
| 187 | /* poll for a descriptor slot */ |
| 188 | if (unlikely(dma_cookie < 0)) { |
| 189 | dma_async_issue_pending(chan); |
| 190 | continue; |
| 191 | } |
| 192 | |
| 193 | len -= copy; |
| 194 | iov[iovec_idx].iov_len -= copy; |
| 195 | iov[iovec_idx].iov_base += copy; |
| 196 | |
| 197 | if (!len) |
| 198 | return dma_cookie; |
| 199 | |
| 200 | kdata += copy; |
| 201 | iov_byte_offset = 0; |
| 202 | page_idx++; |
| 203 | } |
| 204 | iovec_idx++; |
| 205 | } |
| 206 | |
| 207 | /* really bad if we ever run out of iovecs */ |
| 208 | BUG(); |
| 209 | return -EFAULT; |
| 210 | } |
| 211 | |
| 212 | dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, |
| 213 | struct dma_pinned_list *pinned_list, struct page *page, |
| 214 | unsigned int offset, size_t len) |
| 215 | { |
| 216 | int iov_byte_offset; |
| 217 | int copy; |
| 218 | dma_cookie_t dma_cookie = 0; |
| 219 | int iovec_idx; |
| 220 | int page_idx; |
| 221 | int err; |
| 222 | |
| 223 | /* this needs as-yet-unimplemented buf-to-buff, so punt. */ |
| 224 | /* TODO: use dma for this */ |
| 225 | if (!chan || !pinned_list) { |
| 226 | u8 *vaddr = kmap(page); |
| 227 | err = memcpy_toiovec(iov, vaddr + offset, len); |
| 228 | kunmap(page); |
| 229 | return err; |
| 230 | } |
| 231 | |
| 232 | iovec_idx = 0; |
| 233 | while (iovec_idx < pinned_list->nr_iovecs) { |
| 234 | struct dma_page_list *page_list; |
| 235 | |
| 236 | /* skip already used-up iovecs */ |
| 237 | while (!iov[iovec_idx].iov_len) |
| 238 | iovec_idx++; |
| 239 | |
| 240 | page_list = &pinned_list->page_list[iovec_idx]; |
| 241 | |
| 242 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); |
| 243 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) |
| 244 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; |
| 245 | |
| 246 | /* break up copies to not cross page boundary */ |
| 247 | while (iov[iovec_idx].iov_len) { |
| 248 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); |
| 249 | copy = min_t(int, copy, iov[iovec_idx].iov_len); |
| 250 | |
| 251 | dma_cookie = dma_async_memcpy_pg_to_pg(chan, |
| 252 | page_list->pages[page_idx], |
| 253 | iov_byte_offset, |
| 254 | page, |
| 255 | offset, |
| 256 | copy); |
| 257 | /* poll for a descriptor slot */ |
| 258 | if (unlikely(dma_cookie < 0)) { |
| 259 | dma_async_issue_pending(chan); |
| 260 | continue; |
| 261 | } |
| 262 | |
| 263 | len -= copy; |
| 264 | iov[iovec_idx].iov_len -= copy; |
| 265 | iov[iovec_idx].iov_base += copy; |
| 266 | |
| 267 | if (!len) |
| 268 | return dma_cookie; |
| 269 | |
| 270 | offset += copy; |
| 271 | iov_byte_offset = 0; |
| 272 | page_idx++; |
| 273 | } |
| 274 | iovec_idx++; |
| 275 | } |
| 276 | |
| 277 | /* really bad if we ever run out of iovecs */ |
| 278 | BUG(); |
| 279 | return -EFAULT; |
| 280 | } |
| 281 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9
