Root/
1 | /* |
2 | * Copyright (C) 2008 Advanced Micro Devices, Inc. |
3 | * |
4 | * Author: Joerg Roedel <joerg.roedel@amd.com> |
5 | * |
6 | * This program is free software; you can redistribute it and/or modify it |
7 | * under the terms of the GNU General Public License version 2 as published |
8 | * by the Free Software Foundation. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * You should have received a copy of the GNU General Public License |
16 | * along with this program; if not, write to the Free Software |
17 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
18 | */ |
19 | |
20 | #include <linux/scatterlist.h> |
21 | #include <linux/dma-mapping.h> |
22 | #include <linux/stacktrace.h> |
23 | #include <linux/dma-debug.h> |
24 | #include <linux/spinlock.h> |
25 | #include <linux/debugfs.h> |
26 | #include <linux/uaccess.h> |
27 | #include <linux/device.h> |
28 | #include <linux/types.h> |
29 | #include <linux/sched.h> |
30 | #include <linux/ctype.h> |
31 | #include <linux/list.h> |
32 | #include <linux/slab.h> |
33 | |
34 | #include <asm/sections.h> |
35 | |
36 | #define HASH_SIZE 1024ULL |
37 | #define HASH_FN_SHIFT 13 |
38 | #define HASH_FN_MASK (HASH_SIZE - 1) |
39 | |
40 | enum { |
41 | dma_debug_single, |
42 | dma_debug_page, |
43 | dma_debug_sg, |
44 | dma_debug_coherent, |
45 | }; |
46 | |
47 | #define DMA_DEBUG_STACKTRACE_ENTRIES 5 |
48 | |
49 | struct dma_debug_entry { |
50 | struct list_head list; |
51 | struct device *dev; |
52 | int type; |
53 | phys_addr_t paddr; |
54 | u64 dev_addr; |
55 | u64 size; |
56 | int direction; |
57 | int sg_call_ents; |
58 | int sg_mapped_ents; |
59 | #ifdef CONFIG_STACKTRACE |
60 | struct stack_trace stacktrace; |
61 | unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; |
62 | #endif |
63 | }; |
64 | |
65 | struct hash_bucket { |
66 | struct list_head list; |
67 | spinlock_t lock; |
68 | } ____cacheline_aligned_in_smp; |
69 | |
70 | /* Hash list to save the allocated dma addresses */ |
71 | static struct hash_bucket dma_entry_hash[HASH_SIZE]; |
72 | /* List of pre-allocated dma_debug_entry's */ |
73 | static LIST_HEAD(free_entries); |
74 | /* Lock for the list above */ |
75 | static DEFINE_SPINLOCK(free_entries_lock); |
76 | |
77 | /* Global disable flag - will be set in case of an error */ |
78 | static bool global_disable __read_mostly; |
79 | |
80 | /* Global error count */ |
81 | static u32 error_count; |
82 | |
83 | /* Global error show enable*/ |
84 | static u32 show_all_errors __read_mostly; |
85 | /* Number of errors to show */ |
86 | static u32 show_num_errors = 1; |
87 | |
88 | static u32 num_free_entries; |
89 | static u32 min_free_entries; |
90 | static u32 nr_total_entries; |
91 | |
92 | /* number of preallocated entries requested by kernel cmdline */ |
93 | static u32 req_entries; |
94 | |
95 | /* debugfs dentry's for the stuff above */ |
96 | static struct dentry *dma_debug_dent __read_mostly; |
97 | static struct dentry *global_disable_dent __read_mostly; |
98 | static struct dentry *error_count_dent __read_mostly; |
99 | static struct dentry *show_all_errors_dent __read_mostly; |
100 | static struct dentry *show_num_errors_dent __read_mostly; |
101 | static struct dentry *num_free_entries_dent __read_mostly; |
102 | static struct dentry *min_free_entries_dent __read_mostly; |
103 | static struct dentry *filter_dent __read_mostly; |
104 | |
105 | /* per-driver filter related state */ |
106 | |
107 | #define NAME_MAX_LEN 64 |
108 | |
109 | static char current_driver_name[NAME_MAX_LEN] __read_mostly; |
110 | static struct device_driver *current_driver __read_mostly; |
111 | |
112 | static DEFINE_RWLOCK(driver_name_lock); |
113 | |
114 | static const char *type2name[4] = { "single", "page", |
115 | "scather-gather", "coherent" }; |
116 | |
117 | static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE", |
118 | "DMA_FROM_DEVICE", "DMA_NONE" }; |
119 | |
120 | /* little merge helper - remove it after the merge window */ |
121 | #ifndef BUS_NOTIFY_UNBOUND_DRIVER |
122 | #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005 |
123 | #endif |
124 | |
125 | /* |
126 | * The access to some variables in this macro is racy. We can't use atomic_t |
127 | * here because all these variables are exported to debugfs. Some of them even |
128 | * writeable. This is also the reason why a lock won't help much. But anyway, |
129 | * the races are no big deal. Here is why: |
130 | * |
131 | * error_count: the addition is racy, but the worst thing that can happen is |
132 | * that we don't count some errors |
133 | * show_num_errors: the subtraction is racy. Also no big deal because in |
134 | * worst case this will result in one warning more in the |
135 | * system log than the user configured. This variable is |
136 | * writeable via debugfs. |
137 | */ |
138 | static inline void dump_entry_trace(struct dma_debug_entry *entry) |
139 | { |
140 | #ifdef CONFIG_STACKTRACE |
141 | if (entry) { |
142 | pr_warning("Mapped at:\n"); |
143 | print_stack_trace(&entry->stacktrace, 0); |
144 | } |
145 | #endif |
146 | } |
147 | |
148 | static bool driver_filter(struct device *dev) |
149 | { |
150 | struct device_driver *drv; |
151 | unsigned long flags; |
152 | bool ret; |
153 | |
154 | /* driver filter off */ |
155 | if (likely(!current_driver_name[0])) |
156 | return true; |
157 | |
158 | /* driver filter on and initialized */ |
159 | if (current_driver && dev && dev->driver == current_driver) |
160 | return true; |
161 | |
162 | /* driver filter on, but we can't filter on a NULL device... */ |
163 | if (!dev) |
164 | return false; |
165 | |
166 | if (current_driver || !current_driver_name[0]) |
167 | return false; |
168 | |
169 | /* driver filter on but not yet initialized */ |
170 | drv = get_driver(dev->driver); |
171 | if (!drv) |
172 | return false; |
173 | |
174 | /* lock to protect against change of current_driver_name */ |
175 | read_lock_irqsave(&driver_name_lock, flags); |
176 | |
177 | ret = false; |
178 | if (drv->name && |
179 | strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) { |
180 | current_driver = drv; |
181 | ret = true; |
182 | } |
183 | |
184 | read_unlock_irqrestore(&driver_name_lock, flags); |
185 | put_driver(drv); |
186 | |
187 | return ret; |
188 | } |
189 | |
190 | #define err_printk(dev, entry, format, arg...) do { \ |
191 | error_count += 1; \ |
192 | if (driver_filter(dev) && \ |
193 | (show_all_errors || show_num_errors > 0)) { \ |
194 | WARN(1, "%s %s: " format, \ |
195 | dev ? dev_driver_string(dev) : "NULL", \ |
196 | dev ? dev_name(dev) : "NULL", ## arg); \ |
197 | dump_entry_trace(entry); \ |
198 | } \ |
199 | if (!show_all_errors && show_num_errors > 0) \ |
200 | show_num_errors -= 1; \ |
201 | } while (0); |
202 | |
203 | /* |
204 | * Hash related functions |
205 | * |
206 | * Every DMA-API request is saved into a struct dma_debug_entry. To |
207 | * have quick access to these structs they are stored into a hash. |
208 | */ |
209 | static int hash_fn(struct dma_debug_entry *entry) |
210 | { |
211 | /* |
212 | * Hash function is based on the dma address. |
213 | * We use bits 20-27 here as the index into the hash |
214 | */ |
215 | return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; |
216 | } |
217 | |
218 | /* |
219 | * Request exclusive access to a hash bucket for a given dma_debug_entry. |
220 | */ |
221 | static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, |
222 | unsigned long *flags) |
223 | { |
224 | int idx = hash_fn(entry); |
225 | unsigned long __flags; |
226 | |
227 | spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags); |
228 | *flags = __flags; |
229 | return &dma_entry_hash[idx]; |
230 | } |
231 | |
232 | /* |
233 | * Give up exclusive access to the hash bucket |
234 | */ |
235 | static void put_hash_bucket(struct hash_bucket *bucket, |
236 | unsigned long *flags) |
237 | { |
238 | unsigned long __flags = *flags; |
239 | |
240 | spin_unlock_irqrestore(&bucket->lock, __flags); |
241 | } |
242 | |
243 | /* |
244 | * Search a given entry in the hash bucket list |
245 | */ |
246 | static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket, |
247 | struct dma_debug_entry *ref) |
248 | { |
249 | struct dma_debug_entry *entry, *ret = NULL; |
250 | int matches = 0, match_lvl, last_lvl = 0; |
251 | |
252 | list_for_each_entry(entry, &bucket->list, list) { |
253 | if ((entry->dev_addr != ref->dev_addr) || |
254 | (entry->dev != ref->dev)) |
255 | continue; |
256 | |
257 | /* |
258 | * Some drivers map the same physical address multiple |
259 | * times. Without a hardware IOMMU this results in the |
260 | * same device addresses being put into the dma-debug |
261 | * hash multiple times too. This can result in false |
262 | * positives being reported. Therefore we implement a |
263 | * best-fit algorithm here which returns the entry from |
264 | * the hash which fits best to the reference value |
265 | * instead of the first-fit. |
266 | */ |
267 | matches += 1; |
268 | match_lvl = 0; |
269 | entry->size == ref->size ? ++match_lvl : 0; |
270 | entry->type == ref->type ? ++match_lvl : 0; |
271 | entry->direction == ref->direction ? ++match_lvl : 0; |
272 | entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; |
273 | |
274 | if (match_lvl == 4) { |
275 | /* perfect-fit - return the result */ |
276 | return entry; |
277 | } else if (match_lvl > last_lvl) { |
278 | /* |
279 | * We found an entry that fits better then the |
280 | * previous one |
281 | */ |
282 | last_lvl = match_lvl; |
283 | ret = entry; |
284 | } |
285 | } |
286 | |
287 | /* |
288 | * If we have multiple matches but no perfect-fit, just return |
289 | * NULL. |
290 | */ |
291 | ret = (matches == 1) ? ret : NULL; |
292 | |
293 | return ret; |
294 | } |
295 | |
296 | /* |
297 | * Add an entry to a hash bucket |
298 | */ |
299 | static void hash_bucket_add(struct hash_bucket *bucket, |
300 | struct dma_debug_entry *entry) |
301 | { |
302 | list_add_tail(&entry->list, &bucket->list); |
303 | } |
304 | |
305 | /* |
306 | * Remove entry from a hash bucket list |
307 | */ |
308 | static void hash_bucket_del(struct dma_debug_entry *entry) |
309 | { |
310 | list_del(&entry->list); |
311 | } |
312 | |
313 | /* |
314 | * Dump mapping entries for debugging purposes |
315 | */ |
316 | void debug_dma_dump_mappings(struct device *dev) |
317 | { |
318 | int idx; |
319 | |
320 | for (idx = 0; idx < HASH_SIZE; idx++) { |
321 | struct hash_bucket *bucket = &dma_entry_hash[idx]; |
322 | struct dma_debug_entry *entry; |
323 | unsigned long flags; |
324 | |
325 | spin_lock_irqsave(&bucket->lock, flags); |
326 | |
327 | list_for_each_entry(entry, &bucket->list, list) { |
328 | if (!dev || dev == entry->dev) { |
329 | dev_info(entry->dev, |
330 | "%s idx %d P=%Lx D=%Lx L=%Lx %s\n", |
331 | type2name[entry->type], idx, |
332 | (unsigned long long)entry->paddr, |
333 | entry->dev_addr, entry->size, |
334 | dir2name[entry->direction]); |
335 | } |
336 | } |
337 | |
338 | spin_unlock_irqrestore(&bucket->lock, flags); |
339 | } |
340 | } |
341 | EXPORT_SYMBOL(debug_dma_dump_mappings); |
342 | |
343 | /* |
344 | * Wrapper function for adding an entry to the hash. |
345 | * This function takes care of locking itself. |
346 | */ |
347 | static void add_dma_entry(struct dma_debug_entry *entry) |
348 | { |
349 | struct hash_bucket *bucket; |
350 | unsigned long flags; |
351 | |
352 | bucket = get_hash_bucket(entry, &flags); |
353 | hash_bucket_add(bucket, entry); |
354 | put_hash_bucket(bucket, &flags); |
355 | } |
356 | |
357 | static struct dma_debug_entry *__dma_entry_alloc(void) |
358 | { |
359 | struct dma_debug_entry *entry; |
360 | |
361 | entry = list_entry(free_entries.next, struct dma_debug_entry, list); |
362 | list_del(&entry->list); |
363 | memset(entry, 0, sizeof(*entry)); |
364 | |
365 | num_free_entries -= 1; |
366 | if (num_free_entries < min_free_entries) |
367 | min_free_entries = num_free_entries; |
368 | |
369 | return entry; |
370 | } |
371 | |
372 | /* struct dma_entry allocator |
373 | * |
374 | * The next two functions implement the allocator for |
375 | * struct dma_debug_entries. |
376 | */ |
377 | static struct dma_debug_entry *dma_entry_alloc(void) |
378 | { |
379 | struct dma_debug_entry *entry = NULL; |
380 | unsigned long flags; |
381 | |
382 | spin_lock_irqsave(&free_entries_lock, flags); |
383 | |
384 | if (list_empty(&free_entries)) { |
385 | pr_err("DMA-API: debugging out of memory - disabling\n"); |
386 | global_disable = true; |
387 | goto out; |
388 | } |
389 | |
390 | entry = __dma_entry_alloc(); |
391 | |
392 | #ifdef CONFIG_STACKTRACE |
393 | entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES; |
394 | entry->stacktrace.entries = entry->st_entries; |
395 | entry->stacktrace.skip = 2; |
396 | save_stack_trace(&entry->stacktrace); |
397 | #endif |
398 | |
399 | out: |
400 | spin_unlock_irqrestore(&free_entries_lock, flags); |
401 | |
402 | return entry; |
403 | } |
404 | |
405 | static void dma_entry_free(struct dma_debug_entry *entry) |
406 | { |
407 | unsigned long flags; |
408 | |
409 | /* |
410 | * add to beginning of the list - this way the entries are |
411 | * more likely cache hot when they are reallocated. |
412 | */ |
413 | spin_lock_irqsave(&free_entries_lock, flags); |
414 | list_add(&entry->list, &free_entries); |
415 | num_free_entries += 1; |
416 | spin_unlock_irqrestore(&free_entries_lock, flags); |
417 | } |
418 | |
419 | int dma_debug_resize_entries(u32 num_entries) |
420 | { |
421 | int i, delta, ret = 0; |
422 | unsigned long flags; |
423 | struct dma_debug_entry *entry; |
424 | LIST_HEAD(tmp); |
425 | |
426 | spin_lock_irqsave(&free_entries_lock, flags); |
427 | |
428 | if (nr_total_entries < num_entries) { |
429 | delta = num_entries - nr_total_entries; |
430 | |
431 | spin_unlock_irqrestore(&free_entries_lock, flags); |
432 | |
433 | for (i = 0; i < delta; i++) { |
434 | entry = kzalloc(sizeof(*entry), GFP_KERNEL); |
435 | if (!entry) |
436 | break; |
437 | |
438 | list_add_tail(&entry->list, &tmp); |
439 | } |
440 | |
441 | spin_lock_irqsave(&free_entries_lock, flags); |
442 | |
443 | list_splice(&tmp, &free_entries); |
444 | nr_total_entries += i; |
445 | num_free_entries += i; |
446 | } else { |
447 | delta = nr_total_entries - num_entries; |
448 | |
449 | for (i = 0; i < delta && !list_empty(&free_entries); i++) { |
450 | entry = __dma_entry_alloc(); |
451 | kfree(entry); |
452 | } |
453 | |
454 | nr_total_entries -= i; |
455 | } |
456 | |
457 | if (nr_total_entries != num_entries) |
458 | ret = 1; |
459 | |
460 | spin_unlock_irqrestore(&free_entries_lock, flags); |
461 | |
462 | return ret; |
463 | } |
464 | EXPORT_SYMBOL(dma_debug_resize_entries); |
465 | |
466 | /* |
467 | * DMA-API debugging init code |
468 | * |
469 | * The init code does two things: |
470 | * 1. Initialize core data structures |
471 | * 2. Preallocate a given number of dma_debug_entry structs |
472 | */ |
473 | |
474 | static int prealloc_memory(u32 num_entries) |
475 | { |
476 | struct dma_debug_entry *entry, *next_entry; |
477 | int i; |
478 | |
479 | for (i = 0; i < num_entries; ++i) { |
480 | entry = kzalloc(sizeof(*entry), GFP_KERNEL); |
481 | if (!entry) |
482 | goto out_err; |
483 | |
484 | list_add_tail(&entry->list, &free_entries); |
485 | } |
486 | |
487 | num_free_entries = num_entries; |
488 | min_free_entries = num_entries; |
489 | |
490 | pr_info("DMA-API: preallocated %d debug entries\n", num_entries); |
491 | |
492 | return 0; |
493 | |
494 | out_err: |
495 | |
496 | list_for_each_entry_safe(entry, next_entry, &free_entries, list) { |
497 | list_del(&entry->list); |
498 | kfree(entry); |
499 | } |
500 | |
501 | return -ENOMEM; |
502 | } |
503 | |
504 | static ssize_t filter_read(struct file *file, char __user *user_buf, |
505 | size_t count, loff_t *ppos) |
506 | { |
507 | char buf[NAME_MAX_LEN + 1]; |
508 | unsigned long flags; |
509 | int len; |
510 | |
511 | if (!current_driver_name[0]) |
512 | return 0; |
513 | |
514 | /* |
515 | * We can't copy to userspace directly because current_driver_name can |
516 | * only be read under the driver_name_lock with irqs disabled. So |
517 | * create a temporary copy first. |
518 | */ |
519 | read_lock_irqsave(&driver_name_lock, flags); |
520 | len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name); |
521 | read_unlock_irqrestore(&driver_name_lock, flags); |
522 | |
523 | return simple_read_from_buffer(user_buf, count, ppos, buf, len); |
524 | } |
525 | |
526 | static ssize_t filter_write(struct file *file, const char __user *userbuf, |
527 | size_t count, loff_t *ppos) |
528 | { |
529 | char buf[NAME_MAX_LEN]; |
530 | unsigned long flags; |
531 | size_t len; |
532 | int i; |
533 | |
534 | /* |
535 | * We can't copy from userspace directly. Access to |
536 | * current_driver_name is protected with a write_lock with irqs |
537 | * disabled. Since copy_from_user can fault and may sleep we |
538 | * need to copy to temporary buffer first |
539 | */ |
540 | len = min(count, (size_t)(NAME_MAX_LEN - 1)); |
541 | if (copy_from_user(buf, userbuf, len)) |
542 | return -EFAULT; |
543 | |
544 | buf[len] = 0; |
545 | |
546 | write_lock_irqsave(&driver_name_lock, flags); |
547 | |
548 | /* |
549 | * Now handle the string we got from userspace very carefully. |
550 | * The rules are: |
551 | * - only use the first token we got |
552 | * - token delimiter is everything looking like a space |
553 | * character (' ', '\n', '\t' ...) |
554 | * |
555 | */ |
556 | if (!isalnum(buf[0])) { |
557 | /* |
558 | * If the first character userspace gave us is not |
559 | * alphanumerical then assume the filter should be |
560 | * switched off. |
561 | */ |
562 | if (current_driver_name[0]) |
563 | pr_info("DMA-API: switching off dma-debug driver filter\n"); |
564 | current_driver_name[0] = 0; |
565 | current_driver = NULL; |
566 | goto out_unlock; |
567 | } |
568 | |
569 | /* |
570 | * Now parse out the first token and use it as the name for the |
571 | * driver to filter for. |
572 | */ |
573 | for (i = 0; i < NAME_MAX_LEN - 1; ++i) { |
574 | current_driver_name[i] = buf[i]; |
575 | if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0) |
576 | break; |
577 | } |
578 | current_driver_name[i] = 0; |
579 | current_driver = NULL; |
580 | |
581 | pr_info("DMA-API: enable driver filter for driver [%s]\n", |
582 | current_driver_name); |
583 | |
584 | out_unlock: |
585 | write_unlock_irqrestore(&driver_name_lock, flags); |
586 | |
587 | return count; |
588 | } |
589 | |
590 | static const struct file_operations filter_fops = { |
591 | .read = filter_read, |
592 | .write = filter_write, |
593 | }; |
594 | |
595 | static int dma_debug_fs_init(void) |
596 | { |
597 | dma_debug_dent = debugfs_create_dir("dma-api", NULL); |
598 | if (!dma_debug_dent) { |
599 | pr_err("DMA-API: can not create debugfs directory\n"); |
600 | return -ENOMEM; |
601 | } |
602 | |
603 | global_disable_dent = debugfs_create_bool("disabled", 0444, |
604 | dma_debug_dent, |
605 | (u32 *)&global_disable); |
606 | if (!global_disable_dent) |
607 | goto out_err; |
608 | |
609 | error_count_dent = debugfs_create_u32("error_count", 0444, |
610 | dma_debug_dent, &error_count); |
611 | if (!error_count_dent) |
612 | goto out_err; |
613 | |
614 | show_all_errors_dent = debugfs_create_u32("all_errors", 0644, |
615 | dma_debug_dent, |
616 | &show_all_errors); |
617 | if (!show_all_errors_dent) |
618 | goto out_err; |
619 | |
620 | show_num_errors_dent = debugfs_create_u32("num_errors", 0644, |
621 | dma_debug_dent, |
622 | &show_num_errors); |
623 | if (!show_num_errors_dent) |
624 | goto out_err; |
625 | |
626 | num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444, |
627 | dma_debug_dent, |
628 | &num_free_entries); |
629 | if (!num_free_entries_dent) |
630 | goto out_err; |
631 | |
632 | min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444, |
633 | dma_debug_dent, |
634 | &min_free_entries); |
635 | if (!min_free_entries_dent) |
636 | goto out_err; |
637 | |
638 | filter_dent = debugfs_create_file("driver_filter", 0644, |
639 | dma_debug_dent, NULL, &filter_fops); |
640 | if (!filter_dent) |
641 | goto out_err; |
642 | |
643 | return 0; |
644 | |
645 | out_err: |
646 | debugfs_remove_recursive(dma_debug_dent); |
647 | |
648 | return -ENOMEM; |
649 | } |
650 | |
651 | static int device_dma_allocations(struct device *dev) |
652 | { |
653 | struct dma_debug_entry *entry; |
654 | unsigned long flags; |
655 | int count = 0, i; |
656 | |
657 | local_irq_save(flags); |
658 | |
659 | for (i = 0; i < HASH_SIZE; ++i) { |
660 | spin_lock(&dma_entry_hash[i].lock); |
661 | list_for_each_entry(entry, &dma_entry_hash[i].list, list) { |
662 | if (entry->dev == dev) |
663 | count += 1; |
664 | } |
665 | spin_unlock(&dma_entry_hash[i].lock); |
666 | } |
667 | |
668 | local_irq_restore(flags); |
669 | |
670 | return count; |
671 | } |
672 | |
673 | static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data) |
674 | { |
675 | struct device *dev = data; |
676 | int count; |
677 | |
678 | if (global_disable) |
679 | return 0; |
680 | |
681 | switch (action) { |
682 | case BUS_NOTIFY_UNBOUND_DRIVER: |
683 | count = device_dma_allocations(dev); |
684 | if (count == 0) |
685 | break; |
686 | err_printk(dev, NULL, "DMA-API: device driver has pending " |
687 | "DMA allocations while released from device " |
688 | "[count=%d]\n", count); |
689 | break; |
690 | default: |
691 | break; |
692 | } |
693 | |
694 | return 0; |
695 | } |
696 | |
697 | void dma_debug_add_bus(struct bus_type *bus) |
698 | { |
699 | struct notifier_block *nb; |
700 | |
701 | if (global_disable) |
702 | return; |
703 | |
704 | nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); |
705 | if (nb == NULL) { |
706 | pr_err("dma_debug_add_bus: out of memory\n"); |
707 | return; |
708 | } |
709 | |
710 | nb->notifier_call = dma_debug_device_change; |
711 | |
712 | bus_register_notifier(bus, nb); |
713 | } |
714 | |
715 | /* |
716 | * Let the architectures decide how many entries should be preallocated. |
717 | */ |
718 | void dma_debug_init(u32 num_entries) |
719 | { |
720 | int i; |
721 | |
722 | if (global_disable) |
723 | return; |
724 | |
725 | for (i = 0; i < HASH_SIZE; ++i) { |
726 | INIT_LIST_HEAD(&dma_entry_hash[i].list); |
727 | spin_lock_init(&dma_entry_hash[i].lock); |
728 | } |
729 | |
730 | if (dma_debug_fs_init() != 0) { |
731 | pr_err("DMA-API: error creating debugfs entries - disabling\n"); |
732 | global_disable = true; |
733 | |
734 | return; |
735 | } |
736 | |
737 | if (req_entries) |
738 | num_entries = req_entries; |
739 | |
740 | if (prealloc_memory(num_entries) != 0) { |
741 | pr_err("DMA-API: debugging out of memory error - disabled\n"); |
742 | global_disable = true; |
743 | |
744 | return; |
745 | } |
746 | |
747 | nr_total_entries = num_free_entries; |
748 | |
749 | pr_info("DMA-API: debugging enabled by kernel config\n"); |
750 | } |
751 | |
752 | static __init int dma_debug_cmdline(char *str) |
753 | { |
754 | if (!str) |
755 | return -EINVAL; |
756 | |
757 | if (strncmp(str, "off", 3) == 0) { |
758 | pr_info("DMA-API: debugging disabled on kernel command line\n"); |
759 | global_disable = true; |
760 | } |
761 | |
762 | return 0; |
763 | } |
764 | |
765 | static __init int dma_debug_entries_cmdline(char *str) |
766 | { |
767 | int res; |
768 | |
769 | if (!str) |
770 | return -EINVAL; |
771 | |
772 | res = get_option(&str, &req_entries); |
773 | |
774 | if (!res) |
775 | req_entries = 0; |
776 | |
777 | return 0; |
778 | } |
779 | |
780 | __setup("dma_debug=", dma_debug_cmdline); |
781 | __setup("dma_debug_entries=", dma_debug_entries_cmdline); |
782 | |
783 | static void check_unmap(struct dma_debug_entry *ref) |
784 | { |
785 | struct dma_debug_entry *entry; |
786 | struct hash_bucket *bucket; |
787 | unsigned long flags; |
788 | |
789 | if (dma_mapping_error(ref->dev, ref->dev_addr)) { |
790 | err_printk(ref->dev, NULL, "DMA-API: device driver tries " |
791 | "to free an invalid DMA memory address\n"); |
792 | return; |
793 | } |
794 | |
795 | bucket = get_hash_bucket(ref, &flags); |
796 | entry = hash_bucket_find(bucket, ref); |
797 | |
798 | if (!entry) { |
799 | err_printk(ref->dev, NULL, "DMA-API: device driver tries " |
800 | "to free DMA memory it has not allocated " |
801 | "[device address=0x%016llx] [size=%llu bytes]\n", |
802 | ref->dev_addr, ref->size); |
803 | goto out; |
804 | } |
805 | |
806 | if (ref->size != entry->size) { |
807 | err_printk(ref->dev, entry, "DMA-API: device driver frees " |
808 | "DMA memory with different size " |
809 | "[device address=0x%016llx] [map size=%llu bytes] " |
810 | "[unmap size=%llu bytes]\n", |
811 | ref->dev_addr, entry->size, ref->size); |
812 | } |
813 | |
814 | if (ref->type != entry->type) { |
815 | err_printk(ref->dev, entry, "DMA-API: device driver frees " |
816 | "DMA memory with wrong function " |
817 | "[device address=0x%016llx] [size=%llu bytes] " |
818 | "[mapped as %s] [unmapped as %s]\n", |
819 | ref->dev_addr, ref->size, |
820 | type2name[entry->type], type2name[ref->type]); |
821 | } else if ((entry->type == dma_debug_coherent) && |
822 | (ref->paddr != entry->paddr)) { |
823 | err_printk(ref->dev, entry, "DMA-API: device driver frees " |
824 | "DMA memory with different CPU address " |
825 | "[device address=0x%016llx] [size=%llu bytes] " |
826 | "[cpu alloc address=0x%016llx] " |
827 | "[cpu free address=0x%016llx]", |
828 | ref->dev_addr, ref->size, |
829 | (unsigned long long)entry->paddr, |
830 | (unsigned long long)ref->paddr); |
831 | } |
832 | |
833 | if (ref->sg_call_ents && ref->type == dma_debug_sg && |
834 | ref->sg_call_ents != entry->sg_call_ents) { |
835 | err_printk(ref->dev, entry, "DMA-API: device driver frees " |
836 | "DMA sg list with different entry count " |
837 | "[map count=%d] [unmap count=%d]\n", |
838 | entry->sg_call_ents, ref->sg_call_ents); |
839 | } |
840 | |
841 | /* |
842 | * This may be no bug in reality - but most implementations of the |
843 | * DMA API don't handle this properly, so check for it here |
844 | */ |
845 | if (ref->direction != entry->direction) { |
846 | err_printk(ref->dev, entry, "DMA-API: device driver frees " |
847 | "DMA memory with different direction " |
848 | "[device address=0x%016llx] [size=%llu bytes] " |
849 | "[mapped with %s] [unmapped with %s]\n", |
850 | ref->dev_addr, ref->size, |
851 | dir2name[entry->direction], |
852 | dir2name[ref->direction]); |
853 | } |
854 | |
855 | hash_bucket_del(entry); |
856 | dma_entry_free(entry); |
857 | |
858 | out: |
859 | put_hash_bucket(bucket, &flags); |
860 | } |
861 | |
862 | static void check_for_stack(struct device *dev, void *addr) |
863 | { |
864 | if (object_is_on_stack(addr)) |
865 | err_printk(dev, NULL, "DMA-API: device driver maps memory from" |
866 | "stack [addr=%p]\n", addr); |
867 | } |
868 | |
869 | static inline bool overlap(void *addr, unsigned long len, void *start, void *end) |
870 | { |
871 | unsigned long a1 = (unsigned long)addr; |
872 | unsigned long b1 = a1 + len; |
873 | unsigned long a2 = (unsigned long)start; |
874 | unsigned long b2 = (unsigned long)end; |
875 | |
876 | return !(b1 <= a2 || a1 >= b2); |
877 | } |
878 | |
879 | static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len) |
880 | { |
881 | if (overlap(addr, len, _text, _etext) || |
882 | overlap(addr, len, __start_rodata, __end_rodata)) |
883 | err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len); |
884 | } |
885 | |
886 | static void check_sync(struct device *dev, |
887 | struct dma_debug_entry *ref, |
888 | bool to_cpu) |
889 | { |
890 | struct dma_debug_entry *entry; |
891 | struct hash_bucket *bucket; |
892 | unsigned long flags; |
893 | |
894 | bucket = get_hash_bucket(ref, &flags); |
895 | |
896 | entry = hash_bucket_find(bucket, ref); |
897 | |
898 | if (!entry) { |
899 | err_printk(dev, NULL, "DMA-API: device driver tries " |
900 | "to sync DMA memory it has not allocated " |
901 | "[device address=0x%016llx] [size=%llu bytes]\n", |
902 | (unsigned long long)ref->dev_addr, ref->size); |
903 | goto out; |
904 | } |
905 | |
906 | if (ref->size > entry->size) { |
907 | err_printk(dev, entry, "DMA-API: device driver syncs" |
908 | " DMA memory outside allocated range " |
909 | "[device address=0x%016llx] " |
910 | "[allocation size=%llu bytes] " |
911 | "[sync offset+size=%llu]\n", |
912 | entry->dev_addr, entry->size, |
913 | ref->size); |
914 | } |
915 | |
916 | if (entry->direction == DMA_BIDIRECTIONAL) |
917 | goto out; |
918 | |
919 | if (ref->direction != entry->direction) { |
920 | err_printk(dev, entry, "DMA-API: device driver syncs " |
921 | "DMA memory with different direction " |
922 | "[device address=0x%016llx] [size=%llu bytes] " |
923 | "[mapped with %s] [synced with %s]\n", |
924 | (unsigned long long)ref->dev_addr, entry->size, |
925 | dir2name[entry->direction], |
926 | dir2name[ref->direction]); |
927 | } |
928 | |
929 | if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && |
930 | !(ref->direction == DMA_TO_DEVICE)) |
931 | err_printk(dev, entry, "DMA-API: device driver syncs " |
932 | "device read-only DMA memory for cpu " |
933 | "[device address=0x%016llx] [size=%llu bytes] " |
934 | "[mapped with %s] [synced with %s]\n", |
935 | (unsigned long long)ref->dev_addr, entry->size, |
936 | dir2name[entry->direction], |
937 | dir2name[ref->direction]); |
938 | |
939 | if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && |
940 | !(ref->direction == DMA_FROM_DEVICE)) |
941 | err_printk(dev, entry, "DMA-API: device driver syncs " |
942 | "device write-only DMA memory to device " |
943 | "[device address=0x%016llx] [size=%llu bytes] " |
944 | "[mapped with %s] [synced with %s]\n", |
945 | (unsigned long long)ref->dev_addr, entry->size, |
946 | dir2name[entry->direction], |
947 | dir2name[ref->direction]); |
948 | |
949 | out: |
950 | put_hash_bucket(bucket, &flags); |
951 | } |
952 | |
953 | void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, |
954 | size_t size, int direction, dma_addr_t dma_addr, |
955 | bool map_single) |
956 | { |
957 | struct dma_debug_entry *entry; |
958 | |
959 | if (unlikely(global_disable)) |
960 | return; |
961 | |
962 | if (unlikely(dma_mapping_error(dev, dma_addr))) |
963 | return; |
964 | |
965 | entry = dma_entry_alloc(); |
966 | if (!entry) |
967 | return; |
968 | |
969 | entry->dev = dev; |
970 | entry->type = dma_debug_page; |
971 | entry->paddr = page_to_phys(page) + offset; |
972 | entry->dev_addr = dma_addr; |
973 | entry->size = size; |
974 | entry->direction = direction; |
975 | |
976 | if (map_single) |
977 | entry->type = dma_debug_single; |
978 | |
979 | if (!PageHighMem(page)) { |
980 | void *addr = page_address(page) + offset; |
981 | |
982 | check_for_stack(dev, addr); |
983 | check_for_illegal_area(dev, addr, size); |
984 | } |
985 | |
986 | add_dma_entry(entry); |
987 | } |
988 | EXPORT_SYMBOL(debug_dma_map_page); |
989 | |
990 | void debug_dma_unmap_page(struct device *dev, dma_addr_t addr, |
991 | size_t size, int direction, bool map_single) |
992 | { |
993 | struct dma_debug_entry ref = { |
994 | .type = dma_debug_page, |
995 | .dev = dev, |
996 | .dev_addr = addr, |
997 | .size = size, |
998 | .direction = direction, |
999 | }; |
1000 | |
1001 | if (unlikely(global_disable)) |
1002 | return; |
1003 | |
1004 | if (map_single) |
1005 | ref.type = dma_debug_single; |
1006 | |
1007 | check_unmap(&ref); |
1008 | } |
1009 | EXPORT_SYMBOL(debug_dma_unmap_page); |
1010 | |
1011 | void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, |
1012 | int nents, int mapped_ents, int direction) |
1013 | { |
1014 | struct dma_debug_entry *entry; |
1015 | struct scatterlist *s; |
1016 | int i; |
1017 | |
1018 | if (unlikely(global_disable)) |
1019 | return; |
1020 | |
1021 | for_each_sg(sg, s, mapped_ents, i) { |
1022 | entry = dma_entry_alloc(); |
1023 | if (!entry) |
1024 | return; |
1025 | |
1026 | entry->type = dma_debug_sg; |
1027 | entry->dev = dev; |
1028 | entry->paddr = sg_phys(s); |
1029 | entry->size = sg_dma_len(s); |
1030 | entry->dev_addr = sg_dma_address(s); |
1031 | entry->direction = direction; |
1032 | entry->sg_call_ents = nents; |
1033 | entry->sg_mapped_ents = mapped_ents; |
1034 | |
1035 | if (!PageHighMem(sg_page(s))) { |
1036 | check_for_stack(dev, sg_virt(s)); |
1037 | check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s)); |
1038 | } |
1039 | |
1040 | add_dma_entry(entry); |
1041 | } |
1042 | } |
1043 | EXPORT_SYMBOL(debug_dma_map_sg); |
1044 | |
1045 | static int get_nr_mapped_entries(struct device *dev, |
1046 | struct dma_debug_entry *ref) |
1047 | { |
1048 | struct dma_debug_entry *entry; |
1049 | struct hash_bucket *bucket; |
1050 | unsigned long flags; |
1051 | int mapped_ents; |
1052 | |
1053 | bucket = get_hash_bucket(ref, &flags); |
1054 | entry = hash_bucket_find(bucket, ref); |
1055 | mapped_ents = 0; |
1056 | |
1057 | if (entry) |
1058 | mapped_ents = entry->sg_mapped_ents; |
1059 | put_hash_bucket(bucket, &flags); |
1060 | |
1061 | return mapped_ents; |
1062 | } |
1063 | |
1064 | void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, |
1065 | int nelems, int dir) |
1066 | { |
1067 | struct scatterlist *s; |
1068 | int mapped_ents = 0, i; |
1069 | |
1070 | if (unlikely(global_disable)) |
1071 | return; |
1072 | |
1073 | for_each_sg(sglist, s, nelems, i) { |
1074 | |
1075 | struct dma_debug_entry ref = { |
1076 | .type = dma_debug_sg, |
1077 | .dev = dev, |
1078 | .paddr = sg_phys(s), |
1079 | .dev_addr = sg_dma_address(s), |
1080 | .size = sg_dma_len(s), |
1081 | .direction = dir, |
1082 | .sg_call_ents = nelems, |
1083 | }; |
1084 | |
1085 | if (mapped_ents && i >= mapped_ents) |
1086 | break; |
1087 | |
1088 | if (!i) |
1089 | mapped_ents = get_nr_mapped_entries(dev, &ref); |
1090 | |
1091 | check_unmap(&ref); |
1092 | } |
1093 | } |
1094 | EXPORT_SYMBOL(debug_dma_unmap_sg); |
1095 | |
1096 | void debug_dma_alloc_coherent(struct device *dev, size_t size, |
1097 | dma_addr_t dma_addr, void *virt) |
1098 | { |
1099 | struct dma_debug_entry *entry; |
1100 | |
1101 | if (unlikely(global_disable)) |
1102 | return; |
1103 | |
1104 | if (unlikely(virt == NULL)) |
1105 | return; |
1106 | |
1107 | entry = dma_entry_alloc(); |
1108 | if (!entry) |
1109 | return; |
1110 | |
1111 | entry->type = dma_debug_coherent; |
1112 | entry->dev = dev; |
1113 | entry->paddr = virt_to_phys(virt); |
1114 | entry->size = size; |
1115 | entry->dev_addr = dma_addr; |
1116 | entry->direction = DMA_BIDIRECTIONAL; |
1117 | |
1118 | add_dma_entry(entry); |
1119 | } |
1120 | EXPORT_SYMBOL(debug_dma_alloc_coherent); |
1121 | |
1122 | void debug_dma_free_coherent(struct device *dev, size_t size, |
1123 | void *virt, dma_addr_t addr) |
1124 | { |
1125 | struct dma_debug_entry ref = { |
1126 | .type = dma_debug_coherent, |
1127 | .dev = dev, |
1128 | .paddr = virt_to_phys(virt), |
1129 | .dev_addr = addr, |
1130 | .size = size, |
1131 | .direction = DMA_BIDIRECTIONAL, |
1132 | }; |
1133 | |
1134 | if (unlikely(global_disable)) |
1135 | return; |
1136 | |
1137 | check_unmap(&ref); |
1138 | } |
1139 | EXPORT_SYMBOL(debug_dma_free_coherent); |
1140 | |
1141 | void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, |
1142 | size_t size, int direction) |
1143 | { |
1144 | struct dma_debug_entry ref; |
1145 | |
1146 | if (unlikely(global_disable)) |
1147 | return; |
1148 | |
1149 | ref.type = dma_debug_single; |
1150 | ref.dev = dev; |
1151 | ref.dev_addr = dma_handle; |
1152 | ref.size = size; |
1153 | ref.direction = direction; |
1154 | ref.sg_call_ents = 0; |
1155 | |
1156 | check_sync(dev, &ref, true); |
1157 | } |
1158 | EXPORT_SYMBOL(debug_dma_sync_single_for_cpu); |
1159 | |
1160 | void debug_dma_sync_single_for_device(struct device *dev, |
1161 | dma_addr_t dma_handle, size_t size, |
1162 | int direction) |
1163 | { |
1164 | struct dma_debug_entry ref; |
1165 | |
1166 | if (unlikely(global_disable)) |
1167 | return; |
1168 | |
1169 | ref.type = dma_debug_single; |
1170 | ref.dev = dev; |
1171 | ref.dev_addr = dma_handle; |
1172 | ref.size = size; |
1173 | ref.direction = direction; |
1174 | ref.sg_call_ents = 0; |
1175 | |
1176 | check_sync(dev, &ref, false); |
1177 | } |
1178 | EXPORT_SYMBOL(debug_dma_sync_single_for_device); |
1179 | |
1180 | void debug_dma_sync_single_range_for_cpu(struct device *dev, |
1181 | dma_addr_t dma_handle, |
1182 | unsigned long offset, size_t size, |
1183 | int direction) |
1184 | { |
1185 | struct dma_debug_entry ref; |
1186 | |
1187 | if (unlikely(global_disable)) |
1188 | return; |
1189 | |
1190 | ref.type = dma_debug_single; |
1191 | ref.dev = dev; |
1192 | ref.dev_addr = dma_handle; |
1193 | ref.size = offset + size; |
1194 | ref.direction = direction; |
1195 | ref.sg_call_ents = 0; |
1196 | |
1197 | check_sync(dev, &ref, true); |
1198 | } |
1199 | EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu); |
1200 | |
1201 | void debug_dma_sync_single_range_for_device(struct device *dev, |
1202 | dma_addr_t dma_handle, |
1203 | unsigned long offset, |
1204 | size_t size, int direction) |
1205 | { |
1206 | struct dma_debug_entry ref; |
1207 | |
1208 | if (unlikely(global_disable)) |
1209 | return; |
1210 | |
1211 | ref.type = dma_debug_single; |
1212 | ref.dev = dev; |
1213 | ref.dev_addr = dma_handle; |
1214 | ref.size = offset + size; |
1215 | ref.direction = direction; |
1216 | ref.sg_call_ents = 0; |
1217 | |
1218 | check_sync(dev, &ref, false); |
1219 | } |
1220 | EXPORT_SYMBOL(debug_dma_sync_single_range_for_device); |
1221 | |
1222 | void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, |
1223 | int nelems, int direction) |
1224 | { |
1225 | struct scatterlist *s; |
1226 | int mapped_ents = 0, i; |
1227 | |
1228 | if (unlikely(global_disable)) |
1229 | return; |
1230 | |
1231 | for_each_sg(sg, s, nelems, i) { |
1232 | |
1233 | struct dma_debug_entry ref = { |
1234 | .type = dma_debug_sg, |
1235 | .dev = dev, |
1236 | .paddr = sg_phys(s), |
1237 | .dev_addr = sg_dma_address(s), |
1238 | .size = sg_dma_len(s), |
1239 | .direction = direction, |
1240 | .sg_call_ents = nelems, |
1241 | }; |
1242 | |
1243 | if (!i) |
1244 | mapped_ents = get_nr_mapped_entries(dev, &ref); |
1245 | |
1246 | if (i >= mapped_ents) |
1247 | break; |
1248 | |
1249 | check_sync(dev, &ref, true); |
1250 | } |
1251 | } |
1252 | EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu); |
1253 | |
1254 | void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, |
1255 | int nelems, int direction) |
1256 | { |
1257 | struct scatterlist *s; |
1258 | int mapped_ents = 0, i; |
1259 | |
1260 | if (unlikely(global_disable)) |
1261 | return; |
1262 | |
1263 | for_each_sg(sg, s, nelems, i) { |
1264 | |
1265 | struct dma_debug_entry ref = { |
1266 | .type = dma_debug_sg, |
1267 | .dev = dev, |
1268 | .paddr = sg_phys(s), |
1269 | .dev_addr = sg_dma_address(s), |
1270 | .size = sg_dma_len(s), |
1271 | .direction = direction, |
1272 | .sg_call_ents = nelems, |
1273 | }; |
1274 | if (!i) |
1275 | mapped_ents = get_nr_mapped_entries(dev, &ref); |
1276 | |
1277 | if (i >= mapped_ents) |
1278 | break; |
1279 | |
1280 | check_sync(dev, &ref, false); |
1281 | } |
1282 | } |
1283 | EXPORT_SYMBOL(debug_dma_sync_sg_for_device); |
1284 | |
1285 | static int __init dma_debug_driver_setup(char *str) |
1286 | { |
1287 | int i; |
1288 | |
1289 | for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) { |
1290 | current_driver_name[i] = *str; |
1291 | if (*str == 0) |
1292 | break; |
1293 | } |
1294 | |
1295 | if (current_driver_name[0]) |
1296 | pr_info("DMA-API: enable driver filter for driver [%s]\n", |
1297 | current_driver_name); |
1298 | |
1299 | |
1300 | return 1; |
1301 | } |
1302 | __setup("dma_debug_driver=", dma_debug_driver_setup); |
1303 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9