Root/
1 | /* |
2 | * Copyright 2012 Tilera Corporation. All Rights Reserved. |
3 | * |
4 | * This program is free software; you can redistribute it and/or |
5 | * modify it under the terms of the GNU General Public License |
6 | * as published by the Free Software Foundation, version 2. |
7 | * |
8 | * This program is distributed in the hope that it will be useful, but |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or |
11 | * NON INFRINGEMENT. See the GNU General Public License for |
12 | * more details. |
13 | */ |
14 | |
15 | #include <linux/io.h> |
16 | #include <linux/atomic.h> |
17 | #include <linux/module.h> |
18 | #include <gxio/dma_queue.h> |
19 | |
20 | /* Wait for a memory read to complete. */ |
21 | #define wait_for_value(val) \ |
22 | __asm__ __volatile__("move %0, %0" :: "r"(val)) |
23 | |
24 | /* The index is in the low 16. */ |
25 | #define DMA_QUEUE_INDEX_MASK ((1 << 16) - 1) |
26 | |
27 | /* |
28 | * The hardware descriptor-ring type. |
29 | * This matches the types used by mpipe (MPIPE_EDMA_POST_REGION_VAL_t) |
30 | * and trio (TRIO_PUSH_DMA_REGION_VAL_t or TRIO_PULL_DMA_REGION_VAL_t). |
31 | * See those types for more documentation on the individual fields. |
32 | */ |
33 | typedef union { |
34 | struct { |
35 | #ifndef __BIG_ENDIAN__ |
36 | uint64_t ring_idx:16; |
37 | uint64_t count:16; |
38 | uint64_t gen:1; |
39 | uint64_t __reserved:31; |
40 | #else |
41 | uint64_t __reserved:31; |
42 | uint64_t gen:1; |
43 | uint64_t count:16; |
44 | uint64_t ring_idx:16; |
45 | #endif |
46 | }; |
47 | uint64_t word; |
48 | } __gxio_ring_t; |
49 | |
50 | void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue, |
51 | void *post_region_addr, unsigned int num_entries) |
52 | { |
53 | /* |
54 | * Limit 65536 entry rings to 65535 credits because we only have a |
55 | * 16 bit completion counter. |
56 | */ |
57 | int64_t credits = (num_entries < 65536) ? num_entries : 65535; |
58 | |
59 | memset(dma_queue, 0, sizeof(*dma_queue)); |
60 | |
61 | dma_queue->post_region_addr = post_region_addr; |
62 | dma_queue->hw_complete_count = 0; |
63 | dma_queue->credits_and_next_index = credits << DMA_QUEUE_CREDIT_SHIFT; |
64 | } |
65 | |
66 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_init); |
67 | |
68 | void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue) |
69 | { |
70 | __gxio_ring_t val; |
71 | uint64_t count; |
72 | uint64_t delta; |
73 | uint64_t new_count; |
74 | |
75 | /* |
76 | * Read the 64-bit completion count without touching the cache, so |
77 | * we later avoid having to evict any sharers of this cache line |
78 | * when we update it below. |
79 | */ |
80 | uint64_t orig_hw_complete_count = |
81 | cmpxchg(&dma_queue->hw_complete_count, |
82 | -1, -1); |
83 | |
84 | /* Make sure the load completes before we access the hardware. */ |
85 | wait_for_value(orig_hw_complete_count); |
86 | |
87 | /* Read the 16-bit count of how many packets it has completed. */ |
88 | val.word = __gxio_mmio_read(dma_queue->post_region_addr); |
89 | count = val.count; |
90 | |
91 | /* |
92 | * Calculate the number of completions since we last updated the |
93 | * 64-bit counter. It's safe to ignore the high bits because the |
94 | * maximum credit value is 65535. |
95 | */ |
96 | delta = (count - orig_hw_complete_count) & 0xffff; |
97 | if (delta == 0) |
98 | return; |
99 | |
100 | /* |
101 | * Try to write back the count, advanced by delta. If we race with |
102 | * another thread, this might fail, in which case we return |
103 | * immediately on the assumption that some credits are (or at least |
104 | * were) available. |
105 | */ |
106 | new_count = orig_hw_complete_count + delta; |
107 | if (cmpxchg(&dma_queue->hw_complete_count, |
108 | orig_hw_complete_count, |
109 | new_count) != orig_hw_complete_count) |
110 | return; |
111 | |
112 | /* |
113 | * We succeeded in advancing the completion count; add back the |
114 | * corresponding number of egress credits. |
115 | */ |
116 | __insn_fetchadd(&dma_queue->credits_and_next_index, |
117 | (delta << DMA_QUEUE_CREDIT_SHIFT)); |
118 | } |
119 | |
120 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_update_credits); |
121 | |
122 | /* |
123 | * A separate 'blocked' method for put() so that backtraces and |
124 | * profiles will clearly indicate that we're wasting time spinning on |
125 | * egress availability rather than actually posting commands. |
126 | */ |
127 | int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue, |
128 | int64_t modifier) |
129 | { |
130 | int backoff = 16; |
131 | int64_t old; |
132 | |
133 | do { |
134 | int i; |
135 | /* Back off to avoid spamming memory networks. */ |
136 | for (i = backoff; i > 0; i--) |
137 | __insn_mfspr(SPR_PASS); |
138 | |
139 | /* Check credits again. */ |
140 | __gxio_dma_queue_update_credits(dma_queue); |
141 | old = __insn_fetchaddgez(&dma_queue->credits_and_next_index, |
142 | modifier); |
143 | |
144 | /* Calculate bounded exponential backoff for next iteration. */ |
145 | if (backoff < 256) |
146 | backoff *= 2; |
147 | } while (old + modifier < 0); |
148 | |
149 | return old; |
150 | } |
151 | |
152 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_wait_for_credits); |
153 | |
154 | int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue, |
155 | unsigned int num, int wait) |
156 | { |
157 | return __gxio_dma_queue_reserve(dma_queue, num, wait != 0, true); |
158 | } |
159 | |
160 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_reserve_aux); |
161 | |
162 | int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue, |
163 | int64_t completion_slot, int update) |
164 | { |
165 | if (update) { |
166 | if (ACCESS_ONCE(dma_queue->hw_complete_count) > |
167 | completion_slot) |
168 | return 1; |
169 | |
170 | __gxio_dma_queue_update_credits(dma_queue); |
171 | } |
172 | |
173 | return ACCESS_ONCE(dma_queue->hw_complete_count) > completion_slot; |
174 | } |
175 | |
176 | EXPORT_SYMBOL_GPL(__gxio_dma_queue_is_complete); |
177 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9