Root/
Source at commit b05a5adf03613de371c77c3235f7d970d7cd0c71 created 13 years 1 month ago. By Lars-Peter Clausen, NAND: Optimize reading the eec data for the JZ4740 (evil hack) | |
---|---|
1 | /* |
2 | * Functions related to interrupt-poll handling in the block layer. This |
3 | * is similar to NAPI for network devices. |
4 | */ |
5 | #include <linux/kernel.h> |
6 | #include <linux/module.h> |
7 | #include <linux/init.h> |
8 | #include <linux/bio.h> |
9 | #include <linux/blkdev.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/cpu.h> |
12 | #include <linux/blk-iopoll.h> |
13 | #include <linux/delay.h> |
14 | |
15 | #include "blk.h" |
16 | |
17 | int blk_iopoll_enabled = 1; |
18 | EXPORT_SYMBOL(blk_iopoll_enabled); |
19 | |
20 | static unsigned int blk_iopoll_budget __read_mostly = 256; |
21 | |
22 | static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); |
23 | |
24 | /** |
25 | * blk_iopoll_sched - Schedule a run of the iopoll handler |
26 | * @iop: The parent iopoll structure |
27 | * |
28 | * Description: |
29 | * Add this blk_iopoll structure to the pending poll list and trigger the |
30 | * raise of the blk iopoll softirq. The driver must already have gotten a |
31 | * successful return from blk_iopoll_sched_prep() before calling this. |
32 | **/ |
33 | void blk_iopoll_sched(struct blk_iopoll *iop) |
34 | { |
35 | unsigned long flags; |
36 | |
37 | local_irq_save(flags); |
38 | list_add_tail(&iop->list, &__get_cpu_var(blk_cpu_iopoll)); |
39 | __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); |
40 | local_irq_restore(flags); |
41 | } |
42 | EXPORT_SYMBOL(blk_iopoll_sched); |
43 | |
44 | /** |
45 | * __blk_iopoll_complete - Mark this @iop as un-polled again |
46 | * @iop: The parent iopoll structure |
47 | * |
48 | * Description: |
49 | * See blk_iopoll_complete(). This function must be called with interrupts |
50 | * disabled. |
51 | **/ |
52 | void __blk_iopoll_complete(struct blk_iopoll *iop) |
53 | { |
54 | list_del(&iop->list); |
55 | smp_mb__before_clear_bit(); |
56 | clear_bit_unlock(IOPOLL_F_SCHED, &iop->state); |
57 | } |
58 | EXPORT_SYMBOL(__blk_iopoll_complete); |
59 | |
60 | /** |
61 | * blk_iopoll_complete - Mark this @iop as un-polled again |
62 | * @iop: The parent iopoll structure |
63 | * |
64 | * Description: |
65 | * If a driver consumes less than the assigned budget in its run of the |
66 | * iopoll handler, it'll end the polled mode by calling this function. The |
67 | * iopoll handler will not be invoked again before blk_iopoll_sched_prep() |
68 | * is called. |
69 | **/ |
70 | void blk_iopoll_complete(struct blk_iopoll *iopoll) |
71 | { |
72 | unsigned long flags; |
73 | |
74 | local_irq_save(flags); |
75 | __blk_iopoll_complete(iopoll); |
76 | local_irq_restore(flags); |
77 | } |
78 | EXPORT_SYMBOL(blk_iopoll_complete); |
79 | |
80 | static void blk_iopoll_softirq(struct softirq_action *h) |
81 | { |
82 | struct list_head *list = &__get_cpu_var(blk_cpu_iopoll); |
83 | int rearm = 0, budget = blk_iopoll_budget; |
84 | unsigned long start_time = jiffies; |
85 | |
86 | local_irq_disable(); |
87 | |
88 | while (!list_empty(list)) { |
89 | struct blk_iopoll *iop; |
90 | int work, weight; |
91 | |
92 | /* |
93 | * If softirq window is exhausted then punt. |
94 | */ |
95 | if (budget <= 0 || time_after(jiffies, start_time)) { |
96 | rearm = 1; |
97 | break; |
98 | } |
99 | |
100 | local_irq_enable(); |
101 | |
102 | /* Even though interrupts have been re-enabled, this |
103 | * access is safe because interrupts can only add new |
104 | * entries to the tail of this list, and only ->poll() |
105 | * calls can remove this head entry from the list. |
106 | */ |
107 | iop = list_entry(list->next, struct blk_iopoll, list); |
108 | |
109 | weight = iop->weight; |
110 | work = 0; |
111 | if (test_bit(IOPOLL_F_SCHED, &iop->state)) |
112 | work = iop->poll(iop, weight); |
113 | |
114 | budget -= work; |
115 | |
116 | local_irq_disable(); |
117 | |
118 | /* |
119 | * Drivers must not modify the iopoll state, if they |
120 | * consume their assigned weight (or more, some drivers can't |
121 | * easily just stop processing, they have to complete an |
122 | * entire mask of commands).In such cases this code |
123 | * still "owns" the iopoll instance and therefore can |
124 | * move the instance around on the list at-will. |
125 | */ |
126 | if (work >= weight) { |
127 | if (blk_iopoll_disable_pending(iop)) |
128 | __blk_iopoll_complete(iop); |
129 | else |
130 | list_move_tail(&iop->list, list); |
131 | } |
132 | } |
133 | |
134 | if (rearm) |
135 | __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); |
136 | |
137 | local_irq_enable(); |
138 | } |
139 | |
140 | /** |
141 | * blk_iopoll_disable - Disable iopoll on this @iop |
142 | * @iop: The parent iopoll structure |
143 | * |
144 | * Description: |
145 | * Disable io polling and wait for any pending callbacks to have completed. |
146 | **/ |
147 | void blk_iopoll_disable(struct blk_iopoll *iop) |
148 | { |
149 | set_bit(IOPOLL_F_DISABLE, &iop->state); |
150 | while (test_and_set_bit(IOPOLL_F_SCHED, &iop->state)) |
151 | msleep(1); |
152 | clear_bit(IOPOLL_F_DISABLE, &iop->state); |
153 | } |
154 | EXPORT_SYMBOL(blk_iopoll_disable); |
155 | |
156 | /** |
157 | * blk_iopoll_enable - Enable iopoll on this @iop |
158 | * @iop: The parent iopoll structure |
159 | * |
160 | * Description: |
161 | * Enable iopoll on this @iop. Note that the handler run will not be |
162 | * scheduled, it will only mark it as active. |
163 | **/ |
164 | void blk_iopoll_enable(struct blk_iopoll *iop) |
165 | { |
166 | BUG_ON(!test_bit(IOPOLL_F_SCHED, &iop->state)); |
167 | smp_mb__before_clear_bit(); |
168 | clear_bit_unlock(IOPOLL_F_SCHED, &iop->state); |
169 | } |
170 | EXPORT_SYMBOL(blk_iopoll_enable); |
171 | |
172 | /** |
173 | * blk_iopoll_init - Initialize this @iop |
174 | * @iop: The parent iopoll structure |
175 | * @weight: The default weight (or command completion budget) |
176 | * @poll_fn: The handler to invoke |
177 | * |
178 | * Description: |
179 | * Initialize this blk_iopoll structure. Before being actively used, the |
180 | * driver must call blk_iopoll_enable(). |
181 | **/ |
182 | void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn) |
183 | { |
184 | memset(iop, 0, sizeof(*iop)); |
185 | INIT_LIST_HEAD(&iop->list); |
186 | iop->weight = weight; |
187 | iop->poll = poll_fn; |
188 | set_bit(IOPOLL_F_SCHED, &iop->state); |
189 | } |
190 | EXPORT_SYMBOL(blk_iopoll_init); |
191 | |
192 | static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self, |
193 | unsigned long action, void *hcpu) |
194 | { |
195 | /* |
196 | * If a CPU goes away, splice its entries to the current CPU |
197 | * and trigger a run of the softirq |
198 | */ |
199 | if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { |
200 | int cpu = (unsigned long) hcpu; |
201 | |
202 | local_irq_disable(); |
203 | list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), |
204 | &__get_cpu_var(blk_cpu_iopoll)); |
205 | __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); |
206 | local_irq_enable(); |
207 | } |
208 | |
209 | return NOTIFY_OK; |
210 | } |
211 | |
212 | static struct notifier_block __cpuinitdata blk_iopoll_cpu_notifier = { |
213 | .notifier_call = blk_iopoll_cpu_notify, |
214 | }; |
215 | |
216 | static __init int blk_iopoll_setup(void) |
217 | { |
218 | int i; |
219 | |
220 | for_each_possible_cpu(i) |
221 | INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); |
222 | |
223 | open_softirq(BLOCK_IOPOLL_SOFTIRQ, blk_iopoll_softirq); |
224 | register_hotcpu_notifier(&blk_iopoll_cpu_notifier); |
225 | return 0; |
226 | } |
227 | subsys_initcall(blk_iopoll_setup); |
228 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9