Root/
1 | #ifndef _RDS_IW_H |
2 | #define _RDS_IW_H |
3 | |
4 | #include <rdma/ib_verbs.h> |
5 | #include <rdma/rdma_cm.h> |
6 | #include "rds.h" |
7 | #include "rdma_transport.h" |
8 | |
9 | #define RDS_FASTREG_SIZE 20 |
10 | #define RDS_FASTREG_POOL_SIZE 2048 |
11 | |
12 | #define RDS_IW_MAX_SGE 8 |
13 | #define RDS_IW_RECV_SGE 2 |
14 | |
15 | #define RDS_IW_DEFAULT_RECV_WR 1024 |
16 | #define RDS_IW_DEFAULT_SEND_WR 256 |
17 | |
18 | #define RDS_IW_SUPPORTED_PROTOCOLS 0x00000003 /* minor versions supported */ |
19 | |
20 | extern struct list_head rds_iw_devices; |
21 | |
22 | /* |
23 | * IB posts RDS_FRAG_SIZE fragments of pages to the receive queues to |
24 | * try and minimize the amount of memory tied up both the device and |
25 | * socket receive queues. |
26 | */ |
27 | /* page offset of the final full frag that fits in the page */ |
28 | #define RDS_PAGE_LAST_OFF (((PAGE_SIZE / RDS_FRAG_SIZE) - 1) * RDS_FRAG_SIZE) |
29 | struct rds_page_frag { |
30 | struct list_head f_item; |
31 | struct page *f_page; |
32 | unsigned long f_offset; |
33 | dma_addr_t f_mapped; |
34 | }; |
35 | |
36 | struct rds_iw_incoming { |
37 | struct list_head ii_frags; |
38 | struct rds_incoming ii_inc; |
39 | }; |
40 | |
41 | struct rds_iw_connect_private { |
42 | /* Add new fields at the end, and don't permute existing fields. */ |
43 | __be32 dp_saddr; |
44 | __be32 dp_daddr; |
45 | u8 dp_protocol_major; |
46 | u8 dp_protocol_minor; |
47 | __be16 dp_protocol_minor_mask; /* bitmask */ |
48 | __be32 dp_reserved1; |
49 | __be64 dp_ack_seq; |
50 | __be32 dp_credit; /* non-zero enables flow ctl */ |
51 | }; |
52 | |
53 | struct rds_iw_scatterlist { |
54 | struct scatterlist *list; |
55 | unsigned int len; |
56 | int dma_len; |
57 | unsigned int dma_npages; |
58 | unsigned int bytes; |
59 | }; |
60 | |
61 | struct rds_iw_mapping { |
62 | spinlock_t m_lock; /* protect the mapping struct */ |
63 | struct list_head m_list; |
64 | struct rds_iw_mr *m_mr; |
65 | uint32_t m_rkey; |
66 | struct rds_iw_scatterlist m_sg; |
67 | }; |
68 | |
69 | struct rds_iw_send_work { |
70 | struct rds_message *s_rm; |
71 | |
72 | /* We should really put these into a union: */ |
73 | struct rds_rdma_op *s_op; |
74 | struct rds_iw_mapping *s_mapping; |
75 | struct ib_mr *s_mr; |
76 | struct ib_fast_reg_page_list *s_page_list; |
77 | unsigned char s_remap_count; |
78 | |
79 | struct ib_send_wr s_wr; |
80 | struct ib_sge s_sge[RDS_IW_MAX_SGE]; |
81 | unsigned long s_queued; |
82 | }; |
83 | |
84 | struct rds_iw_recv_work { |
85 | struct rds_iw_incoming *r_iwinc; |
86 | struct rds_page_frag *r_frag; |
87 | struct ib_recv_wr r_wr; |
88 | struct ib_sge r_sge[2]; |
89 | }; |
90 | |
91 | struct rds_iw_work_ring { |
92 | u32 w_nr; |
93 | u32 w_alloc_ptr; |
94 | u32 w_alloc_ctr; |
95 | u32 w_free_ptr; |
96 | atomic_t w_free_ctr; |
97 | }; |
98 | |
99 | struct rds_iw_device; |
100 | |
101 | struct rds_iw_connection { |
102 | |
103 | struct list_head iw_node; |
104 | struct rds_iw_device *rds_iwdev; |
105 | struct rds_connection *conn; |
106 | |
107 | /* alphabet soup, IBTA style */ |
108 | struct rdma_cm_id *i_cm_id; |
109 | struct ib_pd *i_pd; |
110 | struct ib_mr *i_mr; |
111 | struct ib_cq *i_send_cq; |
112 | struct ib_cq *i_recv_cq; |
113 | |
114 | /* tx */ |
115 | struct rds_iw_work_ring i_send_ring; |
116 | struct rds_message *i_rm; |
117 | struct rds_header *i_send_hdrs; |
118 | u64 i_send_hdrs_dma; |
119 | struct rds_iw_send_work *i_sends; |
120 | |
121 | /* rx */ |
122 | struct mutex i_recv_mutex; |
123 | struct rds_iw_work_ring i_recv_ring; |
124 | struct rds_iw_incoming *i_iwinc; |
125 | u32 i_recv_data_rem; |
126 | struct rds_header *i_recv_hdrs; |
127 | u64 i_recv_hdrs_dma; |
128 | struct rds_iw_recv_work *i_recvs; |
129 | struct rds_page_frag i_frag; |
130 | u64 i_ack_recv; /* last ACK received */ |
131 | |
132 | /* sending acks */ |
133 | unsigned long i_ack_flags; |
134 | #ifdef KERNEL_HAS_ATOMIC64 |
135 | atomic64_t i_ack_next; /* next ACK to send */ |
136 | #else |
137 | spinlock_t i_ack_lock; /* protect i_ack_next */ |
138 | u64 i_ack_next; /* next ACK to send */ |
139 | #endif |
140 | struct rds_header *i_ack; |
141 | struct ib_send_wr i_ack_wr; |
142 | struct ib_sge i_ack_sge; |
143 | u64 i_ack_dma; |
144 | unsigned long i_ack_queued; |
145 | |
146 | /* Flow control related information |
147 | * |
148 | * Our algorithm uses a pair variables that we need to access |
149 | * atomically - one for the send credits, and one posted |
150 | * recv credits we need to transfer to remote. |
151 | * Rather than protect them using a slow spinlock, we put both into |
152 | * a single atomic_t and update it using cmpxchg |
153 | */ |
154 | atomic_t i_credits; |
155 | |
156 | /* Protocol version specific information */ |
157 | unsigned int i_flowctl:1; /* enable/disable flow ctl */ |
158 | unsigned int i_dma_local_lkey:1; |
159 | unsigned int i_fastreg_posted:1; /* fastreg posted on this connection */ |
160 | /* Batched completions */ |
161 | unsigned int i_unsignaled_wrs; |
162 | long i_unsignaled_bytes; |
163 | }; |
164 | |
165 | /* This assumes that atomic_t is at least 32 bits */ |
166 | #define IB_GET_SEND_CREDITS(v) ((v) & 0xffff) |
167 | #define IB_GET_POST_CREDITS(v) ((v) >> 16) |
168 | #define IB_SET_SEND_CREDITS(v) ((v) & 0xffff) |
169 | #define IB_SET_POST_CREDITS(v) ((v) << 16) |
170 | |
171 | struct rds_iw_cm_id { |
172 | struct list_head list; |
173 | struct rdma_cm_id *cm_id; |
174 | }; |
175 | |
176 | struct rds_iw_device { |
177 | struct list_head list; |
178 | struct list_head cm_id_list; |
179 | struct list_head conn_list; |
180 | struct ib_device *dev; |
181 | struct ib_pd *pd; |
182 | struct ib_mr *mr; |
183 | struct rds_iw_mr_pool *mr_pool; |
184 | int max_sge; |
185 | unsigned int max_wrs; |
186 | unsigned int dma_local_lkey:1; |
187 | spinlock_t spinlock; /* protect the above */ |
188 | }; |
189 | |
190 | /* bits for i_ack_flags */ |
191 | #define IB_ACK_IN_FLIGHT 0 |
192 | #define IB_ACK_REQUESTED 1 |
193 | |
194 | /* Magic WR_ID for ACKs */ |
195 | #define RDS_IW_ACK_WR_ID ((u64)0xffffffffffffffffULL) |
196 | #define RDS_IW_FAST_REG_WR_ID ((u64)0xefefefefefefefefULL) |
197 | #define RDS_IW_LOCAL_INV_WR_ID ((u64)0xdfdfdfdfdfdfdfdfULL) |
198 | |
199 | struct rds_iw_statistics { |
200 | uint64_t s_iw_connect_raced; |
201 | uint64_t s_iw_listen_closed_stale; |
202 | uint64_t s_iw_tx_cq_call; |
203 | uint64_t s_iw_tx_cq_event; |
204 | uint64_t s_iw_tx_ring_full; |
205 | uint64_t s_iw_tx_throttle; |
206 | uint64_t s_iw_tx_sg_mapping_failure; |
207 | uint64_t s_iw_tx_stalled; |
208 | uint64_t s_iw_tx_credit_updates; |
209 | uint64_t s_iw_rx_cq_call; |
210 | uint64_t s_iw_rx_cq_event; |
211 | uint64_t s_iw_rx_ring_empty; |
212 | uint64_t s_iw_rx_refill_from_cq; |
213 | uint64_t s_iw_rx_refill_from_thread; |
214 | uint64_t s_iw_rx_alloc_limit; |
215 | uint64_t s_iw_rx_credit_updates; |
216 | uint64_t s_iw_ack_sent; |
217 | uint64_t s_iw_ack_send_failure; |
218 | uint64_t s_iw_ack_send_delayed; |
219 | uint64_t s_iw_ack_send_piggybacked; |
220 | uint64_t s_iw_ack_received; |
221 | uint64_t s_iw_rdma_mr_alloc; |
222 | uint64_t s_iw_rdma_mr_free; |
223 | uint64_t s_iw_rdma_mr_used; |
224 | uint64_t s_iw_rdma_mr_pool_flush; |
225 | uint64_t s_iw_rdma_mr_pool_wait; |
226 | uint64_t s_iw_rdma_mr_pool_depleted; |
227 | }; |
228 | |
229 | extern struct workqueue_struct *rds_iw_wq; |
230 | |
231 | /* |
232 | * Fake ib_dma_sync_sg_for_{cpu,device} as long as ib_verbs.h |
233 | * doesn't define it. |
234 | */ |
235 | static inline void rds_iw_dma_sync_sg_for_cpu(struct ib_device *dev, |
236 | struct scatterlist *sg, unsigned int sg_dma_len, int direction) |
237 | { |
238 | unsigned int i; |
239 | |
240 | for (i = 0; i < sg_dma_len; ++i) { |
241 | ib_dma_sync_single_for_cpu(dev, |
242 | ib_sg_dma_address(dev, &sg[i]), |
243 | ib_sg_dma_len(dev, &sg[i]), |
244 | direction); |
245 | } |
246 | } |
247 | #define ib_dma_sync_sg_for_cpu rds_iw_dma_sync_sg_for_cpu |
248 | |
249 | static inline void rds_iw_dma_sync_sg_for_device(struct ib_device *dev, |
250 | struct scatterlist *sg, unsigned int sg_dma_len, int direction) |
251 | { |
252 | unsigned int i; |
253 | |
254 | for (i = 0; i < sg_dma_len; ++i) { |
255 | ib_dma_sync_single_for_device(dev, |
256 | ib_sg_dma_address(dev, &sg[i]), |
257 | ib_sg_dma_len(dev, &sg[i]), |
258 | direction); |
259 | } |
260 | } |
261 | #define ib_dma_sync_sg_for_device rds_iw_dma_sync_sg_for_device |
262 | |
263 | static inline u32 rds_iw_local_dma_lkey(struct rds_iw_connection *ic) |
264 | { |
265 | return ic->i_dma_local_lkey ? ic->i_cm_id->device->local_dma_lkey : ic->i_mr->lkey; |
266 | } |
267 | |
268 | /* ib.c */ |
269 | extern struct rds_transport rds_iw_transport; |
270 | extern void rds_iw_add_one(struct ib_device *device); |
271 | extern void rds_iw_remove_one(struct ib_device *device); |
272 | extern struct ib_client rds_iw_client; |
273 | |
274 | extern unsigned int fastreg_pool_size; |
275 | extern unsigned int fastreg_message_size; |
276 | |
277 | extern spinlock_t iw_nodev_conns_lock; |
278 | extern struct list_head iw_nodev_conns; |
279 | |
280 | /* ib_cm.c */ |
281 | int rds_iw_conn_alloc(struct rds_connection *conn, gfp_t gfp); |
282 | void rds_iw_conn_free(void *arg); |
283 | int rds_iw_conn_connect(struct rds_connection *conn); |
284 | void rds_iw_conn_shutdown(struct rds_connection *conn); |
285 | void rds_iw_state_change(struct sock *sk); |
286 | int __init rds_iw_listen_init(void); |
287 | void rds_iw_listen_stop(void); |
288 | void __rds_iw_conn_error(struct rds_connection *conn, const char *, ...); |
289 | int rds_iw_cm_handle_connect(struct rdma_cm_id *cm_id, |
290 | struct rdma_cm_event *event); |
291 | int rds_iw_cm_initiate_connect(struct rdma_cm_id *cm_id); |
292 | void rds_iw_cm_connect_complete(struct rds_connection *conn, |
293 | struct rdma_cm_event *event); |
294 | |
295 | |
296 | #define rds_iw_conn_error(conn, fmt...) \ |
297 | __rds_iw_conn_error(conn, KERN_WARNING "RDS/IW: " fmt) |
298 | |
299 | /* ib_rdma.c */ |
300 | int rds_iw_update_cm_id(struct rds_iw_device *rds_iwdev, struct rdma_cm_id *cm_id); |
301 | void rds_iw_add_conn(struct rds_iw_device *rds_iwdev, struct rds_connection *conn); |
302 | void rds_iw_remove_conn(struct rds_iw_device *rds_iwdev, struct rds_connection *conn); |
303 | void __rds_iw_destroy_conns(struct list_head *list, spinlock_t *list_lock); |
304 | static inline void rds_iw_destroy_nodev_conns(void) |
305 | { |
306 | __rds_iw_destroy_conns(&iw_nodev_conns, &iw_nodev_conns_lock); |
307 | } |
308 | static inline void rds_iw_destroy_conns(struct rds_iw_device *rds_iwdev) |
309 | { |
310 | __rds_iw_destroy_conns(&rds_iwdev->conn_list, &rds_iwdev->spinlock); |
311 | } |
312 | struct rds_iw_mr_pool *rds_iw_create_mr_pool(struct rds_iw_device *); |
313 | void rds_iw_get_mr_info(struct rds_iw_device *rds_iwdev, struct rds_info_rdma_connection *iinfo); |
314 | void rds_iw_destroy_mr_pool(struct rds_iw_mr_pool *); |
315 | void *rds_iw_get_mr(struct scatterlist *sg, unsigned long nents, |
316 | struct rds_sock *rs, u32 *key_ret); |
317 | void rds_iw_sync_mr(void *trans_private, int dir); |
318 | void rds_iw_free_mr(void *trans_private, int invalidate); |
319 | void rds_iw_flush_mrs(void); |
320 | void rds_iw_remove_cm_id(struct rds_iw_device *rds_iwdev, struct rdma_cm_id *cm_id); |
321 | |
322 | /* ib_recv.c */ |
323 | int __init rds_iw_recv_init(void); |
324 | void rds_iw_recv_exit(void); |
325 | int rds_iw_recv(struct rds_connection *conn); |
326 | int rds_iw_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp, |
327 | gfp_t page_gfp, int prefill); |
328 | void rds_iw_inc_purge(struct rds_incoming *inc); |
329 | void rds_iw_inc_free(struct rds_incoming *inc); |
330 | int rds_iw_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov, |
331 | size_t size); |
332 | void rds_iw_recv_cq_comp_handler(struct ib_cq *cq, void *context); |
333 | void rds_iw_recv_init_ring(struct rds_iw_connection *ic); |
334 | void rds_iw_recv_clear_ring(struct rds_iw_connection *ic); |
335 | void rds_iw_recv_init_ack(struct rds_iw_connection *ic); |
336 | void rds_iw_attempt_ack(struct rds_iw_connection *ic); |
337 | void rds_iw_ack_send_complete(struct rds_iw_connection *ic); |
338 | u64 rds_iw_piggyb_ack(struct rds_iw_connection *ic); |
339 | |
340 | /* ib_ring.c */ |
341 | void rds_iw_ring_init(struct rds_iw_work_ring *ring, u32 nr); |
342 | void rds_iw_ring_resize(struct rds_iw_work_ring *ring, u32 nr); |
343 | u32 rds_iw_ring_alloc(struct rds_iw_work_ring *ring, u32 val, u32 *pos); |
344 | void rds_iw_ring_free(struct rds_iw_work_ring *ring, u32 val); |
345 | void rds_iw_ring_unalloc(struct rds_iw_work_ring *ring, u32 val); |
346 | int rds_iw_ring_empty(struct rds_iw_work_ring *ring); |
347 | int rds_iw_ring_low(struct rds_iw_work_ring *ring); |
348 | u32 rds_iw_ring_oldest(struct rds_iw_work_ring *ring); |
349 | u32 rds_iw_ring_completed(struct rds_iw_work_ring *ring, u32 wr_id, u32 oldest); |
350 | extern wait_queue_head_t rds_iw_ring_empty_wait; |
351 | |
352 | /* ib_send.c */ |
353 | void rds_iw_xmit_complete(struct rds_connection *conn); |
354 | int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm, |
355 | unsigned int hdr_off, unsigned int sg, unsigned int off); |
356 | void rds_iw_send_cq_comp_handler(struct ib_cq *cq, void *context); |
357 | void rds_iw_send_init_ring(struct rds_iw_connection *ic); |
358 | void rds_iw_send_clear_ring(struct rds_iw_connection *ic); |
359 | int rds_iw_xmit_rdma(struct rds_connection *conn, struct rds_rdma_op *op); |
360 | void rds_iw_send_add_credits(struct rds_connection *conn, unsigned int credits); |
361 | void rds_iw_advertise_credits(struct rds_connection *conn, unsigned int posted); |
362 | int rds_iw_send_grab_credits(struct rds_iw_connection *ic, u32 wanted, |
363 | u32 *adv_credits, int need_posted, int max_posted); |
364 | |
365 | /* ib_stats.c */ |
366 | DECLARE_PER_CPU(struct rds_iw_statistics, rds_iw_stats); |
367 | #define rds_iw_stats_inc(member) rds_stats_inc_which(rds_iw_stats, member) |
368 | unsigned int rds_iw_stats_info_copy(struct rds_info_iterator *iter, |
369 | unsigned int avail); |
370 | |
371 | /* ib_sysctl.c */ |
372 | int __init rds_iw_sysctl_init(void); |
373 | void rds_iw_sysctl_exit(void); |
374 | extern unsigned long rds_iw_sysctl_max_send_wr; |
375 | extern unsigned long rds_iw_sysctl_max_recv_wr; |
376 | extern unsigned long rds_iw_sysctl_max_unsig_wrs; |
377 | extern unsigned long rds_iw_sysctl_max_unsig_bytes; |
378 | extern unsigned long rds_iw_sysctl_max_recv_allocation; |
379 | extern unsigned int rds_iw_sysctl_flow_control; |
380 | extern ctl_table rds_iw_sysctl_table[]; |
381 | |
382 | /* |
383 | * Helper functions for getting/setting the header and data SGEs in |
384 | * RDS packets (not RDMA) |
385 | */ |
386 | static inline struct ib_sge * |
387 | rds_iw_header_sge(struct rds_iw_connection *ic, struct ib_sge *sge) |
388 | { |
389 | return &sge[0]; |
390 | } |
391 | |
392 | static inline struct ib_sge * |
393 | rds_iw_data_sge(struct rds_iw_connection *ic, struct ib_sge *sge) |
394 | { |
395 | return &sge[1]; |
396 | } |
397 | |
398 | #endif |
399 |
Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master
Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9