1/*
2 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses.  You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 *     Redistribution and use in source and binary forms, with or
12 *     without modification, are permitted provided that the following
13 *     conditions are met:
14 *
15 *      - Redistributions of source code must retain the above
16 *        copyright notice, this list of conditions and the following
17 *        disclaimer.
18 *
19 *      - Redistributions in binary form must reproduce the above
20 *        copyright notice, this list of conditions and the following
21 *        disclaimer in the documentation and/or other materials
22 *        provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#ifndef MLX4_H
35#define MLX4_H
36
37#include <stddef.h>
38
39#include <infiniband/driver.h>
40#include <infiniband/arch.h>
41
42#ifdef HAVE_VALGRIND_MEMCHECK_H
43
44#  include <valgrind/memcheck.h>
45
46#  if !defined(VALGRIND_MAKE_MEM_DEFINED) || !defined(VALGRIND_MAKE_MEM_UNDEFINED)
47#    warning "Valgrind support requested, but VALGRIND_MAKE_MEM_(UN)DEFINED not available"
48#  endif
49
50#endif /* HAVE_VALGRIND_MEMCHECK_H */
51
52#ifndef VALGRIND_MAKE_MEM_DEFINED
53#  define VALGRIND_MAKE_MEM_DEFINED(addr,len)
54#endif
55
56#ifndef VALGRIND_MAKE_MEM_UNDEFINED
57#  define VALGRIND_MAKE_MEM_UNDEFINED(addr,len)
58#endif
59
60#ifndef rmb
61#  define rmb() mb()
62#endif
63
64#ifndef wmb
65#  define wmb() mb()
66#endif
67
68#ifndef wc_wmb
69
70#if defined(__i386__)
71#define wc_wmb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory")
72#elif defined(__x86_64__)
73#define wc_wmb() asm volatile("sfence" ::: "memory")
74#elif defined(__ia64__)
75#define wc_wmb() asm volatile("fwb" ::: "memory")
76#else
77#define wc_wmb() wmb()
78#endif
79
80#endif
81
82#ifndef HAVE_IBV_MORE_OPS
83#undef HAVE_IBV_XRC_OPS
84#undef HAVE_IBV_CREATE_QP_EXP
85#endif
86
87#define HIDDEN		__attribute__((visibility ("hidden")))
88
89#define PFX		"mlx4: "
90
91#ifndef max
92#define max(a,b) \
93	({ typeof (a) _a = (a); \
94	   typeof (b) _b = (b); \
95	   _a > _b ? _a : _b; })
96#endif
97
98#ifndef min
99#define min(a,b) \
100	({ typeof (a) _a = (a); \
101	   typeof (b) _b = (b); \
102	   _a < _b ? _a : _b; })
103#endif
104
105enum {
106	MLX4_CQ_ENTRY_SIZE		= 0x20
107};
108
109enum {
110	MLX4_STAT_RATE_OFFSET		= 5
111};
112
113enum {
114	MLX4_QP_TABLE_BITS		= 8,
115	MLX4_QP_TABLE_SIZE		= 1 << MLX4_QP_TABLE_BITS,
116	MLX4_QP_TABLE_MASK		= MLX4_QP_TABLE_SIZE - 1
117};
118
119enum {
120	MLX4_XRC_SRQ_TABLE_BITS		= 8,
121	MLX4_XRC_SRQ_TABLE_SIZE		= 1 << MLX4_XRC_SRQ_TABLE_BITS,
122	MLX4_XRC_SRQ_TABLE_MASK		= MLX4_XRC_SRQ_TABLE_SIZE - 1
123};
124
125enum {
126	MLX4_XRC_QPN_BIT		= (1 << 23)
127};
128
129enum mlx4_db_type {
130	MLX4_DB_TYPE_CQ,
131	MLX4_DB_TYPE_RQ,
132	MLX4_NUM_DB_TYPE
133};
134
135enum {
136	MLX4_OPCODE_NOP			= 0x00,
137	MLX4_OPCODE_SEND_INVAL		= 0x01,
138	MLX4_OPCODE_RDMA_WRITE		= 0x08,
139	MLX4_OPCODE_RDMA_WRITE_IMM	= 0x09,
140	MLX4_OPCODE_SEND		= 0x0a,
141	MLX4_OPCODE_SEND_IMM		= 0x0b,
142	MLX4_OPCODE_LSO			= 0x0e,
143	MLX4_OPCODE_RDMA_READ		= 0x10,
144	MLX4_OPCODE_ATOMIC_CS		= 0x11,
145	MLX4_OPCODE_ATOMIC_FA		= 0x12,
146	MLX4_OPCODE_ATOMIC_MASK_CS	= 0x14,
147	MLX4_OPCODE_ATOMIC_MASK_FA	= 0x15,
148	MLX4_OPCODE_BIND_MW		= 0x18,
149	MLX4_OPCODE_FMR			= 0x19,
150	MLX4_OPCODE_LOCAL_INVAL		= 0x1b,
151	MLX4_OPCODE_CONFIG_CMD		= 0x1f,
152
153	MLX4_RECV_OPCODE_RDMA_WRITE_IMM	= 0x00,
154	MLX4_RECV_OPCODE_SEND		= 0x01,
155	MLX4_RECV_OPCODE_SEND_IMM	= 0x02,
156	MLX4_RECV_OPCODE_SEND_INVAL	= 0x03,
157
158	MLX4_CQE_OPCODE_ERROR		= 0x1e,
159	MLX4_CQE_OPCODE_RESIZE		= 0x16,
160};
161
162enum {
163	MLX4_MAX_WQE_SIZE = 1008
164};
165
166struct mlx4_device {
167	struct ibv_device		ibv_dev;
168	int				page_size;
169};
170
171struct mlx4_db_page;
172
173struct mlx4_context {
174	struct ibv_context		ibv_ctx;
175
176	void			       *uar;
177	pthread_spinlock_t		uar_lock;
178
179	void			       *bf_page;
180	int				bf_buf_size;
181	int				bf_offset;
182	pthread_spinlock_t		bf_lock;
183
184	struct {
185		struct mlx4_qp	      **table;
186		int			refcnt;
187	}				qp_table[MLX4_QP_TABLE_SIZE];
188	pthread_mutex_t			qp_table_mutex;
189	int				num_qps;
190	int				qp_table_shift;
191	int				qp_table_mask;
192	int				max_qp_wr;
193	int				max_sge;
194	int				max_cqe;
195
196	struct {
197		struct mlx4_srq       **table;
198		int			refcnt;
199	}				xrc_srq_table[MLX4_XRC_SRQ_TABLE_SIZE];
200	pthread_mutex_t			xrc_srq_table_mutex;
201	int				num_xrc_srqs;
202	int				xrc_srq_table_shift;
203	int				xrc_srq_table_mask;
204
205	struct mlx4_db_page	       *db_list[MLX4_NUM_DB_TYPE];
206	pthread_mutex_t			db_list_mutex;
207};
208
209struct mlx4_buf {
210	void			       *buf;
211	size_t				length;
212};
213
214struct mlx4_pd {
215	struct ibv_pd			ibv_pd;
216	uint32_t			pdn;
217};
218
219struct mlx4_cq {
220	struct ibv_cq			ibv_cq;
221	struct mlx4_buf			buf;
222	struct mlx4_buf			resize_buf;
223	pthread_spinlock_t		lock;
224	uint32_t			cqn;
225	uint32_t			cons_index;
226	uint32_t		       *set_ci_db;
227	uint32_t		       *arm_db;
228	int				arm_sn;
229};
230
231struct mlx4_srq {
232	struct ibv_srq			ibv_srq;
233	struct mlx4_buf			buf;
234	pthread_spinlock_t		lock;
235	uint64_t		       *wrid;
236	uint32_t			srqn;
237	int				max;
238	int				max_gs;
239	int				wqe_shift;
240	int				head;
241	int				tail;
242	uint32_t		       *db;
243	uint16_t			counter;
244};
245
246struct mlx4_wq {
247	uint64_t		       *wrid;
248	pthread_spinlock_t		lock;
249	int				wqe_cnt;
250	int				max_post;
251	unsigned			head;
252	unsigned			tail;
253	int				max_gs;
254	int				wqe_shift;
255	int				offset;
256};
257
258struct mlx4_qp {
259	struct ibv_qp			ibv_qp;
260	struct mlx4_buf			buf;
261	int				max_inline_data;
262	int				buf_size;
263
264	uint32_t			doorbell_qpn;
265	uint32_t			sq_signal_bits;
266	int				sq_spare_wqes;
267	struct mlx4_wq			sq;
268
269	uint32_t		       *db;
270	struct mlx4_wq			rq;
271};
272
273struct mlx4_av {
274	uint32_t			port_pd;
275	uint8_t				reserved1;
276	uint8_t				g_slid;
277	uint16_t			dlid;
278	uint8_t				reserved2;
279	uint8_t				gid_index;
280	uint8_t				stat_rate;
281	uint8_t				hop_limit;
282	uint32_t			sl_tclass_flowlabel;
283	uint8_t				dgid[16];
284	uint8_t				mac[8];
285};
286
287struct mlx4_ah {
288	struct ibv_ah			ibv_ah;
289	struct mlx4_av			av;
290	uint16_t			vlan;
291	uint8_t				mac[6];
292	uint8_t				tagged;
293};
294
295struct mlx4_xrc_domain {
296	struct ibv_xrc_domain		ibv_xrcd;
297	uint32_t			xrcdn;
298};
299
300static inline unsigned long align(unsigned long val, unsigned long align)
301{
302	return (val + align - 1) & ~(align - 1);
303}
304
305#define to_mxxx(xxx, type)						\
306	((struct mlx4_##type *)					\
307	 ((void *) ib##xxx - offsetof(struct mlx4_##type, ibv_##xxx)))
308
309static inline struct mlx4_device *to_mdev(struct ibv_device *ibdev)
310{
311	return to_mxxx(dev, device);
312}
313
314static inline struct mlx4_context *to_mctx(struct ibv_context *ibctx)
315{
316	return to_mxxx(ctx, context);
317}
318
319static inline struct mlx4_pd *to_mpd(struct ibv_pd *ibpd)
320{
321	return to_mxxx(pd, pd);
322}
323
324static inline struct mlx4_cq *to_mcq(struct ibv_cq *ibcq)
325{
326	return to_mxxx(cq, cq);
327}
328
329static inline struct mlx4_srq *to_msrq(struct ibv_srq *ibsrq)
330{
331	return to_mxxx(srq, srq);
332}
333
334static inline struct mlx4_qp *to_mqp(struct ibv_qp *ibqp)
335{
336	return to_mxxx(qp, qp);
337}
338
339static inline struct mlx4_ah *to_mah(struct ibv_ah *ibah)
340{
341	return to_mxxx(ah, ah);
342}
343
344#ifdef HAVE_IBV_XRC_OPS
345static inline struct mlx4_xrc_domain *to_mxrcd(struct ibv_xrc_domain *ibxrcd)
346{
347	return to_mxxx(xrcd, xrc_domain);
348}
349#endif
350
351int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size);
352void mlx4_free_buf(struct mlx4_buf *buf);
353
354uint32_t *mlx4_alloc_db(struct mlx4_context *context, enum mlx4_db_type type);
355void mlx4_free_db(struct mlx4_context *context, enum mlx4_db_type type, uint32_t *db);
356
357int mlx4_query_device(struct ibv_context *context,
358		       struct ibv_device_attr *attr);
359int mlx4_query_port(struct ibv_context *context, uint8_t port,
360		     struct ibv_port_attr *attr);
361
362struct ibv_pd *mlx4_alloc_pd(struct ibv_context *context);
363int mlx4_free_pd(struct ibv_pd *pd);
364
365struct ibv_mr *mlx4_reg_mr(struct ibv_pd *pd, void *addr,
366			    size_t length, enum ibv_access_flags access);
367int mlx4_dereg_mr(struct ibv_mr *mr);
368
369struct ibv_cq *mlx4_create_cq(struct ibv_context *context, int cqe,
370			       struct ibv_comp_channel *channel,
371			       int comp_vector);
372int mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent);
373int mlx4_resize_cq(struct ibv_cq *cq, int cqe);
374int mlx4_destroy_cq(struct ibv_cq *cq);
375int mlx4_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
376int mlx4_arm_cq(struct ibv_cq *cq, int solicited);
377void mlx4_cq_event(struct ibv_cq *cq);
378void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
379void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
380int mlx4_get_outstanding_cqes(struct mlx4_cq *cq);
381void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int new_cqe);
382
383struct ibv_srq *mlx4_create_srq(struct ibv_pd *pd,
384				 struct ibv_srq_init_attr *attr);
385int mlx4_modify_srq(struct ibv_srq *srq,
386		     struct ibv_srq_attr *attr,
387		     enum ibv_srq_attr_mask mask);
388int mlx4_query_srq(struct ibv_srq *srq,
389			   struct ibv_srq_attr *attr);
390int mlx4_destroy_srq(struct ibv_srq *srq);
391int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
392			struct mlx4_srq *srq);
393void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);
394int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
395		       struct ibv_recv_wr *wr,
396		       struct ibv_recv_wr **bad_wr);
397struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
398int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,
399		       struct mlx4_srq *srq);
400void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
401
402struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
403int mlx4_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
404		   enum ibv_qp_attr_mask attr_mask,
405		   struct ibv_qp_init_attr *init_attr);
406int mlx4_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
407		    enum ibv_qp_attr_mask attr_mask);
408int mlx4_destroy_qp(struct ibv_qp *qp);
409void mlx4_init_qp_indices(struct mlx4_qp *qp);
410void mlx4_qp_init_sq_ownership(struct mlx4_qp *qp);
411int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
412			  struct ibv_send_wr **bad_wr);
413int mlx4_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
414			  struct ibv_recv_wr **bad_wr);
415void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
416			   struct mlx4_qp *qp);
417int num_inline_segs(int data, enum ibv_qp_type type);
418int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
419		       enum ibv_qp_type type, struct mlx4_qp *qp);
420void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
421		       enum ibv_qp_type type);
422struct mlx4_qp *mlx4_find_qp(struct mlx4_context *ctx, uint32_t qpn);
423int mlx4_store_qp(struct mlx4_context *ctx, uint32_t qpn, struct mlx4_qp *qp);
424void mlx4_clear_qp(struct mlx4_context *ctx, uint32_t qpn);
425struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
426int mlx4_destroy_ah(struct ibv_ah *ah);
427int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr,
428		   struct mlx4_ah *ah);
429void mlx4_free_av(struct mlx4_ah *ah);
430#ifdef HAVE_IBV_XRC_OPS
431struct ibv_srq *mlx4_create_xrc_srq(struct ibv_pd *pd,
432				    struct ibv_xrc_domain *xrc_domain,
433				    struct ibv_cq *xrc_cq,
434				    struct ibv_srq_init_attr *attr);
435struct ibv_xrc_domain *mlx4_open_xrc_domain(struct ibv_context *context,
436					    int fd, int oflag);
437
438int mlx4_close_xrc_domain(struct ibv_xrc_domain *d);
439int mlx4_create_xrc_rcv_qp(struct ibv_qp_init_attr *init_attr,
440			   uint32_t *xrc_qp_num);
441int mlx4_modify_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
442			   uint32_t xrc_qp_num,
443			   struct ibv_qp_attr *attr,
444			   int attr_mask);
445int mlx4_query_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
446			  uint32_t xrc_qp_num,
447			  struct ibv_qp_attr *attr,
448			  int attr_mask,
449			  struct ibv_qp_init_attr *init_attr);
450int mlx4_reg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
451			uint32_t xrc_qp_num);
452int mlx4_unreg_xrc_rcv_qp(struct ibv_xrc_domain *xrc_domain,
453			uint32_t xrc_qp_num);
454#endif
455
456
457#endif /* MLX4_H */
458