1/*
2 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses.  You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 *     Redistribution and use in source and binary forms, with or
12 *     without modification, are permitted provided that the following
13 *     conditions are met:
14 *
15 *      - Redistributions of source code must retain the above
16 *        copyright notice, this list of conditions and the following
17 *        disclaimer.
18 *
19 *      - Redistributions in binary form must reproduce the above
20 *        copyright notice, this list of conditions and the following
21 *        disclaimer in the documentation and/or other materials
22 *        provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#ifndef MTHCA_H
35#define MTHCA_H
36
37#include <stddef.h>
38
39#include <infiniband/driver.h>
40#include <infiniband/arch.h>
41
42#ifdef HAVE_VALGRIND_MEMCHECK_H
43
44#  include <valgrind/memcheck.h>
45
46#  if !defined(VALGRIND_MAKE_MEM_DEFINED) || !defined(VALGRIND_MAKE_MEM_UNDEFINED)
47#    warning "Valgrind support requested, but VALGRIND_MAKE_MEM_(UN)DEFINED not available"
48#  endif
49
50#endif /* HAVE_VALGRIND_MEMCHECK_H */
51
52#ifndef VALGRIND_MAKE_MEM_DEFINED
53#  define VALGRIND_MAKE_MEM_DEFINED(addr,len)
54#endif
55
56#ifndef VALGRIND_MAKE_MEM_UNDEFINED
57#  define VALGRIND_MAKE_MEM_UNDEFINED(addr,len)
58#endif
59
60#ifndef rmb
61#  define rmb() mb()
62#endif
63
64#ifndef wmb
65#  define wmb() mb()
66#endif
67
68#define HIDDEN		__attribute__((visibility ("hidden")))
69
70#define PFX		"mthca: "
71
72enum mthca_hca_type {
73	MTHCA_TAVOR,
74	MTHCA_ARBEL
75};
76
77enum {
78	MTHCA_CQ_ENTRY_SIZE = 0x20
79};
80
81enum {
82	MTHCA_QP_TABLE_BITS = 8,
83	MTHCA_QP_TABLE_SIZE = 1 << MTHCA_QP_TABLE_BITS,
84	MTHCA_QP_TABLE_MASK = MTHCA_QP_TABLE_SIZE - 1
85};
86
87enum {
88	MTHCA_DB_REC_PAGE_SIZE = 4096,
89	MTHCA_DB_REC_PER_PAGE  = MTHCA_DB_REC_PAGE_SIZE / 8
90};
91
92enum mthca_db_type {
93	MTHCA_DB_TYPE_INVALID   = 0x0,
94	MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
95	MTHCA_DB_TYPE_CQ_ARM    = 0x2,
96	MTHCA_DB_TYPE_SQ        = 0x3,
97	MTHCA_DB_TYPE_RQ        = 0x4,
98	MTHCA_DB_TYPE_SRQ       = 0x5,
99	MTHCA_DB_TYPE_GROUP_SEP = 0x7
100};
101
102enum {
103	MTHCA_OPCODE_NOP            = 0x00,
104	MTHCA_OPCODE_RDMA_WRITE     = 0x08,
105	MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09,
106	MTHCA_OPCODE_SEND           = 0x0a,
107	MTHCA_OPCODE_SEND_IMM       = 0x0b,
108	MTHCA_OPCODE_RDMA_READ      = 0x10,
109	MTHCA_OPCODE_ATOMIC_CS      = 0x11,
110	MTHCA_OPCODE_ATOMIC_FA      = 0x12,
111	MTHCA_OPCODE_BIND_MW        = 0x18,
112	MTHCA_OPCODE_INVALID        = 0xff
113};
114
115struct mthca_ah_page;
116
117struct mthca_device {
118	struct ibv_device   ibv_dev;
119	enum mthca_hca_type hca_type;
120	int                 page_size;
121};
122
123struct mthca_db_table;
124
125struct mthca_context {
126	struct ibv_context     ibv_ctx;
127	void                  *uar;
128	pthread_spinlock_t     uar_lock;
129	struct mthca_db_table *db_tab;
130	struct ibv_pd         *pd;
131	struct {
132		struct mthca_qp	**table;
133		int		  refcnt;
134	}		       qp_table[MTHCA_QP_TABLE_SIZE];
135	pthread_mutex_t        qp_table_mutex;
136	int                    num_qps;
137	int		       qp_table_shift;
138	int		       qp_table_mask;
139};
140
141struct mthca_buf {
142	void		       *buf;
143	size_t			length;
144};
145
146struct mthca_pd {
147	struct ibv_pd         ibv_pd;
148	struct mthca_ah_page *ah_list;
149	pthread_mutex_t       ah_mutex;
150	uint32_t              pdn;
151};
152
153struct mthca_cq {
154	struct ibv_cq  	   ibv_cq;
155	struct mthca_buf   buf;
156	pthread_spinlock_t lock;
157	struct ibv_mr  	  *mr;
158	uint32_t       	   cqn;
159	uint32_t       	   cons_index;
160
161	/* Next fields are mem-free only */
162	int                set_ci_db_index;
163	uint32_t          *set_ci_db;
164	int                arm_db_index;
165	uint32_t          *arm_db;
166	int                arm_sn;
167};
168
169struct mthca_srq {
170	struct ibv_srq     ibv_srq;
171	struct mthca_buf   buf;
172	void           	  *last;
173	pthread_spinlock_t lock;
174	struct ibv_mr 	  *mr;
175	uint64_t      	  *wrid;
176	uint32_t       	   srqn;
177	int            	   max;
178	int            	   max_gs;
179	int            	   wqe_shift;
180	int            	   first_free;
181	int            	   last_free;
182	int                buf_size;
183
184	/* Next fields are mem-free only */
185	int           	   db_index;
186	uint32_t      	  *db;
187	uint16_t      	   counter;
188};
189
190struct mthca_wq {
191	pthread_spinlock_t lock;
192	int            	   max;
193	unsigned       	   next_ind;
194	unsigned       	   last_comp;
195	unsigned       	   head;
196	unsigned       	   tail;
197	void           	  *last;
198	int            	   max_gs;
199	int            	   wqe_shift;
200
201	/* Next fields are mem-free only */
202	int                db_index;
203	uint32_t          *db;
204};
205
206struct mthca_qp {
207	struct ibv_qp    ibv_qp;
208	struct mthca_buf buf;
209	uint64_t        *wrid;
210	int              send_wqe_offset;
211	int              max_inline_data;
212	int              buf_size;
213	struct mthca_wq  sq;
214	struct mthca_wq  rq;
215	struct ibv_mr   *mr;
216	int              sq_sig_all;
217};
218
219struct mthca_av {
220	uint32_t port_pd;
221	uint8_t  reserved1;
222	uint8_t  g_slid;
223	uint16_t dlid;
224	uint8_t  reserved2;
225	uint8_t  gid_index;
226	uint8_t  msg_sr;
227	uint8_t  hop_limit;
228	uint32_t sl_tclass_flowlabel;
229	uint32_t dgid[4];
230};
231
232struct mthca_ah {
233	struct ibv_ah         ibv_ah;
234	struct mthca_av      *av;
235	struct mthca_ah_page *page;
236	uint32_t              key;
237};
238
239static inline unsigned long align(unsigned long val, unsigned long align)
240{
241	return (val + align - 1) & ~(align - 1);
242}
243
244static inline uintptr_t db_align(uint32_t *db)
245{
246	return (uintptr_t) db & ~((uintptr_t) MTHCA_DB_REC_PAGE_SIZE - 1);
247}
248
249#define to_mxxx(xxx, type)						\
250	((struct mthca_##type *)					\
251	 ((void *) ib##xxx - offsetof(struct mthca_##type, ibv_##xxx)))
252
253static inline struct mthca_device *to_mdev(struct ibv_device *ibdev)
254{
255	return to_mxxx(dev, device);
256}
257
258static inline struct mthca_context *to_mctx(struct ibv_context *ibctx)
259{
260	return to_mxxx(ctx, context);
261}
262
263static inline struct mthca_pd *to_mpd(struct ibv_pd *ibpd)
264{
265	return to_mxxx(pd, pd);
266}
267
268static inline struct mthca_cq *to_mcq(struct ibv_cq *ibcq)
269{
270	return to_mxxx(cq, cq);
271}
272
273static inline struct mthca_srq *to_msrq(struct ibv_srq *ibsrq)
274{
275	return to_mxxx(srq, srq);
276}
277
278static inline struct mthca_qp *to_mqp(struct ibv_qp *ibqp)
279{
280	return to_mxxx(qp, qp);
281}
282
283static inline struct mthca_ah *to_mah(struct ibv_ah *ibah)
284{
285	return to_mxxx(ah, ah);
286}
287
288static inline int mthca_is_memfree(struct ibv_context *ibctx)
289{
290	return to_mdev(ibctx->device)->hca_type == MTHCA_ARBEL;
291}
292
293int mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size);
294void mthca_free_buf(struct mthca_buf *buf);
295
296int mthca_alloc_db(struct mthca_db_table *db_tab, enum mthca_db_type type,
297		   uint32_t **db);
298void mthca_set_db_qn(uint32_t *db, enum mthca_db_type type, uint32_t qn);
299void mthca_free_db(struct mthca_db_table *db_tab, enum mthca_db_type type, int db_index);
300struct mthca_db_table *mthca_alloc_db_tab(int uarc_size);
301void mthca_free_db_tab(struct mthca_db_table *db_tab);
302
303int mthca_query_device(struct ibv_context *context,
304		       struct ibv_device_attr *attr);
305int mthca_query_port(struct ibv_context *context, uint8_t port,
306		     struct ibv_port_attr *attr);
307
308struct ibv_pd *mthca_alloc_pd(struct ibv_context *context);
309int mthca_free_pd(struct ibv_pd *pd);
310
311struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr,
312			    size_t length, enum ibv_access_flags access);
313int mthca_dereg_mr(struct ibv_mr *mr);
314
315struct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe,
316			       struct ibv_comp_channel *channel,
317			       int comp_vector);
318int mthca_resize_cq(struct ibv_cq *cq, int cqe);
319int mthca_destroy_cq(struct ibv_cq *cq);
320int mthca_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
321int mthca_tavor_arm_cq(struct ibv_cq *cq, int solicited);
322int mthca_arbel_arm_cq(struct ibv_cq *cq, int solicited);
323void mthca_arbel_cq_event(struct ibv_cq *cq);
324void __mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq);
325void mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq);
326void mthca_cq_resize_copy_cqes(struct mthca_cq *cq, void *buf, int new_cqe);
327int mthca_alloc_cq_buf(struct mthca_device *dev, struct mthca_buf *buf, int nent);
328
329struct ibv_srq *mthca_create_srq(struct ibv_pd *pd,
330				 struct ibv_srq_init_attr *attr);
331int mthca_modify_srq(struct ibv_srq *srq,
332		     struct ibv_srq_attr *attr,
333		     enum ibv_srq_attr_mask mask);
334int mthca_query_srq(struct ibv_srq *srq,
335			   struct ibv_srq_attr *attr);
336int mthca_destroy_srq(struct ibv_srq *srq);
337int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
338			struct mthca_srq *srq);
339void mthca_free_srq_wqe(struct mthca_srq *srq, int ind);
340int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq,
341			      struct ibv_recv_wr *wr,
342			      struct ibv_recv_wr **bad_wr);
343int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq,
344			      struct ibv_recv_wr *wr,
345			      struct ibv_recv_wr **bad_wr);
346
347struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
348int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
349		   enum ibv_qp_attr_mask attr_mask,
350		   struct ibv_qp_init_attr *init_attr);
351int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
352		    enum ibv_qp_attr_mask attr_mask);
353int mthca_destroy_qp(struct ibv_qp *qp);
354void mthca_init_qp_indices(struct mthca_qp *qp);
355int mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
356			  struct ibv_send_wr **bad_wr);
357int mthca_tavor_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
358			  struct ibv_recv_wr **bad_wr);
359int mthca_arbel_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
360			  struct ibv_send_wr **bad_wr);
361int mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
362			  struct ibv_recv_wr **bad_wr);
363int mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
364		       enum ibv_qp_type type, struct mthca_qp *qp);
365struct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn);
366int mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct mthca_qp *qp);
367void mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn);
368int mthca_free_err_wqe(struct mthca_qp *qp, int is_send,
369		       int index, int *dbd, uint32_t *new_wqe);
370struct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr);
371int mthca_destroy_ah(struct ibv_ah *ah);
372int mthca_alloc_av(struct mthca_pd *pd, struct ibv_ah_attr *attr,
373		   struct mthca_ah *ah);
374void mthca_free_av(struct mthca_ah *ah);
375int mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
376int mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid);
377
378#endif /* MTHCA_H */
379