1/* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34#ifndef MTHCA_H 35#define MTHCA_H 36 37#include <stddef.h> 38 39#include <infiniband/driver.h> 40#include <infiniband/arch.h> 41 42#ifdef HAVE_VALGRIND_MEMCHECK_H 43 44# include <valgrind/memcheck.h> 45 46# if !defined(VALGRIND_MAKE_MEM_DEFINED) || !defined(VALGRIND_MAKE_MEM_UNDEFINED) 47# warning "Valgrind support requested, but VALGRIND_MAKE_MEM_(UN)DEFINED not available" 48# endif 49 50#endif /* HAVE_VALGRIND_MEMCHECK_H */ 51 52#ifndef VALGRIND_MAKE_MEM_DEFINED 53# define VALGRIND_MAKE_MEM_DEFINED(addr,len) 54#endif 55 56#ifndef VALGRIND_MAKE_MEM_UNDEFINED 57# define VALGRIND_MAKE_MEM_UNDEFINED(addr,len) 58#endif 59 60#ifndef rmb 61# define rmb() mb() 62#endif 63 64#ifndef wmb 65# define wmb() mb() 66#endif 67 68#define HIDDEN __attribute__((visibility ("hidden"))) 69 70#define PFX "mthca: " 71 72enum mthca_hca_type { 73 MTHCA_TAVOR, 74 MTHCA_ARBEL 75}; 76 77enum { 78 MTHCA_CQ_ENTRY_SIZE = 0x20 79}; 80 81enum { 82 MTHCA_QP_TABLE_BITS = 8, 83 MTHCA_QP_TABLE_SIZE = 1 << MTHCA_QP_TABLE_BITS, 84 MTHCA_QP_TABLE_MASK = MTHCA_QP_TABLE_SIZE - 1 85}; 86 87enum { 88 MTHCA_DB_REC_PAGE_SIZE = 4096, 89 MTHCA_DB_REC_PER_PAGE = MTHCA_DB_REC_PAGE_SIZE / 8 90}; 91 92enum mthca_db_type { 93 MTHCA_DB_TYPE_INVALID = 0x0, 94 MTHCA_DB_TYPE_CQ_SET_CI = 0x1, 95 MTHCA_DB_TYPE_CQ_ARM = 0x2, 96 MTHCA_DB_TYPE_SQ = 0x3, 97 MTHCA_DB_TYPE_RQ = 0x4, 98 MTHCA_DB_TYPE_SRQ = 0x5, 99 MTHCA_DB_TYPE_GROUP_SEP = 0x7 100}; 101 102enum { 103 MTHCA_OPCODE_NOP = 0x00, 104 MTHCA_OPCODE_RDMA_WRITE = 0x08, 105 MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09, 106 MTHCA_OPCODE_SEND = 0x0a, 107 MTHCA_OPCODE_SEND_IMM = 0x0b, 108 MTHCA_OPCODE_RDMA_READ = 0x10, 109 MTHCA_OPCODE_ATOMIC_CS = 0x11, 110 MTHCA_OPCODE_ATOMIC_FA = 0x12, 111 MTHCA_OPCODE_BIND_MW = 0x18, 112 MTHCA_OPCODE_INVALID = 0xff 113}; 114 115struct mthca_ah_page; 116 117struct mthca_device { 118 struct ibv_device ibv_dev; 119 enum mthca_hca_type hca_type; 120 int page_size; 121}; 122 123struct mthca_db_table; 124 125struct mthca_context { 126 struct ibv_context ibv_ctx; 127 void *uar; 128 pthread_spinlock_t uar_lock; 129 struct mthca_db_table *db_tab; 130 struct ibv_pd *pd; 131 struct { 132 struct mthca_qp **table; 133 int refcnt; 134 } qp_table[MTHCA_QP_TABLE_SIZE]; 135 pthread_mutex_t qp_table_mutex; 136 int num_qps; 137 int qp_table_shift; 138 int qp_table_mask; 139}; 140 141struct mthca_buf { 142 void *buf; 143 size_t length; 144}; 145 146struct mthca_pd { 147 struct ibv_pd ibv_pd; 148 struct mthca_ah_page *ah_list; 149 pthread_mutex_t ah_mutex; 150 uint32_t pdn; 151}; 152 153struct mthca_cq { 154 struct ibv_cq ibv_cq; 155 struct mthca_buf buf; 156 pthread_spinlock_t lock; 157 struct ibv_mr *mr; 158 uint32_t cqn; 159 uint32_t cons_index; 160 161 /* Next fields are mem-free only */ 162 int set_ci_db_index; 163 uint32_t *set_ci_db; 164 int arm_db_index; 165 uint32_t *arm_db; 166 int arm_sn; 167}; 168 169struct mthca_srq { 170 struct ibv_srq ibv_srq; 171 struct mthca_buf buf; 172 void *last; 173 pthread_spinlock_t lock; 174 struct ibv_mr *mr; 175 uint64_t *wrid; 176 uint32_t srqn; 177 int max; 178 int max_gs; 179 int wqe_shift; 180 int first_free; 181 int last_free; 182 int buf_size; 183 184 /* Next fields are mem-free only */ 185 int db_index; 186 uint32_t *db; 187 uint16_t counter; 188}; 189 190struct mthca_wq { 191 pthread_spinlock_t lock; 192 int max; 193 unsigned next_ind; 194 unsigned last_comp; 195 unsigned head; 196 unsigned tail; 197 void *last; 198 int max_gs; 199 int wqe_shift; 200 201 /* Next fields are mem-free only */ 202 int db_index; 203 uint32_t *db; 204}; 205 206struct mthca_qp { 207 struct ibv_qp ibv_qp; 208 struct mthca_buf buf; 209 uint64_t *wrid; 210 int send_wqe_offset; 211 int max_inline_data; 212 int buf_size; 213 struct mthca_wq sq; 214 struct mthca_wq rq; 215 struct ibv_mr *mr; 216 int sq_sig_all; 217}; 218 219struct mthca_av { 220 uint32_t port_pd; 221 uint8_t reserved1; 222 uint8_t g_slid; 223 uint16_t dlid; 224 uint8_t reserved2; 225 uint8_t gid_index; 226 uint8_t msg_sr; 227 uint8_t hop_limit; 228 uint32_t sl_tclass_flowlabel; 229 uint32_t dgid[4]; 230}; 231 232struct mthca_ah { 233 struct ibv_ah ibv_ah; 234 struct mthca_av *av; 235 struct mthca_ah_page *page; 236 uint32_t key; 237}; 238 239static inline unsigned long align(unsigned long val, unsigned long align) 240{ 241 return (val + align - 1) & ~(align - 1); 242} 243 244static inline uintptr_t db_align(uint32_t *db) 245{ 246 return (uintptr_t) db & ~((uintptr_t) MTHCA_DB_REC_PAGE_SIZE - 1); 247} 248 249#define to_mxxx(xxx, type) \ 250 ((struct mthca_##type *) \ 251 ((void *) ib##xxx - offsetof(struct mthca_##type, ibv_##xxx))) 252 253static inline struct mthca_device *to_mdev(struct ibv_device *ibdev) 254{ 255 return to_mxxx(dev, device); 256} 257 258static inline struct mthca_context *to_mctx(struct ibv_context *ibctx) 259{ 260 return to_mxxx(ctx, context); 261} 262 263static inline struct mthca_pd *to_mpd(struct ibv_pd *ibpd) 264{ 265 return to_mxxx(pd, pd); 266} 267 268static inline struct mthca_cq *to_mcq(struct ibv_cq *ibcq) 269{ 270 return to_mxxx(cq, cq); 271} 272 273static inline struct mthca_srq *to_msrq(struct ibv_srq *ibsrq) 274{ 275 return to_mxxx(srq, srq); 276} 277 278static inline struct mthca_qp *to_mqp(struct ibv_qp *ibqp) 279{ 280 return to_mxxx(qp, qp); 281} 282 283static inline struct mthca_ah *to_mah(struct ibv_ah *ibah) 284{ 285 return to_mxxx(ah, ah); 286} 287 288static inline int mthca_is_memfree(struct ibv_context *ibctx) 289{ 290 return to_mdev(ibctx->device)->hca_type == MTHCA_ARBEL; 291} 292 293int mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size); 294void mthca_free_buf(struct mthca_buf *buf); 295 296int mthca_alloc_db(struct mthca_db_table *db_tab, enum mthca_db_type type, 297 uint32_t **db); 298void mthca_set_db_qn(uint32_t *db, enum mthca_db_type type, uint32_t qn); 299void mthca_free_db(struct mthca_db_table *db_tab, enum mthca_db_type type, int db_index); 300struct mthca_db_table *mthca_alloc_db_tab(int uarc_size); 301void mthca_free_db_tab(struct mthca_db_table *db_tab); 302 303int mthca_query_device(struct ibv_context *context, 304 struct ibv_device_attr *attr); 305int mthca_query_port(struct ibv_context *context, uint8_t port, 306 struct ibv_port_attr *attr); 307 308struct ibv_pd *mthca_alloc_pd(struct ibv_context *context); 309int mthca_free_pd(struct ibv_pd *pd); 310 311struct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr, 312 size_t length, enum ibv_access_flags access); 313int mthca_dereg_mr(struct ibv_mr *mr); 314 315struct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe, 316 struct ibv_comp_channel *channel, 317 int comp_vector); 318int mthca_resize_cq(struct ibv_cq *cq, int cqe); 319int mthca_destroy_cq(struct ibv_cq *cq); 320int mthca_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); 321int mthca_tavor_arm_cq(struct ibv_cq *cq, int solicited); 322int mthca_arbel_arm_cq(struct ibv_cq *cq, int solicited); 323void mthca_arbel_cq_event(struct ibv_cq *cq); 324void __mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq); 325void mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq); 326void mthca_cq_resize_copy_cqes(struct mthca_cq *cq, void *buf, int new_cqe); 327int mthca_alloc_cq_buf(struct mthca_device *dev, struct mthca_buf *buf, int nent); 328 329struct ibv_srq *mthca_create_srq(struct ibv_pd *pd, 330 struct ibv_srq_init_attr *attr); 331int mthca_modify_srq(struct ibv_srq *srq, 332 struct ibv_srq_attr *attr, 333 enum ibv_srq_attr_mask mask); 334int mthca_query_srq(struct ibv_srq *srq, 335 struct ibv_srq_attr *attr); 336int mthca_destroy_srq(struct ibv_srq *srq); 337int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 338 struct mthca_srq *srq); 339void mthca_free_srq_wqe(struct mthca_srq *srq, int ind); 340int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq, 341 struct ibv_recv_wr *wr, 342 struct ibv_recv_wr **bad_wr); 343int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, 344 struct ibv_recv_wr *wr, 345 struct ibv_recv_wr **bad_wr); 346 347struct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); 348int mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 349 enum ibv_qp_attr_mask attr_mask, 350 struct ibv_qp_init_attr *init_attr); 351int mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 352 enum ibv_qp_attr_mask attr_mask); 353int mthca_destroy_qp(struct ibv_qp *qp); 354void mthca_init_qp_indices(struct mthca_qp *qp); 355int mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 356 struct ibv_send_wr **bad_wr); 357int mthca_tavor_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 358 struct ibv_recv_wr **bad_wr); 359int mthca_arbel_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 360 struct ibv_send_wr **bad_wr); 361int mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 362 struct ibv_recv_wr **bad_wr); 363int mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap, 364 enum ibv_qp_type type, struct mthca_qp *qp); 365struct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn); 366int mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct mthca_qp *qp); 367void mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn); 368int mthca_free_err_wqe(struct mthca_qp *qp, int is_send, 369 int index, int *dbd, uint32_t *new_wqe); 370struct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); 371int mthca_destroy_ah(struct ibv_ah *ah); 372int mthca_alloc_av(struct mthca_pd *pd, struct ibv_ah_attr *attr, 373 struct mthca_ah *ah); 374void mthca_free_av(struct mthca_ah *ah); 375int mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); 376int mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); 377 378#endif /* MTHCA_H */ 379