iw_cxgbe.h revision 308304
1316958Sdchagin/* 259243Sobrien * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 359243Sobrien * 459243Sobrien * This software is available to you under a choice of one of two 559243Sobrien * licenses. You may choose to be licensed under the terms of the GNU 659243Sobrien * General Public License (GPL) Version 2, available from the file 759243Sobrien * COPYING in the main directory of this source tree, or the 859243Sobrien * OpenIB.org BSD license below: 959243Sobrien * 1059243Sobrien * Redistribution and use in source and binary forms, with or 1159243Sobrien * without modification, are permitted provided that the following 1259243Sobrien * conditions are met: 1359243Sobrien * 1459243Sobrien * - Redistributions of source code must retain the above 1559243Sobrien * copyright notice, this list of conditions and the following 1659243Sobrien * disclaimer. 17100616Smp * - Redistributions in binary form must reproduce the above 1859243Sobrien * copyright notice, this list of conditions and the following 1959243Sobrien * disclaimer in the documentation and/or other materials 2059243Sobrien * provided with the distribution. 2159243Sobrien * 2259243Sobrien * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 2359243Sobrien * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2459243Sobrien * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 2559243Sobrien * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 2659243Sobrien * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 2759243Sobrien * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 2859243Sobrien * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 2959243Sobrien * SOFTWARE. 3059243Sobrien * 3159243Sobrien * $FreeBSD: stable/10/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h 308304 2016-11-04 18:45:06Z jhb $ 3259243Sobrien */ 3359243Sobrien#ifndef __IW_CXGB4_H__ 3459243Sobrien#define __IW_CXGB4_H__ 35316958Sdchagin 3659243Sobrien#include <linux/list.h> 3759243Sobrien#include <linux/spinlock.h> 3859243Sobrien#include <linux/idr.h> 3969408Sache#include <linux/completion.h> 4059243Sobrien#include <linux/netdevice.h> 4169408Sache#include <linux/sched.h> 4259243Sobrien#include <linux/pci.h> 4359243Sobrien#include <linux/dma-mapping.h> 4459243Sobrien#include <linux/wait.h> 4559243Sobrien#include <linux/kref.h> 4659243Sobrien#include <linux/timer.h> 4759243Sobrien#include <linux/io.h> 4859243Sobrien 4959243Sobrien#include <asm/byteorder.h> 5059243Sobrien 5159243Sobrien#include <netinet/in.h> 5259243Sobrien#include <netinet/toecore.h> 5359243Sobrien 5459243Sobrien#include <rdma/ib_verbs.h> 5559243Sobrien#include <rdma/iw_cm.h> 5659243Sobrien 5759243Sobrien#undef prefetch 5859243Sobrien 5959243Sobrien#include "common/common.h" 6059243Sobrien#include "common/t4_msg.h" 6159243Sobrien#include "common/t4_regs.h" 6259243Sobrien#include "common/t4_tcb.h" 6359243Sobrien#include "t4_l2t.h" 6459243Sobrien 6559243Sobrien#define DRV_NAME "iw_cxgbe" 6659243Sobrien#define MOD DRV_NAME ":" 6759243Sobrien#define KTR_IW_CXGBE KTR_SPARE3 6859243Sobrien 6959243Sobrienextern int c4iw_debug; 7059243Sobrien#define PDBG(fmt, args...) \ 7159243Sobriendo { \ 7259243Sobrien if (c4iw_debug) \ 7359243Sobrien printf(MOD fmt, ## args); \ 7459243Sobrien} while (0) 7559243Sobrien 7659243Sobrien#include "t4.h" 7759243Sobrien 7859243Sobrienstatic inline void *cplhdr(struct mbuf *m) 7959243Sobrien{ 80231990Smp return mtod(m, void*); 8159243Sobrien} 8259243Sobrien 8359243Sobrien#define PBL_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.pbl.start) 8459243Sobrien#define RQT_OFF(rdev_p, a) ((a) - (rdev_p)->adap->vres.rq.start) 8559243Sobrien 8659243Sobrien#define C4IW_ID_TABLE_F_RANDOM 1 /* Pseudo-randomize the id's returned */ 8759243Sobrien#define C4IW_ID_TABLE_F_EMPTY 2 /* Table is initially empty */ 8859243Sobrien 8959243Sobrienstruct c4iw_id_table { 9059243Sobrien u32 flags; 9159243Sobrien u32 start; /* logical minimal id */ 9259243Sobrien u32 last; /* hint for find */ 9359243Sobrien u32 max; 9459243Sobrien spinlock_t lock; 9559243Sobrien unsigned long *table; 96145479Smp}; 9759243Sobrien 98145479Smpstruct c4iw_resource { 9959243Sobrien struct c4iw_id_table tpt_table; 10059243Sobrien struct c4iw_id_table qid_table; 10159243Sobrien struct c4iw_id_table pdid_table; 10259243Sobrien}; 10359243Sobrien 10459243Sobrienstruct c4iw_qid_list { 10559243Sobrien struct list_head entry; 10659243Sobrien u32 qid; 10759243Sobrien}; 10859243Sobrien 10959243Sobrienstruct c4iw_dev_ucontext { 11059243Sobrien struct list_head qpids; 11159243Sobrien struct list_head cqids; 11259243Sobrien struct mutex lock; 11359243Sobrien}; 11459243Sobrien 11559243Sobrienenum c4iw_rdev_flags { 11659243Sobrien T4_FATAL_ERROR = (1<<0), 11759243Sobrien}; 11859243Sobrien 11959243Sobrienstruct c4iw_stat { 12059243Sobrien u64 total; 12159243Sobrien u64 cur; 12259243Sobrien u64 max; 12359243Sobrien u64 fail; 12459243Sobrien}; 12559243Sobrien 12659243Sobrienstruct c4iw_stats { 12759243Sobrien struct mutex lock; 12859243Sobrien struct c4iw_stat qid; 12959243Sobrien struct c4iw_stat pd; 13059243Sobrien struct c4iw_stat stag; 13159243Sobrien struct c4iw_stat pbl; 13259243Sobrien struct c4iw_stat rqt; 13359243Sobrien u64 db_full; 13459243Sobrien u64 db_empty; 13559243Sobrien u64 db_drop; 13659243Sobrien u64 db_state_transitions; 13759243Sobrien}; 13859243Sobrien 13959243Sobrienstruct c4iw_rdev { 14059243Sobrien struct adapter *adap; 14159243Sobrien struct c4iw_resource resource; 14259243Sobrien unsigned long qpshift; 14359243Sobrien u32 qpmask; 144231990Smp unsigned long cqshift; 145167465Smp u32 cqmask; 146167465Smp struct c4iw_dev_ucontext uctx; 147167465Smp struct gen_pool *pbl_pool; 14859243Sobrien struct gen_pool *rqt_pool; 14959243Sobrien u32 flags; 150167465Smp struct c4iw_stats stats; 15159243Sobrien}; 152231990Smp 153100616Smpstatic inline int c4iw_fatal_error(struct c4iw_rdev *rdev) 154231990Smp{ 155100616Smp return rdev->flags & T4_FATAL_ERROR; 15659243Sobrien} 15759243Sobrien 15859243Sobrienstatic inline int c4iw_num_stags(struct c4iw_rdev *rdev) 15959243Sobrien{ 16059243Sobrien return min((int)T4_MAX_NUM_STAG, (int)(rdev->adap->vres.stag.size >> 5)); 16159243Sobrien} 16259243Sobrien 16359243Sobrien#define C4IW_WR_TO (10*HZ) 16459243Sobrien 165100616Smpstruct c4iw_wr_wait { 166100616Smp int ret; 167167465Smp atomic_t completion; 16859243Sobrien}; 169167465Smp 17059243Sobrienstatic inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp) 17159243Sobrien{ 17259243Sobrien wr_waitp->ret = 0; 17359243Sobrien atomic_set(&wr_waitp->completion, 0); 17459243Sobrien} 17559243Sobrien 17659243Sobrienstatic inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret) 177167465Smp{ 17859243Sobrien wr_waitp->ret = ret; 17959243Sobrien atomic_set(&wr_waitp->completion, 1); 18059243Sobrien wakeup(wr_waitp); 18159243Sobrien} 18259243Sobrien 18359243Sobrienstatic inline int 18459243Sobrienc4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp, 18559243Sobrien u32 hwtid, u32 qpid, const char *func) 18659243Sobrien{ 18759243Sobrien struct adapter *sc = rdev->adap; 188167465Smp unsigned to = C4IW_WR_TO; 18959243Sobrien 19059243Sobrien while (!atomic_read(&wr_waitp->completion)) { 19159243Sobrien tsleep(wr_waitp, 0, "c4iw_wait", to); 19259243Sobrien if (SIGPENDING(curthread)) { 19359243Sobrien printf("%s - Device %s not responding - " 19459243Sobrien "tid %u qpid %u\n", func, 19559243Sobrien device_get_nameunit(sc->dev), hwtid, qpid); 19659243Sobrien if (c4iw_fatal_error(rdev)) { 197100616Smp wr_waitp->ret = -EIO; 198167465Smp break; 19959243Sobrien } 200167465Smp to = to << 2; 20159243Sobrien } 20259243Sobrien } 20359243Sobrien if (wr_waitp->ret) 20459243Sobrien CTR4(KTR_IW_CXGBE, "%s: FW reply %d tid %u qpid %u", 20559243Sobrien device_get_nameunit(sc->dev), wr_waitp->ret, hwtid, qpid); 20659243Sobrien return (wr_waitp->ret); 20759243Sobrien} 20859243Sobrien 20959243Sobrienenum db_state { 210167465Smp NORMAL = 0, 211167465Smp FLOW_CONTROL = 1, 21259243Sobrien RECOVERY = 2 213167465Smp}; 214167465Smp 21559243Sobrienstruct c4iw_dev { 21659243Sobrien struct ib_device ibdev; 21759243Sobrien struct c4iw_rdev rdev; 21859243Sobrien u32 device_cap_flags; 21959243Sobrien struct idr cqidr; 22059243Sobrien struct idr qpidr; 22159243Sobrien struct idr mmidr; 22259243Sobrien spinlock_t lock; 22359243Sobrien struct dentry *debugfs_root; 22459243Sobrien enum db_state db_state; 22559243Sobrien int qpcnt; 22659243Sobrien}; 22759243Sobrien 22859243Sobrienstatic inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev) 22959243Sobrien{ 23059243Sobrien return container_of(ibdev, struct c4iw_dev, ibdev); 23159243Sobrien} 23259243Sobrien 23359243Sobrienstatic inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev) 23459243Sobrien{ 23559243Sobrien return container_of(rdev, struct c4iw_dev, rdev); 23659243Sobrien} 23759243Sobrien 23859243Sobrienstatic inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid) 23959243Sobrien{ 24059243Sobrien return idr_find(&rhp->cqidr, cqid); 241167465Smp} 242167465Smp 243167465Smpstatic inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid) 244167465Smp{ 245167465Smp return idr_find(&rhp->qpidr, qpid); 246167465Smp} 247167465Smp 248167465Smpstatic inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid) 249167465Smp{ 25059243Sobrien return idr_find(&rhp->mmidr, mmid); 25159243Sobrien} 25259243Sobrien 25359243Sobrienstatic inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr, 25459243Sobrien void *handle, u32 id, int lock) 255100616Smp{ 256231990Smp int ret; 25759243Sobrien int newid; 258231990Smp 25959243Sobrien do { 260167465Smp if (!idr_pre_get(idr, lock ? GFP_KERNEL : GFP_ATOMIC)) 261167465Smp return -ENOMEM; 262167465Smp if (lock) 26359243Sobrien spin_lock_irq(&rhp->lock); 26459243Sobrien ret = idr_get_new_above(idr, handle, id, &newid); 26559243Sobrien BUG_ON(!ret && newid != id); 26659243Sobrien if (lock) 267231990Smp spin_unlock_irq(&rhp->lock); 268231990Smp } while (ret == -EAGAIN); 269231990Smp 27059243Sobrien return ret; 27159243Sobrien} 27259243Sobrien 27359243Sobrienstatic inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr, 27459243Sobrien void *handle, u32 id) 27559243Sobrien{ 27659243Sobrien return _insert_handle(rhp, idr, handle, id, 1); 27759243Sobrien} 27859243Sobrien 27959243Sobrienstatic inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr, 28059243Sobrien void *handle, u32 id) 28159243Sobrien{ 28259243Sobrien return _insert_handle(rhp, idr, handle, id, 0); 28359243Sobrien} 28459243Sobrien 28559243Sobrienstatic inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr, 28659243Sobrien u32 id, int lock) 28759243Sobrien{ 28859243Sobrien if (lock) 28959243Sobrien spin_lock_irq(&rhp->lock); 29059243Sobrien idr_remove(idr, id); 29159243Sobrien if (lock) 29259243Sobrien spin_unlock_irq(&rhp->lock); 29359243Sobrien} 29459243Sobrien 295167465Smpstatic inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id) 296167465Smp{ 297167465Smp _remove_handle(rhp, idr, id, 1); 29859243Sobrien} 29959243Sobrien 30059243Sobrienstatic inline void remove_handle_nolock(struct c4iw_dev *rhp, 30159243Sobrien struct idr *idr, u32 id) 302167465Smp{ 303167465Smp _remove_handle(rhp, idr, id, 0); 304167465Smp} 30559243Sobrien 306167465Smpstruct c4iw_pd { 30759243Sobrien struct ib_pd ibpd; 30859243Sobrien u32 pdid; 30959243Sobrien struct c4iw_dev *rhp; 31059243Sobrien}; 31159243Sobrien 31259243Sobrienstatic inline struct c4iw_pd *to_c4iw_pd(struct ib_pd *ibpd) 31359243Sobrien{ 31459243Sobrien return container_of(ibpd, struct c4iw_pd, ibpd); 31559243Sobrien} 31659243Sobrien 31759243Sobrienstruct tpt_attributes { 318167465Smp u64 len; 319231990Smp u64 va_fbo; 320231990Smp enum fw_ri_mem_perms perms; 321231990Smp u32 stag; 322231990Smp u32 pdid; 323231990Smp u32 qpid; 324167465Smp u32 pbl_addr; 325167465Smp u32 pbl_size; 326167465Smp u32 state:1; 32759243Sobrien u32 type:2; 328167465Smp u32 rsvd:1; 32959243Sobrien u32 remote_invaliate_disable:1; 33059243Sobrien u32 zbva:1; 33159243Sobrien u32 mw_bind_enable:1; 33259243Sobrien u32 page_size:5; 33359243Sobrien}; 334167465Smp 33559243Sobrienstruct c4iw_mr { 33659243Sobrien struct ib_mr ibmr; 33759243Sobrien struct ib_umem *umem; 33859243Sobrien struct c4iw_dev *rhp; 33959243Sobrien u64 kva; 34059243Sobrien struct tpt_attributes attr; 34159243Sobrien}; 342167465Smp 34359243Sobrienstatic inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr) 34459243Sobrien{ 34559243Sobrien return container_of(ibmr, struct c4iw_mr, ibmr); 34659243Sobrien} 34759243Sobrien 34859243Sobrienstruct c4iw_mw { 34959243Sobrien struct ib_mw ibmw; 35059243Sobrien struct c4iw_dev *rhp; 35159243Sobrien u64 kva; 35259243Sobrien struct tpt_attributes attr; 35359243Sobrien}; 35459243Sobrien 35559243Sobrienstatic inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw) 35659243Sobrien{ 35759243Sobrien return container_of(ibmw, struct c4iw_mw, ibmw); 358167465Smp} 35959243Sobrien 360145479Smpstruct c4iw_fr_page_list { 361145479Smp struct ib_fast_reg_page_list ibpl; 362145479Smp DECLARE_PCI_UNMAP_ADDR(mapping); 36359243Sobrien dma_addr_t dma_addr; 36459243Sobrien struct c4iw_dev *dev; 36559243Sobrien int size; 36659243Sobrien}; 36759243Sobrien 36859243Sobrienstatic inline struct c4iw_fr_page_list *to_c4iw_fr_page_list( 36959243Sobrien struct ib_fast_reg_page_list *ibpl) 37059243Sobrien{ 37159243Sobrien return container_of(ibpl, struct c4iw_fr_page_list, ibpl); 37259243Sobrien} 37359243Sobrien 37459243Sobrienstruct c4iw_cq { 37559243Sobrien struct ib_cq ibcq; 37659243Sobrien struct c4iw_dev *rhp; 37759243Sobrien struct t4_cq cq; 37859243Sobrien spinlock_t lock; 37959243Sobrien spinlock_t comp_handler_lock; 38059243Sobrien atomic_t refcnt; 38159243Sobrien wait_queue_head_t wait; 38259243Sobrien}; 38359243Sobrien 38459243Sobrienstatic inline struct c4iw_cq *to_c4iw_cq(struct ib_cq *ibcq) 38559243Sobrien{ 38659243Sobrien return container_of(ibcq, struct c4iw_cq, ibcq); 38759243Sobrien} 38859243Sobrien 38959243Sobrienstruct c4iw_mpa_attributes { 39059243Sobrien u8 initiator; 39159243Sobrien u8 recv_marker_enabled; 39259243Sobrien u8 xmit_marker_enabled; 39359243Sobrien u8 crc_enabled; 39459243Sobrien u8 enhanced_rdma_conn; 39559243Sobrien u8 version; 39659243Sobrien u8 p2p_type; 39759243Sobrien}; 39859243Sobrien 39959243Sobrienstruct c4iw_qp_attributes { 40059243Sobrien u32 scq; 40159243Sobrien u32 rcq; 40259243Sobrien u32 sq_num_entries; 40359243Sobrien u32 rq_num_entries; 40469408Sache u32 sq_max_sges; 40559243Sobrien u32 sq_max_sges_rdma_write; 40669408Sache u32 rq_max_sges; 40759243Sobrien u32 state; 40859243Sobrien u8 enable_rdma_read; 40959243Sobrien u8 enable_rdma_write; 41059243Sobrien u8 enable_bind; 411167465Smp u8 enable_mmid0_fastreg; 41259243Sobrien u32 max_ord; 413167465Smp u32 max_ird; 414145479Smp u32 pd; 415145479Smp u32 next_state; 416167465Smp char terminate_buffer[52]; 417167465Smp u32 terminate_msg_len; 418167465Smp u8 is_terminate_local; 419167465Smp struct c4iw_mpa_attributes mpa_attr; 42059243Sobrien struct c4iw_ep *llp_stream_handle; 42159243Sobrien u8 layer_etype; 42259243Sobrien u8 ecode; 423167465Smp u16 sq_db_inc; 42459243Sobrien u16 rq_db_inc; 42559243Sobrien}; 426167465Smp 42759243Sobrienstruct c4iw_qp { 428167465Smp struct ib_qp ibqp; 429167465Smp struct c4iw_dev *rhp; 430167465Smp struct c4iw_ep *ep; 431167465Smp struct c4iw_qp_attributes attr; 43259243Sobrien struct t4_wq wq; 433167465Smp spinlock_t lock; 43459243Sobrien struct mutex mutex; 43559243Sobrien atomic_t refcnt; 43659243Sobrien wait_queue_head_t wait; 43759243Sobrien struct timer_list timer; 43859243Sobrien}; 43959243Sobrien 44059243Sobrienstatic inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) 44159243Sobrien{ 44259243Sobrien return container_of(ibqp, struct c4iw_qp, ibqp); 44359243Sobrien} 44459243Sobrien 44559243Sobrienstruct c4iw_ucontext { 44659243Sobrien struct ib_ucontext ibucontext; 44759243Sobrien struct c4iw_dev_ucontext uctx; 44859243Sobrien u32 key; 44959243Sobrien spinlock_t mmap_lock; 45059243Sobrien struct list_head mmaps; 45159243Sobrien}; 45259243Sobrien 453100616Smpstatic inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c) 45459243Sobrien{ 45559243Sobrien return container_of(c, struct c4iw_ucontext, ibucontext); 45659243Sobrien} 45759243Sobrien 45859243Sobrienstruct c4iw_mm_entry { 45959243Sobrien struct list_head entry; 46059243Sobrien u64 addr; 46159243Sobrien u32 key; 46259243Sobrien unsigned len; 46359243Sobrien}; 46459243Sobrien 46559243Sobrienstatic inline struct c4iw_mm_entry *remove_mmap(struct c4iw_ucontext *ucontext, 46659243Sobrien u32 key, unsigned len) 46759243Sobrien{ 46859243Sobrien struct list_head *pos, *nxt; 46959243Sobrien struct c4iw_mm_entry *mm; 47059243Sobrien 47159243Sobrien spin_lock(&ucontext->mmap_lock); 47259243Sobrien list_for_each_safe(pos, nxt, &ucontext->mmaps) { 473167465Smp 47459243Sobrien mm = list_entry(pos, struct c4iw_mm_entry, entry); 47559243Sobrien if (mm->key == key && mm->len == len) { 47659243Sobrien list_del_init(&mm->entry); 47759243Sobrien spin_unlock(&ucontext->mmap_lock); 47859243Sobrien CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", 47959243Sobrien __func__, key, (unsigned long long) mm->addr, 48059243Sobrien mm->len); 48159243Sobrien return mm; 48259243Sobrien } 48359243Sobrien } 484167465Smp spin_unlock(&ucontext->mmap_lock); 48559243Sobrien return NULL; 48659243Sobrien} 48759243Sobrien 48859243Sobrienstatic inline void insert_mmap(struct c4iw_ucontext *ucontext, 48959243Sobrien struct c4iw_mm_entry *mm) 49059243Sobrien{ 49159243Sobrien spin_lock(&ucontext->mmap_lock); 49259243Sobrien CTR4(KTR_IW_CXGBE, "%s key 0x%x addr 0x%llx len %d", __func__, mm->key, 49359243Sobrien (unsigned long long) mm->addr, mm->len); 49459243Sobrien list_add_tail(&mm->entry, &ucontext->mmaps); 49559243Sobrien spin_unlock(&ucontext->mmap_lock); 49659243Sobrien} 49759243Sobrien 49859243Sobrienenum c4iw_qp_attr_mask { 49959243Sobrien C4IW_QP_ATTR_NEXT_STATE = 1 << 0, 50059243Sobrien C4IW_QP_ATTR_SQ_DB = 1<<1, 50159243Sobrien C4IW_QP_ATTR_RQ_DB = 1<<2, 50259243Sobrien C4IW_QP_ATTR_ENABLE_RDMA_READ = 1 << 7, 50359243Sobrien C4IW_QP_ATTR_ENABLE_RDMA_WRITE = 1 << 8, 50459243Sobrien C4IW_QP_ATTR_ENABLE_RDMA_BIND = 1 << 9, 50559243Sobrien C4IW_QP_ATTR_MAX_ORD = 1 << 11, 50659243Sobrien C4IW_QP_ATTR_MAX_IRD = 1 << 12, 50759243Sobrien C4IW_QP_ATTR_LLP_STREAM_HANDLE = 1 << 22, 50859243Sobrien C4IW_QP_ATTR_STREAM_MSG_BUFFER = 1 << 23, 50959243Sobrien C4IW_QP_ATTR_MPA_ATTR = 1 << 24, 51059243Sobrien C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE = 1 << 25, 51159243Sobrien C4IW_QP_ATTR_VALID_MODIFY = (C4IW_QP_ATTR_ENABLE_RDMA_READ | 51259243Sobrien C4IW_QP_ATTR_ENABLE_RDMA_WRITE | 51359243Sobrien C4IW_QP_ATTR_MAX_ORD | 51459243Sobrien C4IW_QP_ATTR_MAX_IRD | 51559243Sobrien C4IW_QP_ATTR_LLP_STREAM_HANDLE | 51659243Sobrien C4IW_QP_ATTR_STREAM_MSG_BUFFER | 517167465Smp C4IW_QP_ATTR_MPA_ATTR | 51859243Sobrien C4IW_QP_ATTR_QP_CONTEXT_ACTIVATE) 51959243Sobrien}; 52059243Sobrien 52159243Sobrienint c4iw_modify_qp(struct c4iw_dev *rhp, 52259243Sobrien struct c4iw_qp *qhp, 52359243Sobrien enum c4iw_qp_attr_mask mask, 52459243Sobrien struct c4iw_qp_attributes *attrs, 52559243Sobrien int internal); 52659243Sobrien 527167465Smpenum c4iw_qp_state { 52859243Sobrien C4IW_QP_STATE_IDLE, 529167465Smp C4IW_QP_STATE_RTS, 530167465Smp C4IW_QP_STATE_ERROR, 531167465Smp C4IW_QP_STATE_TERMINATE, 53259243Sobrien C4IW_QP_STATE_CLOSING, 533167465Smp C4IW_QP_STATE_TOT 534167465Smp}; 535167465Smp 536167465Smpstatic inline int c4iw_convert_state(enum ib_qp_state ib_state) 537167465Smp{ 538167465Smp switch (ib_state) { 539167465Smp case IB_QPS_RESET: 540167465Smp case IB_QPS_INIT: 541167465Smp return C4IW_QP_STATE_IDLE; 542167465Smp case IB_QPS_RTS: 543167465Smp return C4IW_QP_STATE_RTS; 544167465Smp case IB_QPS_SQD: 545167465Smp return C4IW_QP_STATE_CLOSING; 546167465Smp case IB_QPS_SQE: 547167465Smp return C4IW_QP_STATE_TERMINATE; 548167465Smp case IB_QPS_ERR: 549167465Smp return C4IW_QP_STATE_ERROR; 550167465Smp default: 55159243Sobrien return -1; 552167465Smp } 553167465Smp} 554167465Smp 555167465Smpstatic inline int to_ib_qp_state(int c4iw_qp_state) 556167465Smp{ 557167465Smp switch (c4iw_qp_state) { 558167465Smp case C4IW_QP_STATE_IDLE: 559167465Smp return IB_QPS_INIT; 560167465Smp case C4IW_QP_STATE_RTS: 561167465Smp return IB_QPS_RTS; 56259243Sobrien case C4IW_QP_STATE_CLOSING: 563167465Smp return IB_QPS_SQD; 564167465Smp case C4IW_QP_STATE_TERMINATE: 565167465Smp return IB_QPS_SQE; 566167465Smp case C4IW_QP_STATE_ERROR: 567167465Smp return IB_QPS_ERR; 568167465Smp } 56959243Sobrien return IB_QPS_ERR; 57059243Sobrien} 57159243Sobrien 57259243Sobrienstatic inline u32 c4iw_ib_to_tpt_access(int a) 57359243Sobrien{ 57459243Sobrien return (a & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) | 57559243Sobrien (a & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0) | 57659243Sobrien (a & IB_ACCESS_LOCAL_WRITE ? FW_RI_MEM_ACCESS_LOCAL_WRITE : 0) | 57759243Sobrien FW_RI_MEM_ACCESS_LOCAL_READ; 57859243Sobrien} 579167465Smp 580167465Smpstatic inline u32 c4iw_ib_to_tpt_bind_access(int acc) 581167465Smp{ 58259243Sobrien return (acc & IB_ACCESS_REMOTE_WRITE ? FW_RI_MEM_ACCESS_REM_WRITE : 0) | 583167465Smp (acc & IB_ACCESS_REMOTE_READ ? FW_RI_MEM_ACCESS_REM_READ : 0); 58459243Sobrien} 585167465Smp 58659243Sobrienenum c4iw_mmid_state { 587167465Smp C4IW_STAG_STATE_VALID, 588167465Smp C4IW_STAG_STATE_INVALID 589167465Smp}; 590167465Smp 59159243Sobrien#define C4IW_NODE_DESC "iw_cxgbe Chelsio Communications" 592167465Smp 593167465Smp#define MPA_KEY_REQ "MPA ID Req Frame" 594167465Smp#define MPA_KEY_REP "MPA ID Rep Frame" 595167465Smp 596167465Smp#define MPA_MAX_PRIVATE_DATA 256 59759243Sobrien#define MPA_ENHANCED_RDMA_CONN 0x10 59859243Sobrien#define MPA_REJECT 0x20 599167465Smp#define MPA_CRC 0x40 600167465Smp#define MPA_MARKERS 0x80 601167465Smp#define MPA_FLAGS_MASK 0xE0 60259243Sobrien 603167465Smp#define MPA_V2_PEER2PEER_MODEL 0x8000 60459243Sobrien#define MPA_V2_ZERO_LEN_FPDU_RTR 0x4000 605167465Smp#define MPA_V2_RDMA_WRITE_RTR 0x8000 606167465Smp#define MPA_V2_RDMA_READ_RTR 0x4000 607167465Smp#define MPA_V2_IRD_ORD_MASK 0x3FFF 608167465Smp 609316958Sdchagin/* Fixme: Use atomic_read for kref.count as same as Linux */ 610167465Smp#define c4iw_put_ep(ep) { \ 611316958Sdchagin CTR4(KTR_IW_CXGBE, "put_ep (%s:%u) ep %p, refcnt %d", \ 612316958Sdchagin __func__, __LINE__, ep, (ep)->kref.count); \ 61369408Sache WARN_ON((ep)->kref.count < 1); \ 614167465Smp kref_put(&((ep)->kref), _c4iw_free_ep); \ 61569408Sache} 616167465Smp 61759243Sobrien/* Fixme: Use atomic_read for kref.count as same as Linux */ 618167465Smp#define c4iw_get_ep(ep) { \ 61959243Sobrien CTR4(KTR_IW_CXGBE, "get_ep (%s:%u) ep %p, refcnt %d", \ 62059243Sobrien __func__, __LINE__, ep, (ep)->kref.count); \ 62159243Sobrien kref_get(&((ep)->kref)); \ 622167465Smp} 62359243Sobrien 62459243Sobrienvoid _c4iw_free_ep(struct kref *kref); 625100616Smp 62659243Sobrienstruct mpa_message { 62759243Sobrien u8 key[16]; 628100616Smp u8 flags; 62959243Sobrien u8 revision; 630100616Smp __be16 private_data_size; 631100616Smp u8 private_data[0]; 63259243Sobrien}; 63359243Sobrien 63459243Sobrienstruct mpa_v2_conn_params { 63559243Sobrien __be16 ird; 63659243Sobrien __be16 ord; 63759243Sobrien}; 638167465Smp 63959243Sobrienstruct terminate_message { 64059243Sobrien u8 layer_etype; 64159243Sobrien u8 ecode; 64259243Sobrien __be16 hdrct_rsvd; 64359243Sobrien u8 len_hdrs[0]; 644145479Smp}; 64559243Sobrien 64659243Sobrien#define TERM_MAX_LENGTH (sizeof(struct terminate_message) + 2 + 18 + 28) 64759243Sobrien 64859243Sobrienenum c4iw_layers_types { 64969408Sache LAYER_RDMAP = 0x00, 65059243Sobrien LAYER_DDP = 0x10, 65159243Sobrien LAYER_MPA = 0x20, 65269408Sache RDMAP_LOCAL_CATA = 0x00, 65359243Sobrien RDMAP_REMOTE_PROT = 0x01, 65459243Sobrien RDMAP_REMOTE_OP = 0x02, 65559243Sobrien DDP_LOCAL_CATA = 0x00, 65659243Sobrien DDP_TAGGED_ERR = 0x01, 65759243Sobrien DDP_UNTAGGED_ERR = 0x02, 65859243Sobrien DDP_LLP = 0x03 65959243Sobrien}; 66059243Sobrien 66159243Sobrienenum c4iw_rdma_ecodes { 66259243Sobrien RDMAP_INV_STAG = 0x00, 66359243Sobrien RDMAP_BASE_BOUNDS = 0x01, 66459243Sobrien RDMAP_ACC_VIOL = 0x02, 66559243Sobrien RDMAP_STAG_NOT_ASSOC = 0x03, 66659243Sobrien RDMAP_TO_WRAP = 0x04, 66759243Sobrien RDMAP_INV_VERS = 0x05, 66859243Sobrien RDMAP_INV_OPCODE = 0x06, 66959243Sobrien RDMAP_STREAM_CATA = 0x07, 67059243Sobrien RDMAP_GLOBAL_CATA = 0x08, 67159243Sobrien RDMAP_CANT_INV_STAG = 0x09, 67259243Sobrien RDMAP_UNSPECIFIED = 0xff 67369408Sache}; 67469408Sache 675100616Smpenum c4iw_ddp_ecodes { 67659243Sobrien DDPT_INV_STAG = 0x00, 67759243Sobrien DDPT_BASE_BOUNDS = 0x01, 67859243Sobrien DDPT_STAG_NOT_ASSOC = 0x02, 67959243Sobrien DDPT_TO_WRAP = 0x03, 68059243Sobrien DDPT_INV_VERS = 0x04, 68159243Sobrien DDPU_INV_QN = 0x01, 68259243Sobrien DDPU_INV_MSN_NOBUF = 0x02, 68359243Sobrien DDPU_INV_MSN_RANGE = 0x03, 68459243Sobrien DDPU_INV_MO = 0x04, 68559243Sobrien DDPU_MSG_TOOBIG = 0x05, 68659243Sobrien DDPU_INV_VERS = 0x06 68759243Sobrien}; 68859243Sobrien 68959243Sobrienenum c4iw_mpa_ecodes { 69059243Sobrien MPA_CRC_ERR = 0x02, 691167465Smp MPA_MARKER_ERR = 0x03, 692167465Smp MPA_LOCAL_CATA = 0x05, 693167465Smp MPA_INSUFF_IRD = 0x06, 69459243Sobrien MPA_NOMATCH_RTR = 0x07, 69559243Sobrien}; 69659243Sobrien 69759243Sobrienenum c4iw_ep_state { 69859243Sobrien IDLE = 0, 69959243Sobrien LISTEN, 70059243Sobrien CONNECTING, 701100616Smp MPA_REQ_WAIT, 70259243Sobrien MPA_REQ_SENT, 70359243Sobrien MPA_REQ_RCVD, 70459243Sobrien MPA_REP_SENT, 70559243Sobrien FPDU_MODE, 70659243Sobrien ABORTING, 707167465Smp CLOSING, 70859243Sobrien MORIBUND, 70959243Sobrien DEAD, 71059243Sobrien}; 711167465Smp 71259243Sobrienenum c4iw_ep_flags { 71359243Sobrien PEER_ABORT_IN_PROGRESS = 0, 71459243Sobrien ABORT_REQ_IN_PROGRESS = 1, 71569408Sache RELEASE_RESOURCES = 2, 71659243Sobrien CLOSE_SENT = 3, 71769408Sache TIMEOUT = 4 71859243Sobrien}; 71959243Sobrien 72059243Sobrienenum c4iw_ep_history { 72159243Sobrien ACT_OPEN_REQ = 0, 72259243Sobrien ACT_OFLD_CONN = 1, 72359243Sobrien ACT_OPEN_RPL = 2, 72459243Sobrien ACT_ESTAB = 3, 72569408Sache PASS_ACCEPT_REQ = 4, 72659243Sobrien PASS_ESTAB = 5, 72769408Sache ABORT_UPCALL = 6, 72869408Sache ESTAB_UPCALL = 7, 72959243Sobrien CLOSE_UPCALL = 8, 73059243Sobrien ULP_ACCEPT = 9, 731231990Smp ULP_REJECT = 10, 732231990Smp TIMEDOUT = 11, 73359243Sobrien PEER_ABORT = 12, 73459243Sobrien PEER_CLOSE = 13, 735167465Smp CONNREQ_UPCALL = 14, 736131962Smp ABORT_CONN = 15, 737231990Smp DISCONN_UPCALL = 16, 738131962Smp EP_DISC_CLOSE = 17, 739231990Smp EP_DISC_ABORT = 18, 740167465Smp CONN_RPL_UPCALL = 19, 741167465Smp ACT_RETRY_NOMEM = 20, 742167465Smp ACT_RETRY_INUSE = 21 743167465Smp}; 744167465Smp 745131962Smpstruct c4iw_ep_common { 746167465Smp TAILQ_ENTRY(c4iw_ep_common) entry; /* Work queue attachment */ 747167465Smp struct iw_cm_id *cm_id; 74859243Sobrien struct c4iw_qp *qp; 74969408Sache struct c4iw_dev *dev; 75059243Sobrien enum c4iw_ep_state state; 75159243Sobrien struct kref kref; 75259243Sobrien struct mutex mutex; 75359243Sobrien struct sockaddr_in local_addr; 75459243Sobrien struct sockaddr_in remote_addr; 75559243Sobrien struct c4iw_wr_wait wr_wait; 75659243Sobrien unsigned long flags; 75759243Sobrien unsigned long history; 75859243Sobrien int rpl_err; 75959243Sobrien int rpl_done; 76059243Sobrien struct thread *thread; 76169408Sache struct socket *so; 76259243Sobrien}; 763167465Smp 76459243Sobrienstruct c4iw_listen_ep { 76559243Sobrien struct c4iw_ep_common com; 76659243Sobrien unsigned int stid; 76759243Sobrien int backlog; 76859243Sobrien}; 769167465Smp 77059243Sobrienstruct c4iw_ep { 77159243Sobrien struct c4iw_ep_common com; 77259243Sobrien struct c4iw_ep *parent_ep; 77359243Sobrien struct timer_list timer; 77459243Sobrien struct list_head entry; 775167465Smp unsigned int atid; 776167465Smp u32 hwtid; 77759243Sobrien u32 snd_seq; 77859243Sobrien u32 rcv_seq; 77959243Sobrien struct l2t_entry *l2t; 78059243Sobrien struct dst_entry *dst; 78159243Sobrien struct c4iw_mpa_attributes mpa_attr; 782167465Smp u8 mpa_pkt[sizeof(struct mpa_message) + MPA_MAX_PRIVATE_DATA]; 78359243Sobrien unsigned int mpa_pkt_len; 78459243Sobrien u32 ird; 78559243Sobrien u32 ord; 78659243Sobrien u32 smac_idx; 78759243Sobrien u32 tx_chan; 78859243Sobrien u32 mtu; 78959243Sobrien u16 mss; 79059243Sobrien u16 emss; 79159243Sobrien u16 plen; 79259243Sobrien u16 rss_qid; 79359243Sobrien u16 txq_idx; 79459243Sobrien u16 ctrlq_idx; 79559243Sobrien u8 tos; 79659243Sobrien u8 retry_with_mpa_v1; 79759243Sobrien u8 tried_with_mpa_v1; 79859243Sobrien}; 79959243Sobrien 80059243Sobrienstatic inline struct c4iw_ep *to_ep(struct iw_cm_id *cm_id) 80159243Sobrien{ 80259243Sobrien return cm_id->provider_data; 80359243Sobrien} 80469408Sache 805167465Smpstatic inline struct c4iw_listen_ep *to_listen_ep(struct iw_cm_id *cm_id) 80659243Sobrien{ 807145479Smp return cm_id->provider_data; 80859243Sobrien} 80959243Sobrien 81059243Sobrienstatic inline int compute_wscale(int win) 81159243Sobrien{ 81259243Sobrien int wscale = 0; 81359243Sobrien 81459243Sobrien while (wscale < 14 && (65535<<wscale) < win) 815167465Smp wscale++; 81659243Sobrien return wscale; 817231990Smp} 818145479Smp 819145479Smpu32 c4iw_id_alloc(struct c4iw_id_table *alloc); 820145479Smpvoid c4iw_id_free(struct c4iw_id_table *alloc, u32 obj); 821231990Smpint c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num, 82259243Sobrien u32 reserved, u32 flags); 82359243Sobrienvoid c4iw_id_table_free(struct c4iw_id_table *alloc); 824100616Smp 825231990Smptypedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct mbuf *m); 82659243Sobrien 827231990Smpint c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 82859243Sobrien struct l2t_entry *l2t); 82959243Sobrienu32 c4iw_get_resource(struct c4iw_id_table *id_table); 830231990Smpvoid c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry); 831231990Smpint c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid); 832231990Smpint c4iw_init_ctrl_qp(struct c4iw_rdev *rdev); 833231990Smpint c4iw_pblpool_create(struct c4iw_rdev *rdev); 83459243Sobrienint c4iw_rqtpool_create(struct c4iw_rdev *rdev); 83559243Sobrienvoid c4iw_pblpool_destroy(struct c4iw_rdev *rdev); 83659243Sobrienvoid c4iw_rqtpool_destroy(struct c4iw_rdev *rdev); 83759243Sobrienvoid c4iw_destroy_resource(struct c4iw_resource *rscp); 83859243Sobrienint c4iw_destroy_ctrl_qp(struct c4iw_rdev *rdev); 83959243Sobrienint c4iw_register_device(struct c4iw_dev *dev); 84059243Sobrienvoid c4iw_unregister_device(struct c4iw_dev *dev); 84159243Sobrienint __init c4iw_cm_init(void); 84259243Sobrienvoid __exit c4iw_cm_term(void); 84359243Sobrienvoid c4iw_release_dev_ucontext(struct c4iw_rdev *rdev, 84459243Sobrien struct c4iw_dev_ucontext *uctx); 84559243Sobrienvoid c4iw_init_dev_ucontext(struct c4iw_rdev *rdev, 84659243Sobrien struct c4iw_dev_ucontext *uctx); 84759243Sobrienint c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); 848167465Smpint c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 84959243Sobrien struct ib_send_wr **bad_wr); 85059243Sobrienint c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 85159243Sobrien struct ib_recv_wr **bad_wr); 85259243Sobrienint c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw, 85359243Sobrien struct ib_mw_bind *mw_bind); 854167465Smpint c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param); 85559243Sobrienint c4iw_create_listen(struct iw_cm_id *cm_id, int backlog); 85659243Sobrienint c4iw_destroy_listen(struct iw_cm_id *cm_id); 85759243Sobrienint c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param); 85859243Sobrienint c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len); 85959243Sobrienvoid c4iw_qp_add_ref(struct ib_qp *qp); 86059243Sobrienvoid c4iw_qp_rem_ref(struct ib_qp *qp); 86159243Sobrienvoid c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list); 862231990Smpstruct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl( 863231990Smp struct ib_device *device, 864231990Smp int page_list_len); 865231990Smpstruct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth); 866231990Smpint c4iw_dealloc_mw(struct ib_mw *mw); 867167465Smpstruct ib_mw *c4iw_alloc_mw(struct ib_pd *pd); 86859243Sobrienstruct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 86959243Sobrien virt, int acc, struct ib_udata *udata, int mr_id); 87059243Sobrienstruct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc); 87159243Sobrienstruct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd, 87259243Sobrien struct ib_phys_buf *buffer_list, 87359243Sobrien int num_phys_buf, 87459243Sobrien int acc, 87559243Sobrien u64 *iova_start); 87659243Sobrienint c4iw_reregister_phys_mem(struct ib_mr *mr, 87759243Sobrien int mr_rereg_mask, 87859243Sobrien struct ib_pd *pd, 87959243Sobrien struct ib_phys_buf *buffer_list, 88059243Sobrien int num_phys_buf, 88169408Sache int acc, u64 *iova_start); 88259243Sobrienint c4iw_dereg_mr(struct ib_mr *ib_mr); 88359243Sobrienint c4iw_destroy_cq(struct ib_cq *ib_cq); 88459243Sobrienstruct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int entries, 88559243Sobrien int vector, 88659243Sobrien struct ib_ucontext *ib_context, 88759243Sobrien struct ib_udata *udata); 88859243Sobrienint c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata); 889167465Smpint c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 89059243Sobrienint c4iw_destroy_qp(struct ib_qp *ib_qp); 89159243Sobrienstruct ib_qp *c4iw_create_qp(struct ib_pd *pd, 89259243Sobrien struct ib_qp_init_attr *attrs, 89359243Sobrien struct ib_udata *udata); 89459243Sobrienint c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 895167465Smp int attr_mask, struct ib_udata *udata); 896167465Smpint c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 897167465Smp int attr_mask, struct ib_qp_init_attr *init_attr); 89859243Sobrienstruct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn); 899167465Smpu32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size); 90059243Sobrienvoid c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size); 90159243Sobrienu32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size); 90259243Sobrienvoid c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size); 903167465Smpint c4iw_ofld_send(struct c4iw_rdev *rdev, struct mbuf *m); 90459243Sobrienvoid c4iw_flush_hw_cq(struct t4_cq *cq); 90559243Sobrienvoid c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count); 90659243Sobrienvoid c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count); 90759243Sobrienint c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp); 90859243Sobrienint c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count); 90959243Sobrienint c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count); 91059243Sobrienint c4iw_ev_handler(struct sge_iq *, const struct rsp_ctrl *); 91159243Sobrienu16 c4iw_rqes_posted(struct c4iw_qp *qhp); 91269408Sacheint c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe); 91359243Sobrienu32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx); 914167465Smpvoid c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid, 915167465Smp struct c4iw_dev_ucontext *uctx); 916167465Smpu32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx); 917167465Smpvoid c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid, 918167465Smp struct c4iw_dev_ucontext *uctx); 919167465Smpvoid c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe); 920167465Smp 921167465Smpextern struct cxgb4_client t4c_client; 922167465Smpextern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS]; 923167465Smpextern int c4iw_max_read_depth; 924167465Smp 925167465Smp#include <sys/blist.h> 926167465Smpstruct gen_pool { 927167465Smp blist_t gen_list; 92859243Sobrien daddr_t gen_base; 929167465Smp int gen_chunk_shift; 93059243Sobrien struct mutex gen_lock; 931167465Smp}; 932145479Smp 933167465Smpstatic __inline struct gen_pool * 934145479Smpgen_pool_create(daddr_t base, u_int chunk_shift, u_int len) 935145479Smp{ 936167465Smp struct gen_pool *gp; 93759243Sobrien 93859243Sobrien gp = malloc(sizeof(struct gen_pool), M_DEVBUF, M_NOWAIT); 93959243Sobrien if (gp == NULL) 94059243Sobrien return (NULL); 94159243Sobrien 94259243Sobrien memset(gp, 0, sizeof(struct gen_pool)); 94359243Sobrien gp->gen_list = blist_create(len >> chunk_shift, M_NOWAIT); 94459243Sobrien if (gp->gen_list == NULL) { 945167465Smp free(gp, M_DEVBUF); 946167465Smp return (NULL); 947167465Smp } 94859243Sobrien blist_free(gp->gen_list, 0, len >> chunk_shift); 94959243Sobrien gp->gen_base = base; 95059243Sobrien gp->gen_chunk_shift = chunk_shift; 95159243Sobrien //mutex_init(&gp->gen_lock, "genpool", NULL, MTX_DUPOK|MTX_DEF); 95259243Sobrien mutex_init(&gp->gen_lock); 95359243Sobrien 95459243Sobrien return (gp); 95559243Sobrien} 95659243Sobrien 95759243Sobrienstatic __inline unsigned long 95859243Sobriengen_pool_alloc(struct gen_pool *gp, int size) 95959243Sobrien{ 96059243Sobrien int chunks; 96159243Sobrien daddr_t blkno; 96259243Sobrien 96359243Sobrien chunks = (size + (1<<gp->gen_chunk_shift) - 1) >> gp->gen_chunk_shift; 96459243Sobrien mutex_lock(&gp->gen_lock); 96559243Sobrien blkno = blist_alloc(gp->gen_list, chunks); 96659243Sobrien mutex_unlock(&gp->gen_lock); 96759243Sobrien 96859243Sobrien if (blkno == SWAPBLK_NONE) 96959243Sobrien return (0); 97059243Sobrien 97159243Sobrien return (gp->gen_base + ((1 << gp->gen_chunk_shift) * blkno)); 97259243Sobrien} 97359243Sobrien 97459243Sobrienstatic __inline void 97559243Sobriengen_pool_free(struct gen_pool *gp, daddr_t address, int size) 97659243Sobrien{ 97759243Sobrien int chunks; 97859243Sobrien daddr_t blkno; 97959243Sobrien 98059243Sobrien chunks = (size + (1<<gp->gen_chunk_shift) - 1) >> gp->gen_chunk_shift; 98159243Sobrien blkno = (address - gp->gen_base) / (1 << gp->gen_chunk_shift); 98259243Sobrien mutex_lock(&gp->gen_lock); 983167465Smp blist_free(gp->gen_list, blkno, chunks); 984167465Smp mutex_unlock(&gp->gen_lock); 985167465Smp} 98659243Sobrien 98759243Sobrienstatic __inline void 98859243Sobriengen_pool_destroy(struct gen_pool *gp) 98969408Sache{ 99059243Sobrien blist_destroy(gp->gen_list); 99159243Sobrien free(gp, M_DEVBUF); 99259243Sobrien} 99359243Sobrien 99459243Sobrien#if defined(__i386__) || defined(__amd64__) 99559243Sobrien#define L1_CACHE_BYTES 128 99659243Sobrien#else 99759243Sobrien#define L1_CACHE_BYTES 32 99859243Sobrien#endif 999167465Smp 1000167465Smpstatic inline 1001167465Smpint idr_for_each(struct idr *idp, 100259243Sobrien int (*fn)(int id, void *p, void *data), void *data) 100359243Sobrien{ 100459243Sobrien int n, id, max, error = 0; 100569408Sache struct idr_layer *p; 100659243Sobrien struct idr_layer *pa[MAX_LEVEL]; 100759243Sobrien struct idr_layer **paa = &pa[0]; 1008167465Smp 100959243Sobrien n = idp->layers * IDR_BITS; 101059243Sobrien p = idp->top; 1011145479Smp max = 1 << n; 1012145479Smp 1013145479Smp id = 0; 101459243Sobrien while (id < max) { 101559243Sobrien while (n > 0 && p) { 1016100616Smp n -= IDR_BITS; 101759243Sobrien *paa++ = p; 101859243Sobrien p = p->ary[(id >> n) & IDR_MASK]; 101959243Sobrien } 102059243Sobrien 1021167465Smp if (p) { 102259243Sobrien error = fn(id, (void *)p, data); 102359243Sobrien if (error) 102459243Sobrien break; 1025167465Smp } 102659243Sobrien 1027167465Smp id += 1 << n; 102859243Sobrien while (n < fls(id)) { 102959243Sobrien n += IDR_BITS; 103059243Sobrien p = *--paa; 103159243Sobrien } 103259243Sobrien } 103359243Sobrien 103459243Sobrien return error; 1035167465Smp} 1036167465Smp 103759243Sobrienvoid c4iw_cm_init_cpl(struct adapter *); 103859243Sobrienvoid c4iw_cm_term_cpl(struct adapter *); 103959243Sobrien 1040167465Smpvoid your_reg_device(struct c4iw_dev *dev); 1041167465Smp 104259243Sobrien#define SGE_CTRLQ_NUM 0 104359243Sobrien 104459243Sobrien#endif 104559243Sobrien