1 2/************************************************************************** 3 4Copyright (c) 2007, Chelsio Inc. 5All rights reserved. 6 7Redistribution and use in source and binary forms, with or without 8modification, are permitted provided that the following conditions are met: 9 10 1. Redistributions of source code must retain the above copyright notice, 11 this list of conditions and the following disclaimer. 12 13 2. Neither the name of the Chelsio Corporation nor the names of its 14 contributors may be used to endorse or promote products derived from 15 this software without specific prior written permission. 16 17THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27POSSIBILITY OF SUCH DAMAGE. 28 29***************************************************************************/ 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD$"); 32 33#include "opt_inet.h" 34 35#ifdef TCP_OFFLOAD 36#include <sys/param.h> 37#include <sys/systm.h> 38#include <sys/kernel.h> 39#include <sys/bus.h> 40#include <sys/pciio.h> 41#include <sys/conf.h> 42#include <machine/bus.h> 43#include <machine/resource.h> 44#include <sys/bus_dma.h> 45#include <sys/rman.h> 46#include <sys/ioccom.h> 47#include <sys/mbuf.h> 48#include <sys/mutex.h> 49#include <sys/rwlock.h> 50#include <sys/linker.h> 51#include <sys/firmware.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/smp.h> 55#include <sys/sysctl.h> 56#include <sys/syslog.h> 57#include <sys/queue.h> 58#include <sys/taskqueue.h> 59#include <sys/proc.h> 60#include <sys/queue.h> 61#include <sys/libkern.h> 62 63#include <netinet/in.h> 64#include <rdma/ib_verbs.h> 65#include <rdma/ib_umem.h> 66#include <rdma/ib_user_verbs.h> 67#include <linux/idr.h> 68#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 69 70#include <cxgb_include.h> 71#include <ulp/iw_cxgb/iw_cxgb_wr.h> 72#include <ulp/iw_cxgb/iw_cxgb_hal.h> 73#include <ulp/iw_cxgb/iw_cxgb_provider.h> 74#include <ulp/iw_cxgb/iw_cxgb_cm.h> 75#include <ulp/iw_cxgb/iw_cxgb.h> 76#include <ulp/iw_cxgb/iw_cxgb_resource.h> 77#include <ulp/iw_cxgb/iw_cxgb_user.h> 78 79/* 80 * Get one cq entry from cxio and map it to openib. 81 * 82 * Returns: 83 * 0 cqe returned 84 * -ENOBUFS EMPTY; 85 * -EAGAIN caller must try again 86 * any other neg errno fatal error 87 */ 88static int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp, 89 struct ib_wc *wc) 90{ 91 struct iwch_qp *qhp = NULL; 92 struct t3_cqe cqe, *rd_cqe; 93 struct t3_wq *wq; 94 u32 credit = 0; 95 u8 cqe_flushed; 96 u64 cookie; 97 int ret = 1; 98 99 rd_cqe = cxio_next_cqe(&chp->cq); 100 101 if (!rd_cqe) 102 return 0; 103 104 qhp = get_qhp(rhp, CQE_QPID(*rd_cqe)); 105 if (!qhp) 106 wq = NULL; 107 else { 108 mtx_lock(&qhp->lock); 109 wq = &(qhp->wq); 110 } 111 ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, 112 &credit); 113 if (t3a_device(chp->rhp) && credit) { 114 CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__, 115 credit, chp->cq.cqid); 116 cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit); 117 } 118 119 if (ret) { 120 ret = -EAGAIN; 121 goto out; 122 } 123 ret = 1; 124 125 wc->wr_id = cookie; 126 wc->qp = &qhp->ibqp; 127 wc->vendor_err = CQE_STATUS(cqe); 128 129 CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x", 130 CQE_QPID(cqe), CQE_TYPE(cqe), 131 CQE_OPCODE(cqe), CQE_STATUS(cqe)); 132 CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx", 133 CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie); 134 135 if (CQE_TYPE(cqe) == 0) { 136 if (!CQE_STATUS(cqe)) 137 wc->byte_len = CQE_LEN(cqe); 138 else 139 wc->byte_len = 0; 140 wc->opcode = IB_WC_RECV; 141 } else { 142 switch (CQE_OPCODE(cqe)) { 143 case T3_RDMA_WRITE: 144 wc->opcode = IB_WC_RDMA_WRITE; 145 break; 146 case T3_READ_REQ: 147 wc->opcode = IB_WC_RDMA_READ; 148 wc->byte_len = CQE_LEN(cqe); 149 break; 150 case T3_SEND: 151 case T3_SEND_WITH_SE: 152 wc->opcode = IB_WC_SEND; 153 break; 154 case T3_BIND_MW: 155 wc->opcode = IB_WC_BIND_MW; 156 break; 157 158 /* these aren't supported yet */ 159 case T3_SEND_WITH_INV: 160 case T3_SEND_WITH_SE_INV: 161 case T3_LOCAL_INV: 162 case T3_FAST_REGISTER: 163 default: 164 log(LOG_ERR, "Unexpected opcode %d " 165 "in the CQE received for QPID=0x%0x\n", 166 CQE_OPCODE(cqe), CQE_QPID(cqe)); 167 ret = -EINVAL; 168 goto out; 169 } 170 } 171 172 if (cqe_flushed) 173 wc->status = IB_WC_WR_FLUSH_ERR; 174 else { 175 176 switch (CQE_STATUS(cqe)) { 177 case TPT_ERR_SUCCESS: 178 wc->status = IB_WC_SUCCESS; 179 break; 180 case TPT_ERR_STAG: 181 wc->status = IB_WC_LOC_ACCESS_ERR; 182 break; 183 case TPT_ERR_PDID: 184 wc->status = IB_WC_LOC_PROT_ERR; 185 break; 186 case TPT_ERR_QPID: 187 case TPT_ERR_ACCESS: 188 wc->status = IB_WC_LOC_ACCESS_ERR; 189 break; 190 case TPT_ERR_WRAP: 191 wc->status = IB_WC_GENERAL_ERR; 192 break; 193 case TPT_ERR_BOUND: 194 wc->status = IB_WC_LOC_LEN_ERR; 195 break; 196 case TPT_ERR_INVALIDATE_SHARED_MR: 197 case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 198 wc->status = IB_WC_MW_BIND_ERR; 199 break; 200 case TPT_ERR_CRC: 201 case TPT_ERR_MARKER: 202 case TPT_ERR_PDU_LEN_ERR: 203 case TPT_ERR_OUT_OF_RQE: 204 case TPT_ERR_DDP_VERSION: 205 case TPT_ERR_RDMA_VERSION: 206 case TPT_ERR_DDP_QUEUE_NUM: 207 case TPT_ERR_MSN: 208 case TPT_ERR_TBIT: 209 case TPT_ERR_MO: 210 case TPT_ERR_MSN_RANGE: 211 case TPT_ERR_IRD_OVERFLOW: 212 case TPT_ERR_OPCODE: 213 wc->status = IB_WC_FATAL_ERR; 214 break; 215 case TPT_ERR_SWFLUSH: 216 wc->status = IB_WC_WR_FLUSH_ERR; 217 break; 218 default: 219 log(LOG_ERR, "Unexpected cqe_status 0x%x for " 220 "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe)); 221 ret = -EINVAL; 222 } 223 } 224out: 225 if (wq) 226 mtx_unlock(&qhp->lock); 227 return ret; 228} 229 230int iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 231{ 232 struct iwch_dev *rhp; 233 struct iwch_cq *chp; 234 int npolled; 235 int err = 0; 236 237 chp = to_iwch_cq(ibcq); 238 rhp = chp->rhp; 239 240 mtx_lock(&chp->lock); 241 for (npolled = 0; npolled < num_entries; ++npolled) { 242#ifdef DEBUG 243 int i=0; 244#endif 245 246 /* 247 * Because T3 can post CQEs that are _not_ associated 248 * with a WR, we might have to poll again after removing 249 * one of these. 250 */ 251 do { 252 err = iwch_poll_cq_one(rhp, chp, wc + npolled); 253#ifdef DEBUG 254 PANIC_IF(++i > 1000); 255#endif 256 } while (err == -EAGAIN); 257 if (err <= 0) 258 break; 259 } 260 mtx_unlock(&chp->lock); 261 262 if (err < 0) { 263 return err; 264 } else { 265 return npolled; 266 } 267} 268#endif 269