1178786Skmacy 2178786Skmacy/************************************************************************** 3178786Skmacy 4178786SkmacyCopyright (c) 2007, Chelsio Inc. 5178786SkmacyAll rights reserved. 6178786Skmacy 7178786SkmacyRedistribution and use in source and binary forms, with or without 8178786Skmacymodification, are permitted provided that the following conditions are met: 9178786Skmacy 10178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 11178786Skmacy this list of conditions and the following disclaimer. 12178786Skmacy 13178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 14178786Skmacy contributors may be used to endorse or promote products derived from 15178786Skmacy this software without specific prior written permission. 16178786Skmacy 17178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 28178786Skmacy 29178786Skmacy***************************************************************************/ 30178786Skmacy#include <sys/cdefs.h> 31178786Skmacy__FBSDID("$FreeBSD$"); 32178786Skmacy 33237263Snp#include "opt_inet.h" 34237263Snp 35237263Snp#ifdef TCP_OFFLOAD 36178786Skmacy#include <sys/param.h> 37178786Skmacy#include <sys/systm.h> 38178786Skmacy#include <sys/kernel.h> 39178786Skmacy#include <sys/bus.h> 40178786Skmacy#include <sys/pciio.h> 41178786Skmacy#include <sys/conf.h> 42178786Skmacy#include <machine/bus.h> 43178786Skmacy#include <machine/resource.h> 44178786Skmacy#include <sys/bus_dma.h> 45178786Skmacy#include <sys/rman.h> 46178786Skmacy#include <sys/ioccom.h> 47178786Skmacy#include <sys/mbuf.h> 48178786Skmacy#include <sys/mutex.h> 49178786Skmacy#include <sys/rwlock.h> 50178786Skmacy#include <sys/linker.h> 51178786Skmacy#include <sys/firmware.h> 52178786Skmacy#include <sys/socket.h> 53178786Skmacy#include <sys/sockio.h> 54178786Skmacy#include <sys/smp.h> 55178786Skmacy#include <sys/sysctl.h> 56178786Skmacy#include <sys/syslog.h> 57178786Skmacy#include <sys/queue.h> 58178786Skmacy#include <sys/taskqueue.h> 59178786Skmacy#include <sys/proc.h> 60178786Skmacy#include <sys/queue.h> 61178786Skmacy#include <sys/libkern.h> 62178786Skmacy 63178786Skmacy#include <netinet/in.h> 64237263Snp#include <rdma/ib_verbs.h> 65237263Snp#include <rdma/ib_umem.h> 66237263Snp#include <rdma/ib_user_verbs.h> 67237263Snp#include <linux/idr.h> 68237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 69178786Skmacy 70178786Skmacy#include <cxgb_include.h> 71178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 72178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 73178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 74178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 75178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 76178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_resource.h> 77178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_user.h> 78178786Skmacy 79178786Skmacy/* 80178786Skmacy * Get one cq entry from cxio and map it to openib. 81178786Skmacy * 82178786Skmacy * Returns: 83178786Skmacy * 0 cqe returned 84178786Skmacy * -ENOBUFS EMPTY; 85178786Skmacy * -EAGAIN caller must try again 86178786Skmacy * any other neg errno fatal error 87178786Skmacy */ 88178786Skmacystatic int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp, 89178786Skmacy struct ib_wc *wc) 90178786Skmacy{ 91178786Skmacy struct iwch_qp *qhp = NULL; 92178786Skmacy struct t3_cqe cqe, *rd_cqe; 93178786Skmacy struct t3_wq *wq; 94178786Skmacy u32 credit = 0; 95178786Skmacy u8 cqe_flushed; 96178786Skmacy u64 cookie; 97178786Skmacy int ret = 1; 98178786Skmacy 99178786Skmacy rd_cqe = cxio_next_cqe(&chp->cq); 100178786Skmacy 101178786Skmacy if (!rd_cqe) 102178786Skmacy return 0; 103178786Skmacy 104178786Skmacy qhp = get_qhp(rhp, CQE_QPID(*rd_cqe)); 105178786Skmacy if (!qhp) 106178786Skmacy wq = NULL; 107178786Skmacy else { 108178786Skmacy mtx_lock(&qhp->lock); 109178786Skmacy wq = &(qhp->wq); 110178786Skmacy } 111178786Skmacy ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, 112178786Skmacy &credit); 113178786Skmacy if (t3a_device(chp->rhp) && credit) { 114178786Skmacy CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__, 115178786Skmacy credit, chp->cq.cqid); 116178786Skmacy cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit); 117178786Skmacy } 118178786Skmacy 119178786Skmacy if (ret) { 120178786Skmacy ret = -EAGAIN; 121178786Skmacy goto out; 122178786Skmacy } 123178786Skmacy ret = 1; 124178786Skmacy 125178786Skmacy wc->wr_id = cookie; 126178786Skmacy wc->qp = &qhp->ibqp; 127178786Skmacy wc->vendor_err = CQE_STATUS(cqe); 128178786Skmacy 129178786Skmacy CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x", 130178786Skmacy CQE_QPID(cqe), CQE_TYPE(cqe), 131178786Skmacy CQE_OPCODE(cqe), CQE_STATUS(cqe)); 132178786Skmacy CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx", 133178786Skmacy CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie); 134178786Skmacy 135178786Skmacy if (CQE_TYPE(cqe) == 0) { 136178786Skmacy if (!CQE_STATUS(cqe)) 137178786Skmacy wc->byte_len = CQE_LEN(cqe); 138178786Skmacy else 139178786Skmacy wc->byte_len = 0; 140178786Skmacy wc->opcode = IB_WC_RECV; 141178786Skmacy } else { 142178786Skmacy switch (CQE_OPCODE(cqe)) { 143178786Skmacy case T3_RDMA_WRITE: 144178786Skmacy wc->opcode = IB_WC_RDMA_WRITE; 145178786Skmacy break; 146178786Skmacy case T3_READ_REQ: 147178786Skmacy wc->opcode = IB_WC_RDMA_READ; 148178786Skmacy wc->byte_len = CQE_LEN(cqe); 149178786Skmacy break; 150178786Skmacy case T3_SEND: 151178786Skmacy case T3_SEND_WITH_SE: 152178786Skmacy wc->opcode = IB_WC_SEND; 153178786Skmacy break; 154178786Skmacy case T3_BIND_MW: 155178786Skmacy wc->opcode = IB_WC_BIND_MW; 156178786Skmacy break; 157178786Skmacy 158178786Skmacy /* these aren't supported yet */ 159178786Skmacy case T3_SEND_WITH_INV: 160178786Skmacy case T3_SEND_WITH_SE_INV: 161178786Skmacy case T3_LOCAL_INV: 162178786Skmacy case T3_FAST_REGISTER: 163178786Skmacy default: 164178786Skmacy log(LOG_ERR, "Unexpected opcode %d " 165178786Skmacy "in the CQE received for QPID=0x%0x\n", 166178786Skmacy CQE_OPCODE(cqe), CQE_QPID(cqe)); 167178786Skmacy ret = -EINVAL; 168178786Skmacy goto out; 169178786Skmacy } 170178786Skmacy } 171178786Skmacy 172178786Skmacy if (cqe_flushed) 173178786Skmacy wc->status = IB_WC_WR_FLUSH_ERR; 174178786Skmacy else { 175178786Skmacy 176178786Skmacy switch (CQE_STATUS(cqe)) { 177178786Skmacy case TPT_ERR_SUCCESS: 178178786Skmacy wc->status = IB_WC_SUCCESS; 179178786Skmacy break; 180178786Skmacy case TPT_ERR_STAG: 181178786Skmacy wc->status = IB_WC_LOC_ACCESS_ERR; 182178786Skmacy break; 183178786Skmacy case TPT_ERR_PDID: 184178786Skmacy wc->status = IB_WC_LOC_PROT_ERR; 185178786Skmacy break; 186178786Skmacy case TPT_ERR_QPID: 187178786Skmacy case TPT_ERR_ACCESS: 188178786Skmacy wc->status = IB_WC_LOC_ACCESS_ERR; 189178786Skmacy break; 190178786Skmacy case TPT_ERR_WRAP: 191178786Skmacy wc->status = IB_WC_GENERAL_ERR; 192178786Skmacy break; 193178786Skmacy case TPT_ERR_BOUND: 194178786Skmacy wc->status = IB_WC_LOC_LEN_ERR; 195178786Skmacy break; 196178786Skmacy case TPT_ERR_INVALIDATE_SHARED_MR: 197178786Skmacy case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 198178786Skmacy wc->status = IB_WC_MW_BIND_ERR; 199178786Skmacy break; 200178786Skmacy case TPT_ERR_CRC: 201178786Skmacy case TPT_ERR_MARKER: 202178786Skmacy case TPT_ERR_PDU_LEN_ERR: 203178786Skmacy case TPT_ERR_OUT_OF_RQE: 204178786Skmacy case TPT_ERR_DDP_VERSION: 205178786Skmacy case TPT_ERR_RDMA_VERSION: 206178786Skmacy case TPT_ERR_DDP_QUEUE_NUM: 207178786Skmacy case TPT_ERR_MSN: 208178786Skmacy case TPT_ERR_TBIT: 209178786Skmacy case TPT_ERR_MO: 210178786Skmacy case TPT_ERR_MSN_RANGE: 211178786Skmacy case TPT_ERR_IRD_OVERFLOW: 212178786Skmacy case TPT_ERR_OPCODE: 213178786Skmacy wc->status = IB_WC_FATAL_ERR; 214178786Skmacy break; 215178786Skmacy case TPT_ERR_SWFLUSH: 216178786Skmacy wc->status = IB_WC_WR_FLUSH_ERR; 217178786Skmacy break; 218178786Skmacy default: 219178786Skmacy log(LOG_ERR, "Unexpected cqe_status 0x%x for " 220178786Skmacy "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe)); 221178786Skmacy ret = -EINVAL; 222178786Skmacy } 223178786Skmacy } 224178786Skmacyout: 225178786Skmacy if (wq) 226178786Skmacy mtx_unlock(&qhp->lock); 227178786Skmacy return ret; 228178786Skmacy} 229178786Skmacy 230178786Skmacyint iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 231178786Skmacy{ 232178786Skmacy struct iwch_dev *rhp; 233178786Skmacy struct iwch_cq *chp; 234178786Skmacy int npolled; 235178786Skmacy int err = 0; 236178786Skmacy 237178786Skmacy chp = to_iwch_cq(ibcq); 238178786Skmacy rhp = chp->rhp; 239178786Skmacy 240178786Skmacy mtx_lock(&chp->lock); 241178786Skmacy for (npolled = 0; npolled < num_entries; ++npolled) { 242178786Skmacy#ifdef DEBUG 243178786Skmacy int i=0; 244178786Skmacy#endif 245178786Skmacy 246178786Skmacy /* 247178786Skmacy * Because T3 can post CQEs that are _not_ associated 248178786Skmacy * with a WR, we might have to poll again after removing 249178786Skmacy * one of these. 250178786Skmacy */ 251178786Skmacy do { 252178786Skmacy err = iwch_poll_cq_one(rhp, chp, wc + npolled); 253178786Skmacy#ifdef DEBUG 254178796Skmacy PANIC_IF(++i > 1000); 255178786Skmacy#endif 256178786Skmacy } while (err == -EAGAIN); 257178786Skmacy if (err <= 0) 258178786Skmacy break; 259178786Skmacy } 260178786Skmacy mtx_unlock(&chp->lock); 261178786Skmacy 262178786Skmacy if (err < 0) { 263178786Skmacy return err; 264178786Skmacy } else { 265178786Skmacy return npolled; 266178786Skmacy } 267178786Skmacy} 268237263Snp#endif 269