1256694Snp/* 2256694Snp * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved. 3256694Snp * 4256694Snp * This software is available to you under a choice of one of two 5256694Snp * licenses. You may choose to be licensed under the terms of the GNU 6256694Snp * General Public License (GPL) Version 2, available from the file 7256694Snp * COPYING in the main directory of this source tree, or the 8256694Snp * OpenIB.org BSD license below: 9256694Snp * 10256694Snp * Redistribution and use in source and binary forms, with or 11256694Snp * without modification, are permitted provided that the following 12256694Snp * conditions are met: 13256694Snp * 14256694Snp * - Redistributions of source code must retain the above 15256694Snp * copyright notice, this list of conditions and the following 16256694Snp * disclaimer. 17256694Snp * 18256694Snp * - Redistributions in binary form must reproduce the above 19256694Snp * copyright notice, this list of conditions and the following 20256694Snp * disclaimer in the documentation and/or other materials 21256694Snp * provided with the distribution. 22256694Snp * 23256694Snp * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24256694Snp * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25256694Snp * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26256694Snp * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27256694Snp * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28256694Snp * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29256694Snp * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30256694Snp * SOFTWARE. 31256694Snp */ 32256694Snp#include <sys/cdefs.h> 33256694Snp__FBSDID("$FreeBSD$"); 34256694Snp 35256694Snp#include "opt_inet.h" 36256694Snp 37256694Snp#ifdef TCP_OFFLOAD 38256694Snp#include <sys/types.h> 39256694Snp#include <sys/malloc.h> 40256694Snp#include <sys/socket.h> 41256694Snp#include <sys/socketvar.h> 42256694Snp#include <sys/sockio.h> 43256694Snp#include <sys/taskqueue.h> 44256694Snp#include <netinet/in.h> 45256694Snp#include <net/neighbour.h> 46256694Snp#include <net/route.h> 47256694Snp 48256694Snp#include <netinet/in_systm.h> 49256694Snp#include <netinet/in_pcb.h> 50256694Snp#include <netinet/ip.h> 51256694Snp#include <netinet/ip_var.h> 52256694Snp#include <netinet/tcp_var.h> 53256694Snp#include <netinet/tcp.h> 54256694Snp#include <netinet/tcpip.h> 55256694Snp 56256694Snp#include <netinet/toecore.h> 57256694Snp 58256694Snpstruct sge_iq; 59256694Snpstruct rss_header; 60256694Snp#include <linux/types.h> 61256694Snp#include "offload.h" 62256694Snp#include "tom/t4_tom.h" 63256694Snp 64256694Snp#include "iw_cxgbe.h" 65256694Snp#include "user.h" 66256694Snp 67256694Snpextern int db_delay_usecs; 68256694Snpextern int db_fc_threshold; 69256694Snpstatic void creds(struct toepcb *toep, size_t wrsize); 70256694Snp 71256694Snp 72256694Snpstatic void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state) 73256694Snp{ 74256694Snp unsigned long flag; 75256694Snp spin_lock_irqsave(&qhp->lock, flag); 76256694Snp qhp->attr.state = state; 77256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 78256694Snp} 79256694Snp 80256694Snpstatic void dealloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq) 81256694Snp{ 82256694Snp 83256694Snp contigfree(sq->queue, sq->memsize, M_DEVBUF); 84256694Snp} 85256694Snp 86256694Snpstatic void dealloc_sq(struct c4iw_rdev *rdev, struct t4_sq *sq) 87256694Snp{ 88256694Snp 89256694Snp dealloc_host_sq(rdev, sq); 90256694Snp} 91256694Snp 92256694Snpstatic int alloc_host_sq(struct c4iw_rdev *rdev, struct t4_sq *sq) 93256694Snp{ 94256694Snp sq->queue = contigmalloc(sq->memsize, M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 95256694Snp 4096, 0); 96256694Snp 97256694Snp if (sq->queue) 98256694Snp sq->dma_addr = vtophys(sq->queue); 99256694Snp else 100256694Snp return -ENOMEM; 101256694Snp sq->phys_addr = vtophys(sq->queue); 102256694Snp pci_unmap_addr_set(sq, mapping, sq->dma_addr); 103256694Snp CTR4(KTR_IW_CXGBE, "%s sq %p dma_addr %p phys_addr %p", __func__, 104256694Snp sq->queue, sq->dma_addr, sq->phys_addr); 105256694Snp return 0; 106256694Snp} 107256694Snp 108256694Snpstatic int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, 109256694Snp struct c4iw_dev_ucontext *uctx) 110256694Snp{ 111256694Snp /* 112256694Snp * uP clears EQ contexts when the connection exits rdma mode, 113256694Snp * so no need to post a RESET WR for these EQs. 114256694Snp */ 115256694Snp contigfree(wq->rq.queue, wq->rq.memsize, M_DEVBUF); 116256694Snp dealloc_sq(rdev, &wq->sq); 117256694Snp c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); 118256694Snp kfree(wq->rq.sw_rq); 119256694Snp kfree(wq->sq.sw_sq); 120256694Snp c4iw_put_qpid(rdev, wq->rq.qid, uctx); 121256694Snp c4iw_put_qpid(rdev, wq->sq.qid, uctx); 122256694Snp return 0; 123256694Snp} 124256694Snp 125256694Snpstatic int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, 126256694Snp struct t4_cq *rcq, struct t4_cq *scq, 127256694Snp struct c4iw_dev_ucontext *uctx) 128256694Snp{ 129256694Snp struct adapter *sc = rdev->adap; 130256694Snp int user = (uctx != &rdev->uctx); 131256694Snp struct fw_ri_res_wr *res_wr; 132256694Snp struct fw_ri_res *res; 133256694Snp int wr_len; 134256694Snp struct c4iw_wr_wait wr_wait; 135256694Snp int ret; 136256694Snp int eqsize; 137256694Snp struct wrqe *wr; 138256694Snp 139256694Snp wq->sq.qid = c4iw_get_qpid(rdev, uctx); 140256694Snp if (!wq->sq.qid) 141256694Snp return -ENOMEM; 142256694Snp 143256694Snp wq->rq.qid = c4iw_get_qpid(rdev, uctx); 144256694Snp if (!wq->rq.qid) 145256694Snp goto err1; 146256694Snp 147256694Snp if (!user) { 148256694Snp wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq, 149256694Snp GFP_KERNEL); 150256694Snp if (!wq->sq.sw_sq) 151256694Snp goto err2; 152256694Snp 153256694Snp wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq, 154256694Snp GFP_KERNEL); 155256694Snp if (!wq->rq.sw_rq) 156256694Snp goto err3; 157256694Snp } 158256694Snp 159256694Snp /* RQT must be a power of 2. */ 160256694Snp wq->rq.rqt_size = roundup_pow_of_two(wq->rq.size); 161256694Snp wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size); 162256694Snp if (!wq->rq.rqt_hwaddr) 163256694Snp goto err4; 164256694Snp 165256694Snp if (alloc_host_sq(rdev, &wq->sq)) 166256694Snp goto err5; 167256694Snp 168256694Snp memset(wq->sq.queue, 0, wq->sq.memsize); 169256694Snp pci_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); 170256694Snp 171256694Snp wq->rq.queue = contigmalloc(wq->rq.memsize, 172256694Snp M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0); 173256694Snp if (wq->rq.queue) 174256694Snp wq->rq.dma_addr = vtophys(wq->rq.queue); 175256694Snp else 176256694Snp goto err6; 177256694Snp CTR5(KTR_IW_CXGBE, 178256694Snp "%s sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx", __func__, 179256694Snp wq->sq.queue, (unsigned long long)vtophys(wq->sq.queue), 180256694Snp wq->rq.queue, (unsigned long long)vtophys(wq->rq.queue)); 181256694Snp memset(wq->rq.queue, 0, wq->rq.memsize); 182256694Snp pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); 183256694Snp 184256694Snp wq->db = (void *)((unsigned long)rman_get_virtual(sc->regs_res) + 185256694Snp MYPF_REG(SGE_PF_KDOORBELL)); 186256694Snp wq->gts = (void *)((unsigned long)rman_get_virtual(rdev->adap->regs_res) 187256694Snp + MYPF_REG(SGE_PF_GTS)); 188256694Snp if (user) { 189256694Snp wq->sq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) + 190256694Snp (wq->sq.qid << rdev->qpshift)); 191256694Snp wq->sq.udb &= PAGE_MASK; 192256694Snp wq->rq.udb = (u64)((char*)rman_get_virtual(rdev->adap->udbs_res) + 193256694Snp (wq->rq.qid << rdev->qpshift)); 194256694Snp wq->rq.udb &= PAGE_MASK; 195256694Snp } 196256694Snp wq->rdev = rdev; 197256694Snp wq->rq.msn = 1; 198256694Snp 199256694Snp /* build fw_ri_res_wr */ 200256694Snp wr_len = sizeof *res_wr + 2 * sizeof *res; 201256694Snp 202256694Snp wr = alloc_wrqe(wr_len, &sc->sge.mgmtq); 203256694Snp if (wr == NULL) 204256694Snp return (0); 205256694Snp res_wr = wrtod(wr); 206256694Snp 207256694Snp memset(res_wr, 0, wr_len); 208256694Snp res_wr->op_nres = cpu_to_be32( 209256694Snp V_FW_WR_OP(FW_RI_RES_WR) | 210256694Snp V_FW_RI_RES_WR_NRES(2) | 211256694Snp F_FW_WR_COMPL); 212256694Snp res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16)); 213256694Snp res_wr->cookie = (unsigned long) &wr_wait; 214256694Snp res = res_wr->res; 215256694Snp res->u.sqrq.restype = FW_RI_RES_TYPE_SQ; 216256694Snp res->u.sqrq.op = FW_RI_RES_OP_WRITE; 217256694Snp 218256694Snp /* eqsize is the number of 64B entries plus the status page size. */ 219256694Snp eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + spg_creds; 220256694Snp 221256694Snp res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( 222256694Snp V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ 223256694Snp V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ 224256694Snp V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ 225256694Snp V_FW_RI_RES_WR_IQID(scq->cqid)); 226256694Snp res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( 227256694Snp V_FW_RI_RES_WR_DCAEN(0) | 228256694Snp V_FW_RI_RES_WR_DCACPU(0) | 229256694Snp V_FW_RI_RES_WR_FBMIN(2) | 230256694Snp V_FW_RI_RES_WR_FBMAX(2) | 231256694Snp V_FW_RI_RES_WR_CIDXFTHRESHO(0) | 232256694Snp V_FW_RI_RES_WR_CIDXFTHRESH(0) | 233256694Snp V_FW_RI_RES_WR_EQSIZE(eqsize)); 234256694Snp res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid); 235256694Snp res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr); 236256694Snp res++; 237256694Snp res->u.sqrq.restype = FW_RI_RES_TYPE_RQ; 238256694Snp res->u.sqrq.op = FW_RI_RES_OP_WRITE; 239256694Snp 240256694Snp /* eqsize is the number of 64B entries plus the status page size. */ 241256694Snp eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + spg_creds ; 242256694Snp res->u.sqrq.fetchszm_to_iqid = cpu_to_be32( 243256694Snp V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */ 244256694Snp V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */ 245256694Snp V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */ 246256694Snp V_FW_RI_RES_WR_IQID(rcq->cqid)); 247256694Snp res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( 248256694Snp V_FW_RI_RES_WR_DCAEN(0) | 249256694Snp V_FW_RI_RES_WR_DCACPU(0) | 250256694Snp V_FW_RI_RES_WR_FBMIN(2) | 251256694Snp V_FW_RI_RES_WR_FBMAX(2) | 252256694Snp V_FW_RI_RES_WR_CIDXFTHRESHO(0) | 253256694Snp V_FW_RI_RES_WR_CIDXFTHRESH(0) | 254256694Snp V_FW_RI_RES_WR_EQSIZE(eqsize)); 255256694Snp res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid); 256256694Snp res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr); 257256694Snp 258256694Snp c4iw_init_wr_wait(&wr_wait); 259256694Snp 260256694Snp t4_wrq_tx(sc, wr); 261256694Snp ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid, __func__); 262256694Snp if (ret) 263256694Snp goto err7; 264256694Snp 265256694Snp CTR6(KTR_IW_CXGBE, 266256694Snp "%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx", 267256694Snp __func__, wq->sq.qid, wq->rq.qid, wq->db, 268256694Snp (unsigned long long)wq->sq.udb, (unsigned long long)wq->rq.udb); 269256694Snp 270256694Snp return 0; 271256694Snperr7: 272256694Snp contigfree(wq->rq.queue, wq->rq.memsize, M_DEVBUF); 273256694Snperr6: 274256694Snp dealloc_sq(rdev, &wq->sq); 275256694Snperr5: 276256694Snp c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); 277256694Snperr4: 278256694Snp kfree(wq->rq.sw_rq); 279256694Snperr3: 280256694Snp kfree(wq->sq.sw_sq); 281256694Snperr2: 282256694Snp c4iw_put_qpid(rdev, wq->rq.qid, uctx); 283256694Snperr1: 284256694Snp c4iw_put_qpid(rdev, wq->sq.qid, uctx); 285256694Snp return -ENOMEM; 286256694Snp} 287256694Snp 288256694Snpstatic int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, 289256694Snp struct ib_send_wr *wr, int max, u32 *plenp) 290256694Snp{ 291256694Snp u8 *dstp, *srcp; 292256694Snp u32 plen = 0; 293256694Snp int i; 294256694Snp int rem, len; 295256694Snp 296256694Snp dstp = (u8 *)immdp->data; 297256694Snp for (i = 0; i < wr->num_sge; i++) { 298256694Snp if ((plen + wr->sg_list[i].length) > max) 299256694Snp return -EMSGSIZE; 300256694Snp srcp = (u8 *)(unsigned long)wr->sg_list[i].addr; 301256694Snp plen += wr->sg_list[i].length; 302256694Snp rem = wr->sg_list[i].length; 303256694Snp while (rem) { 304256694Snp if (dstp == (u8 *)&sq->queue[sq->size]) 305256694Snp dstp = (u8 *)sq->queue; 306256694Snp if (rem <= (u8 *)&sq->queue[sq->size] - dstp) 307256694Snp len = rem; 308256694Snp else 309256694Snp len = (u8 *)&sq->queue[sq->size] - dstp; 310256694Snp memcpy(dstp, srcp, len); 311256694Snp dstp += len; 312256694Snp srcp += len; 313256694Snp rem -= len; 314256694Snp } 315256694Snp } 316256694Snp len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp); 317256694Snp if (len) 318256694Snp memset(dstp, 0, len); 319256694Snp immdp->op = FW_RI_DATA_IMMD; 320256694Snp immdp->r1 = 0; 321256694Snp immdp->r2 = 0; 322256694Snp immdp->immdlen = cpu_to_be32(plen); 323256694Snp *plenp = plen; 324256694Snp return 0; 325256694Snp} 326256694Snp 327256694Snpstatic int build_isgl(__be64 *queue_start, __be64 *queue_end, 328256694Snp struct fw_ri_isgl *isglp, struct ib_sge *sg_list, 329256694Snp int num_sge, u32 *plenp) 330256694Snp 331256694Snp{ 332256694Snp int i; 333256694Snp u32 plen = 0; 334256694Snp __be64 *flitp = (__be64 *)isglp->sge; 335256694Snp 336256694Snp for (i = 0; i < num_sge; i++) { 337256694Snp if ((plen + sg_list[i].length) < plen) 338256694Snp return -EMSGSIZE; 339256694Snp plen += sg_list[i].length; 340256694Snp *flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) | 341256694Snp sg_list[i].length); 342256694Snp if (++flitp == queue_end) 343256694Snp flitp = queue_start; 344256694Snp *flitp = cpu_to_be64(sg_list[i].addr); 345256694Snp if (++flitp == queue_end) 346256694Snp flitp = queue_start; 347256694Snp } 348256694Snp *flitp = (__force __be64)0; 349256694Snp isglp->op = FW_RI_DATA_ISGL; 350256694Snp isglp->r1 = 0; 351256694Snp isglp->nsge = cpu_to_be16(num_sge); 352256694Snp isglp->r2 = 0; 353256694Snp if (plenp) 354256694Snp *plenp = plen; 355256694Snp return 0; 356256694Snp} 357256694Snp 358256694Snpstatic int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, 359256694Snp struct ib_send_wr *wr, u8 *len16) 360256694Snp{ 361256694Snp u32 plen; 362256694Snp int size; 363256694Snp int ret; 364256694Snp 365256694Snp if (wr->num_sge > T4_MAX_SEND_SGE) 366256694Snp return -EINVAL; 367256694Snp switch (wr->opcode) { 368256694Snp case IB_WR_SEND: 369256694Snp if (wr->send_flags & IB_SEND_SOLICITED) 370256694Snp wqe->send.sendop_pkd = cpu_to_be32( 371256694Snp V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE)); 372256694Snp else 373256694Snp wqe->send.sendop_pkd = cpu_to_be32( 374256694Snp V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND)); 375256694Snp wqe->send.stag_inv = 0; 376256694Snp break; 377256694Snp case IB_WR_SEND_WITH_INV: 378256694Snp if (wr->send_flags & IB_SEND_SOLICITED) 379256694Snp wqe->send.sendop_pkd = cpu_to_be32( 380256694Snp V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE_INV)); 381256694Snp else 382256694Snp wqe->send.sendop_pkd = cpu_to_be32( 383256694Snp V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_INV)); 384256694Snp wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); 385256694Snp break; 386256694Snp 387256694Snp default: 388256694Snp return -EINVAL; 389256694Snp } 390256694Snp 391256694Snp plen = 0; 392256694Snp if (wr->num_sge) { 393256694Snp if (wr->send_flags & IB_SEND_INLINE) { 394256694Snp ret = build_immd(sq, wqe->send.u.immd_src, wr, 395256694Snp T4_MAX_SEND_INLINE, &plen); 396256694Snp if (ret) 397256694Snp return ret; 398256694Snp size = sizeof wqe->send + sizeof(struct fw_ri_immd) + 399256694Snp plen; 400256694Snp } else { 401256694Snp ret = build_isgl((__be64 *)sq->queue, 402256694Snp (__be64 *)&sq->queue[sq->size], 403256694Snp wqe->send.u.isgl_src, 404256694Snp wr->sg_list, wr->num_sge, &plen); 405256694Snp if (ret) 406256694Snp return ret; 407256694Snp size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + 408256694Snp wr->num_sge * sizeof(struct fw_ri_sge); 409256694Snp } 410256694Snp } else { 411256694Snp wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD; 412256694Snp wqe->send.u.immd_src[0].r1 = 0; 413256694Snp wqe->send.u.immd_src[0].r2 = 0; 414256694Snp wqe->send.u.immd_src[0].immdlen = 0; 415256694Snp size = sizeof wqe->send + sizeof(struct fw_ri_immd); 416256694Snp plen = 0; 417256694Snp } 418256694Snp *len16 = DIV_ROUND_UP(size, 16); 419256694Snp wqe->send.plen = cpu_to_be32(plen); 420256694Snp return 0; 421256694Snp} 422256694Snp 423256694Snpstatic int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, 424256694Snp struct ib_send_wr *wr, u8 *len16) 425256694Snp{ 426256694Snp u32 plen; 427256694Snp int size; 428256694Snp int ret; 429256694Snp 430256694Snp if (wr->num_sge > T4_MAX_SEND_SGE) 431256694Snp return -EINVAL; 432256694Snp wqe->write.r2 = 0; 433256694Snp wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); 434256694Snp wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); 435256694Snp if (wr->num_sge) { 436256694Snp if (wr->send_flags & IB_SEND_INLINE) { 437256694Snp ret = build_immd(sq, wqe->write.u.immd_src, wr, 438256694Snp T4_MAX_WRITE_INLINE, &plen); 439256694Snp if (ret) 440256694Snp return ret; 441256694Snp size = sizeof wqe->write + sizeof(struct fw_ri_immd) + 442256694Snp plen; 443256694Snp } else { 444256694Snp ret = build_isgl((__be64 *)sq->queue, 445256694Snp (__be64 *)&sq->queue[sq->size], 446256694Snp wqe->write.u.isgl_src, 447256694Snp wr->sg_list, wr->num_sge, &plen); 448256694Snp if (ret) 449256694Snp return ret; 450256694Snp size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + 451256694Snp wr->num_sge * sizeof(struct fw_ri_sge); 452256694Snp } 453256694Snp } else { 454256694Snp wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD; 455256694Snp wqe->write.u.immd_src[0].r1 = 0; 456256694Snp wqe->write.u.immd_src[0].r2 = 0; 457256694Snp wqe->write.u.immd_src[0].immdlen = 0; 458256694Snp size = sizeof wqe->write + sizeof(struct fw_ri_immd); 459256694Snp plen = 0; 460256694Snp } 461256694Snp *len16 = DIV_ROUND_UP(size, 16); 462256694Snp wqe->write.plen = cpu_to_be32(plen); 463256694Snp return 0; 464256694Snp} 465256694Snp 466256694Snpstatic int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) 467256694Snp{ 468256694Snp if (wr->num_sge > 1) 469256694Snp return -EINVAL; 470256694Snp if (wr->num_sge) { 471256694Snp wqe->read.stag_src = cpu_to_be32(wr->wr.rdma.rkey); 472256694Snp wqe->read.to_src_hi = cpu_to_be32((u32)(wr->wr.rdma.remote_addr 473256694Snp >> 32)); 474256694Snp wqe->read.to_src_lo = cpu_to_be32((u32)wr->wr.rdma.remote_addr); 475256694Snp wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey); 476256694Snp wqe->read.plen = cpu_to_be32(wr->sg_list[0].length); 477256694Snp wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr 478256694Snp >> 32)); 479256694Snp wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr)); 480256694Snp } else { 481256694Snp wqe->read.stag_src = cpu_to_be32(2); 482256694Snp wqe->read.to_src_hi = 0; 483256694Snp wqe->read.to_src_lo = 0; 484256694Snp wqe->read.stag_sink = cpu_to_be32(2); 485256694Snp wqe->read.plen = 0; 486256694Snp wqe->read.to_sink_hi = 0; 487256694Snp wqe->read.to_sink_lo = 0; 488256694Snp } 489256694Snp wqe->read.r2 = 0; 490256694Snp wqe->read.r5 = 0; 491256694Snp *len16 = DIV_ROUND_UP(sizeof wqe->read, 16); 492256694Snp return 0; 493256694Snp} 494256694Snp 495256694Snpstatic int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, 496256694Snp struct ib_recv_wr *wr, u8 *len16) 497256694Snp{ 498256694Snp int ret; 499256694Snp 500256694Snp ret = build_isgl((__be64 *)qhp->wq.rq.queue, 501256694Snp (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size], 502256694Snp &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); 503256694Snp if (ret) 504256694Snp return ret; 505256694Snp *len16 = DIV_ROUND_UP(sizeof wqe->recv + 506256694Snp wr->num_sge * sizeof(struct fw_ri_sge), 16); 507256694Snp return 0; 508256694Snp} 509256694Snp 510256694Snpstatic int build_fastreg(struct t4_sq *sq, union t4_wr *wqe, 511256694Snp struct ib_send_wr *wr, u8 *len16) 512256694Snp{ 513256694Snp 514256694Snp struct fw_ri_immd *imdp; 515256694Snp __be64 *p; 516256694Snp int i; 517256694Snp int pbllen = roundup(wr->wr.fast_reg.page_list_len * sizeof(u64), 32); 518256694Snp int rem; 519256694Snp 520256694Snp if (wr->wr.fast_reg.page_list_len > T4_MAX_FR_DEPTH) 521256694Snp return -EINVAL; 522256694Snp 523256694Snp wqe->fr.qpbinde_to_dcacpu = 0; 524256694Snp wqe->fr.pgsz_shift = wr->wr.fast_reg.page_shift - 12; 525256694Snp wqe->fr.addr_type = FW_RI_VA_BASED_TO; 526256694Snp wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->wr.fast_reg.access_flags); 527256694Snp wqe->fr.len_hi = 0; 528256694Snp wqe->fr.len_lo = cpu_to_be32(wr->wr.fast_reg.length); 529256694Snp wqe->fr.stag = cpu_to_be32(wr->wr.fast_reg.rkey); 530256694Snp wqe->fr.va_hi = cpu_to_be32(wr->wr.fast_reg.iova_start >> 32); 531256694Snp wqe->fr.va_lo_fbo = cpu_to_be32(wr->wr.fast_reg.iova_start & 532256694Snp 0xffffffff); 533256694Snp WARN_ON(pbllen > T4_MAX_FR_IMMD); 534256694Snp imdp = (struct fw_ri_immd *)(&wqe->fr + 1); 535256694Snp imdp->op = FW_RI_DATA_IMMD; 536256694Snp imdp->r1 = 0; 537256694Snp imdp->r2 = 0; 538256694Snp imdp->immdlen = cpu_to_be32(pbllen); 539256694Snp p = (__be64 *)(imdp + 1); 540256694Snp rem = pbllen; 541256694Snp for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) { 542256694Snp *p = cpu_to_be64((u64)wr->wr.fast_reg.page_list->page_list[i]); 543256694Snp rem -= sizeof *p; 544256694Snp if (++p == (__be64 *)&sq->queue[sq->size]) 545256694Snp p = (__be64 *)sq->queue; 546256694Snp } 547256694Snp BUG_ON(rem < 0); 548256694Snp while (rem) { 549256694Snp *p = 0; 550256694Snp rem -= sizeof *p; 551256694Snp if (++p == (__be64 *)&sq->queue[sq->size]) 552256694Snp p = (__be64 *)sq->queue; 553256694Snp } 554256694Snp *len16 = DIV_ROUND_UP(sizeof wqe->fr + sizeof *imdp + pbllen, 16); 555256694Snp return 0; 556256694Snp} 557256694Snp 558256694Snpstatic int build_inv_stag(union t4_wr *wqe, struct ib_send_wr *wr, 559256694Snp u8 *len16) 560256694Snp{ 561256694Snp wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); 562256694Snp wqe->inv.r2 = 0; 563256694Snp *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16); 564256694Snp return 0; 565256694Snp} 566256694Snp 567256694Snpvoid c4iw_qp_add_ref(struct ib_qp *qp) 568256694Snp{ 569256694Snp CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp); 570256694Snp atomic_inc(&(to_c4iw_qp(qp)->refcnt)); 571256694Snp} 572256694Snp 573256694Snpvoid c4iw_qp_rem_ref(struct ib_qp *qp) 574256694Snp{ 575256694Snp CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp); 576256694Snp if (atomic_dec_and_test(&(to_c4iw_qp(qp)->refcnt))) 577256694Snp wake_up(&(to_c4iw_qp(qp)->wait)); 578256694Snp} 579256694Snp 580256694Snpint c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 581256694Snp struct ib_send_wr **bad_wr) 582256694Snp{ 583256694Snp int err = 0; 584256694Snp u8 len16 = 0; 585256694Snp enum fw_wr_opcodes fw_opcode = 0; 586256694Snp enum fw_ri_wr_flags fw_flags; 587256694Snp struct c4iw_qp *qhp; 588256694Snp union t4_wr *wqe; 589256694Snp u32 num_wrs; 590256694Snp struct t4_swsqe *swsqe; 591256694Snp unsigned long flag; 592256694Snp u16 idx = 0; 593256694Snp 594256694Snp qhp = to_c4iw_qp(ibqp); 595256694Snp spin_lock_irqsave(&qhp->lock, flag); 596256694Snp if (t4_wq_in_error(&qhp->wq)) { 597256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 598256694Snp return -EINVAL; 599256694Snp } 600256694Snp num_wrs = t4_sq_avail(&qhp->wq); 601256694Snp if (num_wrs == 0) { 602256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 603256694Snp return -ENOMEM; 604256694Snp } 605256694Snp while (wr) { 606256694Snp if (num_wrs == 0) { 607256694Snp err = -ENOMEM; 608256694Snp *bad_wr = wr; 609256694Snp break; 610256694Snp } 611256694Snp wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue + 612256694Snp qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE); 613256694Snp 614256694Snp fw_flags = 0; 615256694Snp if (wr->send_flags & IB_SEND_SOLICITED) 616256694Snp fw_flags |= FW_RI_SOLICITED_EVENT_FLAG; 617256694Snp if (wr->send_flags & IB_SEND_SIGNALED) 618256694Snp fw_flags |= FW_RI_COMPLETION_FLAG; 619256694Snp swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx]; 620256694Snp switch (wr->opcode) { 621256694Snp case IB_WR_SEND_WITH_INV: 622256694Snp case IB_WR_SEND: 623256694Snp if (wr->send_flags & IB_SEND_FENCE) 624256694Snp fw_flags |= FW_RI_READ_FENCE_FLAG; 625256694Snp fw_opcode = FW_RI_SEND_WR; 626256694Snp if (wr->opcode == IB_WR_SEND) 627256694Snp swsqe->opcode = FW_RI_SEND; 628256694Snp else 629256694Snp swsqe->opcode = FW_RI_SEND_WITH_INV; 630256694Snp err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); 631256694Snp break; 632256694Snp case IB_WR_RDMA_WRITE: 633256694Snp fw_opcode = FW_RI_RDMA_WRITE_WR; 634256694Snp swsqe->opcode = FW_RI_RDMA_WRITE; 635256694Snp err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16); 636256694Snp break; 637256694Snp case IB_WR_RDMA_READ: 638256694Snp case IB_WR_RDMA_READ_WITH_INV: 639256694Snp fw_opcode = FW_RI_RDMA_READ_WR; 640256694Snp swsqe->opcode = FW_RI_READ_REQ; 641256694Snp if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) 642256694Snp fw_flags = FW_RI_RDMA_READ_INVALIDATE; 643256694Snp else 644256694Snp fw_flags = 0; 645256694Snp err = build_rdma_read(wqe, wr, &len16); 646256694Snp if (err) 647256694Snp break; 648256694Snp swsqe->read_len = wr->sg_list[0].length; 649256694Snp if (!qhp->wq.sq.oldest_read) 650256694Snp qhp->wq.sq.oldest_read = swsqe; 651256694Snp break; 652256694Snp case IB_WR_FAST_REG_MR: 653256694Snp fw_opcode = FW_RI_FR_NSMR_WR; 654256694Snp swsqe->opcode = FW_RI_FAST_REGISTER; 655256694Snp err = build_fastreg(&qhp->wq.sq, wqe, wr, &len16); 656256694Snp break; 657256694Snp case IB_WR_LOCAL_INV: 658256694Snp if (wr->send_flags & IB_SEND_FENCE) 659256694Snp fw_flags |= FW_RI_LOCAL_FENCE_FLAG; 660256694Snp fw_opcode = FW_RI_INV_LSTAG_WR; 661256694Snp swsqe->opcode = FW_RI_LOCAL_INV; 662256694Snp err = build_inv_stag(wqe, wr, &len16); 663256694Snp break; 664256694Snp default: 665256694Snp CTR2(KTR_IW_CXGBE, "%s post of type =%d TBD!", __func__, 666256694Snp wr->opcode); 667256694Snp err = -EINVAL; 668256694Snp } 669256694Snp if (err) { 670256694Snp *bad_wr = wr; 671256694Snp break; 672256694Snp } 673256694Snp swsqe->idx = qhp->wq.sq.pidx; 674256694Snp swsqe->complete = 0; 675256694Snp swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED); 676256694Snp swsqe->wr_id = wr->wr_id; 677256694Snp 678256694Snp init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); 679256694Snp 680256694Snp CTR5(KTR_IW_CXGBE, 681256694Snp "%s cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u", 682256694Snp __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx, 683256694Snp swsqe->opcode, swsqe->read_len); 684256694Snp wr = wr->next; 685256694Snp num_wrs--; 686256694Snp t4_sq_produce(&qhp->wq, len16); 687256694Snp idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 688256694Snp } 689256694Snp if (t4_wq_db_enabled(&qhp->wq)) 690256694Snp t4_ring_sq_db(&qhp->wq, idx); 691256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 692256694Snp return err; 693256694Snp} 694256694Snp 695256694Snpint c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 696256694Snp struct ib_recv_wr **bad_wr) 697256694Snp{ 698256694Snp int err = 0; 699256694Snp struct c4iw_qp *qhp; 700256694Snp union t4_recv_wr *wqe; 701256694Snp u32 num_wrs; 702256694Snp u8 len16 = 0; 703256694Snp unsigned long flag; 704256694Snp u16 idx = 0; 705256694Snp 706256694Snp qhp = to_c4iw_qp(ibqp); 707256694Snp spin_lock_irqsave(&qhp->lock, flag); 708256694Snp if (t4_wq_in_error(&qhp->wq)) { 709256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 710256694Snp return -EINVAL; 711256694Snp } 712256694Snp num_wrs = t4_rq_avail(&qhp->wq); 713256694Snp if (num_wrs == 0) { 714256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 715256694Snp return -ENOMEM; 716256694Snp } 717256694Snp while (wr) { 718256694Snp if (wr->num_sge > T4_MAX_RECV_SGE) { 719256694Snp err = -EINVAL; 720256694Snp *bad_wr = wr; 721256694Snp break; 722256694Snp } 723256694Snp wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue + 724256694Snp qhp->wq.rq.wq_pidx * 725256694Snp T4_EQ_ENTRY_SIZE); 726256694Snp if (num_wrs) 727256694Snp err = build_rdma_recv(qhp, wqe, wr, &len16); 728256694Snp else 729256694Snp err = -ENOMEM; 730256694Snp if (err) { 731256694Snp *bad_wr = wr; 732256694Snp break; 733256694Snp } 734256694Snp 735256694Snp qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id; 736256694Snp 737256694Snp wqe->recv.opcode = FW_RI_RECV_WR; 738256694Snp wqe->recv.r1 = 0; 739256694Snp wqe->recv.wrid = qhp->wq.rq.pidx; 740256694Snp wqe->recv.r2[0] = 0; 741256694Snp wqe->recv.r2[1] = 0; 742256694Snp wqe->recv.r2[2] = 0; 743256694Snp wqe->recv.len16 = len16; 744256694Snp CTR3(KTR_IW_CXGBE, "%s cookie 0x%llx pidx %u", __func__, 745256694Snp (unsigned long long) wr->wr_id, qhp->wq.rq.pidx); 746256694Snp t4_rq_produce(&qhp->wq, len16); 747256694Snp idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 748256694Snp wr = wr->next; 749256694Snp num_wrs--; 750256694Snp } 751256694Snp if (t4_wq_db_enabled(&qhp->wq)) 752256694Snp t4_ring_rq_db(&qhp->wq, idx); 753256694Snp spin_unlock_irqrestore(&qhp->lock, flag); 754256694Snp return err; 755256694Snp} 756256694Snp 757256694Snpint c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw, struct ib_mw_bind *mw_bind) 758256694Snp{ 759256694Snp return -ENOSYS; 760256694Snp} 761256694Snp 762256694Snpstatic inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type, 763256694Snp u8 *ecode) 764256694Snp{ 765256694Snp int status; 766256694Snp int tagged; 767256694Snp int opcode; 768256694Snp int rqtype; 769256694Snp int send_inv; 770256694Snp 771256694Snp if (!err_cqe) { 772256694Snp *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 773256694Snp *ecode = 0; 774256694Snp return; 775256694Snp } 776256694Snp 777256694Snp status = CQE_STATUS(err_cqe); 778256694Snp opcode = CQE_OPCODE(err_cqe); 779256694Snp rqtype = RQ_TYPE(err_cqe); 780256694Snp send_inv = (opcode == FW_RI_SEND_WITH_INV) || 781256694Snp (opcode == FW_RI_SEND_WITH_SE_INV); 782256694Snp tagged = (opcode == FW_RI_RDMA_WRITE) || 783256694Snp (rqtype && (opcode == FW_RI_READ_RESP)); 784256694Snp 785256694Snp switch (status) { 786256694Snp case T4_ERR_STAG: 787256694Snp if (send_inv) { 788256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 789256694Snp *ecode = RDMAP_CANT_INV_STAG; 790256694Snp } else { 791256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 792256694Snp *ecode = RDMAP_INV_STAG; 793256694Snp } 794256694Snp break; 795256694Snp case T4_ERR_PDID: 796256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 797256694Snp if ((opcode == FW_RI_SEND_WITH_INV) || 798256694Snp (opcode == FW_RI_SEND_WITH_SE_INV)) 799256694Snp *ecode = RDMAP_CANT_INV_STAG; 800256694Snp else 801256694Snp *ecode = RDMAP_STAG_NOT_ASSOC; 802256694Snp break; 803256694Snp case T4_ERR_QPID: 804256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 805256694Snp *ecode = RDMAP_STAG_NOT_ASSOC; 806256694Snp break; 807256694Snp case T4_ERR_ACCESS: 808256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 809256694Snp *ecode = RDMAP_ACC_VIOL; 810256694Snp break; 811256694Snp case T4_ERR_WRAP: 812256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 813256694Snp *ecode = RDMAP_TO_WRAP; 814256694Snp break; 815256694Snp case T4_ERR_BOUND: 816256694Snp if (tagged) { 817256694Snp *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 818256694Snp *ecode = DDPT_BASE_BOUNDS; 819256694Snp } else { 820256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 821256694Snp *ecode = RDMAP_BASE_BOUNDS; 822256694Snp } 823256694Snp break; 824256694Snp case T4_ERR_INVALIDATE_SHARED_MR: 825256694Snp case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND: 826256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 827256694Snp *ecode = RDMAP_CANT_INV_STAG; 828256694Snp break; 829256694Snp case T4_ERR_ECC: 830256694Snp case T4_ERR_ECC_PSTAG: 831256694Snp case T4_ERR_INTERNAL_ERR: 832256694Snp *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 833256694Snp *ecode = 0; 834256694Snp break; 835256694Snp case T4_ERR_OUT_OF_RQE: 836256694Snp *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 837256694Snp *ecode = DDPU_INV_MSN_NOBUF; 838256694Snp break; 839256694Snp case T4_ERR_PBL_ADDR_BOUND: 840256694Snp *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 841256694Snp *ecode = DDPT_BASE_BOUNDS; 842256694Snp break; 843256694Snp case T4_ERR_CRC: 844256694Snp *layer_type = LAYER_MPA|DDP_LLP; 845256694Snp *ecode = MPA_CRC_ERR; 846256694Snp break; 847256694Snp case T4_ERR_MARKER: 848256694Snp *layer_type = LAYER_MPA|DDP_LLP; 849256694Snp *ecode = MPA_MARKER_ERR; 850256694Snp break; 851256694Snp case T4_ERR_PDU_LEN_ERR: 852256694Snp *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 853256694Snp *ecode = DDPU_MSG_TOOBIG; 854256694Snp break; 855256694Snp case T4_ERR_DDP_VERSION: 856256694Snp if (tagged) { 857256694Snp *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 858256694Snp *ecode = DDPT_INV_VERS; 859256694Snp } else { 860256694Snp *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 861256694Snp *ecode = DDPU_INV_VERS; 862256694Snp } 863256694Snp break; 864256694Snp case T4_ERR_RDMA_VERSION: 865256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 866256694Snp *ecode = RDMAP_INV_VERS; 867256694Snp break; 868256694Snp case T4_ERR_OPCODE: 869256694Snp *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 870256694Snp *ecode = RDMAP_INV_OPCODE; 871256694Snp break; 872256694Snp case T4_ERR_DDP_QUEUE_NUM: 873256694Snp *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 874256694Snp *ecode = DDPU_INV_QN; 875256694Snp break; 876256694Snp case T4_ERR_MSN: 877256694Snp case T4_ERR_MSN_GAP: 878256694Snp case T4_ERR_MSN_RANGE: 879256694Snp case T4_ERR_IRD_OVERFLOW: 880256694Snp *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 881256694Snp *ecode = DDPU_INV_MSN_RANGE; 882256694Snp break; 883256694Snp case T4_ERR_TBIT: 884256694Snp *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 885256694Snp *ecode = 0; 886256694Snp break; 887256694Snp case T4_ERR_MO: 888256694Snp *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 889256694Snp *ecode = DDPU_INV_MO; 890256694Snp break; 891256694Snp default: 892256694Snp *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 893256694Snp *ecode = 0; 894256694Snp break; 895256694Snp } 896256694Snp} 897256694Snp 898256694Snpstatic void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe, 899256694Snp gfp_t gfp) 900256694Snp{ 901256694Snp struct fw_ri_wr *wqe; 902256694Snp struct terminate_message *term; 903256694Snp struct wrqe *wr; 904256694Snp struct socket *so = qhp->ep->com.so; 905256694Snp struct inpcb *inp = sotoinpcb(so); 906256694Snp struct tcpcb *tp = intotcpcb(inp); 907256694Snp struct toepcb *toep = tp->t_toe; 908256694Snp 909256694Snp CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, 910256694Snp qhp->wq.sq.qid, qhp->ep->hwtid); 911256694Snp 912256694Snp wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); 913256694Snp if (wr == NULL) 914256694Snp return; 915256694Snp wqe = wrtod(wr); 916256694Snp 917256694Snp memset(wqe, 0, sizeof *wqe); 918256694Snp wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR)); 919256694Snp wqe->flowid_len16 = cpu_to_be32( 920256694Snp V_FW_WR_FLOWID(qhp->ep->hwtid) | 921256694Snp V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); 922256694Snp 923256694Snp wqe->u.terminate.type = FW_RI_TYPE_TERMINATE; 924256694Snp wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term); 925256694Snp term = (struct terminate_message *)wqe->u.terminate.termmsg; 926256694Snp if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) { 927256694Snp term->layer_etype = qhp->attr.layer_etype; 928256694Snp term->ecode = qhp->attr.ecode; 929256694Snp } else 930256694Snp build_term_codes(err_cqe, &term->layer_etype, &term->ecode); 931256694Snp creds(toep, sizeof(*wqe)); 932256694Snp t4_wrq_tx(qhp->rhp->rdev.adap, wr); 933256694Snp} 934256694Snp 935256694Snp/* Assumes qhp lock is held. */ 936256694Snpstatic void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, 937256694Snp struct c4iw_cq *schp) 938256694Snp{ 939256694Snp int count; 940256694Snp int flushed; 941256694Snp unsigned long flag; 942256694Snp 943256694Snp CTR4(KTR_IW_CXGBE, "%s qhp %p rchp %p schp %p", __func__, qhp, rchp, 944256694Snp schp); 945256694Snp 946256694Snp /* locking hierarchy: cq lock first, then qp lock. */ 947256694Snp spin_lock_irqsave(&rchp->lock, flag); 948256694Snp spin_lock(&qhp->lock); 949256694Snp c4iw_flush_hw_cq(&rchp->cq); 950256694Snp c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count); 951256694Snp flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count); 952256694Snp spin_unlock(&qhp->lock); 953256694Snp spin_unlock_irqrestore(&rchp->lock, flag); 954256694Snp if (flushed) { 955256694Snp spin_lock_irqsave(&rchp->comp_handler_lock, flag); 956256694Snp (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 957256694Snp spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); 958256694Snp } 959256694Snp 960256694Snp /* locking hierarchy: cq lock first, then qp lock. */ 961256694Snp spin_lock_irqsave(&schp->lock, flag); 962256694Snp spin_lock(&qhp->lock); 963256694Snp c4iw_flush_hw_cq(&schp->cq); 964256694Snp c4iw_count_scqes(&schp->cq, &qhp->wq, &count); 965256694Snp flushed = c4iw_flush_sq(&qhp->wq, &schp->cq, count); 966256694Snp spin_unlock(&qhp->lock); 967256694Snp spin_unlock_irqrestore(&schp->lock, flag); 968256694Snp if (flushed) { 969256694Snp spin_lock_irqsave(&schp->comp_handler_lock, flag); 970256694Snp (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); 971256694Snp spin_unlock_irqrestore(&schp->comp_handler_lock, flag); 972256694Snp } 973256694Snp} 974256694Snp 975256694Snpstatic void flush_qp(struct c4iw_qp *qhp) 976256694Snp{ 977256694Snp struct c4iw_cq *rchp, *schp; 978256694Snp unsigned long flag; 979256694Snp 980256694Snp rchp = get_chp(qhp->rhp, qhp->attr.rcq); 981256694Snp schp = get_chp(qhp->rhp, qhp->attr.scq); 982256694Snp 983256694Snp if (qhp->ibqp.uobject) { 984256694Snp t4_set_wq_in_error(&qhp->wq); 985256694Snp t4_set_cq_in_error(&rchp->cq); 986256694Snp spin_lock_irqsave(&rchp->comp_handler_lock, flag); 987256694Snp (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 988256694Snp spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); 989256694Snp if (schp != rchp) { 990256694Snp t4_set_cq_in_error(&schp->cq); 991256694Snp spin_lock_irqsave(&schp->comp_handler_lock, flag); 992256694Snp (*schp->ibcq.comp_handler)(&schp->ibcq, 993256694Snp schp->ibcq.cq_context); 994256694Snp spin_unlock_irqrestore(&schp->comp_handler_lock, flag); 995256694Snp } 996256694Snp return; 997256694Snp } 998256694Snp __flush_qp(qhp, rchp, schp); 999256694Snp} 1000256694Snp 1001256694Snpstatic int 1002256694Snprdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep) 1003256694Snp{ 1004256694Snp struct c4iw_rdev *rdev = &rhp->rdev; 1005256694Snp struct adapter *sc = rdev->adap; 1006256694Snp struct fw_ri_wr *wqe; 1007256694Snp int ret; 1008256694Snp struct wrqe *wr; 1009256694Snp struct socket *so = ep->com.so; 1010256694Snp struct inpcb *inp = sotoinpcb(so); 1011256694Snp struct tcpcb *tp = intotcpcb(inp); 1012256694Snp struct toepcb *toep = tp->t_toe; 1013256694Snp 1014256694Snp KASSERT(rhp == qhp->rhp && ep == qhp->ep, ("%s: EDOOFUS", __func__)); 1015256694Snp 1016256694Snp CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, 1017256694Snp qhp->wq.sq.qid, ep->hwtid); 1018256694Snp 1019256694Snp wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); 1020256694Snp if (wr == NULL) 1021256694Snp return (0); 1022256694Snp wqe = wrtod(wr); 1023256694Snp 1024256694Snp memset(wqe, 0, sizeof *wqe); 1025256694Snp 1026256694Snp wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR) | F_FW_WR_COMPL); 1027256694Snp wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) | 1028256694Snp V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); 1029256694Snp wqe->cookie = (unsigned long) &ep->com.wr_wait; 1030256694Snp wqe->u.fini.type = FW_RI_TYPE_FINI; 1031256694Snp 1032256694Snp c4iw_init_wr_wait(&ep->com.wr_wait); 1033256694Snp 1034256694Snp creds(toep, sizeof(*wqe)); 1035256694Snp t4_wrq_tx(sc, wr); 1036256694Snp 1037256694Snp ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid, 1038256694Snp qhp->wq.sq.qid, __func__); 1039256694Snp return ret; 1040256694Snp} 1041256694Snp 1042256694Snpstatic void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) 1043256694Snp{ 1044256694Snp CTR2(KTR_IW_CXGBE, "%s p2p_type = %d", __func__, p2p_type); 1045256694Snp memset(&init->u, 0, sizeof init->u); 1046256694Snp switch (p2p_type) { 1047256694Snp case FW_RI_INIT_P2PTYPE_RDMA_WRITE: 1048256694Snp init->u.write.opcode = FW_RI_RDMA_WRITE_WR; 1049256694Snp init->u.write.stag_sink = cpu_to_be32(1); 1050256694Snp init->u.write.to_sink = cpu_to_be64(1); 1051256694Snp init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD; 1052256694Snp init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write + 1053256694Snp sizeof(struct fw_ri_immd), 1054256694Snp 16); 1055256694Snp break; 1056256694Snp case FW_RI_INIT_P2PTYPE_READ_REQ: 1057256694Snp init->u.write.opcode = FW_RI_RDMA_READ_WR; 1058256694Snp init->u.read.stag_src = cpu_to_be32(1); 1059256694Snp init->u.read.to_src_lo = cpu_to_be32(1); 1060256694Snp init->u.read.stag_sink = cpu_to_be32(1); 1061256694Snp init->u.read.to_sink_lo = cpu_to_be32(1); 1062256694Snp init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16); 1063256694Snp break; 1064256694Snp } 1065256694Snp} 1066256694Snp 1067256694Snpstatic void 1068256694Snpcreds(struct toepcb *toep, size_t wrsize) 1069256694Snp{ 1070256694Snp struct ofld_tx_sdesc *txsd; 1071256694Snp 1072256694Snp CTR3(KTR_IW_CXGBE, "%s:creB %p %u", __func__, toep , wrsize); 1073256694Snp INP_WLOCK(toep->inp); 1074256694Snp txsd = &toep->txsd[toep->txsd_pidx]; 1075256694Snp txsd->tx_credits = howmany(wrsize, 16); 1076256694Snp txsd->plen = 0; 1077256694Snp KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 1078256694Snp ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 1079256694Snp toep->tx_credits -= txsd->tx_credits; 1080256694Snp if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 1081256694Snp toep->txsd_pidx = 0; 1082256694Snp toep->txsd_avail--; 1083256694Snp INP_WUNLOCK(toep->inp); 1084256694Snp CTR5(KTR_IW_CXGBE, "%s:creE %p %u %u %u", __func__, toep , 1085256694Snp txsd->tx_credits, toep->tx_credits, toep->txsd_pidx); 1086256694Snp} 1087256694Snp 1088256694Snpstatic int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp) 1089256694Snp{ 1090256694Snp struct fw_ri_wr *wqe; 1091256694Snp int ret; 1092256694Snp struct wrqe *wr; 1093256694Snp struct c4iw_ep *ep = qhp->ep; 1094256694Snp struct c4iw_rdev *rdev = &qhp->rhp->rdev; 1095256694Snp struct adapter *sc = rdev->adap; 1096256694Snp struct socket *so = ep->com.so; 1097256694Snp struct inpcb *inp = sotoinpcb(so); 1098256694Snp struct tcpcb *tp = intotcpcb(inp); 1099256694Snp struct toepcb *toep = tp->t_toe; 1100256694Snp 1101256694Snp CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp, 1102256694Snp qhp->wq.sq.qid, ep->hwtid); 1103256694Snp 1104256694Snp wr = alloc_wrqe(sizeof(*wqe), toep->ofld_txq); 1105256694Snp if (wr == NULL) 1106256694Snp return (0); 1107256694Snp wqe = wrtod(wr); 1108256694Snp 1109256694Snp memset(wqe, 0, sizeof *wqe); 1110256694Snp 1111256694Snp wqe->op_compl = cpu_to_be32( 1112256694Snp V_FW_WR_OP(FW_RI_WR) | 1113256694Snp F_FW_WR_COMPL); 1114256694Snp wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) | 1115256694Snp V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); 1116256694Snp 1117256694Snp wqe->cookie = (unsigned long) &ep->com.wr_wait; 1118256694Snp 1119256694Snp wqe->u.init.type = FW_RI_TYPE_INIT; 1120256694Snp wqe->u.init.mpareqbit_p2ptype = 1121256694Snp V_FW_RI_WR_MPAREQBIT(qhp->attr.mpa_attr.initiator) | 1122256694Snp V_FW_RI_WR_P2PTYPE(qhp->attr.mpa_attr.p2p_type); 1123256694Snp wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE; 1124256694Snp if (qhp->attr.mpa_attr.recv_marker_enabled) 1125256694Snp wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE; 1126256694Snp if (qhp->attr.mpa_attr.xmit_marker_enabled) 1127256694Snp wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE; 1128256694Snp if (qhp->attr.mpa_attr.crc_enabled) 1129256694Snp wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE; 1130256694Snp 1131256694Snp wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE | 1132256694Snp FW_RI_QP_RDMA_WRITE_ENABLE | 1133256694Snp FW_RI_QP_BIND_ENABLE; 1134256694Snp if (!qhp->ibqp.uobject) 1135256694Snp wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE | 1136256694Snp FW_RI_QP_STAG0_ENABLE; 1137256694Snp wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq)); 1138256694Snp wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd); 1139256694Snp wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid); 1140256694Snp wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid); 1141256694Snp wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid); 1142256694Snp wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq); 1143256694Snp wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq); 1144256694Snp wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord); 1145256694Snp wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird); 1146256694Snp wqe->u.init.iss = cpu_to_be32(ep->snd_seq); 1147256694Snp wqe->u.init.irs = cpu_to_be32(ep->rcv_seq); 1148256694Snp wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size); 1149256694Snp wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr - 1150256694Snp sc->vres.rq.start); 1151256694Snp if (qhp->attr.mpa_attr.initiator) 1152256694Snp build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init); 1153256694Snp 1154256694Snp c4iw_init_wr_wait(&ep->com.wr_wait); 1155256694Snp 1156256694Snp creds(toep, sizeof(*wqe)); 1157256694Snp t4_wrq_tx(sc, wr); 1158256694Snp 1159256694Snp ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid, 1160256694Snp qhp->wq.sq.qid, __func__); 1161256694Snp 1162256694Snp toep->ulp_mode = ULP_MODE_RDMA; 1163256694Snp 1164256694Snp return ret; 1165256694Snp} 1166256694Snp 1167256694Snpint c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, 1168256694Snp enum c4iw_qp_attr_mask mask, 1169256694Snp struct c4iw_qp_attributes *attrs, 1170256694Snp int internal) 1171256694Snp{ 1172256694Snp int ret = 0; 1173256694Snp struct c4iw_qp_attributes newattr = qhp->attr; 1174256694Snp int disconnect = 0; 1175256694Snp int terminate = 0; 1176256694Snp int abort = 0; 1177256694Snp int free = 0; 1178256694Snp struct c4iw_ep *ep = NULL; 1179256694Snp 1180256694Snp CTR5(KTR_IW_CXGBE, "%s qhp %p sqid 0x%x rqid 0x%x ep %p", __func__, qhp, 1181256694Snp qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep); 1182256694Snp CTR3(KTR_IW_CXGBE, "%s state %d -> %d", __func__, qhp->attr.state, 1183256694Snp (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 1184256694Snp 1185256694Snp mutex_lock(&qhp->mutex); 1186256694Snp 1187256694Snp /* Process attr changes if in IDLE */ 1188256694Snp if (mask & C4IW_QP_ATTR_VALID_MODIFY) { 1189256694Snp if (qhp->attr.state != C4IW_QP_STATE_IDLE) { 1190256694Snp ret = -EIO; 1191256694Snp goto out; 1192256694Snp } 1193256694Snp if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ) 1194256694Snp newattr.enable_rdma_read = attrs->enable_rdma_read; 1195256694Snp if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE) 1196256694Snp newattr.enable_rdma_write = attrs->enable_rdma_write; 1197256694Snp if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND) 1198256694Snp newattr.enable_bind = attrs->enable_bind; 1199256694Snp if (mask & C4IW_QP_ATTR_MAX_ORD) { 1200256694Snp if (attrs->max_ord > c4iw_max_read_depth) { 1201256694Snp ret = -EINVAL; 1202256694Snp goto out; 1203256694Snp } 1204256694Snp newattr.max_ord = attrs->max_ord; 1205256694Snp } 1206256694Snp if (mask & C4IW_QP_ATTR_MAX_IRD) { 1207256694Snp if (attrs->max_ird > c4iw_max_read_depth) { 1208256694Snp ret = -EINVAL; 1209256694Snp goto out; 1210256694Snp } 1211256694Snp newattr.max_ird = attrs->max_ird; 1212256694Snp } 1213256694Snp qhp->attr = newattr; 1214256694Snp } 1215256694Snp 1216256694Snp if (!(mask & C4IW_QP_ATTR_NEXT_STATE)) 1217256694Snp goto out; 1218256694Snp if (qhp->attr.state == attrs->next_state) 1219256694Snp goto out; 1220256694Snp 1221256694Snp switch (qhp->attr.state) { 1222256694Snp case C4IW_QP_STATE_IDLE: 1223256694Snp switch (attrs->next_state) { 1224256694Snp case C4IW_QP_STATE_RTS: 1225256694Snp if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) { 1226256694Snp ret = -EINVAL; 1227256694Snp goto out; 1228256694Snp } 1229256694Snp if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) { 1230256694Snp ret = -EINVAL; 1231256694Snp goto out; 1232256694Snp } 1233256694Snp qhp->attr.mpa_attr = attrs->mpa_attr; 1234256694Snp qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 1235256694Snp qhp->ep = qhp->attr.llp_stream_handle; 1236256694Snp set_state(qhp, C4IW_QP_STATE_RTS); 1237256694Snp 1238256694Snp /* 1239256694Snp * Ref the endpoint here and deref when we 1240256694Snp * disassociate the endpoint from the QP. This 1241256694Snp * happens in CLOSING->IDLE transition or *->ERROR 1242256694Snp * transition. 1243256694Snp */ 1244256694Snp c4iw_get_ep(&qhp->ep->com); 1245256694Snp ret = rdma_init(rhp, qhp); 1246256694Snp if (ret) 1247256694Snp goto err; 1248256694Snp break; 1249256694Snp case C4IW_QP_STATE_ERROR: 1250256694Snp set_state(qhp, C4IW_QP_STATE_ERROR); 1251256694Snp flush_qp(qhp); 1252256694Snp break; 1253256694Snp default: 1254256694Snp ret = -EINVAL; 1255256694Snp goto out; 1256256694Snp } 1257256694Snp break; 1258256694Snp case C4IW_QP_STATE_RTS: 1259256694Snp switch (attrs->next_state) { 1260256694Snp case C4IW_QP_STATE_CLOSING: 1261256694Snp //Fixme: Use atomic_read as same as Linux 1262256694Snp BUG_ON(qhp->ep->com.kref.count < 2); 1263256694Snp set_state(qhp, C4IW_QP_STATE_CLOSING); 1264256694Snp ep = qhp->ep; 1265256694Snp if (!internal) { 1266256694Snp abort = 0; 1267256694Snp disconnect = 1; 1268256694Snp c4iw_get_ep(&qhp->ep->com); 1269256694Snp } 1270256694Snp if (qhp->ibqp.uobject) 1271256694Snp t4_set_wq_in_error(&qhp->wq); 1272256694Snp ret = rdma_fini(rhp, qhp, ep); 1273256694Snp if (ret) 1274256694Snp goto err; 1275256694Snp break; 1276256694Snp case C4IW_QP_STATE_TERMINATE: 1277256694Snp set_state(qhp, C4IW_QP_STATE_TERMINATE); 1278256694Snp qhp->attr.layer_etype = attrs->layer_etype; 1279256694Snp qhp->attr.ecode = attrs->ecode; 1280256694Snp if (qhp->ibqp.uobject) 1281256694Snp t4_set_wq_in_error(&qhp->wq); 1282256694Snp ep = qhp->ep; 1283256694Snp if (!internal) 1284256694Snp terminate = 1; 1285256694Snp disconnect = 1; 1286256694Snp c4iw_get_ep(&qhp->ep->com); 1287256694Snp break; 1288256694Snp case C4IW_QP_STATE_ERROR: 1289256694Snp set_state(qhp, C4IW_QP_STATE_ERROR); 1290256694Snp if (qhp->ibqp.uobject) 1291256694Snp t4_set_wq_in_error(&qhp->wq); 1292256694Snp if (!internal) { 1293256694Snp abort = 1; 1294256694Snp disconnect = 1; 1295256694Snp ep = qhp->ep; 1296256694Snp c4iw_get_ep(&qhp->ep->com); 1297256694Snp } 1298256694Snp goto err; 1299256694Snp break; 1300256694Snp default: 1301256694Snp ret = -EINVAL; 1302256694Snp goto out; 1303256694Snp } 1304256694Snp break; 1305256694Snp case C4IW_QP_STATE_CLOSING: 1306256694Snp if (!internal) { 1307256694Snp ret = -EINVAL; 1308256694Snp goto out; 1309256694Snp } 1310256694Snp switch (attrs->next_state) { 1311256694Snp case C4IW_QP_STATE_IDLE: 1312256694Snp flush_qp(qhp); 1313256694Snp set_state(qhp, C4IW_QP_STATE_IDLE); 1314256694Snp qhp->attr.llp_stream_handle = NULL; 1315256694Snp c4iw_put_ep(&qhp->ep->com); 1316256694Snp qhp->ep = NULL; 1317256694Snp wake_up(&qhp->wait); 1318256694Snp break; 1319256694Snp case C4IW_QP_STATE_ERROR: 1320256694Snp goto err; 1321256694Snp default: 1322256694Snp ret = -EINVAL; 1323256694Snp goto err; 1324256694Snp } 1325256694Snp break; 1326256694Snp case C4IW_QP_STATE_ERROR: 1327256694Snp if (attrs->next_state != C4IW_QP_STATE_IDLE) { 1328256694Snp ret = -EINVAL; 1329256694Snp goto out; 1330256694Snp } 1331256694Snp if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) { 1332256694Snp ret = -EINVAL; 1333256694Snp goto out; 1334256694Snp } 1335256694Snp set_state(qhp, C4IW_QP_STATE_IDLE); 1336256694Snp break; 1337256694Snp case C4IW_QP_STATE_TERMINATE: 1338256694Snp if (!internal) { 1339256694Snp ret = -EINVAL; 1340256694Snp goto out; 1341256694Snp } 1342256694Snp goto err; 1343256694Snp break; 1344256694Snp default: 1345256694Snp printf("%s in a bad state %d\n", 1346256694Snp __func__, qhp->attr.state); 1347256694Snp ret = -EINVAL; 1348256694Snp goto err; 1349256694Snp break; 1350256694Snp } 1351256694Snp goto out; 1352256694Snperr: 1353256694Snp CTR3(KTR_IW_CXGBE, "%s disassociating ep %p qpid 0x%x", __func__, 1354256694Snp qhp->ep, qhp->wq.sq.qid); 1355256694Snp 1356256694Snp /* disassociate the LLP connection */ 1357256694Snp qhp->attr.llp_stream_handle = NULL; 1358256694Snp if (!ep) 1359256694Snp ep = qhp->ep; 1360256694Snp qhp->ep = NULL; 1361256694Snp set_state(qhp, C4IW_QP_STATE_ERROR); 1362256694Snp free = 1; 1363256694Snp wake_up(&qhp->wait); 1364256694Snp BUG_ON(!ep); 1365256694Snp flush_qp(qhp); 1366256694Snpout: 1367256694Snp mutex_unlock(&qhp->mutex); 1368256694Snp 1369256694Snp if (terminate) 1370256694Snp post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL); 1371256694Snp 1372256694Snp /* 1373256694Snp * If disconnect is 1, then we need to initiate a disconnect 1374256694Snp * on the EP. This can be a normal close (RTS->CLOSING) or 1375256694Snp * an abnormal close (RTS/CLOSING->ERROR). 1376256694Snp */ 1377256694Snp if (disconnect) { 1378256694Snp c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC : 1379256694Snp GFP_KERNEL); 1380256694Snp c4iw_put_ep(&ep->com); 1381256694Snp } 1382256694Snp 1383256694Snp /* 1384256694Snp * If free is 1, then we've disassociated the EP from the QP 1385256694Snp * and we need to dereference the EP. 1386256694Snp */ 1387256694Snp if (free) 1388256694Snp c4iw_put_ep(&ep->com); 1389256694Snp CTR2(KTR_IW_CXGBE, "%s exit state %d", __func__, qhp->attr.state); 1390256694Snp return ret; 1391256694Snp} 1392256694Snp 1393256694Snpstatic int enable_qp_db(int id, void *p, void *data) 1394256694Snp{ 1395256694Snp struct c4iw_qp *qp = p; 1396256694Snp 1397256694Snp t4_enable_wq_db(&qp->wq); 1398256694Snp return 0; 1399256694Snp} 1400256694Snp 1401256694Snpint c4iw_destroy_qp(struct ib_qp *ib_qp) 1402256694Snp{ 1403256694Snp struct c4iw_dev *rhp; 1404256694Snp struct c4iw_qp *qhp; 1405256694Snp struct c4iw_qp_attributes attrs; 1406256694Snp struct c4iw_ucontext *ucontext; 1407256694Snp 1408256694Snp CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ib_qp); 1409256694Snp qhp = to_c4iw_qp(ib_qp); 1410256694Snp rhp = qhp->rhp; 1411256694Snp 1412256694Snp attrs.next_state = C4IW_QP_STATE_ERROR; 1413256694Snp if (qhp->attr.state == C4IW_QP_STATE_TERMINATE) 1414256694Snp c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1415256694Snp else 1416256694Snp c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1417256694Snp wait_event(qhp->wait, !qhp->ep); 1418256694Snp 1419256694Snp spin_lock_irq(&rhp->lock); 1420256694Snp remove_handle_nolock(rhp, &rhp->qpidr, qhp->wq.sq.qid); 1421256694Snp rhp->qpcnt--; 1422256694Snp BUG_ON(rhp->qpcnt < 0); 1423256694Snp if (rhp->qpcnt <= db_fc_threshold && rhp->db_state == FLOW_CONTROL) { 1424256694Snp rhp->rdev.stats.db_state_transitions++; 1425256694Snp rhp->db_state = NORMAL; 1426256694Snp idr_for_each(&rhp->qpidr, enable_qp_db, NULL); 1427256694Snp } 1428256694Snp spin_unlock_irq(&rhp->lock); 1429256694Snp atomic_dec(&qhp->refcnt); 1430256694Snp wait_event(qhp->wait, !atomic_read(&qhp->refcnt)); 1431256694Snp 1432256694Snp ucontext = ib_qp->uobject ? 1433256694Snp to_c4iw_ucontext(ib_qp->uobject->context) : NULL; 1434256694Snp destroy_qp(&rhp->rdev, &qhp->wq, 1435256694Snp ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 1436256694Snp 1437256694Snp CTR3(KTR_IW_CXGBE, "%s ib_qp %p qpid 0x%0x", __func__, ib_qp, 1438256694Snp qhp->wq.sq.qid); 1439256694Snp kfree(qhp); 1440256694Snp return 0; 1441256694Snp} 1442256694Snp 1443256694Snpstatic int disable_qp_db(int id, void *p, void *data) 1444256694Snp{ 1445256694Snp struct c4iw_qp *qp = p; 1446256694Snp 1447256694Snp t4_disable_wq_db(&qp->wq); 1448256694Snp return 0; 1449256694Snp} 1450256694Snp 1451256694Snpstruct ib_qp * 1452256694Snpc4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, 1453256694Snp struct ib_udata *udata) 1454256694Snp{ 1455256694Snp struct c4iw_dev *rhp; 1456256694Snp struct c4iw_qp *qhp; 1457256694Snp struct c4iw_pd *php; 1458256694Snp struct c4iw_cq *schp; 1459256694Snp struct c4iw_cq *rchp; 1460256694Snp struct c4iw_create_qp_resp uresp; 1461256694Snp int sqsize, rqsize; 1462256694Snp struct c4iw_ucontext *ucontext; 1463256694Snp int ret; 1464256694Snp struct c4iw_mm_entry *mm1, *mm2, *mm3, *mm4; 1465256694Snp 1466256694Snp CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd); 1467256694Snp 1468256694Snp if (attrs->qp_type != IB_QPT_RC) 1469256694Snp return ERR_PTR(-EINVAL); 1470256694Snp 1471256694Snp php = to_c4iw_pd(pd); 1472256694Snp rhp = php->rhp; 1473256694Snp schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid); 1474256694Snp rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid); 1475256694Snp if (!schp || !rchp) 1476256694Snp return ERR_PTR(-EINVAL); 1477256694Snp 1478256694Snp if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE) 1479256694Snp return ERR_PTR(-EINVAL); 1480256694Snp 1481256694Snp rqsize = roundup(attrs->cap.max_recv_wr + 1, 16); 1482256694Snp if (rqsize > T4_MAX_RQ_SIZE) 1483256694Snp return ERR_PTR(-E2BIG); 1484256694Snp 1485256694Snp sqsize = roundup(attrs->cap.max_send_wr + 1, 16); 1486256694Snp if (sqsize > T4_MAX_SQ_SIZE) 1487256694Snp return ERR_PTR(-E2BIG); 1488256694Snp 1489256694Snp ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL; 1490256694Snp 1491256694Snp 1492256694Snp qhp = kzalloc(sizeof(*qhp), GFP_KERNEL); 1493256694Snp if (!qhp) 1494256694Snp return ERR_PTR(-ENOMEM); 1495256694Snp qhp->wq.sq.size = sqsize; 1496256694Snp qhp->wq.sq.memsize = (sqsize + 1) * sizeof *qhp->wq.sq.queue; 1497256694Snp qhp->wq.rq.size = rqsize; 1498256694Snp qhp->wq.rq.memsize = (rqsize + 1) * sizeof *qhp->wq.rq.queue; 1499256694Snp 1500256694Snp if (ucontext) { 1501256694Snp qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE); 1502256694Snp qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE); 1503256694Snp } 1504256694Snp 1505256694Snp CTR5(KTR_IW_CXGBE, "%s sqsize %u sqmemsize %zu rqsize %u rqmemsize %zu", 1506256694Snp __func__, sqsize, qhp->wq.sq.memsize, rqsize, qhp->wq.rq.memsize); 1507256694Snp 1508256694Snp ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq, 1509256694Snp ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 1510256694Snp if (ret) 1511256694Snp goto err1; 1512256694Snp 1513256694Snp attrs->cap.max_recv_wr = rqsize - 1; 1514256694Snp attrs->cap.max_send_wr = sqsize - 1; 1515256694Snp attrs->cap.max_inline_data = T4_MAX_SEND_INLINE; 1516256694Snp 1517256694Snp qhp->rhp = rhp; 1518256694Snp qhp->attr.pd = php->pdid; 1519256694Snp qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid; 1520256694Snp qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid; 1521256694Snp qhp->attr.sq_num_entries = attrs->cap.max_send_wr; 1522256694Snp qhp->attr.rq_num_entries = attrs->cap.max_recv_wr; 1523256694Snp qhp->attr.sq_max_sges = attrs->cap.max_send_sge; 1524256694Snp qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge; 1525256694Snp qhp->attr.rq_max_sges = attrs->cap.max_recv_sge; 1526256694Snp qhp->attr.state = C4IW_QP_STATE_IDLE; 1527256694Snp qhp->attr.next_state = C4IW_QP_STATE_IDLE; 1528256694Snp qhp->attr.enable_rdma_read = 1; 1529256694Snp qhp->attr.enable_rdma_write = 1; 1530256694Snp qhp->attr.enable_bind = 1; 1531256694Snp qhp->attr.max_ord = 1; 1532256694Snp qhp->attr.max_ird = 1; 1533256694Snp spin_lock_init(&qhp->lock); 1534256694Snp mutex_init(&qhp->mutex); 1535256694Snp init_waitqueue_head(&qhp->wait); 1536256694Snp atomic_set(&qhp->refcnt, 1); 1537256694Snp 1538256694Snp spin_lock_irq(&rhp->lock); 1539256694Snp if (rhp->db_state != NORMAL) 1540256694Snp t4_disable_wq_db(&qhp->wq); 1541256694Snp if (++rhp->qpcnt > db_fc_threshold && rhp->db_state == NORMAL) { 1542256694Snp rhp->rdev.stats.db_state_transitions++; 1543256694Snp rhp->db_state = FLOW_CONTROL; 1544256694Snp idr_for_each(&rhp->qpidr, disable_qp_db, NULL); 1545256694Snp } 1546256694Snp ret = insert_handle_nolock(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid); 1547256694Snp spin_unlock_irq(&rhp->lock); 1548256694Snp if (ret) 1549256694Snp goto err2; 1550256694Snp 1551256694Snp if (udata) { 1552256694Snp mm1 = kmalloc(sizeof *mm1, GFP_KERNEL); 1553256694Snp if (!mm1) { 1554256694Snp ret = -ENOMEM; 1555256694Snp goto err3; 1556256694Snp } 1557256694Snp mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); 1558256694Snp if (!mm2) { 1559256694Snp ret = -ENOMEM; 1560256694Snp goto err4; 1561256694Snp } 1562256694Snp mm3 = kmalloc(sizeof *mm3, GFP_KERNEL); 1563256694Snp if (!mm3) { 1564256694Snp ret = -ENOMEM; 1565256694Snp goto err5; 1566256694Snp } 1567256694Snp mm4 = kmalloc(sizeof *mm4, GFP_KERNEL); 1568256694Snp if (!mm4) { 1569256694Snp ret = -ENOMEM; 1570256694Snp goto err6; 1571256694Snp } 1572256694Snp uresp.flags = 0; 1573256694Snp uresp.qid_mask = rhp->rdev.qpmask; 1574256694Snp uresp.sqid = qhp->wq.sq.qid; 1575256694Snp uresp.sq_size = qhp->wq.sq.size; 1576256694Snp uresp.sq_memsize = qhp->wq.sq.memsize; 1577256694Snp uresp.rqid = qhp->wq.rq.qid; 1578256694Snp uresp.rq_size = qhp->wq.rq.size; 1579256694Snp uresp.rq_memsize = qhp->wq.rq.memsize; 1580256694Snp spin_lock(&ucontext->mmap_lock); 1581256694Snp uresp.sq_key = ucontext->key; 1582256694Snp ucontext->key += PAGE_SIZE; 1583256694Snp uresp.rq_key = ucontext->key; 1584256694Snp ucontext->key += PAGE_SIZE; 1585256694Snp uresp.sq_db_gts_key = ucontext->key; 1586256694Snp ucontext->key += PAGE_SIZE; 1587256694Snp uresp.rq_db_gts_key = ucontext->key; 1588256694Snp ucontext->key += PAGE_SIZE; 1589256694Snp spin_unlock(&ucontext->mmap_lock); 1590256694Snp ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); 1591256694Snp if (ret) 1592256694Snp goto err7; 1593256694Snp mm1->key = uresp.sq_key; 1594256694Snp mm1->addr = qhp->wq.sq.phys_addr; 1595256694Snp mm1->len = PAGE_ALIGN(qhp->wq.sq.memsize); 1596256694Snp CTR4(KTR_IW_CXGBE, "%s mm1 %x, %x, %d", __func__, mm1->key, 1597256694Snp mm1->addr, mm1->len); 1598256694Snp insert_mmap(ucontext, mm1); 1599256694Snp mm2->key = uresp.rq_key; 1600256694Snp mm2->addr = vtophys(qhp->wq.rq.queue); 1601256694Snp mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize); 1602256694Snp CTR4(KTR_IW_CXGBE, "%s mm2 %x, %x, %d", __func__, mm2->key, 1603256694Snp mm2->addr, mm2->len); 1604256694Snp insert_mmap(ucontext, mm2); 1605256694Snp mm3->key = uresp.sq_db_gts_key; 1606256694Snp mm3->addr = qhp->wq.sq.udb; 1607256694Snp mm3->len = PAGE_SIZE; 1608256694Snp CTR4(KTR_IW_CXGBE, "%s mm3 %x, %x, %d", __func__, mm3->key, 1609256694Snp mm3->addr, mm3->len); 1610256694Snp insert_mmap(ucontext, mm3); 1611256694Snp mm4->key = uresp.rq_db_gts_key; 1612256694Snp mm4->addr = qhp->wq.rq.udb; 1613256694Snp mm4->len = PAGE_SIZE; 1614256694Snp CTR4(KTR_IW_CXGBE, "%s mm4 %x, %x, %d", __func__, mm4->key, 1615256694Snp mm4->addr, mm4->len); 1616256694Snp insert_mmap(ucontext, mm4); 1617256694Snp } 1618256694Snp qhp->ibqp.qp_num = qhp->wq.sq.qid; 1619256694Snp init_timer(&(qhp->timer)); 1620256694Snp CTR5(KTR_IW_CXGBE, 1621256694Snp "%s qhp %p sq_num_entries %d, rq_num_entries %d qpid 0x%0x", 1622256694Snp __func__, qhp, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, 1623256694Snp qhp->wq.sq.qid); 1624256694Snp return &qhp->ibqp; 1625256694Snperr7: 1626256694Snp kfree(mm4); 1627256694Snperr6: 1628256694Snp kfree(mm3); 1629256694Snperr5: 1630256694Snp kfree(mm2); 1631256694Snperr4: 1632256694Snp kfree(mm1); 1633256694Snperr3: 1634256694Snp remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid); 1635256694Snperr2: 1636256694Snp destroy_qp(&rhp->rdev, &qhp->wq, 1637256694Snp ucontext ? &ucontext->uctx : &rhp->rdev.uctx); 1638256694Snperr1: 1639256694Snp kfree(qhp); 1640256694Snp return ERR_PTR(ret); 1641256694Snp} 1642256694Snp 1643256694Snpint c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1644256694Snp int attr_mask, struct ib_udata *udata) 1645256694Snp{ 1646256694Snp struct c4iw_dev *rhp; 1647256694Snp struct c4iw_qp *qhp; 1648256694Snp enum c4iw_qp_attr_mask mask = 0; 1649256694Snp struct c4iw_qp_attributes attrs; 1650256694Snp 1651256694Snp CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ibqp); 1652256694Snp 1653256694Snp /* iwarp does not support the RTR state */ 1654256694Snp if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR)) 1655256694Snp attr_mask &= ~IB_QP_STATE; 1656256694Snp 1657256694Snp /* Make sure we still have something left to do */ 1658256694Snp if (!attr_mask) 1659256694Snp return 0; 1660256694Snp 1661256694Snp memset(&attrs, 0, sizeof attrs); 1662256694Snp qhp = to_c4iw_qp(ibqp); 1663256694Snp rhp = qhp->rhp; 1664256694Snp 1665256694Snp attrs.next_state = c4iw_convert_state(attr->qp_state); 1666256694Snp attrs.enable_rdma_read = (attr->qp_access_flags & 1667256694Snp IB_ACCESS_REMOTE_READ) ? 1 : 0; 1668256694Snp attrs.enable_rdma_write = (attr->qp_access_flags & 1669256694Snp IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 1670256694Snp attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0; 1671256694Snp 1672256694Snp 1673256694Snp mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0; 1674256694Snp mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ? 1675256694Snp (C4IW_QP_ATTR_ENABLE_RDMA_READ | 1676256694Snp C4IW_QP_ATTR_ENABLE_RDMA_WRITE | 1677256694Snp C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0; 1678256694Snp 1679256694Snp /* 1680256694Snp * Use SQ_PSN and RQ_PSN to pass in IDX_INC values for 1681256694Snp * ringing the queue db when we're in DB_FULL mode. 1682256694Snp */ 1683256694Snp attrs.sq_db_inc = attr->sq_psn; 1684256694Snp attrs.rq_db_inc = attr->rq_psn; 1685256694Snp mask |= (attr_mask & IB_QP_SQ_PSN) ? C4IW_QP_ATTR_SQ_DB : 0; 1686256694Snp mask |= (attr_mask & IB_QP_RQ_PSN) ? C4IW_QP_ATTR_RQ_DB : 0; 1687256694Snp 1688256694Snp return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0); 1689256694Snp} 1690256694Snp 1691256694Snpstruct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn) 1692256694Snp{ 1693256694Snp CTR3(KTR_IW_CXGBE, "%s ib_dev %p qpn 0x%x", __func__, dev, qpn); 1694256694Snp return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn); 1695256694Snp} 1696256694Snp 1697256694Snpint c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1698256694Snp int attr_mask, struct ib_qp_init_attr *init_attr) 1699256694Snp{ 1700256694Snp struct c4iw_qp *qhp = to_c4iw_qp(ibqp); 1701256694Snp 1702256694Snp memset(attr, 0, sizeof *attr); 1703256694Snp memset(init_attr, 0, sizeof *init_attr); 1704256694Snp attr->qp_state = to_ib_qp_state(qhp->attr.state); 1705256694Snp return 0; 1706256694Snp} 1707256694Snp#endif 1708