1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD$"); 31178786Skmacy 32237263Snp#include "opt_inet.h" 33237263Snp 34237263Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/rwlock.h> 48178786Skmacy#include <sys/linker.h> 49178786Skmacy#include <sys/firmware.h> 50178786Skmacy#include <sys/socket.h> 51178786Skmacy#include <sys/socketvar.h> 52178786Skmacy#include <sys/sockio.h> 53178786Skmacy#include <sys/smp.h> 54178786Skmacy#include <sys/sysctl.h> 55178786Skmacy#include <sys/syslog.h> 56178786Skmacy#include <sys/queue.h> 57178786Skmacy#include <sys/taskqueue.h> 58178786Skmacy#include <sys/proc.h> 59178786Skmacy#include <sys/uio.h> 60178786Skmacy 61178786Skmacy#include <net/route.h> 62178786Skmacy#include <netinet/in_systm.h> 63178786Skmacy#include <netinet/in.h> 64178786Skmacy#include <netinet/in_pcb.h> 65178786Skmacy#include <netinet/ip.h> 66178786Skmacy#include <netinet/ip_var.h> 67178786Skmacy#include <netinet/tcp_var.h> 68178786Skmacy#include <netinet/tcp.h> 69178786Skmacy#include <netinet/tcpip.h> 70178786Skmacy 71237263Snp#include <rdma/ib_verbs.h> 72237263Snp#include <linux/idr.h> 73237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 74178786Skmacy 75178786Skmacy#include <cxgb_include.h> 76178786Skmacy#include <ulp/tom/cxgb_tom.h> 77178786Skmacy#include <ulp/tom/cxgb_toepcb.h> 78237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 79237263Snp#include <rdma/ib_verbs.h> 80237263Snp#include <linux/idr.h> 81237263Snp 82178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 83178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 84178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 85178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 86178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 87178786Skmacy 88178786Skmacy#ifdef KTR 89178786Skmacystatic char *states[] = { 90178786Skmacy "idle", 91178786Skmacy "listen", 92178786Skmacy "connecting", 93178786Skmacy "mpa_wait_req", 94178786Skmacy "mpa_req_sent", 95178786Skmacy "mpa_req_rcvd", 96178786Skmacy "mpa_rep_sent", 97178786Skmacy "fpdu_mode", 98178786Skmacy "aborting", 99178786Skmacy "closing", 100178786Skmacy "moribund", 101178786Skmacy "dead", 102178786Skmacy NULL, 103178786Skmacy}; 104178786Skmacy#endif 105178786Skmacy 106237263SnpSYSCTL_NODE(_hw, OID_AUTO, iw_cxgb, CTLFLAG_RD, 0, "iw_cxgb driver parameters"); 107178786Skmacy 108237263Snpstatic int ep_timeout_secs = 60; 109178786SkmacyTUNABLE_INT("hw.iw_cxgb.ep_timeout_secs", &ep_timeout_secs); 110237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, ep_timeout_secs, CTLFLAG_RW, &ep_timeout_secs, 0, 111237263Snp "CM Endpoint operation timeout in seconds (default=60)"); 112178786Skmacy 113178786Skmacystatic int mpa_rev = 1; 114178786SkmacyTUNABLE_INT("hw.iw_cxgb.mpa_rev", &mpa_rev); 115237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, mpa_rev, CTLFLAG_RW, &mpa_rev, 0, 116178786Skmacy "MPA Revision, 0 supports amso1100, 1 is spec compliant. (default=1)"); 117178786Skmacy 118178786Skmacystatic int markers_enabled = 0; 119178786SkmacyTUNABLE_INT("hw.iw_cxgb.markers_enabled", &markers_enabled); 120237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, markers_enabled, CTLFLAG_RW, &markers_enabled, 0, 121178786Skmacy "Enable MPA MARKERS (default(0)=disabled)"); 122178786Skmacy 123178786Skmacystatic int crc_enabled = 1; 124178786SkmacyTUNABLE_INT("hw.iw_cxgb.crc_enabled", &crc_enabled); 125237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, crc_enabled, CTLFLAG_RW, &crc_enabled, 0, 126178786Skmacy "Enable MPA CRC (default(1)=enabled)"); 127178786Skmacy 128178786Skmacystatic int rcv_win = 256 * 1024; 129178786SkmacyTUNABLE_INT("hw.iw_cxgb.rcv_win", &rcv_win); 130237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, rcv_win, CTLFLAG_RW, &rcv_win, 0, 131178786Skmacy "TCP receive window in bytes (default=256KB)"); 132178786Skmacy 133178786Skmacystatic int snd_win = 32 * 1024; 134178786SkmacyTUNABLE_INT("hw.iw_cxgb.snd_win", &snd_win); 135237263SnpSYSCTL_INT(_hw_iw_cxgb, OID_AUTO, snd_win, CTLFLAG_RW, &snd_win, 0, 136178786Skmacy "TCP send window in bytes (default=32KB)"); 137178786Skmacy 138178786Skmacystatic unsigned int nocong = 0; 139178786SkmacyTUNABLE_INT("hw.iw_cxgb.nocong", &nocong); 140237263SnpSYSCTL_UINT(_hw_iw_cxgb, OID_AUTO, nocong, CTLFLAG_RW, &nocong, 0, 141178786Skmacy "Turn off congestion control (default=0)"); 142178786Skmacy 143178786Skmacystatic unsigned int cong_flavor = 1; 144178786SkmacyTUNABLE_INT("hw.iw_cxgb.cong_flavor", &cong_flavor); 145237263SnpSYSCTL_UINT(_hw_iw_cxgb, OID_AUTO, cong_flavor, CTLFLAG_RW, &cong_flavor, 0, 146178786Skmacy "TCP Congestion control flavor (default=1)"); 147178786Skmacy 148178786Skmacystatic void ep_timeout(void *arg); 149178786Skmacystatic void connect_reply_upcall(struct iwch_ep *ep, int status); 150193272Sjhbstatic int iwch_so_upcall(struct socket *so, void *arg, int waitflag); 151178786Skmacy 152178786Skmacy/* 153178786Skmacy * Cruft to offload socket upcalls onto thread. 154178786Skmacy */ 155178786Skmacystatic struct mtx req_lock; 156178786Skmacystatic TAILQ_HEAD(iwch_ep_list, iwch_ep_common) req_list; 157178786Skmacystatic struct task iw_cxgb_task; 158178786Skmacystatic struct taskqueue *iw_cxgb_taskq; 159178786Skmacystatic void process_req(void *ctx, int pending); 160178786Skmacy 161178786Skmacystatic void 162178786Skmacystart_ep_timer(struct iwch_ep *ep) 163178786Skmacy{ 164178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 165178786Skmacy if (callout_pending(&ep->timer)) { 166178786Skmacy CTR2(KTR_IW_CXGB, "%s stopped / restarted timer ep %p", __FUNCTION__, ep); 167178786Skmacy callout_deactivate(&ep->timer); 168178786Skmacy callout_drain(&ep->timer); 169178786Skmacy } else { 170178786Skmacy /* 171178786Skmacy * XXX this looks racy 172178786Skmacy */ 173178786Skmacy get_ep(&ep->com); 174178786Skmacy callout_init(&ep->timer, TRUE); 175178786Skmacy } 176178786Skmacy callout_reset(&ep->timer, ep_timeout_secs * hz, ep_timeout, ep); 177178786Skmacy} 178178786Skmacy 179178786Skmacystatic void 180178786Skmacystop_ep_timer(struct iwch_ep *ep) 181178786Skmacy{ 182178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 183237263Snp if (!callout_pending(&ep->timer)) { 184237263Snp CTR3(KTR_IW_CXGB, "%s timer stopped when its not running! ep %p state %u\n", 185237263Snp __func__, ep, ep->com.state); 186237263Snp return; 187237263Snp } 188178786Skmacy callout_drain(&ep->timer); 189178786Skmacy put_ep(&ep->com); 190178786Skmacy} 191178786Skmacy 192237263Snpstatic int 193237263Snpset_tcpinfo(struct iwch_ep *ep) 194178786Skmacy{ 195237263Snp struct socket *so = ep->com.so; 196237263Snp struct inpcb *inp = sotoinpcb(so); 197237263Snp struct tcpcb *tp; 198237263Snp struct toepcb *toep; 199237263Snp int rc = 0; 200178786Skmacy 201237263Snp INP_WLOCK(inp); 202237263Snp tp = intotcpcb(inp); 203237263Snp 204237263Snp if ((tp->t_flags & TF_TOE) == 0) { 205237263Snp rc = EINVAL; 206237263Snp printf("%s: connection NOT OFFLOADED!\n", __func__); 207237263Snp goto done; 208178786Skmacy } 209237263Snp toep = tp->t_toe; 210178786Skmacy 211237263Snp ep->hwtid = toep->tp_tid; 212237263Snp ep->snd_seq = tp->snd_nxt; 213237263Snp ep->rcv_seq = tp->rcv_nxt; 214237263Snp ep->emss = tp->t_maxseg; 215178786Skmacy if (ep->emss < 128) 216178786Skmacy ep->emss = 128; 217237263Snpdone: 218237263Snp INP_WUNLOCK(inp); 219237263Snp return (rc); 220237263Snp 221178786Skmacy} 222178786Skmacy 223178786Skmacystatic enum iwch_ep_state 224178786Skmacystate_read(struct iwch_ep_common *epc) 225178786Skmacy{ 226178786Skmacy enum iwch_ep_state state; 227178786Skmacy 228178786Skmacy mtx_lock(&epc->lock); 229178786Skmacy state = epc->state; 230178786Skmacy mtx_unlock(&epc->lock); 231178786Skmacy return state; 232178786Skmacy} 233178786Skmacy 234178786Skmacystatic void 235178786Skmacy__state_set(struct iwch_ep_common *epc, enum iwch_ep_state new) 236178786Skmacy{ 237178786Skmacy epc->state = new; 238178786Skmacy} 239178786Skmacy 240178786Skmacystatic void 241178786Skmacystate_set(struct iwch_ep_common *epc, enum iwch_ep_state new) 242178786Skmacy{ 243178786Skmacy 244178786Skmacy mtx_lock(&epc->lock); 245178786Skmacy CTR3(KTR_IW_CXGB, "%s - %s -> %s", __FUNCTION__, states[epc->state], states[new]); 246178786Skmacy __state_set(epc, new); 247178786Skmacy mtx_unlock(&epc->lock); 248178786Skmacy return; 249178786Skmacy} 250178786Skmacy 251178786Skmacystatic void * 252178786Skmacyalloc_ep(int size, int flags) 253178786Skmacy{ 254178786Skmacy struct iwch_ep_common *epc; 255178786Skmacy 256178786Skmacy epc = malloc(size, M_DEVBUF, flags); 257178786Skmacy if (epc) { 258178786Skmacy memset(epc, 0, size); 259178786Skmacy refcount_init(&epc->refcount, 1); 260178786Skmacy mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF|MTX_DUPOK); 261178786Skmacy cv_init(&epc->waitq, "iwch_epc cv"); 262178786Skmacy } 263178786Skmacy CTR2(KTR_IW_CXGB, "%s alloc ep %p", __FUNCTION__, epc); 264178786Skmacy return epc; 265178786Skmacy} 266178786Skmacy 267178786Skmacyvoid __free_ep(struct iwch_ep_common *epc) 268178786Skmacy{ 269178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p state %s", __FUNCTION__, epc, states[state_read(epc)]); 270178786Skmacy KASSERT(!epc->so, ("%s warning ep->so %p \n", __FUNCTION__, epc->so)); 271178786Skmacy KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list!\n", __FUNCTION__, epc)); 272178786Skmacy free(epc, M_DEVBUF); 273178786Skmacy} 274178786Skmacy 275178786Skmacystatic struct rtentry * 276178786Skmacyfind_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 277178786Skmacy __be16 peer_port, u8 tos) 278178786Skmacy{ 279178786Skmacy struct route iproute; 280178786Skmacy struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst; 281178786Skmacy 282178786Skmacy bzero(&iproute, sizeof iproute); 283178786Skmacy dst->sin_family = AF_INET; 284178786Skmacy dst->sin_len = sizeof *dst; 285178786Skmacy dst->sin_addr.s_addr = peer_ip; 286178786Skmacy 287178786Skmacy rtalloc(&iproute); 288178786Skmacy return iproute.ro_rt; 289178786Skmacy} 290178786Skmacy 291178786Skmacystatic void 292237263Snpclose_socket(struct iwch_ep_common *epc, int close) 293178786Skmacy{ 294178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]); 295178786Skmacy SOCK_LOCK(epc->so); 296193272Sjhb soupcall_clear(epc->so, SO_RCV); 297178786Skmacy SOCK_UNLOCK(epc->so); 298237263Snp if (close) 299237263Snp soclose(epc->so); 300237263Snp else 301237263Snp soshutdown(epc->so, SHUT_WR|SHUT_RD); 302178786Skmacy epc->so = NULL; 303178786Skmacy} 304178786Skmacy 305178786Skmacystatic void 306178786Skmacyshutdown_socket(struct iwch_ep_common *epc) 307178786Skmacy{ 308178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]); 309178786Skmacy soshutdown(epc->so, SHUT_WR); 310178786Skmacy} 311178786Skmacy 312178786Skmacystatic void 313178786Skmacyabort_socket(struct iwch_ep *ep) 314178786Skmacy{ 315178786Skmacy struct sockopt sopt; 316178786Skmacy int err; 317178786Skmacy struct linger l; 318178786Skmacy 319178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 320178786Skmacy l.l_onoff = 1; 321178786Skmacy l.l_linger = 0; 322178786Skmacy 323178786Skmacy /* linger_time of 0 forces RST to be sent */ 324178786Skmacy sopt.sopt_dir = SOPT_SET; 325178786Skmacy sopt.sopt_level = SOL_SOCKET; 326178786Skmacy sopt.sopt_name = SO_LINGER; 327178786Skmacy sopt.sopt_val = (caddr_t)&l; 328178786Skmacy sopt.sopt_valsize = sizeof l; 329178786Skmacy sopt.sopt_td = NULL; 330178786Skmacy err = sosetopt(ep->com.so, &sopt); 331178786Skmacy if (err) 332178786Skmacy printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err); 333178786Skmacy} 334178786Skmacy 335178786Skmacystatic void 336178786Skmacysend_mpa_req(struct iwch_ep *ep) 337178786Skmacy{ 338178786Skmacy int mpalen; 339178786Skmacy struct mpa_message *mpa; 340178786Skmacy struct mbuf *m; 341178786Skmacy int err; 342178786Skmacy 343178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p pd_len %d", __FUNCTION__, ep, ep->plen); 344178786Skmacy 345178786Skmacy mpalen = sizeof(*mpa) + ep->plen; 346178786Skmacy m = m_gethdr(mpalen, M_NOWAIT); 347178786Skmacy if (m == NULL) { 348178786Skmacy connect_reply_upcall(ep, -ENOMEM); 349178786Skmacy return; 350178786Skmacy } 351178786Skmacy mpa = mtod(m, struct mpa_message *); 352178786Skmacy m->m_len = mpalen; 353178786Skmacy m->m_pkthdr.len = mpalen; 354178786Skmacy memset(mpa, 0, sizeof(*mpa)); 355178786Skmacy memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 356178786Skmacy mpa->flags = (crc_enabled ? MPA_CRC : 0) | 357178786Skmacy (markers_enabled ? MPA_MARKERS : 0); 358178786Skmacy mpa->private_data_size = htons(ep->plen); 359178786Skmacy mpa->revision = mpa_rev; 360178786Skmacy if (ep->plen) 361178786Skmacy memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen); 362178786Skmacy 363178786Skmacy err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 364178786Skmacy if (err) { 365178786Skmacy m_freem(m); 366178786Skmacy connect_reply_upcall(ep, -ENOMEM); 367178786Skmacy return; 368178786Skmacy } 369178786Skmacy 370178786Skmacy start_ep_timer(ep); 371178786Skmacy state_set(&ep->com, MPA_REQ_SENT); 372178786Skmacy return; 373178786Skmacy} 374178786Skmacy 375178786Skmacystatic int 376178786Skmacysend_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen) 377178786Skmacy{ 378178786Skmacy int mpalen; 379178786Skmacy struct mpa_message *mpa; 380178786Skmacy struct mbuf *m; 381178786Skmacy int err; 382178786Skmacy 383178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p plen %d", __FUNCTION__, ep, plen); 384178786Skmacy 385178786Skmacy mpalen = sizeof(*mpa) + plen; 386178786Skmacy 387178786Skmacy m = m_gethdr(mpalen, M_NOWAIT); 388178786Skmacy if (m == NULL) { 389178786Skmacy printf("%s - cannot alloc mbuf!\n", __FUNCTION__); 390178786Skmacy return (-ENOMEM); 391178786Skmacy } 392178786Skmacy mpa = mtod(m, struct mpa_message *); 393178786Skmacy m->m_len = mpalen; 394178786Skmacy m->m_pkthdr.len = mpalen; 395178786Skmacy memset(mpa, 0, sizeof(*mpa)); 396178786Skmacy memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 397178786Skmacy mpa->flags = MPA_REJECT; 398178786Skmacy mpa->revision = mpa_rev; 399178786Skmacy mpa->private_data_size = htons(plen); 400178786Skmacy if (plen) 401178786Skmacy memcpy(mpa->private_data, pdata, plen); 402178786Skmacy err = sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 403178786Skmacy PANIC_IF(err); 404178786Skmacy return 0; 405178786Skmacy} 406178786Skmacy 407178786Skmacystatic int 408178786Skmacysend_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen) 409178786Skmacy{ 410178786Skmacy int mpalen; 411178786Skmacy struct mpa_message *mpa; 412178786Skmacy struct mbuf *m; 413178786Skmacy 414178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p plen %d", __FUNCTION__, ep, ep->com.so, plen); 415178786Skmacy 416178786Skmacy mpalen = sizeof(*mpa) + plen; 417178786Skmacy 418178786Skmacy m = m_gethdr(mpalen, M_NOWAIT); 419178786Skmacy if (m == NULL) { 420178786Skmacy printf("%s - cannot alloc mbuf!\n", __FUNCTION__); 421178786Skmacy return (-ENOMEM); 422178786Skmacy } 423178786Skmacy mpa = mtod(m, struct mpa_message *); 424178786Skmacy m->m_len = mpalen; 425178786Skmacy m->m_pkthdr.len = mpalen; 426178786Skmacy memset(mpa, 0, sizeof(*mpa)); 427178786Skmacy memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 428178786Skmacy mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 429178786Skmacy (markers_enabled ? MPA_MARKERS : 0); 430178786Skmacy mpa->revision = mpa_rev; 431178786Skmacy mpa->private_data_size = htons(plen); 432178786Skmacy if (plen) 433178786Skmacy memcpy(mpa->private_data, pdata, plen); 434178786Skmacy 435178786Skmacy state_set(&ep->com, MPA_REP_SENT); 436178786Skmacy return sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 437178786Skmacy ep->com.thread); 438178786Skmacy} 439178786Skmacy 440178786Skmacystatic void 441178786Skmacyclose_complete_upcall(struct iwch_ep *ep) 442178786Skmacy{ 443178786Skmacy struct iw_cm_event event; 444178786Skmacy 445178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 446178786Skmacy memset(&event, 0, sizeof(event)); 447178786Skmacy event.event = IW_CM_EVENT_CLOSE; 448178786Skmacy if (ep->com.cm_id) { 449178786Skmacy CTR3(KTR_IW_CXGB, "close complete delivered ep %p cm_id %p tid %d", 450178786Skmacy ep, ep->com.cm_id, ep->hwtid); 451178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 452178786Skmacy ep->com.cm_id->rem_ref(ep->com.cm_id); 453178786Skmacy ep->com.cm_id = NULL; 454178786Skmacy ep->com.qp = NULL; 455178786Skmacy } 456178786Skmacy} 457178786Skmacy 458178786Skmacystatic void 459178786Skmacyabort_connection(struct iwch_ep *ep) 460178786Skmacy{ 461178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 462178786Skmacy state_set(&ep->com, ABORTING); 463178786Skmacy abort_socket(ep); 464237263Snp close_socket(&ep->com, 0); 465178786Skmacy close_complete_upcall(ep); 466178786Skmacy state_set(&ep->com, DEAD); 467178786Skmacy put_ep(&ep->com); 468178786Skmacy} 469178786Skmacy 470178786Skmacystatic void 471178786Skmacypeer_close_upcall(struct iwch_ep *ep) 472178786Skmacy{ 473178786Skmacy struct iw_cm_event event; 474178786Skmacy 475178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 476178786Skmacy memset(&event, 0, sizeof(event)); 477178786Skmacy event.event = IW_CM_EVENT_DISCONNECT; 478178786Skmacy if (ep->com.cm_id) { 479178786Skmacy CTR3(KTR_IW_CXGB, "peer close delivered ep %p cm_id %p tid %d", 480178786Skmacy ep, ep->com.cm_id, ep->hwtid); 481178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 482178786Skmacy } 483178786Skmacy} 484178786Skmacy 485178786Skmacystatic void 486178786Skmacypeer_abort_upcall(struct iwch_ep *ep) 487178786Skmacy{ 488178786Skmacy struct iw_cm_event event; 489178786Skmacy 490178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 491178786Skmacy memset(&event, 0, sizeof(event)); 492178786Skmacy event.event = IW_CM_EVENT_CLOSE; 493178786Skmacy event.status = ECONNRESET; 494178786Skmacy if (ep->com.cm_id) { 495178786Skmacy CTR3(KTR_IW_CXGB, "abort delivered ep %p cm_id %p tid %d", ep, 496178786Skmacy ep->com.cm_id, ep->hwtid); 497178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 498178786Skmacy ep->com.cm_id->rem_ref(ep->com.cm_id); 499178786Skmacy ep->com.cm_id = NULL; 500178786Skmacy ep->com.qp = NULL; 501178786Skmacy } 502178786Skmacy} 503178786Skmacy 504178786Skmacystatic void 505178786Skmacyconnect_reply_upcall(struct iwch_ep *ep, int status) 506178786Skmacy{ 507178786Skmacy struct iw_cm_event event; 508178786Skmacy 509178786Skmacy CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status); 510178786Skmacy memset(&event, 0, sizeof(event)); 511178786Skmacy event.event = IW_CM_EVENT_CONNECT_REPLY; 512178786Skmacy event.status = status; 513178786Skmacy event.local_addr = ep->com.local_addr; 514178786Skmacy event.remote_addr = ep->com.remote_addr; 515178786Skmacy 516178786Skmacy if ((status == 0) || (status == ECONNREFUSED)) { 517178786Skmacy event.private_data_len = ep->plen; 518178786Skmacy event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 519178786Skmacy } 520178786Skmacy if (ep->com.cm_id) { 521178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p tid %d status %d", __FUNCTION__, ep, 522178786Skmacy ep->hwtid, status); 523178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 524178786Skmacy } 525178786Skmacy if (status < 0) { 526178786Skmacy ep->com.cm_id->rem_ref(ep->com.cm_id); 527178786Skmacy ep->com.cm_id = NULL; 528178786Skmacy ep->com.qp = NULL; 529178786Skmacy } 530178786Skmacy} 531178786Skmacy 532178786Skmacystatic void 533178786Skmacyconnect_request_upcall(struct iwch_ep *ep) 534178786Skmacy{ 535178786Skmacy struct iw_cm_event event; 536178786Skmacy 537178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 538178786Skmacy memset(&event, 0, sizeof(event)); 539178786Skmacy event.event = IW_CM_EVENT_CONNECT_REQUEST; 540178786Skmacy event.local_addr = ep->com.local_addr; 541178786Skmacy event.remote_addr = ep->com.remote_addr; 542178786Skmacy event.private_data_len = ep->plen; 543178786Skmacy event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 544178786Skmacy event.provider_data = ep; 545178786Skmacy event.so = ep->com.so; 546237263Snp if (state_read(&ep->parent_ep->com) != DEAD) { 547237263Snp get_ep(&ep->com); 548178786Skmacy ep->parent_ep->com.cm_id->event_handler( 549178786Skmacy ep->parent_ep->com.cm_id, 550178786Skmacy &event); 551237263Snp } 552178786Skmacy put_ep(&ep->parent_ep->com); 553178786Skmacy} 554178786Skmacy 555178786Skmacystatic void 556178786Skmacyestablished_upcall(struct iwch_ep *ep) 557178786Skmacy{ 558178786Skmacy struct iw_cm_event event; 559178786Skmacy 560178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 561178786Skmacy memset(&event, 0, sizeof(event)); 562178786Skmacy event.event = IW_CM_EVENT_ESTABLISHED; 563178786Skmacy if (ep->com.cm_id) { 564178786Skmacy CTR3(KTR_IW_CXGB, "%s ep %p tid %d", __FUNCTION__, ep, ep->hwtid); 565178786Skmacy ep->com.cm_id->event_handler(ep->com.cm_id, &event); 566178786Skmacy } 567178786Skmacy} 568178786Skmacy 569178786Skmacystatic void 570178786Skmacyprocess_mpa_reply(struct iwch_ep *ep) 571178786Skmacy{ 572178786Skmacy struct mpa_message *mpa; 573178786Skmacy u16 plen; 574178786Skmacy struct iwch_qp_attributes attrs; 575178786Skmacy enum iwch_qp_attr_mask mask; 576178786Skmacy int err; 577178786Skmacy struct mbuf *top, *m; 578178786Skmacy int flags = MSG_DONTWAIT; 579178786Skmacy struct uio uio; 580178786Skmacy int len; 581178786Skmacy 582178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 583178786Skmacy 584178786Skmacy /* 585178786Skmacy * Stop mpa timer. If it expired, then the state has 586178786Skmacy * changed and we bail since ep_timeout already aborted 587178786Skmacy * the connection. 588178786Skmacy */ 589178786Skmacy stop_ep_timer(ep); 590178786Skmacy if (state_read(&ep->com) != MPA_REQ_SENT) 591178786Skmacy return; 592178786Skmacy 593178786Skmacy uio.uio_resid = len = 1000000; 594178786Skmacy uio.uio_td = ep->com.thread; 595178786Skmacy err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 596178786Skmacy if (err) { 597178786Skmacy if (err == EWOULDBLOCK) { 598178786Skmacy start_ep_timer(ep); 599178786Skmacy return; 600178786Skmacy } 601178786Skmacy err = -err; 602178786Skmacy goto err; 603178786Skmacy } 604178786Skmacy 605178786Skmacy if (ep->com.so->so_rcv.sb_mb) { 606178786Skmacy printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 607178786Skmacy __FUNCTION__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 608178786Skmacy } 609178786Skmacy 610178786Skmacy m = top; 611178786Skmacy do { 612178786Skmacy /* 613178786Skmacy * If we get more than the supported amount of private data 614178786Skmacy * then we must fail this connection. 615178786Skmacy */ 616178786Skmacy if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 617178786Skmacy err = (-EINVAL); 618178786Skmacy goto err; 619178786Skmacy } 620178786Skmacy 621178786Skmacy /* 622178786Skmacy * copy the new data into our accumulation buffer. 623178786Skmacy */ 624178786Skmacy m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 625178786Skmacy ep->mpa_pkt_len += m->m_len; 626178786Skmacy if (!m->m_next) 627178786Skmacy m = m->m_nextpkt; 628178786Skmacy else 629178786Skmacy m = m->m_next; 630178786Skmacy } while (m); 631178786Skmacy 632178786Skmacy m_freem(top); 633178786Skmacy 634178786Skmacy /* 635178786Skmacy * if we don't even have the mpa message, then bail. 636178786Skmacy */ 637178786Skmacy if (ep->mpa_pkt_len < sizeof(*mpa)) 638178786Skmacy return; 639178786Skmacy mpa = (struct mpa_message *)ep->mpa_pkt; 640178786Skmacy 641178786Skmacy /* Validate MPA header. */ 642178786Skmacy if (mpa->revision != mpa_rev) { 643178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision); 644178786Skmacy err = EPROTO; 645178786Skmacy goto err; 646178786Skmacy } 647178786Skmacy if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 648178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key); 649178786Skmacy err = EPROTO; 650178786Skmacy goto err; 651178786Skmacy } 652178786Skmacy 653178786Skmacy plen = ntohs(mpa->private_data_size); 654178786Skmacy 655178786Skmacy /* 656178786Skmacy * Fail if there's too much private data. 657178786Skmacy */ 658178786Skmacy if (plen > MPA_MAX_PRIVATE_DATA) { 659178786Skmacy CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen); 660178786Skmacy err = EPROTO; 661178786Skmacy goto err; 662178786Skmacy } 663178786Skmacy 664178786Skmacy /* 665178786Skmacy * If plen does not account for pkt size 666178786Skmacy */ 667178786Skmacy if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 668178786Skmacy CTR2(KTR_IW_CXGB, "%s pkt too big %d", __FUNCTION__, ep->mpa_pkt_len); 669178786Skmacy err = EPROTO; 670178786Skmacy goto err; 671178786Skmacy } 672178786Skmacy 673178786Skmacy ep->plen = (u8) plen; 674178786Skmacy 675178786Skmacy /* 676178786Skmacy * If we don't have all the pdata yet, then bail. 677178786Skmacy * We'll continue process when more data arrives. 678178786Skmacy */ 679178786Skmacy if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 680178786Skmacy return; 681178786Skmacy 682178786Skmacy if (mpa->flags & MPA_REJECT) { 683178786Skmacy err = ECONNREFUSED; 684178786Skmacy goto err; 685178786Skmacy } 686178786Skmacy 687178786Skmacy /* 688178786Skmacy * If we get here we have accumulated the entire mpa 689178786Skmacy * start reply message including private data. And 690178786Skmacy * the MPA header is valid. 691178786Skmacy */ 692178786Skmacy CTR1(KTR_IW_CXGB, "%s mpa rpl looks good!", __FUNCTION__); 693178786Skmacy state_set(&ep->com, FPDU_MODE); 694237263Snp ep->mpa_attr.initiator = 1; 695178786Skmacy ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 696178786Skmacy ep->mpa_attr.recv_marker_enabled = markers_enabled; 697178786Skmacy ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 698178786Skmacy ep->mpa_attr.version = mpa_rev; 699178786Skmacy if (set_tcpinfo(ep)) { 700178786Skmacy printf("%s set_tcpinfo error\n", __FUNCTION__); 701178786Skmacy goto err; 702178786Skmacy } 703178786Skmacy CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, " 704178786Skmacy "xmit_marker_enabled=%d, version=%d", __FUNCTION__, 705178786Skmacy ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 706178786Skmacy ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 707178786Skmacy 708178786Skmacy attrs.mpa_attr = ep->mpa_attr; 709178786Skmacy attrs.max_ird = ep->ird; 710178786Skmacy attrs.max_ord = ep->ord; 711178786Skmacy attrs.llp_stream_handle = ep; 712178786Skmacy attrs.next_state = IWCH_QP_STATE_RTS; 713178786Skmacy 714178786Skmacy mask = IWCH_QP_ATTR_NEXT_STATE | 715178786Skmacy IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR | 716178786Skmacy IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD; 717178786Skmacy 718178786Skmacy /* bind QP and TID with INIT_WR */ 719178786Skmacy err = iwch_modify_qp(ep->com.qp->rhp, 720178786Skmacy ep->com.qp, mask, &attrs, 1); 721178786Skmacy if (!err) 722178786Skmacy goto out; 723178786Skmacyerr: 724178786Skmacy abort_connection(ep); 725178786Skmacyout: 726178786Skmacy connect_reply_upcall(ep, err); 727178786Skmacy return; 728178786Skmacy} 729178786Skmacy 730178786Skmacystatic void 731178786Skmacyprocess_mpa_request(struct iwch_ep *ep) 732178786Skmacy{ 733178786Skmacy struct mpa_message *mpa; 734178786Skmacy u16 plen; 735178786Skmacy int flags = MSG_DONTWAIT; 736178786Skmacy struct mbuf *top, *m; 737178786Skmacy int err; 738178786Skmacy struct uio uio; 739178786Skmacy int len; 740178786Skmacy 741178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 742178786Skmacy 743178786Skmacy /* 744178786Skmacy * Stop mpa timer. If it expired, then the state has 745178786Skmacy * changed and we bail since ep_timeout already aborted 746178786Skmacy * the connection. 747178786Skmacy */ 748178786Skmacy stop_ep_timer(ep); 749178786Skmacy if (state_read(&ep->com) != MPA_REQ_WAIT) 750178786Skmacy return; 751178786Skmacy 752178786Skmacy uio.uio_resid = len = 1000000; 753178786Skmacy uio.uio_td = ep->com.thread; 754178786Skmacy err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 755178786Skmacy if (err) { 756178786Skmacy if (err == EWOULDBLOCK) { 757178786Skmacy start_ep_timer(ep); 758178786Skmacy return; 759178786Skmacy } 760178786Skmacy err = -err; 761178786Skmacy goto err; 762178786Skmacy } 763178786Skmacy 764178786Skmacy m = top; 765178786Skmacy do { 766178786Skmacy 767178786Skmacy /* 768178786Skmacy * If we get more than the supported amount of private data 769178786Skmacy * then we must fail this connection. 770178786Skmacy */ 771178786Skmacy if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 772178786Skmacy CTR2(KTR_IW_CXGB, "%s mpa message too big %d", __FUNCTION__, 773178786Skmacy ep->mpa_pkt_len + m->m_len); 774178786Skmacy goto err; 775178786Skmacy } 776178786Skmacy 777178786Skmacy 778178786Skmacy /* 779178786Skmacy * Copy the new data into our accumulation buffer. 780178786Skmacy */ 781178786Skmacy m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 782178786Skmacy ep->mpa_pkt_len += m->m_len; 783178786Skmacy 784178786Skmacy if (!m->m_next) 785178786Skmacy m = m->m_nextpkt; 786178786Skmacy else 787178786Skmacy m = m->m_next; 788178786Skmacy } while (m); 789178786Skmacy 790178786Skmacy m_freem(top); 791178786Skmacy 792178786Skmacy /* 793178786Skmacy * If we don't even have the mpa message, then bail. 794178786Skmacy * We'll continue process when more data arrives. 795178786Skmacy */ 796178786Skmacy if (ep->mpa_pkt_len < sizeof(*mpa)) { 797178786Skmacy start_ep_timer(ep); 798178786Skmacy CTR2(KTR_IW_CXGB, "%s not enough header %d...waiting...", __FUNCTION__, 799178786Skmacy ep->mpa_pkt_len); 800178786Skmacy return; 801178786Skmacy } 802178786Skmacy mpa = (struct mpa_message *) ep->mpa_pkt; 803178786Skmacy 804178786Skmacy /* 805178786Skmacy * Validate MPA Header. 806178786Skmacy */ 807178786Skmacy if (mpa->revision != mpa_rev) { 808178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa rev %d", __FUNCTION__, mpa->revision); 809178786Skmacy goto err; 810178786Skmacy } 811178786Skmacy 812178786Skmacy if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 813178786Skmacy CTR2(KTR_IW_CXGB, "%s bad mpa key |%16s|", __FUNCTION__, mpa->key); 814178786Skmacy goto err; 815178786Skmacy } 816178786Skmacy 817178786Skmacy plen = ntohs(mpa->private_data_size); 818178786Skmacy 819178786Skmacy /* 820178786Skmacy * Fail if there's too much private data. 821178786Skmacy */ 822178786Skmacy if (plen > MPA_MAX_PRIVATE_DATA) { 823178786Skmacy CTR2(KTR_IW_CXGB, "%s plen too big %d", __FUNCTION__, plen); 824178786Skmacy goto err; 825178786Skmacy } 826178786Skmacy 827178786Skmacy /* 828178786Skmacy * If plen does not account for pkt size 829178786Skmacy */ 830178786Skmacy if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 831178786Skmacy CTR2(KTR_IW_CXGB, "%s more data after private data %d", __FUNCTION__, 832178786Skmacy ep->mpa_pkt_len); 833178786Skmacy goto err; 834178786Skmacy } 835178786Skmacy ep->plen = (u8) plen; 836178786Skmacy 837178786Skmacy /* 838178786Skmacy * If we don't have all the pdata yet, then bail. 839178786Skmacy */ 840178786Skmacy if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 841178786Skmacy start_ep_timer(ep); 842178786Skmacy CTR2(KTR_IW_CXGB, "%s more mpa msg to come %d", __FUNCTION__, 843178786Skmacy ep->mpa_pkt_len); 844178786Skmacy return; 845178786Skmacy } 846178786Skmacy 847178786Skmacy /* 848178786Skmacy * If we get here we have accumulated the entire mpa 849178786Skmacy * start reply message including private data. 850178786Skmacy */ 851237263Snp ep->mpa_attr.initiator = 0; 852178786Skmacy ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 853178786Skmacy ep->mpa_attr.recv_marker_enabled = markers_enabled; 854178786Skmacy ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 855178786Skmacy ep->mpa_attr.version = mpa_rev; 856178786Skmacy if (set_tcpinfo(ep)) { 857178786Skmacy printf("%s set_tcpinfo error\n", __FUNCTION__); 858178786Skmacy goto err; 859178786Skmacy } 860178786Skmacy CTR5(KTR_IW_CXGB, "%s - crc_enabled=%d, recv_marker_enabled=%d, " 861178786Skmacy "xmit_marker_enabled=%d, version=%d", __FUNCTION__, 862178786Skmacy ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 863178786Skmacy ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 864178786Skmacy 865178786Skmacy state_set(&ep->com, MPA_REQ_RCVD); 866178786Skmacy 867178786Skmacy /* drive upcall */ 868178786Skmacy connect_request_upcall(ep); 869178786Skmacy return; 870178786Skmacyerr: 871178786Skmacy abort_connection(ep); 872178786Skmacy return; 873178786Skmacy} 874178786Skmacy 875178786Skmacystatic void 876178786Skmacyprocess_peer_close(struct iwch_ep *ep) 877178786Skmacy{ 878178786Skmacy struct iwch_qp_attributes attrs; 879178786Skmacy int disconnect = 1; 880178786Skmacy int release = 0; 881178786Skmacy 882178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 883178786Skmacy 884178786Skmacy mtx_lock(&ep->com.lock); 885178786Skmacy switch (ep->com.state) { 886178786Skmacy case MPA_REQ_WAIT: 887178786Skmacy __state_set(&ep->com, CLOSING); 888178786Skmacy break; 889178786Skmacy case MPA_REQ_SENT: 890178786Skmacy __state_set(&ep->com, CLOSING); 891178786Skmacy connect_reply_upcall(ep, -ECONNRESET); 892178786Skmacy break; 893178786Skmacy case MPA_REQ_RCVD: 894178786Skmacy 895178786Skmacy /* 896178786Skmacy * We're gonna mark this puppy DEAD, but keep 897178786Skmacy * the reference on it until the ULP accepts or 898178786Skmacy * rejects the CR. 899178786Skmacy */ 900178786Skmacy __state_set(&ep->com, CLOSING); 901178786Skmacy break; 902178786Skmacy case MPA_REP_SENT: 903178786Skmacy __state_set(&ep->com, CLOSING); 904178786Skmacy break; 905178786Skmacy case FPDU_MODE: 906178786Skmacy start_ep_timer(ep); 907178786Skmacy __state_set(&ep->com, CLOSING); 908178786Skmacy attrs.next_state = IWCH_QP_STATE_CLOSING; 909178786Skmacy iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, 910178786Skmacy IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); 911178786Skmacy peer_close_upcall(ep); 912178786Skmacy break; 913178786Skmacy case ABORTING: 914178786Skmacy disconnect = 0; 915178786Skmacy break; 916178786Skmacy case CLOSING: 917178786Skmacy __state_set(&ep->com, MORIBUND); 918178786Skmacy disconnect = 0; 919178786Skmacy break; 920178786Skmacy case MORIBUND: 921178786Skmacy stop_ep_timer(ep); 922178786Skmacy if (ep->com.cm_id && ep->com.qp) { 923178786Skmacy attrs.next_state = IWCH_QP_STATE_IDLE; 924178786Skmacy iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, 925178786Skmacy IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); 926178786Skmacy } 927237263Snp close_socket(&ep->com, 0); 928178786Skmacy close_complete_upcall(ep); 929178786Skmacy __state_set(&ep->com, DEAD); 930178786Skmacy release = 1; 931178786Skmacy disconnect = 0; 932178786Skmacy break; 933178786Skmacy case DEAD: 934178786Skmacy disconnect = 0; 935178786Skmacy break; 936178786Skmacy default: 937178786Skmacy PANIC_IF(1); 938178786Skmacy } 939178786Skmacy mtx_unlock(&ep->com.lock); 940178786Skmacy if (disconnect) 941178786Skmacy iwch_ep_disconnect(ep, 0, M_NOWAIT); 942178786Skmacy if (release) 943178786Skmacy put_ep(&ep->com); 944178786Skmacy return; 945178786Skmacy} 946178786Skmacy 947178786Skmacystatic void 948178786Skmacyprocess_conn_error(struct iwch_ep *ep) 949178786Skmacy{ 950178786Skmacy struct iwch_qp_attributes attrs; 951178786Skmacy int ret; 952178786Skmacy 953237263Snp mtx_lock(&ep->com.lock); 954237263Snp CTR3(KTR_IW_CXGB, "%s ep %p state %u", __func__, ep, ep->com.state); 955237263Snp switch (ep->com.state) { 956178786Skmacy case MPA_REQ_WAIT: 957178786Skmacy stop_ep_timer(ep); 958178786Skmacy break; 959178786Skmacy case MPA_REQ_SENT: 960178786Skmacy stop_ep_timer(ep); 961178786Skmacy connect_reply_upcall(ep, -ECONNRESET); 962178786Skmacy break; 963178786Skmacy case MPA_REP_SENT: 964178786Skmacy ep->com.rpl_err = ECONNRESET; 965178786Skmacy CTR1(KTR_IW_CXGB, "waking up ep %p", ep); 966178786Skmacy break; 967178786Skmacy case MPA_REQ_RCVD: 968178786Skmacy 969178786Skmacy /* 970178786Skmacy * We're gonna mark this puppy DEAD, but keep 971178786Skmacy * the reference on it until the ULP accepts or 972178786Skmacy * rejects the CR. 973178786Skmacy */ 974178786Skmacy break; 975178786Skmacy case MORIBUND: 976178786Skmacy case CLOSING: 977178786Skmacy stop_ep_timer(ep); 978178786Skmacy /*FALLTHROUGH*/ 979178786Skmacy case FPDU_MODE: 980178786Skmacy if (ep->com.cm_id && ep->com.qp) { 981178786Skmacy attrs.next_state = IWCH_QP_STATE_ERROR; 982178786Skmacy ret = iwch_modify_qp(ep->com.qp->rhp, 983178786Skmacy ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 984178786Skmacy &attrs, 1); 985178786Skmacy if (ret) 986178786Skmacy log(LOG_ERR, 987178786Skmacy "%s - qp <- error failed!\n", 988178786Skmacy __FUNCTION__); 989178786Skmacy } 990178786Skmacy peer_abort_upcall(ep); 991178786Skmacy break; 992178786Skmacy case ABORTING: 993178786Skmacy break; 994178786Skmacy case DEAD: 995237263Snp mtx_unlock(&ep->com.lock); 996178786Skmacy CTR2(KTR_IW_CXGB, "%s so_error %d IN DEAD STATE!!!!", __FUNCTION__, 997178786Skmacy ep->com.so->so_error); 998178786Skmacy return; 999178786Skmacy default: 1000178786Skmacy PANIC_IF(1); 1001178786Skmacy break; 1002178786Skmacy } 1003178786Skmacy 1004237263Snp if (ep->com.state != ABORTING) { 1005237263Snp close_socket(&ep->com, 0); 1006237263Snp __state_set(&ep->com, DEAD); 1007178786Skmacy put_ep(&ep->com); 1008178786Skmacy } 1009237263Snp mtx_unlock(&ep->com.lock); 1010178786Skmacy return; 1011178786Skmacy} 1012178786Skmacy 1013178786Skmacystatic void 1014178786Skmacyprocess_close_complete(struct iwch_ep *ep) 1015178786Skmacy{ 1016178786Skmacy struct iwch_qp_attributes attrs; 1017178786Skmacy int release = 0; 1018178786Skmacy 1019178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1020178786Skmacy PANIC_IF(!ep); 1021178786Skmacy 1022178786Skmacy /* The cm_id may be null if we failed to connect */ 1023178786Skmacy mtx_lock(&ep->com.lock); 1024178786Skmacy switch (ep->com.state) { 1025178786Skmacy case CLOSING: 1026178786Skmacy __state_set(&ep->com, MORIBUND); 1027178786Skmacy break; 1028178786Skmacy case MORIBUND: 1029178786Skmacy stop_ep_timer(ep); 1030178786Skmacy if ((ep->com.cm_id) && (ep->com.qp)) { 1031178786Skmacy attrs.next_state = IWCH_QP_STATE_IDLE; 1032178786Skmacy iwch_modify_qp(ep->com.qp->rhp, 1033178786Skmacy ep->com.qp, 1034178786Skmacy IWCH_QP_ATTR_NEXT_STATE, 1035178786Skmacy &attrs, 1); 1036178786Skmacy } 1037237263Snp if (ep->parent_ep) 1038237263Snp close_socket(&ep->com, 1); 1039237263Snp else 1040237263Snp close_socket(&ep->com, 0); 1041178786Skmacy close_complete_upcall(ep); 1042178786Skmacy __state_set(&ep->com, DEAD); 1043178786Skmacy release = 1; 1044178786Skmacy break; 1045178786Skmacy case ABORTING: 1046178786Skmacy break; 1047178786Skmacy case DEAD: 1048178786Skmacy default: 1049178786Skmacy PANIC_IF(1); 1050178786Skmacy break; 1051178786Skmacy } 1052178786Skmacy mtx_unlock(&ep->com.lock); 1053178786Skmacy if (release) 1054178786Skmacy put_ep(&ep->com); 1055178786Skmacy return; 1056178786Skmacy} 1057178786Skmacy 1058178786Skmacy/* 1059178786Skmacy * T3A does 3 things when a TERM is received: 1060178786Skmacy * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet 1061178786Skmacy * 2) generate an async event on the QP with the TERMINATE opcode 1062178786Skmacy * 3) post a TERMINATE opcde cqe into the associated CQ. 1063178786Skmacy * 1064178786Skmacy * For (1), we save the message in the qp for later consumer consumption. 1065178786Skmacy * For (2), we move the QP into TERMINATE, post a QP event and disconnect. 1066178786Skmacy * For (3), we toss the CQE in cxio_poll_cq(). 1067178786Skmacy * 1068178786Skmacy * terminate() handles case (1)... 1069178786Skmacy */ 1070178786Skmacystatic int 1071237263Snpterminate(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1072178786Skmacy{ 1073237263Snp struct adapter *sc = qs->adap; 1074237263Snp struct tom_data *td = sc->tom_softc; 1075237263Snp uint32_t hash = *((uint32_t *)r + 1); 1076237263Snp unsigned int tid = ntohl(hash) >> 8 & 0xfffff; 1077237263Snp struct toepcb *toep = lookup_tid(&td->tid_maps, tid); 1078237263Snp struct socket *so = toep->tp_inp->inp_socket; 1079193272Sjhb struct iwch_ep *ep = so->so_rcv.sb_upcallarg; 1080178786Skmacy 1081237263Snp if (state_read(&ep->com) != FPDU_MODE) 1082237263Snp goto done; 1083237263Snp 1084178786Skmacy m_adj(m, sizeof(struct cpl_rdma_terminate)); 1085237263Snp 1086237263Snp CTR4(KTR_IW_CXGB, "%s: tid %u, ep %p, saved %d bytes", 1087237263Snp __func__, tid, ep, m->m_len); 1088237263Snp 1089178786Skmacy m_copydata(m, 0, m->m_len, ep->com.qp->attr.terminate_buffer); 1090178786Skmacy ep->com.qp->attr.terminate_msg_len = m->m_len; 1091178786Skmacy ep->com.qp->attr.is_terminate_local = 0; 1092237263Snp 1093237263Snpdone: 1094237263Snp m_freem(m); 1095237263Snp return (0); 1096178786Skmacy} 1097178786Skmacy 1098178786Skmacystatic int 1099237263Snpec_status(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 1100178786Skmacy{ 1101237263Snp struct adapter *sc = qs->adap; 1102237263Snp struct tom_data *td = sc->tom_softc; 1103237263Snp struct cpl_rdma_ec_status *rep = mtod(m, void *); 1104237263Snp unsigned int tid = GET_TID(rep); 1105237263Snp struct toepcb *toep = lookup_tid(&td->tid_maps, tid); 1106237263Snp struct socket *so = toep->tp_inp->inp_socket; 1107237263Snp struct iwch_ep *ep = so->so_rcv.sb_upcallarg; 1108178786Skmacy 1109237263Snp if (rep->status) { 1110237263Snp struct iwch_qp_attributes attrs; 1111237263Snp 1112237263Snp CTR1(KTR_IW_CXGB, "%s BAD CLOSE - Aborting", __FUNCTION__); 1113178786Skmacy stop_ep_timer(ep); 1114178786Skmacy attrs.next_state = IWCH_QP_STATE_ERROR; 1115178786Skmacy iwch_modify_qp(ep->com.qp->rhp, 1116237263Snp ep->com.qp, 1117237263Snp IWCH_QP_ATTR_NEXT_STATE, 1118237263Snp &attrs, 1); 1119237263Snp abort_connection(ep); 1120178786Skmacy } 1121237263Snp 1122237263Snp m_freem(m); 1123237263Snp return (0); 1124178786Skmacy} 1125178786Skmacy 1126178786Skmacystatic void 1127178786Skmacyep_timeout(void *arg) 1128178786Skmacy{ 1129178786Skmacy struct iwch_ep *ep = (struct iwch_ep *)arg; 1130178786Skmacy struct iwch_qp_attributes attrs; 1131178786Skmacy int err = 0; 1132237263Snp int abort = 1; 1133178786Skmacy 1134178786Skmacy mtx_lock(&ep->com.lock); 1135178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1136178786Skmacy switch (ep->com.state) { 1137178786Skmacy case MPA_REQ_SENT: 1138237263Snp __state_set(&ep->com, ABORTING); 1139178786Skmacy connect_reply_upcall(ep, -ETIMEDOUT); 1140178786Skmacy break; 1141178786Skmacy case MPA_REQ_WAIT: 1142237263Snp __state_set(&ep->com, ABORTING); 1143178786Skmacy break; 1144178786Skmacy case CLOSING: 1145178786Skmacy case MORIBUND: 1146178786Skmacy if (ep->com.cm_id && ep->com.qp) 1147178786Skmacy err = 1; 1148237263Snp __state_set(&ep->com, ABORTING); 1149178786Skmacy break; 1150178786Skmacy default: 1151237263Snp CTR3(KTR_IW_CXGB, "%s unexpected state ep %p state %u\n", 1152237263Snp __func__, ep, ep->com.state); 1153237263Snp abort = 0; 1154178786Skmacy } 1155178786Skmacy mtx_unlock(&ep->com.lock); 1156178786Skmacy if (err){ 1157178786Skmacy attrs.next_state = IWCH_QP_STATE_ERROR; 1158178786Skmacy iwch_modify_qp(ep->com.qp->rhp, 1159178786Skmacy ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, 1160178786Skmacy &attrs, 1); 1161178786Skmacy } 1162237263Snp if (abort) 1163237263Snp abort_connection(ep); 1164178786Skmacy put_ep(&ep->com); 1165178786Skmacy} 1166178786Skmacy 1167178786Skmacyint 1168178786Skmacyiwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 1169178786Skmacy{ 1170178786Skmacy int err; 1171178786Skmacy struct iwch_ep *ep = to_ep(cm_id); 1172178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1173178786Skmacy 1174178786Skmacy if (state_read(&ep->com) == DEAD) { 1175178786Skmacy put_ep(&ep->com); 1176178786Skmacy return (-ECONNRESET); 1177178786Skmacy } 1178178786Skmacy PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD); 1179178786Skmacy if (mpa_rev == 0) { 1180178786Skmacy abort_connection(ep); 1181178786Skmacy } else { 1182178786Skmacy err = send_mpa_reject(ep, pdata, pdata_len); 1183178786Skmacy err = soshutdown(ep->com.so, 3); 1184178786Skmacy } 1185237263Snp put_ep(&ep->com); 1186178786Skmacy return 0; 1187178786Skmacy} 1188178786Skmacy 1189178786Skmacyint 1190178786Skmacyiwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 1191178786Skmacy{ 1192178786Skmacy int err; 1193178786Skmacy struct iwch_qp_attributes attrs; 1194178786Skmacy enum iwch_qp_attr_mask mask; 1195178786Skmacy struct iwch_ep *ep = to_ep(cm_id); 1196178786Skmacy struct iwch_dev *h = to_iwch_dev(cm_id->device); 1197178786Skmacy struct iwch_qp *qp = get_qhp(h, conn_param->qpn); 1198178786Skmacy 1199178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1200237263Snp if (state_read(&ep->com) == DEAD) { 1201237263Snp err = -ECONNRESET; 1202237263Snp goto err; 1203237263Snp } 1204178786Skmacy 1205178786Skmacy PANIC_IF(state_read(&ep->com) != MPA_REQ_RCVD); 1206178786Skmacy PANIC_IF(!qp); 1207178786Skmacy 1208178786Skmacy if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) || 1209178786Skmacy (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) { 1210178786Skmacy abort_connection(ep); 1211237263Snp err = -EINVAL; 1212237263Snp goto err; 1213178786Skmacy } 1214178786Skmacy 1215178786Skmacy cm_id->add_ref(cm_id); 1216178786Skmacy ep->com.cm_id = cm_id; 1217178786Skmacy ep->com.qp = qp; 1218178786Skmacy 1219178786Skmacy ep->com.rpl_err = 0; 1220178786Skmacy ep->com.rpl_done = 0; 1221178786Skmacy ep->ird = conn_param->ird; 1222178786Skmacy ep->ord = conn_param->ord; 1223178786Skmacy CTR3(KTR_IW_CXGB, "%s ird %d ord %d", __FUNCTION__, ep->ird, ep->ord); 1224178786Skmacy 1225178786Skmacy /* bind QP to EP and move to RTS */ 1226178786Skmacy attrs.mpa_attr = ep->mpa_attr; 1227237263Snp attrs.max_ird = ep->ird; 1228178786Skmacy attrs.max_ord = ep->ord; 1229178786Skmacy attrs.llp_stream_handle = ep; 1230178786Skmacy attrs.next_state = IWCH_QP_STATE_RTS; 1231178786Skmacy 1232178786Skmacy /* bind QP and TID with INIT_WR */ 1233178786Skmacy mask = IWCH_QP_ATTR_NEXT_STATE | 1234178786Skmacy IWCH_QP_ATTR_LLP_STREAM_HANDLE | 1235178786Skmacy IWCH_QP_ATTR_MPA_ATTR | 1236178786Skmacy IWCH_QP_ATTR_MAX_IRD | 1237178786Skmacy IWCH_QP_ATTR_MAX_ORD; 1238178786Skmacy 1239178786Skmacy err = iwch_modify_qp(ep->com.qp->rhp, 1240178786Skmacy ep->com.qp, mask, &attrs, 1); 1241178786Skmacy 1242178786Skmacy if (err) 1243237263Snp goto err1; 1244178786Skmacy 1245178786Skmacy err = send_mpa_reply(ep, conn_param->private_data, 1246178786Skmacy conn_param->private_data_len); 1247178786Skmacy if (err) 1248237263Snp goto err1; 1249178786Skmacy state_set(&ep->com, FPDU_MODE); 1250178786Skmacy established_upcall(ep); 1251178786Skmacy put_ep(&ep->com); 1252178786Skmacy return 0; 1253237263Snperr1: 1254178786Skmacy ep->com.cm_id = NULL; 1255178786Skmacy ep->com.qp = NULL; 1256178786Skmacy cm_id->rem_ref(cm_id); 1257237263Snperr: 1258178786Skmacy put_ep(&ep->com); 1259178786Skmacy return err; 1260178786Skmacy} 1261178786Skmacy 1262178786Skmacystatic int init_sock(struct iwch_ep_common *epc) 1263178786Skmacy{ 1264178786Skmacy int err; 1265178786Skmacy struct sockopt sopt; 1266178786Skmacy int on=1; 1267178786Skmacy 1268193272Sjhb SOCK_LOCK(epc->so); 1269193272Sjhb soupcall_set(epc->so, SO_RCV, iwch_so_upcall, epc); 1270178786Skmacy epc->so->so_state |= SS_NBIO; 1271193272Sjhb SOCK_UNLOCK(epc->so); 1272178786Skmacy sopt.sopt_dir = SOPT_SET; 1273178786Skmacy sopt.sopt_level = IPPROTO_TCP; 1274178786Skmacy sopt.sopt_name = TCP_NODELAY; 1275178786Skmacy sopt.sopt_val = (caddr_t)&on; 1276178786Skmacy sopt.sopt_valsize = sizeof on; 1277178786Skmacy sopt.sopt_td = NULL; 1278178786Skmacy err = sosetopt(epc->so, &sopt); 1279178786Skmacy if (err) 1280178786Skmacy printf("%s can't set TCP_NODELAY err %d\n", __FUNCTION__, err); 1281178786Skmacy 1282178786Skmacy return 0; 1283178786Skmacy} 1284178786Skmacy 1285178786Skmacystatic int 1286178786Skmacyis_loopback_dst(struct iw_cm_id *cm_id) 1287178786Skmacy{ 1288178786Skmacy uint16_t port = cm_id->remote_addr.sin_port; 1289194622Srwatson int ifa_present; 1290178786Skmacy 1291178786Skmacy cm_id->remote_addr.sin_port = 0; 1292194622Srwatson ifa_present = ifa_ifwithaddr_check( 1293194622Srwatson (struct sockaddr *)&cm_id->remote_addr); 1294178786Skmacy cm_id->remote_addr.sin_port = port; 1295194622Srwatson return (ifa_present); 1296178786Skmacy} 1297178786Skmacy 1298178786Skmacyint 1299178786Skmacyiwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 1300178786Skmacy{ 1301178786Skmacy int err = 0; 1302178786Skmacy struct iwch_dev *h = to_iwch_dev(cm_id->device); 1303178786Skmacy struct iwch_ep *ep; 1304178786Skmacy struct rtentry *rt; 1305178786Skmacy struct toedev *tdev; 1306178786Skmacy 1307178786Skmacy if (is_loopback_dst(cm_id)) { 1308178786Skmacy err = -ENOSYS; 1309178786Skmacy goto out; 1310178786Skmacy } 1311178786Skmacy 1312178786Skmacy ep = alloc_ep(sizeof(*ep), M_NOWAIT); 1313178786Skmacy if (!ep) { 1314178786Skmacy printf("%s - cannot alloc ep.\n", __FUNCTION__); 1315178786Skmacy err = (-ENOMEM); 1316178786Skmacy goto out; 1317178786Skmacy } 1318178786Skmacy callout_init(&ep->timer, TRUE); 1319178786Skmacy ep->plen = conn_param->private_data_len; 1320178786Skmacy if (ep->plen) 1321178786Skmacy memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 1322178786Skmacy conn_param->private_data, ep->plen); 1323178786Skmacy ep->ird = conn_param->ird; 1324178786Skmacy ep->ord = conn_param->ord; 1325178786Skmacy 1326178786Skmacy cm_id->add_ref(cm_id); 1327178786Skmacy ep->com.cm_id = cm_id; 1328178786Skmacy ep->com.qp = get_qhp(h, conn_param->qpn); 1329178786Skmacy ep->com.thread = curthread; 1330178786Skmacy PANIC_IF(!ep->com.qp); 1331178786Skmacy CTR4(KTR_IW_CXGB, "%s qpn 0x%x qp %p cm_id %p", __FUNCTION__, conn_param->qpn, 1332178786Skmacy ep->com.qp, cm_id); 1333178786Skmacy 1334178786Skmacy ep->com.so = cm_id->so; 1335178786Skmacy err = init_sock(&ep->com); 1336178786Skmacy if (err) 1337178786Skmacy goto fail2; 1338178786Skmacy 1339178786Skmacy /* find a route */ 1340178786Skmacy rt = find_route(cm_id->local_addr.sin_addr.s_addr, 1341178786Skmacy cm_id->remote_addr.sin_addr.s_addr, 1342178786Skmacy cm_id->local_addr.sin_port, 1343178786Skmacy cm_id->remote_addr.sin_port, IPTOS_LOWDELAY); 1344178786Skmacy if (!rt) { 1345178786Skmacy printf("%s - cannot find route.\n", __FUNCTION__); 1346178786Skmacy err = EHOSTUNREACH; 1347178786Skmacy goto fail2; 1348178786Skmacy } 1349178786Skmacy 1350178786Skmacy if (!(rt->rt_ifp->if_flags & IFCAP_TOE)) { 1351178786Skmacy printf("%s - interface not TOE capable.\n", __FUNCTION__); 1352237263Snp RTFREE(rt); 1353237263Snp goto fail2; 1354178786Skmacy } 1355178786Skmacy tdev = TOEDEV(rt->rt_ifp); 1356178786Skmacy if (tdev == NULL) { 1357178786Skmacy printf("%s - No toedev for interface.\n", __FUNCTION__); 1358237263Snp RTFREE(rt); 1359237263Snp goto fail2; 1360178786Skmacy } 1361178786Skmacy RTFREE(rt); 1362178786Skmacy 1363178786Skmacy state_set(&ep->com, CONNECTING); 1364178786Skmacy ep->com.local_addr = cm_id->local_addr; 1365178786Skmacy ep->com.remote_addr = cm_id->remote_addr; 1366178786Skmacy err = soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 1367178786Skmacy ep->com.thread); 1368178786Skmacy if (!err) 1369178786Skmacy goto out; 1370178786Skmacyfail2: 1371178786Skmacy put_ep(&ep->com); 1372178786Skmacyout: 1373178786Skmacy return err; 1374178786Skmacy} 1375178786Skmacy 1376178786Skmacyint 1377178786Skmacyiwch_create_listen(struct iw_cm_id *cm_id, int backlog) 1378178786Skmacy{ 1379178786Skmacy int err = 0; 1380178786Skmacy struct iwch_listen_ep *ep; 1381178786Skmacy 1382178786Skmacy ep = alloc_ep(sizeof(*ep), M_NOWAIT); 1383178786Skmacy if (!ep) { 1384178786Skmacy printf("%s - cannot alloc ep.\n", __FUNCTION__); 1385178786Skmacy err = ENOMEM; 1386178786Skmacy goto out; 1387178786Skmacy } 1388178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 1389178786Skmacy cm_id->add_ref(cm_id); 1390178786Skmacy ep->com.cm_id = cm_id; 1391178786Skmacy ep->backlog = backlog; 1392178786Skmacy ep->com.local_addr = cm_id->local_addr; 1393178786Skmacy ep->com.thread = curthread; 1394178786Skmacy state_set(&ep->com, LISTEN); 1395178786Skmacy 1396178786Skmacy ep->com.so = cm_id->so; 1397178786Skmacy err = init_sock(&ep->com); 1398178786Skmacy if (err) 1399178786Skmacy goto fail; 1400178786Skmacy 1401178786Skmacy err = solisten(ep->com.so, ep->backlog, ep->com.thread); 1402178786Skmacy if (!err) { 1403178786Skmacy cm_id->provider_data = ep; 1404178786Skmacy goto out; 1405178786Skmacy } 1406237263Snp close_socket(&ep->com, 0); 1407178786Skmacyfail: 1408178786Skmacy cm_id->rem_ref(cm_id); 1409178786Skmacy put_ep(&ep->com); 1410178786Skmacyout: 1411178786Skmacy return err; 1412178786Skmacy} 1413178786Skmacy 1414178786Skmacyint 1415178786Skmacyiwch_destroy_listen(struct iw_cm_id *cm_id) 1416178786Skmacy{ 1417178786Skmacy struct iwch_listen_ep *ep = to_listen_ep(cm_id); 1418178786Skmacy 1419178786Skmacy CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); 1420178786Skmacy 1421178786Skmacy state_set(&ep->com, DEAD); 1422237263Snp close_socket(&ep->com, 0); 1423178786Skmacy cm_id->rem_ref(cm_id); 1424178786Skmacy put_ep(&ep->com); 1425178786Skmacy return 0; 1426178786Skmacy} 1427178786Skmacy 1428178786Skmacyint 1429178786Skmacyiwch_ep_disconnect(struct iwch_ep *ep, int abrupt, int flags) 1430178786Skmacy{ 1431178786Skmacy int close = 0; 1432178786Skmacy 1433178786Skmacy mtx_lock(&ep->com.lock); 1434178786Skmacy 1435178786Skmacy PANIC_IF(!ep); 1436178786Skmacy PANIC_IF(!ep->com.so); 1437178786Skmacy 1438178786Skmacy CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s, abrupt %d", __FUNCTION__, ep, 1439178786Skmacy ep->com.so, states[ep->com.state], abrupt); 1440178786Skmacy 1441178786Skmacy switch (ep->com.state) { 1442178786Skmacy case MPA_REQ_WAIT: 1443178786Skmacy case MPA_REQ_SENT: 1444178786Skmacy case MPA_REQ_RCVD: 1445178786Skmacy case MPA_REP_SENT: 1446178786Skmacy case FPDU_MODE: 1447178786Skmacy close = 1; 1448237263Snp if (abrupt) 1449237263Snp ep->com.state = ABORTING; 1450237263Snp else { 1451237263Snp ep->com.state = CLOSING; 1452237263Snp start_ep_timer(ep); 1453237263Snp } 1454178786Skmacy break; 1455178786Skmacy case CLOSING: 1456178786Skmacy close = 1; 1457237263Snp if (abrupt) { 1458237263Snp stop_ep_timer(ep); 1459237263Snp ep->com.state = ABORTING; 1460237263Snp } else 1461237263Snp ep->com.state = MORIBUND; 1462178786Skmacy break; 1463178786Skmacy case MORIBUND: 1464178786Skmacy case ABORTING: 1465237263Snp case DEAD: 1466237263Snp CTR3(KTR_IW_CXGB, "%s ignoring disconnect ep %p state %u\n", 1467237263Snp __func__, ep, ep->com.state); 1468178786Skmacy break; 1469178786Skmacy default: 1470178786Skmacy panic("unknown state: %d\n", ep->com.state); 1471178786Skmacy break; 1472178786Skmacy } 1473237263Snp 1474178786Skmacy mtx_unlock(&ep->com.lock); 1475178786Skmacy if (close) { 1476178786Skmacy if (abrupt) 1477178786Skmacy abort_connection(ep); 1478237263Snp else { 1479237263Snp if (!ep->parent_ep) 1480237263Snp __state_set(&ep->com, MORIBUND); 1481178786Skmacy shutdown_socket(&ep->com); 1482237263Snp } 1483178786Skmacy } 1484178786Skmacy return 0; 1485178786Skmacy} 1486178786Skmacy 1487178786Skmacystatic void 1488178786Skmacyprocess_data(struct iwch_ep *ep) 1489178786Skmacy{ 1490178786Skmacy struct sockaddr_in *local, *remote; 1491178786Skmacy 1492178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1493178786Skmacy 1494178786Skmacy switch (state_read(&ep->com)) { 1495178786Skmacy case MPA_REQ_SENT: 1496178786Skmacy process_mpa_reply(ep); 1497178786Skmacy break; 1498178786Skmacy case MPA_REQ_WAIT: 1499178786Skmacy 1500178786Skmacy /* 1501178786Skmacy * XXX 1502178786Skmacy * Set local and remote addrs here because when we 1503178786Skmacy * dequeue the newly accepted socket, they aren't set 1504178786Skmacy * yet in the pcb! 1505178786Skmacy */ 1506178786Skmacy in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 1507178786Skmacy in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 1508178786Skmacy CTR3(KTR_IW_CXGB, "%s local %s remote %s", __FUNCTION__, 1509178786Skmacy inet_ntoa(local->sin_addr), 1510178786Skmacy inet_ntoa(remote->sin_addr)); 1511178786Skmacy ep->com.local_addr = *local; 1512178786Skmacy ep->com.remote_addr = *remote; 1513178786Skmacy free(local, M_SONAME); 1514178786Skmacy free(remote, M_SONAME); 1515178786Skmacy process_mpa_request(ep); 1516178786Skmacy break; 1517178786Skmacy default: 1518178786Skmacy if (ep->com.so->so_rcv.sb_cc) 1519178786Skmacy printf("%s Unexpected streaming data." 1520178786Skmacy " ep %p state %d so %p so_state %x so_rcv.sb_cc %u so_rcv.sb_mb %p\n", 1521178786Skmacy __FUNCTION__, ep, state_read(&ep->com), ep->com.so, ep->com.so->so_state, 1522178786Skmacy ep->com.so->so_rcv.sb_cc, ep->com.so->so_rcv.sb_mb); 1523178786Skmacy break; 1524178786Skmacy } 1525178786Skmacy return; 1526178786Skmacy} 1527178786Skmacy 1528178786Skmacystatic void 1529178786Skmacyprocess_connected(struct iwch_ep *ep) 1530178786Skmacy{ 1531178786Skmacy CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); 1532178786Skmacy if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) { 1533178786Skmacy send_mpa_req(ep); 1534178786Skmacy } else { 1535178786Skmacy connect_reply_upcall(ep, -ep->com.so->so_error); 1536237263Snp close_socket(&ep->com, 0); 1537178786Skmacy state_set(&ep->com, DEAD); 1538178786Skmacy put_ep(&ep->com); 1539178786Skmacy } 1540178786Skmacy} 1541178786Skmacy 1542178786Skmacystatic struct socket * 1543178786Skmacydequeue_socket(struct socket *head, struct sockaddr_in **remote, struct iwch_ep *child_ep) 1544178786Skmacy{ 1545178786Skmacy struct socket *so; 1546178786Skmacy 1547178786Skmacy ACCEPT_LOCK(); 1548178786Skmacy so = TAILQ_FIRST(&head->so_comp); 1549178786Skmacy if (!so) { 1550178786Skmacy ACCEPT_UNLOCK(); 1551178786Skmacy return NULL; 1552178786Skmacy } 1553178786Skmacy TAILQ_REMOVE(&head->so_comp, so, so_list); 1554178786Skmacy head->so_qlen--; 1555178786Skmacy SOCK_LOCK(so); 1556178786Skmacy so->so_qstate &= ~SQ_COMP; 1557178786Skmacy so->so_head = NULL; 1558178786Skmacy soref(so); 1559193272Sjhb soupcall_set(so, SO_RCV, iwch_so_upcall, child_ep); 1560178786Skmacy so->so_state |= SS_NBIO; 1561178786Skmacy PANIC_IF(!(so->so_state & SS_ISCONNECTED)); 1562178786Skmacy PANIC_IF(so->so_error); 1563178786Skmacy SOCK_UNLOCK(so); 1564178786Skmacy ACCEPT_UNLOCK(); 1565178786Skmacy soaccept(so, (struct sockaddr **)remote); 1566178786Skmacy return so; 1567178786Skmacy} 1568178786Skmacy 1569178786Skmacystatic void 1570178786Skmacyprocess_newconn(struct iwch_ep *parent_ep) 1571178786Skmacy{ 1572178786Skmacy struct socket *child_so; 1573178786Skmacy struct iwch_ep *child_ep; 1574178786Skmacy struct sockaddr_in *remote; 1575178786Skmacy 1576178786Skmacy CTR3(KTR_IW_CXGB, "%s parent ep %p so %p", __FUNCTION__, parent_ep, parent_ep->com.so); 1577178786Skmacy child_ep = alloc_ep(sizeof(*child_ep), M_NOWAIT); 1578178786Skmacy if (!child_ep) { 1579178786Skmacy log(LOG_ERR, "%s - failed to allocate ep entry!\n", 1580178786Skmacy __FUNCTION__); 1581178786Skmacy return; 1582178786Skmacy } 1583178786Skmacy child_so = dequeue_socket(parent_ep->com.so, &remote, child_ep); 1584178786Skmacy if (!child_so) { 1585178786Skmacy log(LOG_ERR, "%s - failed to dequeue child socket!\n", 1586178786Skmacy __FUNCTION__); 1587178786Skmacy __free_ep(&child_ep->com); 1588178786Skmacy return; 1589178786Skmacy } 1590178786Skmacy CTR3(KTR_IW_CXGB, "%s remote addr %s port %d", __FUNCTION__, 1591178786Skmacy inet_ntoa(remote->sin_addr), ntohs(remote->sin_port)); 1592237263Snp child_ep->com.tdev = parent_ep->com.tdev; 1593237263Snp child_ep->com.local_addr.sin_family = parent_ep->com.local_addr.sin_family; 1594237263Snp child_ep->com.local_addr.sin_port = parent_ep->com.local_addr.sin_port; 1595237263Snp child_ep->com.local_addr.sin_addr.s_addr = parent_ep->com.local_addr.sin_addr.s_addr; 1596237263Snp child_ep->com.local_addr.sin_len = parent_ep->com.local_addr.sin_len; 1597237263Snp child_ep->com.remote_addr.sin_family = remote->sin_family; 1598237263Snp child_ep->com.remote_addr.sin_port = remote->sin_port; 1599237263Snp child_ep->com.remote_addr.sin_addr.s_addr = remote->sin_addr.s_addr; 1600237263Snp child_ep->com.remote_addr.sin_len = remote->sin_len; 1601178786Skmacy child_ep->com.so = child_so; 1602178786Skmacy child_ep->com.cm_id = NULL; 1603178786Skmacy child_ep->com.thread = parent_ep->com.thread; 1604178786Skmacy child_ep->parent_ep = parent_ep; 1605237263Snp 1606178786Skmacy free(remote, M_SONAME); 1607178786Skmacy get_ep(&parent_ep->com); 1608178786Skmacy child_ep->parent_ep = parent_ep; 1609178786Skmacy callout_init(&child_ep->timer, TRUE); 1610178786Skmacy state_set(&child_ep->com, MPA_REQ_WAIT); 1611178786Skmacy start_ep_timer(child_ep); 1612178786Skmacy 1613178786Skmacy /* maybe the request has already been queued up on the socket... */ 1614178786Skmacy process_mpa_request(child_ep); 1615178786Skmacy} 1616178786Skmacy 1617193272Sjhbstatic int 1618178786Skmacyiwch_so_upcall(struct socket *so, void *arg, int waitflag) 1619178786Skmacy{ 1620178786Skmacy struct iwch_ep *ep = arg; 1621178786Skmacy 1622178786Skmacy CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]); 1623178786Skmacy mtx_lock(&req_lock); 1624178786Skmacy if (ep && ep->com.so && !ep->com.entry.tqe_prev) { 1625178786Skmacy get_ep(&ep->com); 1626178786Skmacy TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 1627178786Skmacy taskqueue_enqueue(iw_cxgb_taskq, &iw_cxgb_task); 1628178786Skmacy } 1629178786Skmacy mtx_unlock(&req_lock); 1630193272Sjhb return (SU_OK); 1631178786Skmacy} 1632178786Skmacy 1633178786Skmacystatic void 1634178786Skmacyprocess_socket_event(struct iwch_ep *ep) 1635178786Skmacy{ 1636178786Skmacy int state = state_read(&ep->com); 1637178786Skmacy struct socket *so = ep->com.so; 1638178786Skmacy 1639178786Skmacy CTR6(KTR_IW_CXGB, "%s so %p so state %x ep %p ep state(%d)=%s", __FUNCTION__, so, so->so_state, ep, ep->com.state, states[ep->com.state]); 1640178786Skmacy if (state == CONNECTING) { 1641178786Skmacy process_connected(ep); 1642178786Skmacy return; 1643178786Skmacy } 1644178786Skmacy 1645178786Skmacy if (state == LISTEN) { 1646178786Skmacy process_newconn(ep); 1647178786Skmacy return; 1648178786Skmacy } 1649178786Skmacy 1650178786Skmacy /* connection error */ 1651178786Skmacy if (so->so_error) { 1652178786Skmacy process_conn_error(ep); 1653178786Skmacy return; 1654178786Skmacy } 1655178786Skmacy 1656178786Skmacy /* peer close */ 1657178786Skmacy if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state < CLOSING) { 1658178786Skmacy process_peer_close(ep); 1659178786Skmacy return; 1660178786Skmacy } 1661178786Skmacy 1662178786Skmacy /* close complete */ 1663178786Skmacy if (so->so_state & (SS_ISDISCONNECTED)) { 1664178786Skmacy process_close_complete(ep); 1665178786Skmacy return; 1666178786Skmacy } 1667178786Skmacy 1668178786Skmacy /* rx data */ 1669178786Skmacy process_data(ep); 1670178786Skmacy return; 1671178786Skmacy} 1672178786Skmacy 1673178786Skmacystatic void 1674178786Skmacyprocess_req(void *ctx, int pending) 1675178786Skmacy{ 1676178786Skmacy struct iwch_ep_common *epc; 1677178786Skmacy 1678178786Skmacy CTR1(KTR_IW_CXGB, "%s enter", __FUNCTION__); 1679178786Skmacy mtx_lock(&req_lock); 1680178786Skmacy while (!TAILQ_EMPTY(&req_list)) { 1681178786Skmacy epc = TAILQ_FIRST(&req_list); 1682178786Skmacy TAILQ_REMOVE(&req_list, epc, entry); 1683178786Skmacy epc->entry.tqe_prev = NULL; 1684178786Skmacy mtx_unlock(&req_lock); 1685178786Skmacy if (epc->so) 1686178786Skmacy process_socket_event((struct iwch_ep *)epc); 1687178786Skmacy put_ep(epc); 1688178786Skmacy mtx_lock(&req_lock); 1689178786Skmacy } 1690178786Skmacy mtx_unlock(&req_lock); 1691178786Skmacy} 1692178786Skmacy 1693178786Skmacyint 1694178786Skmacyiwch_cm_init(void) 1695178786Skmacy{ 1696178786Skmacy TAILQ_INIT(&req_list); 1697178786Skmacy mtx_init(&req_lock, "iw_cxgb req_list lock", NULL, MTX_DEF); 1698178786Skmacy iw_cxgb_taskq = taskqueue_create("iw_cxgb_taskq", M_NOWAIT, 1699178786Skmacy taskqueue_thread_enqueue, &iw_cxgb_taskq); 1700178786Skmacy if (iw_cxgb_taskq == NULL) { 1701178786Skmacy printf("failed to allocate iw_cxgb taskqueue\n"); 1702178786Skmacy return (ENOMEM); 1703178786Skmacy } 1704178786Skmacy taskqueue_start_threads(&iw_cxgb_taskq, 1, PI_NET, "iw_cxgb taskq"); 1705178786Skmacy TASK_INIT(&iw_cxgb_task, 0, process_req, NULL); 1706237263Snp return (0); 1707178786Skmacy} 1708178786Skmacy 1709178786Skmacyvoid 1710178786Skmacyiwch_cm_term(void) 1711178786Skmacy{ 1712237263Snp 1713178786Skmacy taskqueue_drain(iw_cxgb_taskq, &iw_cxgb_task); 1714178786Skmacy taskqueue_free(iw_cxgb_taskq); 1715178786Skmacy} 1716178786Skmacy 1717237263Snpvoid 1718237263Snpiwch_cm_init_cpl(struct adapter *sc) 1719237263Snp{ 1720237263Snp 1721237263Snp t3_register_cpl_handler(sc, CPL_RDMA_TERMINATE, terminate); 1722237263Snp t3_register_cpl_handler(sc, CPL_RDMA_EC_STATUS, ec_status); 1723237263Snp} 1724237263Snp 1725237263Snpvoid 1726237263Snpiwch_cm_term_cpl(struct adapter *sc) 1727237263Snp{ 1728237263Snp 1729237263Snp t3_register_cpl_handler(sc, CPL_RDMA_TERMINATE, NULL); 1730237263Snp t3_register_cpl_handler(sc, CPL_RDMA_EC_STATUS, NULL); 1731237263Snp} 1732237263Snp#endif 1733