t4_cpl_io.c revision 286895
1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c 286895 2015-08-18 18:54:45Z np $"); 30 31#include "opt_inet.h" 32 33#ifdef TCP_OFFLOAD 34#include <sys/param.h> 35#include <sys/types.h> 36#include <sys/kernel.h> 37#include <sys/ktr.h> 38#include <sys/module.h> 39#include <sys/protosw.h> 40#include <sys/domain.h> 41#include <sys/socket.h> 42#include <sys/socketvar.h> 43#include <sys/sglist.h> 44#include <netinet/in.h> 45#include <netinet/in_pcb.h> 46#include <netinet/ip.h> 47#include <netinet/ip6.h> 48#include <netinet/tcp_var.h> 49#define TCPSTATES 50#include <netinet/tcp_fsm.h> 51#include <netinet/tcp_seq.h> 52#include <netinet/toecore.h> 53 54#include "common/common.h" 55#include "common/t4_msg.h" 56#include "common/t4_regs.h" 57#include "common/t4_tcb.h" 58#include "tom/t4_tom_l2t.h" 59#include "tom/t4_tom.h" 60 61VNET_DECLARE(int, tcp_do_autosndbuf); 62#define V_tcp_do_autosndbuf VNET(tcp_do_autosndbuf) 63VNET_DECLARE(int, tcp_autosndbuf_inc); 64#define V_tcp_autosndbuf_inc VNET(tcp_autosndbuf_inc) 65VNET_DECLARE(int, tcp_autosndbuf_max); 66#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 67VNET_DECLARE(int, tcp_do_autorcvbuf); 68#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 69VNET_DECLARE(int, tcp_autorcvbuf_inc); 70#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 71VNET_DECLARE(int, tcp_autorcvbuf_max); 72#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 73 74/* 75 * For ULP connections HW may add headers, e.g., for digests, that aren't part 76 * of the messages sent by the host but that are part of the TCP payload and 77 * therefore consume TCP sequence space. Tx connection parameters that 78 * operate in TCP sequence space are affected by the HW additions and need to 79 * compensate for them to accurately track TCP sequence numbers. This array 80 * contains the compensating extra lengths for ULP packets. It is indexed by 81 * a packet's ULP submode. 82 */ 83const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8}; 84 85/* 86 * Return the length of any HW additions that will be made to a Tx packet. 87 * Such additions can happen for some types of ULP packets. 88 */ 89static inline unsigned int 90ulp_extra_len(struct mbuf *m, int *ulp_mode) 91{ 92 struct m_tag *mtag; 93 94 if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL) 95 return (0); 96 *ulp_mode = *((int *)(mtag + 1)); 97 98 return (t4_ulp_extra_len[*ulp_mode & 3]); 99} 100 101void 102send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) 103{ 104 struct wrqe *wr; 105 struct fw_flowc_wr *flowc; 106 unsigned int nparams = ftxp ? 8 : 6, flowclen; 107 struct port_info *pi = toep->port; 108 struct adapter *sc = pi->adapter; 109 unsigned int pfvf = G_FW_VIID_PFN(pi->viid) << S_FW_VIID_PFN; 110 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 111 112 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT), 113 ("%s: flowc for tid %u sent already", __func__, toep->tid)); 114 115 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 116 117 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq); 118 if (wr == NULL) { 119 /* XXX */ 120 panic("%s: allocation failure.", __func__); 121 } 122 flowc = wrtod(wr); 123 memset(flowc, 0, wr->wr_len); 124 125 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 126 V_FW_FLOWC_WR_NPARAMS(nparams)); 127 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 128 V_FW_WR_FLOWID(toep->tid)); 129 130 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 131 flowc->mnemval[0].val = htobe32(pfvf); 132 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 133 flowc->mnemval[1].val = htobe32(pi->tx_chan); 134 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 135 flowc->mnemval[2].val = htobe32(pi->tx_chan); 136 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 137 flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id); 138 if (ftxp) { 139 uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf); 140 141 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 142 flowc->mnemval[4].val = htobe32(ftxp->snd_nxt); 143 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 144 flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt); 145 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 146 flowc->mnemval[6].val = htobe32(sndbuf); 147 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 148 flowc->mnemval[7].val = htobe32(ftxp->mss); 149 150 CTR6(KTR_CXGBE, 151 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x", 152 __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt, 153 ftxp->rcv_nxt); 154 } else { 155 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF; 156 flowc->mnemval[4].val = htobe32(512); 157 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS; 158 flowc->mnemval[5].val = htobe32(512); 159 160 CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid); 161 } 162 163 txsd->tx_credits = howmany(flowclen, 16); 164 txsd->plen = 0; 165 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 166 ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 167 toep->tx_credits -= txsd->tx_credits; 168 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 169 toep->txsd_pidx = 0; 170 toep->txsd_avail--; 171 172 toep->flags |= TPF_FLOWC_WR_SENT; 173 t4_wrq_tx(sc, wr); 174} 175 176void 177send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt) 178{ 179 struct wrqe *wr; 180 struct cpl_abort_req *req; 181 int tid = toep->tid; 182 struct inpcb *inp = toep->inp; 183 struct tcpcb *tp = intotcpcb(inp); /* don't use if INP_DROPPED */ 184 185 INP_WLOCK_ASSERT(inp); 186 187 CTR6(KTR_CXGBE, "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x%s", 188 __func__, toep->tid, 189 inp->inp_flags & INP_DROPPED ? "inp dropped" : 190 tcpstates[tp->t_state], 191 toep->flags, inp->inp_flags, 192 toep->flags & TPF_ABORT_SHUTDOWN ? 193 " (abort already in progress)" : ""); 194 195 if (toep->flags & TPF_ABORT_SHUTDOWN) 196 return; /* abort already in progress */ 197 198 toep->flags |= TPF_ABORT_SHUTDOWN; 199 200 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 201 ("%s: flowc_wr not sent for tid %d.", __func__, tid)); 202 203 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 204 if (wr == NULL) { 205 /* XXX */ 206 panic("%s: allocation failure.", __func__); 207 } 208 req = wrtod(wr); 209 210 INIT_TP_WR_MIT_CPL(req, CPL_ABORT_REQ, tid); 211 if (inp->inp_flags & INP_DROPPED) 212 req->rsvd0 = htobe32(snd_nxt); 213 else 214 req->rsvd0 = htobe32(tp->snd_nxt); 215 req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT); 216 req->cmd = CPL_ABORT_SEND_RST; 217 218 /* 219 * XXX: What's the correct way to tell that the inp hasn't been detached 220 * from its socket? Should I even be flushing the snd buffer here? 221 */ 222 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 223 struct socket *so = inp->inp_socket; 224 225 if (so != NULL) /* because I'm not sure. See comment above */ 226 sbflush(&so->so_snd); 227 } 228 229 t4_l2t_send(sc, wr, toep->l2te); 230} 231 232/* 233 * Called when a connection is established to translate the TCP options 234 * reported by HW to FreeBSD's native format. 235 */ 236static void 237assign_rxopt(struct tcpcb *tp, unsigned int opt) 238{ 239 struct toepcb *toep = tp->t_toe; 240 struct inpcb *inp = tp->t_inpcb; 241 struct adapter *sc = td_adapter(toep->td); 242 int n; 243 244 INP_LOCK_ASSERT(inp); 245 246 if (inp->inp_inc.inc_flags & INC_ISIPV6) 247 n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 248 else 249 n = sizeof(struct ip) + sizeof(struct tcphdr); 250 tp->t_maxseg = tp->t_maxopd = sc->params.mtus[G_TCPOPT_MSS(opt)] - n; 251 252 CTR4(KTR_CXGBE, "%s: tid %d, mtu_idx %u (%u)", __func__, toep->tid, 253 G_TCPOPT_MSS(opt), sc->params.mtus[G_TCPOPT_MSS(opt)]); 254 255 if (G_TCPOPT_TSTAMP(opt)) { 256 tp->t_flags |= TF_RCVD_TSTMP; /* timestamps ok */ 257 tp->ts_recent = 0; /* hmmm */ 258 tp->ts_recent_age = tcp_ts_getticks(); 259 tp->t_maxseg -= TCPOLEN_TSTAMP_APPA; 260 } 261 262 if (G_TCPOPT_SACK(opt)) 263 tp->t_flags |= TF_SACK_PERMIT; /* should already be set */ 264 else 265 tp->t_flags &= ~TF_SACK_PERMIT; /* sack disallowed by peer */ 266 267 if (G_TCPOPT_WSCALE_OK(opt)) 268 tp->t_flags |= TF_RCVD_SCALE; 269 270 /* Doing window scaling? */ 271 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 272 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 273 tp->rcv_scale = tp->request_r_scale; 274 tp->snd_scale = G_TCPOPT_SND_WSCALE(opt); 275 } 276} 277 278/* 279 * Completes some final bits of initialization for just established connections 280 * and changes their state to TCPS_ESTABLISHED. 281 * 282 * The ISNs are from after the exchange of SYNs. i.e., the true ISN + 1. 283 */ 284void 285make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn, 286 uint16_t opt) 287{ 288 struct inpcb *inp = toep->inp; 289 struct socket *so = inp->inp_socket; 290 struct tcpcb *tp = intotcpcb(inp); 291 long bufsize; 292 uint32_t iss = be32toh(snd_isn) - 1; /* true ISS */ 293 uint32_t irs = be32toh(rcv_isn) - 1; /* true IRS */ 294 uint16_t tcpopt = be16toh(opt); 295 struct flowc_tx_params ftxp; 296 297 INP_WLOCK_ASSERT(inp); 298 KASSERT(tp->t_state == TCPS_SYN_SENT || 299 tp->t_state == TCPS_SYN_RECEIVED, 300 ("%s: TCP state %s", __func__, tcpstates[tp->t_state])); 301 302 CTR4(KTR_CXGBE, "%s: tid %d, toep %p, inp %p", 303 __func__, toep->tid, toep, inp); 304 305 tp->t_state = TCPS_ESTABLISHED; 306 tp->t_starttime = ticks; 307 TCPSTAT_INC(tcps_connects); 308 309 tp->irs = irs; 310 tcp_rcvseqinit(tp); 311 tp->rcv_wnd = toep->rx_credits << 10; 312 tp->rcv_adv += tp->rcv_wnd; 313 tp->last_ack_sent = tp->rcv_nxt; 314 315 /* 316 * If we were unable to send all rx credits via opt0, save the remainder 317 * in rx_credits so that they can be handed over with the next credit 318 * update. 319 */ 320 SOCKBUF_LOCK(&so->so_rcv); 321 bufsize = select_rcv_wnd(so); 322 SOCKBUF_UNLOCK(&so->so_rcv); 323 toep->rx_credits = bufsize - tp->rcv_wnd; 324 325 tp->iss = iss; 326 tcp_sendseqinit(tp); 327 tp->snd_una = iss + 1; 328 tp->snd_nxt = iss + 1; 329 tp->snd_max = iss + 1; 330 331 assign_rxopt(tp, tcpopt); 332 333 SOCKBUF_LOCK(&so->so_snd); 334 if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf) 335 bufsize = V_tcp_autosndbuf_max; 336 else 337 bufsize = sbspace(&so->so_snd); 338 SOCKBUF_UNLOCK(&so->so_snd); 339 340 ftxp.snd_nxt = tp->snd_nxt; 341 ftxp.rcv_nxt = tp->rcv_nxt; 342 ftxp.snd_space = bufsize; 343 ftxp.mss = tp->t_maxseg; 344 send_flowc_wr(toep, &ftxp); 345 346 soisconnected(so); 347} 348 349static int 350send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) 351{ 352 struct wrqe *wr; 353 struct cpl_rx_data_ack *req; 354 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 355 356 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits)); 357 358 wr = alloc_wrqe(sizeof(*req), toep->ctrlq); 359 if (wr == NULL) 360 return (0); 361 req = wrtod(wr); 362 363 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid); 364 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits)); 365 366 t4_wrq_tx(sc, wr); 367 return (credits); 368} 369 370void 371t4_rcvd(struct toedev *tod, struct tcpcb *tp) 372{ 373 struct adapter *sc = tod->tod_softc; 374 struct inpcb *inp = tp->t_inpcb; 375 struct socket *so = inp->inp_socket; 376 struct sockbuf *sb = &so->so_rcv; 377 struct toepcb *toep = tp->t_toe; 378 int credits; 379 380 INP_WLOCK_ASSERT(inp); 381 382 SOCKBUF_LOCK(sb); 383 KASSERT(toep->sb_cc >= sb->sb_cc, 384 ("%s: sb %p has more data (%d) than last time (%d).", 385 __func__, sb, sb->sb_cc, toep->sb_cc)); 386 if (toep->ulp_mode == ULP_MODE_ISCSI) { 387 toep->rx_credits += toep->sb_cc; 388 toep->sb_cc = 0; 389 } else { 390 toep->rx_credits += toep->sb_cc - sb->sb_cc; 391 toep->sb_cc = sb->sb_cc; 392 } 393 if (toep->rx_credits > 0 && 394 (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || 395 (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || 396 toep->sb_cc + tp->rcv_wnd < sb->sb_lowat)) { 397 398 credits = send_rx_credits(sc, toep, toep->rx_credits); 399 toep->rx_credits -= credits; 400 tp->rcv_wnd += credits; 401 tp->rcv_adv += credits; 402 } 403 SOCKBUF_UNLOCK(sb); 404} 405 406/* 407 * Close a connection by sending a CPL_CLOSE_CON_REQ message. 408 */ 409static int 410close_conn(struct adapter *sc, struct toepcb *toep) 411{ 412 struct wrqe *wr; 413 struct cpl_close_con_req *req; 414 unsigned int tid = toep->tid; 415 416 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid, 417 toep->flags & TPF_FIN_SENT ? ", IGNORED" : ""); 418 419 if (toep->flags & TPF_FIN_SENT) 420 return (0); 421 422 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 423 ("%s: flowc_wr not sent for tid %u.", __func__, tid)); 424 425 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq); 426 if (wr == NULL) { 427 /* XXX */ 428 panic("%s: allocation failure.", __func__); 429 } 430 req = wrtod(wr); 431 432 req->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | 433 V_FW_WR_IMMDLEN(sizeof(*req) - sizeof(req->wr))); 434 req->wr.wr_mid = htonl(V_FW_WR_LEN16(howmany(sizeof(*req), 16)) | 435 V_FW_WR_FLOWID(tid)); 436 req->wr.wr_lo = cpu_to_be64(0); 437 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 438 req->rsvd = 0; 439 440 toep->flags |= TPF_FIN_SENT; 441 toep->flags &= ~TPF_SEND_FIN; 442 t4_l2t_send(sc, wr, toep->l2te); 443 444 return (0); 445} 446 447#define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16) 448#define MIN_OFLD_TX_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_wr) + 1, 16)) 449 450/* Maximum amount of immediate data we could stuff in a WR */ 451static inline int 452max_imm_payload(int tx_credits) 453{ 454 const int n = 2; /* Use only up to 2 desc for imm. data WR */ 455 456 KASSERT(tx_credits >= 0 && 457 tx_credits <= MAX_OFLD_TX_CREDITS, 458 ("%s: %d credits", __func__, tx_credits)); 459 460 if (tx_credits < MIN_OFLD_TX_CREDITS) 461 return (0); 462 463 if (tx_credits >= (n * EQ_ESIZE) / 16) 464 return ((n * EQ_ESIZE) - sizeof(struct fw_ofld_tx_data_wr)); 465 else 466 return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_wr)); 467} 468 469/* Maximum number of SGL entries we could stuff in a WR */ 470static inline int 471max_dsgl_nsegs(int tx_credits) 472{ 473 int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ 474 int sge_pair_credits = tx_credits - MIN_OFLD_TX_CREDITS; 475 476 KASSERT(tx_credits >= 0 && 477 tx_credits <= MAX_OFLD_TX_CREDITS, 478 ("%s: %d credits", __func__, tx_credits)); 479 480 if (tx_credits < MIN_OFLD_TX_CREDITS) 481 return (0); 482 483 nseg += 2 * (sge_pair_credits * 16 / 24); 484 if ((sge_pair_credits * 16) % 24 == 16) 485 nseg++; 486 487 return (nseg); 488} 489 490static inline void 491write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, 492 unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign) 493{ 494 struct fw_ofld_tx_data_wr *txwr = dst; 495 unsigned int wr_ulp_mode; 496 497 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | 498 V_FW_WR_IMMDLEN(immdlen)); 499 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | 500 V_FW_WR_LEN16(credits)); 501 502 /* for iscsi, the mode & submode setting is per-packet */ 503 if (toep->ulp_mode == ULP_MODE_ISCSI) 504 wr_ulp_mode = V_TX_ULP_MODE(ulp_mode >> 4) | 505 V_TX_ULP_SUBMODE(ulp_mode & 3); 506 else 507 wr_ulp_mode = V_TX_ULP_MODE(toep->ulp_mode); 508 509 txwr->lsodisable_to_flags = htobe32(wr_ulp_mode | V_TX_URG(0) | /*XXX*/ 510 V_TX_SHOVE(shove)); 511 txwr->plen = htobe32(plen); 512 513 if (txalign > 0) { 514 struct tcpcb *tp = intotcpcb(toep->inp); 515 516 if (plen < 2 * tp->t_maxseg || is_10G_port(toep->port)) 517 txwr->lsodisable_to_flags |= 518 htobe32(F_FW_OFLD_TX_DATA_WR_LSODISABLE); 519 else 520 txwr->lsodisable_to_flags |= 521 htobe32(F_FW_OFLD_TX_DATA_WR_ALIGNPLD | 522 (tp->t_flags & TF_NODELAY ? 0 : 523 F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE)); 524 } 525} 526 527/* 528 * Generate a DSGL from a starting mbuf. The total number of segments and the 529 * maximum segments in any one mbuf are provided. 530 */ 531static void 532write_tx_sgl(void *dst, struct mbuf *start, struct mbuf *stop, int nsegs, int n) 533{ 534 struct mbuf *m; 535 struct ulptx_sgl *usgl = dst; 536 int i, j, rc; 537 struct sglist sg; 538 struct sglist_seg segs[n]; 539 540 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__)); 541 542 sglist_init(&sg, n, segs); 543 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 544 V_ULPTX_NSGE(nsegs)); 545 546 i = -1; 547 for (m = start; m != stop; m = m->m_next) { 548 rc = sglist_append(&sg, mtod(m, void *), m->m_len); 549 if (__predict_false(rc != 0)) 550 panic("%s: sglist_append %d", __func__, rc); 551 552 for (j = 0; j < sg.sg_nseg; i++, j++) { 553 if (i < 0) { 554 usgl->len0 = htobe32(segs[j].ss_len); 555 usgl->addr0 = htobe64(segs[j].ss_paddr); 556 } else { 557 usgl->sge[i / 2].len[i & 1] = 558 htobe32(segs[j].ss_len); 559 usgl->sge[i / 2].addr[i & 1] = 560 htobe64(segs[j].ss_paddr); 561 } 562#ifdef INVARIANTS 563 nsegs--; 564#endif 565 } 566 sglist_reset(&sg); 567 } 568 if (i & 1) 569 usgl->sge[i / 2].len[1] = htobe32(0); 570 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, stop %p", 571 __func__, nsegs, start, stop)); 572} 573 574/* 575 * Max number of SGL entries an offload tx work request can have. This is 41 576 * (1 + 40) for a full 512B work request. 577 * fw_ofld_tx_data_wr(16B) + ulptx_sgl(16B, 1) + ulptx_sge_pair(480B, 40) 578 */ 579#define OFLD_SGL_LEN (41) 580 581/* 582 * Send data and/or a FIN to the peer. 583 * 584 * The socket's so_snd buffer consists of a stream of data starting with sb_mb 585 * and linked together with m_next. sb_sndptr, if set, is the last mbuf that 586 * was transmitted. 587 * 588 * drop indicates the number of bytes that should be dropped from the head of 589 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating 590 * contention on the send buffer lock (before this change it used to do 591 * sowwakeup and then t4_push_frames right after that when recovering from tx 592 * stalls). When drop is set this function MUST drop the bytes and wake up any 593 * writers. 594 */ 595void 596t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 597{ 598 struct mbuf *sndptr, *m, *sb_sndptr; 599 struct fw_ofld_tx_data_wr *txwr; 600 struct wrqe *wr; 601 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 602 struct inpcb *inp = toep->inp; 603 struct tcpcb *tp = intotcpcb(inp); 604 struct socket *so = inp->inp_socket; 605 struct sockbuf *sb = &so->so_snd; 606 int tx_credits, shove, compl, space, sowwakeup; 607 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 608 609 INP_WLOCK_ASSERT(inp); 610 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 611 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 612 613 KASSERT(toep->ulp_mode == ULP_MODE_NONE || 614 toep->ulp_mode == ULP_MODE_TCPDDP || 615 toep->ulp_mode == ULP_MODE_RDMA, 616 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); 617 618 /* 619 * This function doesn't resume by itself. Someone else must clear the 620 * flag and call this function. 621 */ 622 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { 623 KASSERT(drop == 0, 624 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); 625 return; 626 } 627 628 do { 629 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 630 max_imm = max_imm_payload(tx_credits); 631 max_nsegs = max_dsgl_nsegs(tx_credits); 632 633 SOCKBUF_LOCK(sb); 634 sowwakeup = drop; 635 if (drop) { 636 sbdrop_locked(sb, drop); 637 drop = 0; 638 } 639 sb_sndptr = sb->sb_sndptr; 640 sndptr = sb_sndptr ? sb_sndptr->m_next : sb->sb_mb; 641 plen = 0; 642 nsegs = 0; 643 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 644 for (m = sndptr; m != NULL; m = m->m_next) { 645 int n = sglist_count(mtod(m, void *), m->m_len); 646 647 nsegs += n; 648 plen += m->m_len; 649 650 /* This mbuf sent us _over_ the nsegs limit, back out */ 651 if (plen > max_imm && nsegs > max_nsegs) { 652 nsegs -= n; 653 plen -= m->m_len; 654 if (plen == 0) { 655 /* Too few credits */ 656 toep->flags |= TPF_TX_SUSPENDED; 657 if (sowwakeup) 658 sowwakeup_locked(so); 659 else 660 SOCKBUF_UNLOCK(sb); 661 SOCKBUF_UNLOCK_ASSERT(sb); 662 return; 663 } 664 break; 665 } 666 667 if (max_nsegs_1mbuf < n) 668 max_nsegs_1mbuf = n; 669 sb_sndptr = m; /* new sb->sb_sndptr if all goes well */ 670 671 /* This mbuf put us right at the max_nsegs limit */ 672 if (plen > max_imm && nsegs == max_nsegs) { 673 m = m->m_next; 674 break; 675 } 676 } 677 678 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); 679 space = sbspace(sb); 680 681 if (space <= sb->sb_hiwat * 3 / 8 && 682 toep->plen_nocompl + plen >= sb->sb_hiwat / 4) 683 compl = 1; 684 else 685 compl = 0; 686 687 if (sb->sb_flags & SB_AUTOSIZE && 688 V_tcp_do_autosndbuf && 689 sb->sb_hiwat < V_tcp_autosndbuf_max && 690 space < sb->sb_hiwat / 8) { 691 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc, 692 V_tcp_autosndbuf_max); 693 694 if (!sbreserve_locked(sb, newsize, so, NULL)) 695 sb->sb_flags &= ~SB_AUTOSIZE; 696 else 697 sowwakeup = 1; /* room available */ 698 } 699 if (sowwakeup) 700 sowwakeup_locked(so); 701 else 702 SOCKBUF_UNLOCK(sb); 703 SOCKBUF_UNLOCK_ASSERT(sb); 704 705 /* nothing to send */ 706 if (plen == 0) { 707 KASSERT(m == NULL, 708 ("%s: nothing to send, but m != NULL", __func__)); 709 break; 710 } 711 712 if (__predict_false(toep->flags & TPF_FIN_SENT)) 713 panic("%s: excess tx.", __func__); 714 715 if (plen <= max_imm) { 716 717 /* Immediate data tx */ 718 719 wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), 720 toep->ofld_txq); 721 if (wr == NULL) { 722 /* XXX: how will we recover from this? */ 723 toep->flags |= TPF_TX_SUSPENDED; 724 return; 725 } 726 txwr = wrtod(wr); 727 credits = howmany(wr->wr_len, 16); 728 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0, 729 sc->tt.tx_align); 730 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 731 nsegs = 0; 732 } else { 733 int wr_len; 734 735 /* DSGL tx */ 736 737 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 738 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 739 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 740 if (wr == NULL) { 741 /* XXX: how will we recover from this? */ 742 toep->flags |= TPF_TX_SUSPENDED; 743 return; 744 } 745 txwr = wrtod(wr); 746 credits = howmany(wr_len, 16); 747 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0, 748 sc->tt.tx_align); 749 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 750 max_nsegs_1mbuf); 751 if (wr_len & 0xf) { 752 uint64_t *pad = (uint64_t *) 753 ((uintptr_t)txwr + wr_len); 754 *pad = 0; 755 } 756 } 757 758 KASSERT(toep->tx_credits >= credits, 759 ("%s: not enough credits", __func__)); 760 761 toep->tx_credits -= credits; 762 toep->tx_nocompl += credits; 763 toep->plen_nocompl += plen; 764 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 765 toep->tx_nocompl >= toep->tx_total / 4) 766 compl = 1; 767 768 if (compl || toep->ulp_mode == ULP_MODE_RDMA) { 769 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 770 toep->tx_nocompl = 0; 771 toep->plen_nocompl = 0; 772 } 773 774 tp->snd_nxt += plen; 775 tp->snd_max += plen; 776 777 SOCKBUF_LOCK(sb); 778 KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); 779 sb->sb_sndptr = sb_sndptr; 780 SOCKBUF_UNLOCK(sb); 781 782 toep->flags |= TPF_TX_DATA_SENT; 783 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) 784 toep->flags |= TPF_TX_SUSPENDED; 785 786 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 787 txsd->plen = plen; 788 txsd->tx_credits = credits; 789 txsd++; 790 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 791 toep->txsd_pidx = 0; 792 txsd = &toep->txsd[0]; 793 } 794 toep->txsd_avail--; 795 796 t4_l2t_send(sc, wr, toep->l2te); 797 } while (m != NULL); 798 799 /* Send a FIN if requested, but only if there's no more data to send */ 800 if (m == NULL && toep->flags & TPF_SEND_FIN) 801 close_conn(sc, toep); 802} 803 804/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */ 805void 806t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 807{ 808 struct mbuf *sndptr, *m = NULL; 809 struct fw_ofld_tx_data_wr *txwr; 810 struct wrqe *wr; 811 unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 812 struct inpcb *inp = toep->inp; 813 struct tcpcb *tp; 814 struct socket *so; 815 struct sockbuf *sb; 816 int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0; 817 int shove, compl; 818 struct ofld_tx_sdesc *txsd; 819 820 INP_WLOCK_ASSERT(inp); 821 if (toep->flags & TPF_ABORT_SHUTDOWN) 822 return; 823 824 tp = intotcpcb(inp); 825 so = inp->inp_socket; 826 sb = &so->so_snd; 827 txsd = &toep->txsd[toep->txsd_pidx]; 828 829 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 830 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 831 832 /* 833 * This function doesn't resume by itself. Someone else must clear the 834 * flag and call this function. 835 */ 836 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) 837 return; 838 839 sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen); 840 if (!qlen) 841 return; 842 843 do { 844 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 845 max_imm = max_imm_payload(tx_credits); 846 max_nsegs = max_dsgl_nsegs(tx_credits); 847 848 if (drop) { 849 t4_cpl_iscsi_callback(toep->td, toep, &drop, 850 CPL_FW4_ACK); 851 drop = 0; 852 } 853 854 plen = 0; 855 nsegs = 0; 856 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 857 for (m = sndptr; m != NULL; m = m->m_next) { 858 int n = sglist_count(mtod(m, void *), m->m_len); 859 860 nsegs += n; 861 plen += m->m_len; 862 863 /* This mbuf sent us _over_ the nsegs limit, return */ 864 if (plen > max_imm && nsegs > max_nsegs) { 865 toep->flags |= TPF_TX_SUSPENDED; 866 return; 867 } 868 869 if (max_nsegs_1mbuf < n) 870 max_nsegs_1mbuf = n; 871 872 /* This mbuf put us right at the max_nsegs limit */ 873 if (plen > max_imm && nsegs == max_nsegs) { 874 toep->flags |= TPF_TX_SUSPENDED; 875 return; 876 } 877 } 878 879 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); 880 /* nothing to send */ 881 if (plen == 0) { 882 KASSERT(m == NULL, 883 ("%s: nothing to send, but m != NULL", __func__)); 884 break; 885 } 886 887 if (__predict_false(toep->flags & TPF_FIN_SENT)) 888 panic("%s: excess tx.", __func__); 889 890 ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode); 891 if (plen <= max_imm) { 892 893 /* Immediate data tx */ 894 wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16), 895 toep->ofld_txq); 896 if (wr == NULL) { 897 /* XXX: how will we recover from this? */ 898 toep->flags |= TPF_TX_SUSPENDED; 899 return; 900 } 901 txwr = wrtod(wr); 902 credits = howmany(wr->wr_len, 16); 903 write_tx_wr(txwr, toep, plen, ulp_len, credits, shove, 904 ulp_mode, 0); 905 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 906 } else { 907 int wr_len; 908 909 /* DSGL tx */ 910 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 911 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 912 wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq); 913 if (wr == NULL) { 914 /* XXX: how will we recover from this? */ 915 toep->flags |= TPF_TX_SUSPENDED; 916 return; 917 } 918 txwr = wrtod(wr); 919 credits = howmany(wr_len, 16); 920 write_tx_wr(txwr, toep, 0, ulp_len, credits, shove, 921 ulp_mode, 0); 922 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 923 max_nsegs_1mbuf); 924 if (wr_len & 0xf) { 925 uint64_t *pad = (uint64_t *) 926 ((uintptr_t)txwr + wr_len); 927 *pad = 0; 928 } 929 } 930 931 KASSERT(toep->tx_credits >= credits, 932 ("%s: not enough credits", __func__)); 933 934 toep->tx_credits -= credits; 935 toep->tx_nocompl += credits; 936 toep->plen_nocompl += plen; 937 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 938 toep->tx_nocompl >= toep->tx_total / 4) 939 compl = 1; 940 941 if (compl) { 942 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 943 toep->tx_nocompl = 0; 944 toep->plen_nocompl = 0; 945 } 946 tp->snd_nxt += ulp_len; 947 tp->snd_max += ulp_len; 948 949 /* goto next mbuf */ 950 sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen); 951 952 toep->flags |= TPF_TX_DATA_SENT; 953 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) { 954 toep->flags |= TPF_TX_SUSPENDED; 955 } 956 957 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 958 txsd->plen = plen; 959 txsd->tx_credits = credits; 960 txsd++; 961 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 962 toep->txsd_pidx = 0; 963 txsd = &toep->txsd[0]; 964 } 965 toep->txsd_avail--; 966 967 t4_l2t_send(sc, wr, toep->l2te); 968 } while (m != NULL); 969 970 /* Send a FIN if requested, but only if there's no more data to send */ 971 if (m == NULL && toep->flags & TPF_SEND_FIN) 972 close_conn(sc, toep); 973} 974 975int 976t4_tod_output(struct toedev *tod, struct tcpcb *tp) 977{ 978 struct adapter *sc = tod->tod_softc; 979#ifdef INVARIANTS 980 struct inpcb *inp = tp->t_inpcb; 981#endif 982 struct toepcb *toep = tp->t_toe; 983 984 INP_WLOCK_ASSERT(inp); 985 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 986 ("%s: inp %p dropped.", __func__, inp)); 987 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 988 989 t4_push_frames(sc, toep, 0); 990 991 return (0); 992} 993 994int 995t4_send_fin(struct toedev *tod, struct tcpcb *tp) 996{ 997 struct adapter *sc = tod->tod_softc; 998#ifdef INVARIANTS 999 struct inpcb *inp = tp->t_inpcb; 1000#endif 1001 struct toepcb *toep = tp->t_toe; 1002 1003 INP_WLOCK_ASSERT(inp); 1004 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1005 ("%s: inp %p dropped.", __func__, inp)); 1006 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1007 1008 toep->flags |= TPF_SEND_FIN; 1009 if (tp->t_state >= TCPS_ESTABLISHED) { 1010 if (toep->ulp_mode == ULP_MODE_ISCSI) 1011 t4_ulp_push_frames(sc, toep, 0); 1012 else 1013 t4_push_frames(sc, toep, 0); 1014 } 1015 1016 return (0); 1017} 1018 1019int 1020t4_send_rst(struct toedev *tod, struct tcpcb *tp) 1021{ 1022 struct adapter *sc = tod->tod_softc; 1023#if defined(INVARIANTS) 1024 struct inpcb *inp = tp->t_inpcb; 1025#endif 1026 struct toepcb *toep = tp->t_toe; 1027 1028 INP_WLOCK_ASSERT(inp); 1029 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 1030 ("%s: inp %p dropped.", __func__, inp)); 1031 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 1032 1033 /* hmmmm */ 1034 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 1035 ("%s: flowc for tid %u [%s] not sent already", 1036 __func__, toep->tid, tcpstates[tp->t_state])); 1037 1038 send_reset(sc, toep, 0); 1039 return (0); 1040} 1041 1042/* 1043 * Peer has sent us a FIN. 1044 */ 1045static int 1046do_peer_close(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1047{ 1048 struct adapter *sc = iq->adapter; 1049 const struct cpl_peer_close *cpl = (const void *)(rss + 1); 1050 unsigned int tid = GET_TID(cpl); 1051 struct toepcb *toep = lookup_tid(sc, tid); 1052 struct inpcb *inp = toep->inp; 1053 struct tcpcb *tp = NULL; 1054 struct socket *so; 1055 struct sockbuf *sb; 1056#ifdef INVARIANTS 1057 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1058#endif 1059 1060 KASSERT(opcode == CPL_PEER_CLOSE, 1061 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1062 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1063 1064 if (__predict_false(toep->flags & TPF_SYNQE)) { 1065#ifdef INVARIANTS 1066 struct synq_entry *synqe = (void *)toep; 1067 1068 INP_WLOCK(synqe->lctx->inp); 1069 if (synqe->flags & TPF_SYNQE_HAS_L2TE) { 1070 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, 1071 ("%s: listen socket closed but tid %u not aborted.", 1072 __func__, tid)); 1073 } else { 1074 /* 1075 * do_pass_accept_req is still running and will 1076 * eventually take care of this tid. 1077 */ 1078 } 1079 INP_WUNLOCK(synqe->lctx->inp); 1080#endif 1081 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1082 toep, toep->flags); 1083 return (0); 1084 } 1085 1086 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1087 1088 INP_INFO_WLOCK(&V_tcbinfo); 1089 INP_WLOCK(inp); 1090 tp = intotcpcb(inp); 1091 1092 CTR5(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__, 1093 tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp); 1094 1095 if (toep->flags & TPF_ABORT_SHUTDOWN) 1096 goto done; 1097 1098 tp->rcv_nxt++; /* FIN */ 1099 1100 so = inp->inp_socket; 1101 sb = &so->so_rcv; 1102 SOCKBUF_LOCK(sb); 1103 if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) { 1104 m = get_ddp_mbuf(be32toh(cpl->rcv_nxt) - tp->rcv_nxt); 1105 tp->rcv_nxt = be32toh(cpl->rcv_nxt); 1106 toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE); 1107 1108 KASSERT(toep->sb_cc >= sb->sb_cc, 1109 ("%s: sb %p has more data (%d) than last time (%d).", 1110 __func__, sb, sb->sb_cc, toep->sb_cc)); 1111 toep->rx_credits += toep->sb_cc - sb->sb_cc; 1112#ifdef USE_DDP_RX_FLOW_CONTROL 1113 toep->rx_credits -= m->m_len; /* adjust for F_RX_FC_DDP */ 1114#endif 1115 sbappendstream_locked(sb, m); 1116 toep->sb_cc = sb->sb_cc; 1117 } 1118 socantrcvmore_locked(so); /* unlocks the sockbuf */ 1119 1120 if (toep->ulp_mode != ULP_MODE_RDMA) { 1121 KASSERT(tp->rcv_nxt == be32toh(cpl->rcv_nxt), 1122 ("%s: rcv_nxt mismatch: %u %u", __func__, tp->rcv_nxt, 1123 be32toh(cpl->rcv_nxt))); 1124 } 1125 1126 switch (tp->t_state) { 1127 case TCPS_SYN_RECEIVED: 1128 tp->t_starttime = ticks; 1129 /* FALLTHROUGH */ 1130 1131 case TCPS_ESTABLISHED: 1132 tp->t_state = TCPS_CLOSE_WAIT; 1133 break; 1134 1135 case TCPS_FIN_WAIT_1: 1136 tp->t_state = TCPS_CLOSING; 1137 break; 1138 1139 case TCPS_FIN_WAIT_2: 1140 tcp_twstart(tp); 1141 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1142 INP_INFO_WUNLOCK(&V_tcbinfo); 1143 1144 INP_WLOCK(inp); 1145 final_cpl_received(toep); 1146 return (0); 1147 1148 default: 1149 log(LOG_ERR, "%s: TID %u received CPL_PEER_CLOSE in state %d\n", 1150 __func__, tid, tp->t_state); 1151 } 1152done: 1153 INP_WUNLOCK(inp); 1154 INP_INFO_WUNLOCK(&V_tcbinfo); 1155 return (0); 1156} 1157 1158/* 1159 * Peer has ACK'd our FIN. 1160 */ 1161static int 1162do_close_con_rpl(struct sge_iq *iq, const struct rss_header *rss, 1163 struct mbuf *m) 1164{ 1165 struct adapter *sc = iq->adapter; 1166 const struct cpl_close_con_rpl *cpl = (const void *)(rss + 1); 1167 unsigned int tid = GET_TID(cpl); 1168 struct toepcb *toep = lookup_tid(sc, tid); 1169 struct inpcb *inp = toep->inp; 1170 struct tcpcb *tp = NULL; 1171 struct socket *so = NULL; 1172#ifdef INVARIANTS 1173 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1174#endif 1175 1176 KASSERT(opcode == CPL_CLOSE_CON_RPL, 1177 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1178 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1179 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1180 1181 INP_INFO_WLOCK(&V_tcbinfo); 1182 INP_WLOCK(inp); 1183 tp = intotcpcb(inp); 1184 1185 CTR4(KTR_CXGBE, "%s: tid %u (%s), toep_flags 0x%x", 1186 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags); 1187 1188 if (toep->flags & TPF_ABORT_SHUTDOWN) 1189 goto done; 1190 1191 so = inp->inp_socket; 1192 tp->snd_una = be32toh(cpl->snd_nxt) - 1; /* exclude FIN */ 1193 1194 switch (tp->t_state) { 1195 case TCPS_CLOSING: /* see TCPS_FIN_WAIT_2 in do_peer_close too */ 1196 tcp_twstart(tp); 1197release: 1198 INP_UNLOCK_ASSERT(inp); /* safe, we have a ref on the inp */ 1199 INP_INFO_WUNLOCK(&V_tcbinfo); 1200 1201 INP_WLOCK(inp); 1202 final_cpl_received(toep); /* no more CPLs expected */ 1203 1204 return (0); 1205 case TCPS_LAST_ACK: 1206 if (tcp_close(tp)) 1207 INP_WUNLOCK(inp); 1208 goto release; 1209 1210 case TCPS_FIN_WAIT_1: 1211 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1212 soisdisconnected(so); 1213 tp->t_state = TCPS_FIN_WAIT_2; 1214 break; 1215 1216 default: 1217 log(LOG_ERR, 1218 "%s: TID %u received CPL_CLOSE_CON_RPL in state %s\n", 1219 __func__, tid, tcpstates[tp->t_state]); 1220 } 1221done: 1222 INP_WUNLOCK(inp); 1223 INP_INFO_WUNLOCK(&V_tcbinfo); 1224 return (0); 1225} 1226 1227void 1228send_abort_rpl(struct adapter *sc, struct sge_wrq *ofld_txq, int tid, 1229 int rst_status) 1230{ 1231 struct wrqe *wr; 1232 struct cpl_abort_rpl *cpl; 1233 1234 wr = alloc_wrqe(sizeof(*cpl), ofld_txq); 1235 if (wr == NULL) { 1236 /* XXX */ 1237 panic("%s: allocation failure.", __func__); 1238 } 1239 cpl = wrtod(wr); 1240 1241 INIT_TP_WR_MIT_CPL(cpl, CPL_ABORT_RPL, tid); 1242 cpl->cmd = rst_status; 1243 1244 t4_wrq_tx(sc, wr); 1245} 1246 1247static int 1248abort_status_to_errno(struct tcpcb *tp, unsigned int abort_reason) 1249{ 1250 switch (abort_reason) { 1251 case CPL_ERR_BAD_SYN: 1252 case CPL_ERR_CONN_RESET: 1253 return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); 1254 case CPL_ERR_XMIT_TIMEDOUT: 1255 case CPL_ERR_PERSIST_TIMEDOUT: 1256 case CPL_ERR_FINWAIT2_TIMEDOUT: 1257 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1258 return (ETIMEDOUT); 1259 default: 1260 return (EIO); 1261 } 1262} 1263 1264int 1265cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *); 1266/* 1267 * tom_cpl_iscsi_callback - 1268 * iscsi and tom would share the following cpl messages, so when any of these 1269 * message is received, after tom is done with processing it, the messages 1270 * needs to be forwarded to iscsi for further processing: 1271 * - CPL_SET_TCB_RPL 1272 * - CPL_RX_DATA_DDP 1273 */ 1274void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *, 1275 unsigned int); 1276 1277struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *); 1278/* 1279 * Check if the handler function is set for a given CPL 1280 * return 0 if the function is NULL or cpl_not_handled, 1 otherwise. 1281 */ 1282int 1283t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode) 1284{ 1285 1286 MPASS(opcode < nitems(sc->cpl_handler)); 1287 1288 return (sc->cpl_handler[opcode] && 1289 sc->cpl_handler[opcode] != cpl_not_handled); 1290} 1291 1292/* 1293 * set the tom_cpl_iscsi_callback function, this function should be used 1294 * whenever both toe and iscsi need to process the same cpl msg. 1295 */ 1296void 1297t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *, 1298 void *, unsigned int)) 1299{ 1300 1301 tom_cpl_iscsi_callback = fp; 1302} 1303 1304void 1305t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, 1306 unsigned int, int *qlen)) 1307{ 1308 1309 tom_queue_iscsi_callback = fp; 1310} 1311 1312int 1313t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m, 1314 unsigned int opcode) 1315{ 1316 struct socket *so; 1317 1318 if (opcode == CPL_FW4_ACK) 1319 so = toep->inp->inp_socket; 1320 else { 1321 INP_WLOCK(toep->inp); 1322 so = toep->inp->inp_socket; 1323 INP_WUNLOCK(toep->inp); 1324 } 1325 1326 if (tom_cpl_iscsi_callback && so) { 1327 if (toep->ulp_mode == ULP_MODE_ISCSI) { 1328 tom_cpl_iscsi_callback(td, so, m, opcode); 1329 return (0); 1330 } 1331 } 1332 1333 return (1); 1334} 1335 1336struct mbuf * 1337t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep, 1338 unsigned int cmd, int *qlen) 1339{ 1340 1341 if (tom_queue_iscsi_callback && so) { 1342 if (toep->ulp_mode == ULP_MODE_ISCSI) 1343 return (tom_queue_iscsi_callback(so, cmd, qlen)); 1344 } 1345 1346 return (NULL); 1347} 1348 1349/* 1350 * TCP RST from the peer, timeout, or some other such critical error. 1351 */ 1352static int 1353do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1354{ 1355 struct adapter *sc = iq->adapter; 1356 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); 1357 unsigned int tid = GET_TID(cpl); 1358 struct toepcb *toep = lookup_tid(sc, tid); 1359 struct sge_wrq *ofld_txq = toep->ofld_txq; 1360 struct inpcb *inp; 1361 struct tcpcb *tp; 1362#ifdef INVARIANTS 1363 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1364#endif 1365 1366 KASSERT(opcode == CPL_ABORT_REQ_RSS, 1367 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1368 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1369 1370 if (toep->flags & TPF_SYNQE) 1371 return (do_abort_req_synqe(iq, rss, m)); 1372 1373 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1374 1375 if (negative_advice(cpl->status)) { 1376 CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)", 1377 __func__, cpl->status, tid, toep->flags); 1378 return (0); /* Ignore negative advice */ 1379 } 1380 1381 inp = toep->inp; 1382 INP_INFO_WLOCK(&V_tcbinfo); /* for tcp_close */ 1383 INP_WLOCK(inp); 1384 1385 tp = intotcpcb(inp); 1386 1387 CTR6(KTR_CXGBE, 1388 "%s: tid %d (%s), toep_flags 0x%x, inp_flags 0x%x, status %d", 1389 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, 1390 inp->inp_flags, cpl->status); 1391 1392 /* 1393 * If we'd initiated an abort earlier the reply to it is responsible for 1394 * cleaning up resources. Otherwise we tear everything down right here 1395 * right now. We owe the T4 a CPL_ABORT_RPL no matter what. 1396 */ 1397 if (toep->flags & TPF_ABORT_SHUTDOWN) { 1398 INP_WUNLOCK(inp); 1399 goto done; 1400 } 1401 toep->flags |= TPF_ABORT_SHUTDOWN; 1402 1403 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 1404 struct socket *so = inp->inp_socket; 1405 1406 if (so != NULL) 1407 so_error_set(so, abort_status_to_errno(tp, 1408 cpl->status)); 1409 tp = tcp_close(tp); 1410 if (tp == NULL) 1411 INP_WLOCK(inp); /* re-acquire */ 1412 } 1413 1414 final_cpl_received(toep); 1415done: 1416 INP_INFO_WUNLOCK(&V_tcbinfo); 1417 send_abort_rpl(sc, ofld_txq, tid, CPL_ABORT_NO_RST); 1418 return (0); 1419} 1420 1421/* 1422 * Reply to the CPL_ABORT_REQ (send_reset) 1423 */ 1424static int 1425do_abort_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1426{ 1427 struct adapter *sc = iq->adapter; 1428 const struct cpl_abort_rpl_rss *cpl = (const void *)(rss + 1); 1429 unsigned int tid = GET_TID(cpl); 1430 struct toepcb *toep = lookup_tid(sc, tid); 1431 struct inpcb *inp = toep->inp; 1432#ifdef INVARIANTS 1433 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1434#endif 1435 1436 KASSERT(opcode == CPL_ABORT_RPL_RSS, 1437 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1438 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1439 1440 if (toep->flags & TPF_SYNQE) 1441 return (do_abort_rpl_synqe(iq, rss, m)); 1442 1443 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1444 1445 CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d", 1446 __func__, tid, toep, inp, cpl->status); 1447 1448 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1449 ("%s: wasn't expecting abort reply", __func__)); 1450 1451 INP_WLOCK(inp); 1452 final_cpl_received(toep); 1453 1454 return (0); 1455} 1456 1457static int 1458do_rx_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1459{ 1460 struct adapter *sc = iq->adapter; 1461 const struct cpl_rx_data *cpl = mtod(m, const void *); 1462 unsigned int tid = GET_TID(cpl); 1463 struct toepcb *toep = lookup_tid(sc, tid); 1464 struct inpcb *inp = toep->inp; 1465 struct tcpcb *tp; 1466 struct socket *so; 1467 struct sockbuf *sb; 1468 int len; 1469 uint32_t ddp_placed = 0; 1470 1471 if (__predict_false(toep->flags & TPF_SYNQE)) { 1472#ifdef INVARIANTS 1473 struct synq_entry *synqe = (void *)toep; 1474 1475 INP_WLOCK(synqe->lctx->inp); 1476 if (synqe->flags & TPF_SYNQE_HAS_L2TE) { 1477 KASSERT(synqe->flags & TPF_ABORT_SHUTDOWN, 1478 ("%s: listen socket closed but tid %u not aborted.", 1479 __func__, tid)); 1480 } else { 1481 /* 1482 * do_pass_accept_req is still running and will 1483 * eventually take care of this tid. 1484 */ 1485 } 1486 INP_WUNLOCK(synqe->lctx->inp); 1487#endif 1488 CTR4(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x)", __func__, tid, 1489 toep, toep->flags); 1490 m_freem(m); 1491 return (0); 1492 } 1493 1494 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1495 1496 /* strip off CPL header */ 1497 m_adj(m, sizeof(*cpl)); 1498 len = m->m_pkthdr.len; 1499 1500 INP_WLOCK(inp); 1501 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1502 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 1503 __func__, tid, len, inp->inp_flags); 1504 INP_WUNLOCK(inp); 1505 m_freem(m); 1506 return (0); 1507 } 1508 1509 tp = intotcpcb(inp); 1510 1511 if (__predict_false(tp->rcv_nxt != be32toh(cpl->seq))) 1512 ddp_placed = be32toh(cpl->seq) - tp->rcv_nxt; 1513 1514 tp->rcv_nxt += len; 1515 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__)); 1516 tp->rcv_wnd -= len; 1517 tp->t_rcvtime = ticks; 1518 1519 so = inp_inpcbtosocket(inp); 1520 sb = &so->so_rcv; 1521 SOCKBUF_LOCK(sb); 1522 1523 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1524 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", 1525 __func__, tid, len); 1526 m_freem(m); 1527 SOCKBUF_UNLOCK(sb); 1528 INP_WUNLOCK(inp); 1529 1530 INP_INFO_WLOCK(&V_tcbinfo); 1531 INP_WLOCK(inp); 1532 tp = tcp_drop(tp, ECONNRESET); 1533 if (tp) 1534 INP_WUNLOCK(inp); 1535 INP_INFO_WUNLOCK(&V_tcbinfo); 1536 1537 return (0); 1538 } 1539 1540 /* receive buffer autosize */ 1541 if (sb->sb_flags & SB_AUTOSIZE && 1542 V_tcp_do_autorcvbuf && 1543 sb->sb_hiwat < V_tcp_autorcvbuf_max && 1544 len > (sbspace(sb) / 8 * 7)) { 1545 unsigned int hiwat = sb->sb_hiwat; 1546 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc, 1547 V_tcp_autorcvbuf_max); 1548 1549 if (!sbreserve_locked(sb, newsize, so, NULL)) 1550 sb->sb_flags &= ~SB_AUTOSIZE; 1551 else 1552 toep->rx_credits += newsize - hiwat; 1553 } 1554 1555 if (toep->ulp_mode == ULP_MODE_TCPDDP) { 1556 int changed = !(toep->ddp_flags & DDP_ON) ^ cpl->ddp_off; 1557 1558 if (changed) { 1559 if (toep->ddp_flags & DDP_SC_REQ) 1560 toep->ddp_flags ^= DDP_ON | DDP_SC_REQ; 1561 else { 1562 KASSERT(cpl->ddp_off == 1, 1563 ("%s: DDP switched on by itself.", 1564 __func__)); 1565 1566 /* Fell out of DDP mode */ 1567 toep->ddp_flags &= ~(DDP_ON | DDP_BUF0_ACTIVE | 1568 DDP_BUF1_ACTIVE); 1569 1570 if (ddp_placed) 1571 insert_ddp_data(toep, ddp_placed); 1572 } 1573 } 1574 1575 if ((toep->ddp_flags & DDP_OK) == 0 && 1576 time_uptime >= toep->ddp_disabled + DDP_RETRY_WAIT) { 1577 toep->ddp_score = DDP_LOW_SCORE; 1578 toep->ddp_flags |= DDP_OK; 1579 CTR3(KTR_CXGBE, "%s: tid %u DDP_OK @ %u", 1580 __func__, tid, time_uptime); 1581 } 1582 1583 if (toep->ddp_flags & DDP_ON) { 1584 1585 /* 1586 * CPL_RX_DATA with DDP on can only be an indicate. Ask 1587 * soreceive to post a buffer or disable DDP. The 1588 * payload that arrived in this indicate is appended to 1589 * the socket buffer as usual. 1590 */ 1591 1592#if 0 1593 CTR5(KTR_CXGBE, 1594 "%s: tid %u (0x%x) DDP indicate (seq 0x%x, len %d)", 1595 __func__, tid, toep->flags, be32toh(cpl->seq), len); 1596#endif 1597 sb->sb_flags |= SB_DDP_INDICATE; 1598 } else if ((toep->ddp_flags & (DDP_OK|DDP_SC_REQ)) == DDP_OK && 1599 tp->rcv_wnd > DDP_RSVD_WIN && len >= sc->tt.ddp_thres) { 1600 1601 /* 1602 * DDP allowed but isn't on (and a request to switch it 1603 * on isn't pending either), and conditions are ripe for 1604 * it to work. Switch it on. 1605 */ 1606 1607 enable_ddp(sc, toep); 1608 } 1609 } 1610 1611 KASSERT(toep->sb_cc >= sb->sb_cc, 1612 ("%s: sb %p has more data (%d) than last time (%d).", 1613 __func__, sb, sb->sb_cc, toep->sb_cc)); 1614 toep->rx_credits += toep->sb_cc - sb->sb_cc; 1615 sbappendstream_locked(sb, m); 1616 toep->sb_cc = sb->sb_cc; 1617 if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) { 1618 int credits; 1619 1620 credits = send_rx_credits(sc, toep, toep->rx_credits); 1621 toep->rx_credits -= credits; 1622 tp->rcv_wnd += credits; 1623 tp->rcv_adv += credits; 1624 } 1625 sorwakeup_locked(so); 1626 SOCKBUF_UNLOCK_ASSERT(sb); 1627 1628 INP_WUNLOCK(inp); 1629 return (0); 1630} 1631 1632#define S_CPL_FW4_ACK_OPCODE 24 1633#define M_CPL_FW4_ACK_OPCODE 0xff 1634#define V_CPL_FW4_ACK_OPCODE(x) ((x) << S_CPL_FW4_ACK_OPCODE) 1635#define G_CPL_FW4_ACK_OPCODE(x) \ 1636 (((x) >> S_CPL_FW4_ACK_OPCODE) & M_CPL_FW4_ACK_OPCODE) 1637 1638#define S_CPL_FW4_ACK_FLOWID 0 1639#define M_CPL_FW4_ACK_FLOWID 0xffffff 1640#define V_CPL_FW4_ACK_FLOWID(x) ((x) << S_CPL_FW4_ACK_FLOWID) 1641#define G_CPL_FW4_ACK_FLOWID(x) \ 1642 (((x) >> S_CPL_FW4_ACK_FLOWID) & M_CPL_FW4_ACK_FLOWID) 1643 1644#define S_CPL_FW4_ACK_CR 24 1645#define M_CPL_FW4_ACK_CR 0xff 1646#define V_CPL_FW4_ACK_CR(x) ((x) << S_CPL_FW4_ACK_CR) 1647#define G_CPL_FW4_ACK_CR(x) (((x) >> S_CPL_FW4_ACK_CR) & M_CPL_FW4_ACK_CR) 1648 1649#define S_CPL_FW4_ACK_SEQVAL 0 1650#define M_CPL_FW4_ACK_SEQVAL 0x1 1651#define V_CPL_FW4_ACK_SEQVAL(x) ((x) << S_CPL_FW4_ACK_SEQVAL) 1652#define G_CPL_FW4_ACK_SEQVAL(x) \ 1653 (((x) >> S_CPL_FW4_ACK_SEQVAL) & M_CPL_FW4_ACK_SEQVAL) 1654#define F_CPL_FW4_ACK_SEQVAL V_CPL_FW4_ACK_SEQVAL(1U) 1655 1656static int 1657do_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1658{ 1659 struct adapter *sc = iq->adapter; 1660 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 1661 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 1662 struct toepcb *toep = lookup_tid(sc, tid); 1663 struct inpcb *inp; 1664 struct tcpcb *tp; 1665 struct socket *so; 1666 uint8_t credits = cpl->credits; 1667 struct ofld_tx_sdesc *txsd; 1668 int plen; 1669#ifdef INVARIANTS 1670 unsigned int opcode = G_CPL_FW4_ACK_OPCODE(be32toh(OPCODE_TID(cpl))); 1671#endif 1672 1673 /* 1674 * Very unusual case: we'd sent a flowc + abort_req for a synq entry and 1675 * now this comes back carrying the credits for the flowc. 1676 */ 1677 if (__predict_false(toep->flags & TPF_SYNQE)) { 1678 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN, 1679 ("%s: credits for a synq entry %p", __func__, toep)); 1680 return (0); 1681 } 1682 1683 inp = toep->inp; 1684 1685 KASSERT(opcode == CPL_FW4_ACK, 1686 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1687 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1688 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__)); 1689 1690 INP_WLOCK(inp); 1691 1692 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) { 1693 INP_WUNLOCK(inp); 1694 return (0); 1695 } 1696 1697 KASSERT((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0, 1698 ("%s: inp_flags 0x%x", __func__, inp->inp_flags)); 1699 1700 tp = intotcpcb(inp); 1701 1702 if (cpl->flags & CPL_FW4_ACK_FLAGS_SEQVAL) { 1703 tcp_seq snd_una = be32toh(cpl->snd_una); 1704 1705#ifdef INVARIANTS 1706 if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { 1707 log(LOG_ERR, 1708 "%s: unexpected seq# %x for TID %u, snd_una %x\n", 1709 __func__, snd_una, toep->tid, tp->snd_una); 1710 } 1711#endif 1712 1713 if (tp->snd_una != snd_una) { 1714 tp->snd_una = snd_una; 1715 tp->ts_recent_age = tcp_ts_getticks(); 1716 } 1717 } 1718 1719 so = inp->inp_socket; 1720 txsd = &toep->txsd[toep->txsd_cidx]; 1721 plen = 0; 1722 while (credits) { 1723 KASSERT(credits >= txsd->tx_credits, 1724 ("%s: too many (or partial) credits", __func__)); 1725 credits -= txsd->tx_credits; 1726 toep->tx_credits += txsd->tx_credits; 1727 plen += txsd->plen; 1728 txsd++; 1729 toep->txsd_avail++; 1730 KASSERT(toep->txsd_avail <= toep->txsd_total, 1731 ("%s: txsd avail > total", __func__)); 1732 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) { 1733 txsd = &toep->txsd[0]; 1734 toep->txsd_cidx = 0; 1735 } 1736 } 1737 1738 if (toep->tx_credits == toep->tx_total) { 1739 toep->tx_nocompl = 0; 1740 toep->plen_nocompl = 0; 1741 } 1742 1743 if (toep->flags & TPF_TX_SUSPENDED && 1744 toep->tx_credits >= toep->tx_total / 4) { 1745 toep->flags &= ~TPF_TX_SUSPENDED; 1746 if (toep->ulp_mode == ULP_MODE_ISCSI) 1747 t4_ulp_push_frames(sc, toep, plen); 1748 else 1749 t4_push_frames(sc, toep, plen); 1750 } else if (plen > 0) { 1751 struct sockbuf *sb = &so->so_snd; 1752 1753 if (toep->ulp_mode == ULP_MODE_ISCSI) 1754 t4_cpl_iscsi_callback(toep->td, toep, &plen, 1755 CPL_FW4_ACK); 1756 else { 1757 SOCKBUF_LOCK(sb); 1758 sbdrop_locked(sb, plen); 1759 sowwakeup_locked(so); 1760 SOCKBUF_UNLOCK_ASSERT(sb); 1761 } 1762 } 1763 1764 INP_WUNLOCK(inp); 1765 1766 return (0); 1767} 1768 1769static int 1770do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1771{ 1772 struct adapter *sc = iq->adapter; 1773 const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); 1774 unsigned int tid = GET_TID(cpl); 1775#ifdef INVARIANTS 1776 unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); 1777#endif 1778 1779 KASSERT(opcode == CPL_SET_TCB_RPL, 1780 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1781 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1782 1783 if (is_ftid(sc, tid)) 1784 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ 1785 else { 1786 struct toepcb *toep = lookup_tid(sc, tid); 1787 1788 t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL); 1789 return (0); 1790 } 1791 1792 CXGBE_UNIMPLEMENTED(__func__); 1793} 1794 1795void 1796t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl, 1797 uint16_t word, uint64_t mask, uint64_t val) 1798{ 1799 struct wrqe *wr; 1800 struct cpl_set_tcb_field *req; 1801 1802 wr = alloc_wrqe(sizeof(*req), ctrl ? toep->ctrlq : toep->ofld_txq); 1803 if (wr == NULL) { 1804 /* XXX */ 1805 panic("%s: allocation failure.", __func__); 1806 } 1807 req = wrtod(wr); 1808 1809 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid); 1810 req->reply_ctrl = htobe16(V_NO_REPLY(1) | 1811 V_QUEUENO(toep->ofld_rxq->iq.abs_id)); 1812 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(0)); 1813 req->mask = htobe64(mask); 1814 req->val = htobe64(val); 1815 1816 t4_wrq_tx(sc, wr); 1817} 1818 1819void 1820t4_init_cpl_io_handlers(struct adapter *sc) 1821{ 1822 1823 t4_register_cpl_handler(sc, CPL_PEER_CLOSE, do_peer_close); 1824 t4_register_cpl_handler(sc, CPL_CLOSE_CON_RPL, do_close_con_rpl); 1825 t4_register_cpl_handler(sc, CPL_ABORT_REQ_RSS, do_abort_req); 1826 t4_register_cpl_handler(sc, CPL_ABORT_RPL_RSS, do_abort_rpl); 1827 t4_register_cpl_handler(sc, CPL_RX_DATA, do_rx_data); 1828 t4_register_cpl_handler(sc, CPL_FW4_ACK, do_fw4_ack); 1829 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, do_set_tcb_rpl); 1830} 1831 1832void 1833t4_uninit_cpl_io_handlers(struct adapter *sc) 1834{ 1835 1836 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 1837} 1838#endif 1839