1237263Snp/*- 2237263Snp * Copyright (c) 2012 Chelsio Communications, Inc. 3237263Snp * All rights reserved. 4237263Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5237263Snp * 6237263Snp * Redistribution and use in source and binary forms, with or without 7237263Snp * modification, are permitted provided that the following conditions 8237263Snp * are met: 9237263Snp * 1. Redistributions of source code must retain the above copyright 10237263Snp * notice, this list of conditions and the following disclaimer. 11237263Snp * 2. Redistributions in binary form must reproduce the above copyright 12237263Snp * notice, this list of conditions and the following disclaimer in the 13237263Snp * documentation and/or other materials provided with the distribution. 14237263Snp * 15237263Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16237263Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17237263Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18237263Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19237263Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20237263Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21237263Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22237263Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23237263Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24237263Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25237263Snp * SUCH DAMAGE. 26237263Snp */ 27237263Snp 28237263Snp#include <sys/cdefs.h> 29237263Snp__FBSDID("$FreeBSD: stable/10/sys/netinet/toecore.c 329982 2018-02-25 11:29:55Z hselasky $"); 30237263Snp 31237263Snp#include "opt_inet.h" 32237263Snp#include "opt_inet6.h" 33237263Snp 34237263Snp#include <sys/param.h> 35237263Snp#include <sys/kernel.h> 36237263Snp#include <sys/systm.h> 37237263Snp#include <sys/mbuf.h> 38237263Snp#include <sys/module.h> 39237263Snp#include <sys/types.h> 40237263Snp#include <sys/sockopt.h> 41237263Snp#include <sys/sysctl.h> 42237263Snp#include <sys/socket.h> 43237263Snp 44329982Shselasky#if defined(KLD_MODULE) || defined(INET) || defined(INET6) 45329982Shselasky 46237263Snp#include <net/ethernet.h> 47237263Snp#include <net/if.h> 48237263Snp#include <net/if_types.h> 49237263Snp#include <net/if_vlan_var.h> 50237263Snp#include <net/if_llatbl.h> 51237263Snp#include <net/route.h> 52237263Snp 53237263Snp#include <netinet/if_ether.h> 54237263Snp#include <netinet/in.h> 55237263Snp#include <netinet/in_pcb.h> 56237263Snp#include <netinet/in_var.h> 57245932Snp#include <netinet6/in6_var.h> 58245916Snp#include <netinet6/in6_pcb.h> 59237263Snp#include <netinet6/nd6.h> 60237263Snp#define TCPSTATES 61237263Snp#include <netinet/tcp.h> 62237263Snp#include <netinet/tcp_fsm.h> 63237263Snp#include <netinet/tcp_timer.h> 64237263Snp#include <netinet/tcp_var.h> 65237263Snp#include <netinet/tcp_syncache.h> 66237263Snp#include <netinet/tcp_offload.h> 67237263Snp#include <netinet/toecore.h> 68237263Snp 69237263Snpstatic struct mtx toedev_lock; 70237263Snpstatic TAILQ_HEAD(, toedev) toedev_list; 71237263Snpstatic eventhandler_tag listen_start_eh; 72237263Snpstatic eventhandler_tag listen_stop_eh; 73237263Snpstatic eventhandler_tag lle_event_eh; 74237263Snpstatic eventhandler_tag route_redirect_eh; 75237263Snp 76237263Snpstatic int 77237263Snptoedev_connect(struct toedev *tod __unused, struct socket *so __unused, 78237263Snp struct rtentry *rt __unused, struct sockaddr *nam __unused) 79237263Snp{ 80237263Snp 81237263Snp return (ENOTSUP); 82237263Snp} 83237263Snp 84237263Snpstatic int 85237263Snptoedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 86237263Snp{ 87237263Snp 88237263Snp return (ENOTSUP); 89237263Snp} 90237263Snp 91237263Snpstatic int 92237263Snptoedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 93237263Snp{ 94237263Snp 95237263Snp return (ENOTSUP); 96237263Snp} 97237263Snp 98237263Snpstatic void 99237263Snptoedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 100237263Snp struct mbuf *m) 101237263Snp{ 102237263Snp 103237263Snp m_freem(m); 104237263Snp return; 105237263Snp} 106237263Snp 107237263Snpstatic void 108237263Snptoedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 109237263Snp{ 110237263Snp 111237263Snp return; 112237263Snp} 113237263Snp 114237263Snpstatic int 115237263Snptoedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 116237263Snp{ 117237263Snp 118237263Snp return (ENOTSUP); 119237263Snp} 120237263Snp 121237263Snpstatic void 122237263Snptoedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 123237263Snp{ 124237263Snp 125237263Snp return; 126237263Snp} 127237263Snp 128237263Snpstatic void 129237263Snptoedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 130237263Snp struct sockaddr *sa __unused, uint8_t *lladdr __unused, 131237263Snp uint16_t vtag __unused) 132237263Snp{ 133237263Snp 134237263Snp return; 135237263Snp} 136237263Snp 137237263Snpstatic void 138237263Snptoedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 139237263Snp struct rtentry *rt0 __unused, struct rtentry *rt1 __unused) 140237263Snp{ 141237263Snp 142237263Snp return; 143237263Snp} 144237263Snp 145237263Snpstatic void 146237263Snptoedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 147237263Snp{ 148237263Snp 149237263Snp return; 150237263Snp} 151237263Snp 152237263Snpstatic void 153237263Snptoedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 154237263Snp{ 155237263Snp 156237263Snp return; 157237263Snp} 158237263Snp 159237263Snpstatic int 160237263Snptoedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 161237263Snp struct mbuf *m) 162237263Snp{ 163237263Snp 164237263Snp m_freem(m); 165237263Snp return (0); 166237263Snp} 167237263Snp 168237263Snpstatic void 169237263Snptoedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 170237263Snp struct socket *so __unused) 171237263Snp{ 172237263Snp 173237263Snp return; 174237263Snp} 175237263Snp 176237263Snpstatic void 177237263Snptoedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 178237263Snp int sopt_dir __unused, int sopt_name __unused) 179237263Snp{ 180237263Snp 181237263Snp return; 182237263Snp} 183237263Snp 184237263Snp/* 185237263Snp * Inform one or more TOE devices about a listening socket. 186237263Snp */ 187237263Snpstatic void 188237263Snptoe_listen_start(struct inpcb *inp, void *arg) 189237263Snp{ 190237263Snp struct toedev *t, *tod; 191237263Snp struct tcpcb *tp; 192237263Snp 193237263Snp INP_WLOCK_ASSERT(inp); 194237263Snp KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 195237263Snp ("%s: inp is not a TCP inp", __func__)); 196237263Snp 197237263Snp if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) 198237263Snp return; 199237263Snp 200237263Snp tp = intotcpcb(inp); 201237263Snp if (tp->t_state != TCPS_LISTEN) 202237263Snp return; 203237263Snp 204237263Snp t = arg; 205237263Snp mtx_lock(&toedev_lock); 206237263Snp TAILQ_FOREACH(tod, &toedev_list, link) { 207237263Snp if (t == NULL || t == tod) 208237263Snp tod->tod_listen_start(tod, tp); 209237263Snp } 210237263Snp mtx_unlock(&toedev_lock); 211237263Snp} 212237263Snp 213237263Snpstatic void 214237263Snptoe_listen_start_event(void *arg __unused, struct tcpcb *tp) 215237263Snp{ 216237263Snp struct inpcb *inp = tp->t_inpcb; 217237263Snp 218237263Snp INP_WLOCK_ASSERT(inp); 219237263Snp KASSERT(tp->t_state == TCPS_LISTEN, 220237263Snp ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 221237263Snp 222237263Snp toe_listen_start(inp, NULL); 223237263Snp} 224237263Snp 225237263Snpstatic void 226237263Snptoe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 227237263Snp{ 228237263Snp struct toedev *tod; 229237263Snp#ifdef INVARIANTS 230237263Snp struct inpcb *inp = tp->t_inpcb; 231237263Snp#endif 232237263Snp 233237263Snp INP_WLOCK_ASSERT(inp); 234237263Snp KASSERT(tp->t_state == TCPS_LISTEN, 235237263Snp ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 236237263Snp 237237263Snp mtx_lock(&toedev_lock); 238237263Snp TAILQ_FOREACH(tod, &toedev_list, link) 239237263Snp tod->tod_listen_stop(tod, tp); 240237263Snp mtx_unlock(&toedev_lock); 241237263Snp} 242237263Snp 243237263Snp/* 244237263Snp * Fill up a freshly allocated toedev struct with reasonable defaults. 245237263Snp */ 246237263Snpvoid 247237263Snpinit_toedev(struct toedev *tod) 248237263Snp{ 249237263Snp 250237263Snp tod->tod_softc = NULL; 251237263Snp 252237263Snp /* 253237263Snp * Provide no-op defaults so that the kernel can call any toedev 254237263Snp * function without having to check whether the TOE driver supplied one 255237263Snp * or not. 256237263Snp */ 257237263Snp tod->tod_connect = toedev_connect; 258237263Snp tod->tod_listen_start = toedev_listen_start; 259237263Snp tod->tod_listen_stop = toedev_listen_stop; 260237263Snp tod->tod_input = toedev_input; 261237263Snp tod->tod_rcvd = toedev_rcvd; 262237263Snp tod->tod_output = toedev_output; 263237263Snp tod->tod_send_rst = toedev_output; 264237263Snp tod->tod_send_fin = toedev_output; 265237263Snp tod->tod_pcb_detach = toedev_pcb_detach; 266237263Snp tod->tod_l2_update = toedev_l2_update; 267237263Snp tod->tod_route_redirect = toedev_route_redirect; 268237263Snp tod->tod_syncache_added = toedev_syncache_added; 269237263Snp tod->tod_syncache_removed = toedev_syncache_removed; 270237263Snp tod->tod_syncache_respond = toedev_syncache_respond; 271237263Snp tod->tod_offload_socket = toedev_offload_socket; 272237263Snp tod->tod_ctloutput = toedev_ctloutput; 273237263Snp} 274237263Snp 275237263Snp/* 276237263Snp * Register an active TOE device with the system. This allows it to receive 277237263Snp * notifications from the kernel. 278237263Snp */ 279237263Snpint 280237263Snpregister_toedev(struct toedev *tod) 281237263Snp{ 282237263Snp struct toedev *t; 283237263Snp 284237263Snp mtx_lock(&toedev_lock); 285237263Snp TAILQ_FOREACH(t, &toedev_list, link) { 286237263Snp if (t == tod) { 287237263Snp mtx_unlock(&toedev_lock); 288237263Snp return (EEXIST); 289237263Snp } 290237263Snp } 291237263Snp 292237263Snp TAILQ_INSERT_TAIL(&toedev_list, tod, link); 293237263Snp registered_toedevs++; 294237263Snp mtx_unlock(&toedev_lock); 295237263Snp 296237263Snp inp_apply_all(toe_listen_start, tod); 297237263Snp 298237263Snp return (0); 299237263Snp} 300237263Snp 301237263Snp/* 302237263Snp * Remove the TOE device from the global list of active TOE devices. It is the 303237263Snp * caller's responsibility to ensure that the TOE device is quiesced prior to 304237263Snp * this call. 305237263Snp */ 306237263Snpint 307237263Snpunregister_toedev(struct toedev *tod) 308237263Snp{ 309237263Snp struct toedev *t, *t2; 310237263Snp int rc = ENODEV; 311237263Snp 312237263Snp mtx_lock(&toedev_lock); 313237263Snp TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 314237263Snp if (t == tod) { 315237263Snp TAILQ_REMOVE(&toedev_list, tod, link); 316237263Snp registered_toedevs--; 317237263Snp rc = 0; 318237263Snp break; 319237263Snp } 320237263Snp } 321237263Snp KASSERT(registered_toedevs >= 0, 322237263Snp ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 323237263Snp mtx_unlock(&toedev_lock); 324237263Snp return (rc); 325237263Snp} 326237263Snp 327237263Snpvoid 328237263Snptoe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 329237263Snp struct inpcb *inp, void *tod, void *todctx) 330237263Snp{ 331237263Snp struct socket *lso = inp->inp_socket; 332237263Snp 333237263Snp INP_WLOCK_ASSERT(inp); 334237263Snp 335237263Snp syncache_add(inc, to, th, inp, &lso, NULL, tod, todctx); 336237263Snp} 337237263Snp 338237263Snpint 339237263Snptoe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 340237263Snp struct tcphdr *th, struct socket **lsop) 341237263Snp{ 342237263Snp 343309108Sjch INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 344237263Snp 345237263Snp return (syncache_expand(inc, to, th, lsop, NULL)); 346237263Snp} 347237263Snp 348237263Snp/* 349237263Snp * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 350237263Snp * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 351237263Snp * in TIME_WAIT may be assassinated freeing it up for re-use. 352237263Snp * 353237263Snp * Note that the TCP header must have been run through tcp_fields_to_host() or 354237263Snp * equivalent. 355237263Snp */ 356237263Snpint 357237263Snptoe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 358237263Snp{ 359237263Snp struct inpcb *inp; 360237263Snp 361245916Snp if (inc->inc_flags & INC_ISIPV6) { 362329982Shselasky#if defined(KLD_MODULE) || defined(INET6) 363245916Snp inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 364245916Snp inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 365245916Snp INPLOOKUP_WLOCKPCB, ifp); 366329982Shselasky#else 367329982Shselasky inp = NULL; 368329982Shselasky#endif 369245916Snp } else { 370329982Shselasky#if defined(KLD_MODULE) || defined(INET) 371245916Snp inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 372245916Snp inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp); 373329982Shselasky#else 374329982Shselasky inp = NULL; 375329982Shselasky#endif 376245916Snp } 377237263Snp if (inp != NULL) { 378237263Snp INP_WLOCK_ASSERT(inp); 379237263Snp 380237263Snp if ((inp->inp_flags & INP_TIMEWAIT) && th != NULL) { 381237263Snp 382309108Sjch INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* for twcheck */ 383237263Snp if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 384237263Snp return (EADDRINUSE); 385237263Snp } else { 386237263Snp INP_WUNLOCK(inp); 387237263Snp return (EADDRINUSE); 388237263Snp } 389237263Snp } 390237263Snp 391237263Snp return (0); 392237263Snp} 393237263Snp 394237263Snpstatic void 395237263Snptoe_lle_event(void *arg __unused, struct llentry *lle, int evt) 396237263Snp{ 397237263Snp struct toedev *tod; 398237263Snp struct ifnet *ifp; 399237263Snp struct sockaddr *sa; 400237263Snp uint8_t *lladdr; 401237263Snp uint16_t vtag; 402237263Snp 403237263Snp LLE_WLOCK_ASSERT(lle); 404237263Snp 405237263Snp ifp = lle->lle_tbl->llt_ifp; 406237263Snp sa = L3_ADDR(lle); 407237263Snp 408237263Snp KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 409237263Snp ("%s: lle_event %d for lle %p but sa %p !INET && !INET6", 410237263Snp __func__, evt, lle, sa)); 411237263Snp 412237263Snp /* 413237263Snp * Not interested if the interface's TOE capability is not enabled. 414237263Snp */ 415237263Snp if ((sa->sa_family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 416237263Snp (sa->sa_family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 417237263Snp return; 418237263Snp 419237263Snp tod = TOEDEV(ifp); 420237263Snp if (tod == NULL) 421237263Snp return; 422237263Snp 423237263Snp vtag = 0xfff; 424237263Snp if (evt != LLENTRY_RESOLVED) { 425237263Snp 426237263Snp /* 427237263Snp * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 428237263Snp * this entry is going to be deleted. 429237263Snp */ 430237263Snp 431237263Snp lladdr = NULL; 432237263Snp } else { 433237263Snp 434237263Snp KASSERT(lle->la_flags & LLE_VALID, 435237263Snp ("%s: %p resolved but not valid?", __func__, lle)); 436237263Snp 437237263Snp lladdr = (uint8_t *)&lle->ll_addr; 438237263Snp#ifdef VLAN_TAG 439237263Snp VLAN_TAG(ifp, &vtag); 440237263Snp#endif 441237263Snp } 442237263Snp 443237263Snp tod->tod_l2_update(tod, ifp, sa, lladdr, vtag); 444237263Snp} 445237263Snp 446237263Snp/* 447237263Snp * XXX: implement. 448237263Snp */ 449237263Snpstatic void 450237263Snptoe_route_redirect_event(void *arg __unused, struct rtentry *rt0, 451237263Snp struct rtentry *rt1, struct sockaddr *sa) 452237263Snp{ 453237263Snp 454237263Snp return; 455237263Snp} 456237263Snp 457245932Snp#ifdef INET6 458237263Snp/* 459245932Snp * XXX: no checks to verify that sa is really a neighbor because we assume it is 460245932Snp * the result of a route lookup and is on-link on the given ifp. 461245932Snp */ 462245932Snpstatic int 463245932Snptoe_nd6_resolve(struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr) 464245932Snp{ 465245932Snp struct llentry *lle; 466245932Snp struct sockaddr_in6 *sin6 = (void *)sa; 467245932Snp int rc, flags = 0; 468245932Snp 469245932Snprestart: 470245932Snp IF_AFDATA_RLOCK(ifp); 471245932Snp lle = lla_lookup(LLTABLE6(ifp), flags, sa); 472245932Snp IF_AFDATA_RUNLOCK(ifp); 473245932Snp if (lle == NULL) { 474245932Snp IF_AFDATA_LOCK(ifp); 475245932Snp lle = nd6_lookup(&sin6->sin6_addr, ND6_CREATE | ND6_EXCLUSIVE, 476245932Snp ifp); 477245932Snp IF_AFDATA_UNLOCK(ifp); 478245932Snp if (lle == NULL) 479245932Snp return (ENOMEM); /* Couldn't create entry in cache. */ 480245932Snp lle->ln_state = ND6_LLINFO_INCOMPLETE; 481245932Snp nd6_llinfo_settimer_locked(lle, 482245932Snp (long)ND_IFINFO(ifp)->retrans * hz / 1000); 483245932Snp LLE_WUNLOCK(lle); 484245932Snp 485245932Snp nd6_ns_output(ifp, NULL, &sin6->sin6_addr, NULL, 0); 486245932Snp 487245932Snp return (EWOULDBLOCK); 488245932Snp } 489245932Snp 490245932Snp if (lle->ln_state == ND6_LLINFO_STALE) { 491245932Snp if ((flags & LLE_EXCLUSIVE) == 0) { 492245932Snp LLE_RUNLOCK(lle); 493245932Snp flags |= LLE_EXCLUSIVE; 494245932Snp goto restart; 495245932Snp } 496245932Snp 497245932Snp LLE_WLOCK_ASSERT(lle); 498245932Snp 499245932Snp lle->la_asked = 0; 500245932Snp lle->ln_state = ND6_LLINFO_DELAY; 501245932Snp nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz); 502245932Snp } 503245932Snp 504245932Snp if (lle->la_flags & LLE_VALID) { 505245932Snp memcpy(lladdr, &lle->ll_addr, ifp->if_addrlen); 506245932Snp rc = 0; 507245932Snp } else 508245932Snp rc = EWOULDBLOCK; 509245932Snp 510245932Snp if (flags & LLE_EXCLUSIVE) 511245932Snp LLE_WUNLOCK(lle); 512245932Snp else 513245932Snp LLE_RUNLOCK(lle); 514245932Snp 515245932Snp return (rc); 516245932Snp} 517245932Snp#endif 518245932Snp 519245932Snp/* 520237263Snp * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 521237263Snp * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 522237263Snp * tod_l2_update will be called later, when the entry is resolved or times out. 523237263Snp */ 524237263Snpint 525237263Snptoe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 526237263Snp uint8_t *lladdr, uint16_t *vtag) 527237263Snp{ 528245932Snp#ifdef INET 529237263Snp struct llentry *lle; 530245932Snp#endif 531237263Snp int rc; 532237263Snp 533237263Snp switch (sa->sa_family) { 534237263Snp#ifdef INET 535237263Snp case AF_INET: 536237263Snp rc = arpresolve(ifp, NULL, NULL, sa, lladdr, &lle); 537237263Snp break; 538237263Snp#endif 539237263Snp#ifdef INET6 540237263Snp case AF_INET6: 541245932Snp rc = toe_nd6_resolve(ifp, sa, lladdr); 542237263Snp break; 543237263Snp#endif 544237263Snp default: 545237263Snp return (EPROTONOSUPPORT); 546237263Snp } 547237263Snp 548237263Snp if (rc == 0) { 549237263Snp#ifdef VLAN_TAG 550237263Snp if (VLAN_TAG(ifp, vtag) != 0) 551237263Snp#endif 552237263Snp *vtag = 0xfff; 553237263Snp } 554237263Snp 555237263Snp return (rc); 556237263Snp} 557237263Snp 558237263Snpvoid 559239511Snptoe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 560237263Snp{ 561237263Snp 562237263Snp INP_WLOCK_ASSERT(inp); 563237263Snp 564237263Snp if (!(inp->inp_flags & INP_DROPPED)) { 565239511Snp struct tcpcb *tp = intotcpcb(inp); 566239511Snp 567239511Snp KASSERT(tp->t_flags & TF_TOE, 568239511Snp ("%s: tp %p not offloaded.", __func__, tp)); 569239511Snp 570237263Snp if (err == EAGAIN) { 571237263Snp 572237263Snp /* 573237263Snp * Temporary failure during offload, take this PCB back. 574237263Snp * Detach from the TOE driver and do the rest of what 575237263Snp * TCP's pru_connect would have done if the connection 576237263Snp * wasn't offloaded. 577237263Snp */ 578237263Snp 579237263Snp tod->tod_pcb_detach(tod, tp); 580237263Snp KASSERT(!(tp->t_flags & TF_TOE), 581237263Snp ("%s: tp %p still offloaded.", __func__, tp)); 582237263Snp tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 583237263Snp (void) tcp_output(tp); 584237263Snp } else { 585237263Snp 586309108Sjch INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 587237263Snp tp = tcp_drop(tp, err); 588237263Snp if (tp == NULL) 589237263Snp INP_WLOCK(inp); /* re-acquire */ 590237263Snp } 591237263Snp } 592237263Snp INP_WLOCK_ASSERT(inp); 593237263Snp} 594237263Snp 595237263Snpstatic int 596237263Snptoecore_load(void) 597237263Snp{ 598237263Snp 599237263Snp mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 600237263Snp TAILQ_INIT(&toedev_list); 601237263Snp 602237263Snp listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 603237263Snp toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 604237263Snp listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 605237263Snp toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 606237263Snp lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 607237263Snp EVENTHANDLER_PRI_ANY); 608237263Snp route_redirect_eh = EVENTHANDLER_REGISTER(route_redirect_event, 609237263Snp toe_route_redirect_event, NULL, EVENTHANDLER_PRI_ANY); 610237263Snp 611237263Snp return (0); 612237263Snp} 613237263Snp 614237263Snpstatic int 615237263Snptoecore_unload(void) 616237263Snp{ 617237263Snp 618237263Snp mtx_lock(&toedev_lock); 619237263Snp if (!TAILQ_EMPTY(&toedev_list)) { 620237263Snp mtx_unlock(&toedev_lock); 621237263Snp return (EBUSY); 622237263Snp } 623237263Snp 624237263Snp EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 625237263Snp EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 626237263Snp EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 627237263Snp EVENTHANDLER_DEREGISTER(route_redirect_event, route_redirect_eh); 628237263Snp 629237263Snp mtx_unlock(&toedev_lock); 630237263Snp mtx_destroy(&toedev_lock); 631237263Snp 632237263Snp return (0); 633237263Snp} 634237263Snp 635237263Snpstatic int 636237263Snptoecore_mod_handler(module_t mod, int cmd, void *arg) 637237263Snp{ 638237263Snp 639237263Snp if (cmd == MOD_LOAD) 640237263Snp return (toecore_load()); 641237263Snp 642237263Snp if (cmd == MOD_UNLOAD) 643237263Snp return (toecore_unload()); 644237263Snp 645237263Snp return (EOPNOTSUPP); 646237263Snp} 647237263Snp 648237263Snpstatic moduledata_t mod_data= { 649237263Snp "toecore", 650237263Snp toecore_mod_handler, 651241394Skevlo 0 652237263Snp}; 653237263Snp 654329982ShselaskyDECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 655329982Shselasky#endif /* defined(KLD_MODULE) || defined(INET) || defined(INET6) */ 656329982Shselasky 657237263SnpMODULE_VERSION(toecore, 1); 658329982Shselasky 659