1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/netinet/toecore.c 329982 2018-02-25 11:29:55Z hselasky $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/kernel.h> 36#include <sys/systm.h> 37#include <sys/mbuf.h> 38#include <sys/module.h> 39#include <sys/types.h> 40#include <sys/sockopt.h> 41#include <sys/sysctl.h> 42#include <sys/socket.h> 43 44#if defined(KLD_MODULE) || defined(INET) || defined(INET6) 45 46#include <net/ethernet.h> 47#include <net/if.h> 48#include <net/if_types.h> 49#include <net/if_vlan_var.h> 50#include <net/if_llatbl.h> 51#include <net/route.h> 52 53#include <netinet/if_ether.h> 54#include <netinet/in.h> 55#include <netinet/in_pcb.h> 56#include <netinet/in_var.h> 57#include <netinet6/in6_var.h> 58#include <netinet6/in6_pcb.h> 59#include <netinet6/nd6.h> 60#define TCPSTATES 61#include <netinet/tcp.h> 62#include <netinet/tcp_fsm.h> 63#include <netinet/tcp_timer.h> 64#include <netinet/tcp_var.h> 65#include <netinet/tcp_syncache.h> 66#include <netinet/tcp_offload.h> 67#include <netinet/toecore.h> 68 69static struct mtx toedev_lock; 70static TAILQ_HEAD(, toedev) toedev_list; 71static eventhandler_tag listen_start_eh; 72static eventhandler_tag listen_stop_eh; 73static eventhandler_tag lle_event_eh; 74static eventhandler_tag route_redirect_eh; 75 76static int 77toedev_connect(struct toedev *tod __unused, struct socket *so __unused, 78 struct rtentry *rt __unused, struct sockaddr *nam __unused) 79{ 80 81 return (ENOTSUP); 82} 83 84static int 85toedev_listen_start(struct toedev *tod __unused, struct tcpcb *tp __unused) 86{ 87 88 return (ENOTSUP); 89} 90 91static int 92toedev_listen_stop(struct toedev *tod __unused, struct tcpcb *tp __unused) 93{ 94 95 return (ENOTSUP); 96} 97 98static void 99toedev_input(struct toedev *tod __unused, struct tcpcb *tp __unused, 100 struct mbuf *m) 101{ 102 103 m_freem(m); 104 return; 105} 106 107static void 108toedev_rcvd(struct toedev *tod __unused, struct tcpcb *tp __unused) 109{ 110 111 return; 112} 113 114static int 115toedev_output(struct toedev *tod __unused, struct tcpcb *tp __unused) 116{ 117 118 return (ENOTSUP); 119} 120 121static void 122toedev_pcb_detach(struct toedev *tod __unused, struct tcpcb *tp __unused) 123{ 124 125 return; 126} 127 128static void 129toedev_l2_update(struct toedev *tod __unused, struct ifnet *ifp __unused, 130 struct sockaddr *sa __unused, uint8_t *lladdr __unused, 131 uint16_t vtag __unused) 132{ 133 134 return; 135} 136 137static void 138toedev_route_redirect(struct toedev *tod __unused, struct ifnet *ifp __unused, 139 struct rtentry *rt0 __unused, struct rtentry *rt1 __unused) 140{ 141 142 return; 143} 144 145static void 146toedev_syncache_added(struct toedev *tod __unused, void *ctx __unused) 147{ 148 149 return; 150} 151 152static void 153toedev_syncache_removed(struct toedev *tod __unused, void *ctx __unused) 154{ 155 156 return; 157} 158 159static int 160toedev_syncache_respond(struct toedev *tod __unused, void *ctx __unused, 161 struct mbuf *m) 162{ 163 164 m_freem(m); 165 return (0); 166} 167 168static void 169toedev_offload_socket(struct toedev *tod __unused, void *ctx __unused, 170 struct socket *so __unused) 171{ 172 173 return; 174} 175 176static void 177toedev_ctloutput(struct toedev *tod __unused, struct tcpcb *tp __unused, 178 int sopt_dir __unused, int sopt_name __unused) 179{ 180 181 return; 182} 183 184/* 185 * Inform one or more TOE devices about a listening socket. 186 */ 187static void 188toe_listen_start(struct inpcb *inp, void *arg) 189{ 190 struct toedev *t, *tod; 191 struct tcpcb *tp; 192 193 INP_WLOCK_ASSERT(inp); 194 KASSERT(inp->inp_pcbinfo == &V_tcbinfo, 195 ("%s: inp is not a TCP inp", __func__)); 196 197 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) 198 return; 199 200 tp = intotcpcb(inp); 201 if (tp->t_state != TCPS_LISTEN) 202 return; 203 204 t = arg; 205 mtx_lock(&toedev_lock); 206 TAILQ_FOREACH(tod, &toedev_list, link) { 207 if (t == NULL || t == tod) 208 tod->tod_listen_start(tod, tp); 209 } 210 mtx_unlock(&toedev_lock); 211} 212 213static void 214toe_listen_start_event(void *arg __unused, struct tcpcb *tp) 215{ 216 struct inpcb *inp = tp->t_inpcb; 217 218 INP_WLOCK_ASSERT(inp); 219 KASSERT(tp->t_state == TCPS_LISTEN, 220 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 221 222 toe_listen_start(inp, NULL); 223} 224 225static void 226toe_listen_stop_event(void *arg __unused, struct tcpcb *tp) 227{ 228 struct toedev *tod; 229#ifdef INVARIANTS 230 struct inpcb *inp = tp->t_inpcb; 231#endif 232 233 INP_WLOCK_ASSERT(inp); 234 KASSERT(tp->t_state == TCPS_LISTEN, 235 ("%s: t_state %s", __func__, tcpstates[tp->t_state])); 236 237 mtx_lock(&toedev_lock); 238 TAILQ_FOREACH(tod, &toedev_list, link) 239 tod->tod_listen_stop(tod, tp); 240 mtx_unlock(&toedev_lock); 241} 242 243/* 244 * Fill up a freshly allocated toedev struct with reasonable defaults. 245 */ 246void 247init_toedev(struct toedev *tod) 248{ 249 250 tod->tod_softc = NULL; 251 252 /* 253 * Provide no-op defaults so that the kernel can call any toedev 254 * function without having to check whether the TOE driver supplied one 255 * or not. 256 */ 257 tod->tod_connect = toedev_connect; 258 tod->tod_listen_start = toedev_listen_start; 259 tod->tod_listen_stop = toedev_listen_stop; 260 tod->tod_input = toedev_input; 261 tod->tod_rcvd = toedev_rcvd; 262 tod->tod_output = toedev_output; 263 tod->tod_send_rst = toedev_output; 264 tod->tod_send_fin = toedev_output; 265 tod->tod_pcb_detach = toedev_pcb_detach; 266 tod->tod_l2_update = toedev_l2_update; 267 tod->tod_route_redirect = toedev_route_redirect; 268 tod->tod_syncache_added = toedev_syncache_added; 269 tod->tod_syncache_removed = toedev_syncache_removed; 270 tod->tod_syncache_respond = toedev_syncache_respond; 271 tod->tod_offload_socket = toedev_offload_socket; 272 tod->tod_ctloutput = toedev_ctloutput; 273} 274 275/* 276 * Register an active TOE device with the system. This allows it to receive 277 * notifications from the kernel. 278 */ 279int 280register_toedev(struct toedev *tod) 281{ 282 struct toedev *t; 283 284 mtx_lock(&toedev_lock); 285 TAILQ_FOREACH(t, &toedev_list, link) { 286 if (t == tod) { 287 mtx_unlock(&toedev_lock); 288 return (EEXIST); 289 } 290 } 291 292 TAILQ_INSERT_TAIL(&toedev_list, tod, link); 293 registered_toedevs++; 294 mtx_unlock(&toedev_lock); 295 296 inp_apply_all(toe_listen_start, tod); 297 298 return (0); 299} 300 301/* 302 * Remove the TOE device from the global list of active TOE devices. It is the 303 * caller's responsibility to ensure that the TOE device is quiesced prior to 304 * this call. 305 */ 306int 307unregister_toedev(struct toedev *tod) 308{ 309 struct toedev *t, *t2; 310 int rc = ENODEV; 311 312 mtx_lock(&toedev_lock); 313 TAILQ_FOREACH_SAFE(t, &toedev_list, link, t2) { 314 if (t == tod) { 315 TAILQ_REMOVE(&toedev_list, tod, link); 316 registered_toedevs--; 317 rc = 0; 318 break; 319 } 320 } 321 KASSERT(registered_toedevs >= 0, 322 ("%s: registered_toedevs (%d) < 0", __func__, registered_toedevs)); 323 mtx_unlock(&toedev_lock); 324 return (rc); 325} 326 327void 328toe_syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th, 329 struct inpcb *inp, void *tod, void *todctx) 330{ 331 struct socket *lso = inp->inp_socket; 332 333 INP_WLOCK_ASSERT(inp); 334 335 syncache_add(inc, to, th, inp, &lso, NULL, tod, todctx); 336} 337 338int 339toe_syncache_expand(struct in_conninfo *inc, struct tcpopt *to, 340 struct tcphdr *th, struct socket **lsop) 341{ 342 343 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 344 345 return (syncache_expand(inc, to, th, lsop, NULL)); 346} 347 348/* 349 * General purpose check to see if a 4-tuple is in use by the kernel. If a TCP 350 * header (presumably for an incoming SYN) is also provided, an existing 4-tuple 351 * in TIME_WAIT may be assassinated freeing it up for re-use. 352 * 353 * Note that the TCP header must have been run through tcp_fields_to_host() or 354 * equivalent. 355 */ 356int 357toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp) 358{ 359 struct inpcb *inp; 360 361 if (inc->inc_flags & INC_ISIPV6) { 362#if defined(KLD_MODULE) || defined(INET6) 363 inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr, 364 inc->inc_fport, &inc->inc6_laddr, inc->inc_lport, 365 INPLOOKUP_WLOCKPCB, ifp); 366#else 367 inp = NULL; 368#endif 369 } else { 370#if defined(KLD_MODULE) || defined(INET) 371 inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport, 372 inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp); 373#else 374 inp = NULL; 375#endif 376 } 377 if (inp != NULL) { 378 INP_WLOCK_ASSERT(inp); 379 380 if ((inp->inp_flags & INP_TIMEWAIT) && th != NULL) { 381 382 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); /* for twcheck */ 383 if (!tcp_twcheck(inp, NULL, th, NULL, 0)) 384 return (EADDRINUSE); 385 } else { 386 INP_WUNLOCK(inp); 387 return (EADDRINUSE); 388 } 389 } 390 391 return (0); 392} 393 394static void 395toe_lle_event(void *arg __unused, struct llentry *lle, int evt) 396{ 397 struct toedev *tod; 398 struct ifnet *ifp; 399 struct sockaddr *sa; 400 uint8_t *lladdr; 401 uint16_t vtag; 402 403 LLE_WLOCK_ASSERT(lle); 404 405 ifp = lle->lle_tbl->llt_ifp; 406 sa = L3_ADDR(lle); 407 408 KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6, 409 ("%s: lle_event %d for lle %p but sa %p !INET && !INET6", 410 __func__, evt, lle, sa)); 411 412 /* 413 * Not interested if the interface's TOE capability is not enabled. 414 */ 415 if ((sa->sa_family == AF_INET && !(ifp->if_capenable & IFCAP_TOE4)) || 416 (sa->sa_family == AF_INET6 && !(ifp->if_capenable & IFCAP_TOE6))) 417 return; 418 419 tod = TOEDEV(ifp); 420 if (tod == NULL) 421 return; 422 423 vtag = 0xfff; 424 if (evt != LLENTRY_RESOLVED) { 425 426 /* 427 * LLENTRY_TIMEDOUT, LLENTRY_DELETED, LLENTRY_EXPIRED all mean 428 * this entry is going to be deleted. 429 */ 430 431 lladdr = NULL; 432 } else { 433 434 KASSERT(lle->la_flags & LLE_VALID, 435 ("%s: %p resolved but not valid?", __func__, lle)); 436 437 lladdr = (uint8_t *)&lle->ll_addr; 438#ifdef VLAN_TAG 439 VLAN_TAG(ifp, &vtag); 440#endif 441 } 442 443 tod->tod_l2_update(tod, ifp, sa, lladdr, vtag); 444} 445 446/* 447 * XXX: implement. 448 */ 449static void 450toe_route_redirect_event(void *arg __unused, struct rtentry *rt0, 451 struct rtentry *rt1, struct sockaddr *sa) 452{ 453 454 return; 455} 456 457#ifdef INET6 458/* 459 * XXX: no checks to verify that sa is really a neighbor because we assume it is 460 * the result of a route lookup and is on-link on the given ifp. 461 */ 462static int 463toe_nd6_resolve(struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr) 464{ 465 struct llentry *lle; 466 struct sockaddr_in6 *sin6 = (void *)sa; 467 int rc, flags = 0; 468 469restart: 470 IF_AFDATA_RLOCK(ifp); 471 lle = lla_lookup(LLTABLE6(ifp), flags, sa); 472 IF_AFDATA_RUNLOCK(ifp); 473 if (lle == NULL) { 474 IF_AFDATA_LOCK(ifp); 475 lle = nd6_lookup(&sin6->sin6_addr, ND6_CREATE | ND6_EXCLUSIVE, 476 ifp); 477 IF_AFDATA_UNLOCK(ifp); 478 if (lle == NULL) 479 return (ENOMEM); /* Couldn't create entry in cache. */ 480 lle->ln_state = ND6_LLINFO_INCOMPLETE; 481 nd6_llinfo_settimer_locked(lle, 482 (long)ND_IFINFO(ifp)->retrans * hz / 1000); 483 LLE_WUNLOCK(lle); 484 485 nd6_ns_output(ifp, NULL, &sin6->sin6_addr, NULL, 0); 486 487 return (EWOULDBLOCK); 488 } 489 490 if (lle->ln_state == ND6_LLINFO_STALE) { 491 if ((flags & LLE_EXCLUSIVE) == 0) { 492 LLE_RUNLOCK(lle); 493 flags |= LLE_EXCLUSIVE; 494 goto restart; 495 } 496 497 LLE_WLOCK_ASSERT(lle); 498 499 lle->la_asked = 0; 500 lle->ln_state = ND6_LLINFO_DELAY; 501 nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz); 502 } 503 504 if (lle->la_flags & LLE_VALID) { 505 memcpy(lladdr, &lle->ll_addr, ifp->if_addrlen); 506 rc = 0; 507 } else 508 rc = EWOULDBLOCK; 509 510 if (flags & LLE_EXCLUSIVE) 511 LLE_WUNLOCK(lle); 512 else 513 LLE_RUNLOCK(lle); 514 515 return (rc); 516} 517#endif 518 519/* 520 * Returns 0 or EWOULDBLOCK on success (any other value is an error). 0 means 521 * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's 522 * tod_l2_update will be called later, when the entry is resolved or times out. 523 */ 524int 525toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa, 526 uint8_t *lladdr, uint16_t *vtag) 527{ 528#ifdef INET 529 struct llentry *lle; 530#endif 531 int rc; 532 533 switch (sa->sa_family) { 534#ifdef INET 535 case AF_INET: 536 rc = arpresolve(ifp, NULL, NULL, sa, lladdr, &lle); 537 break; 538#endif 539#ifdef INET6 540 case AF_INET6: 541 rc = toe_nd6_resolve(ifp, sa, lladdr); 542 break; 543#endif 544 default: 545 return (EPROTONOSUPPORT); 546 } 547 548 if (rc == 0) { 549#ifdef VLAN_TAG 550 if (VLAN_TAG(ifp, vtag) != 0) 551#endif 552 *vtag = 0xfff; 553 } 554 555 return (rc); 556} 557 558void 559toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) 560{ 561 562 INP_WLOCK_ASSERT(inp); 563 564 if (!(inp->inp_flags & INP_DROPPED)) { 565 struct tcpcb *tp = intotcpcb(inp); 566 567 KASSERT(tp->t_flags & TF_TOE, 568 ("%s: tp %p not offloaded.", __func__, tp)); 569 570 if (err == EAGAIN) { 571 572 /* 573 * Temporary failure during offload, take this PCB back. 574 * Detach from the TOE driver and do the rest of what 575 * TCP's pru_connect would have done if the connection 576 * wasn't offloaded. 577 */ 578 579 tod->tod_pcb_detach(tod, tp); 580 KASSERT(!(tp->t_flags & TF_TOE), 581 ("%s: tp %p still offloaded.", __func__, tp)); 582 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); 583 (void) tcp_output(tp); 584 } else { 585 586 INP_INFO_RLOCK_ASSERT(&V_tcbinfo); 587 tp = tcp_drop(tp, err); 588 if (tp == NULL) 589 INP_WLOCK(inp); /* re-acquire */ 590 } 591 } 592 INP_WLOCK_ASSERT(inp); 593} 594 595static int 596toecore_load(void) 597{ 598 599 mtx_init(&toedev_lock, "toedev lock", NULL, MTX_DEF); 600 TAILQ_INIT(&toedev_list); 601 602 listen_start_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_start, 603 toe_listen_start_event, NULL, EVENTHANDLER_PRI_ANY); 604 listen_stop_eh = EVENTHANDLER_REGISTER(tcp_offload_listen_stop, 605 toe_listen_stop_event, NULL, EVENTHANDLER_PRI_ANY); 606 lle_event_eh = EVENTHANDLER_REGISTER(lle_event, toe_lle_event, NULL, 607 EVENTHANDLER_PRI_ANY); 608 route_redirect_eh = EVENTHANDLER_REGISTER(route_redirect_event, 609 toe_route_redirect_event, NULL, EVENTHANDLER_PRI_ANY); 610 611 return (0); 612} 613 614static int 615toecore_unload(void) 616{ 617 618 mtx_lock(&toedev_lock); 619 if (!TAILQ_EMPTY(&toedev_list)) { 620 mtx_unlock(&toedev_lock); 621 return (EBUSY); 622 } 623 624 EVENTHANDLER_DEREGISTER(tcp_offload_listen_start, listen_start_eh); 625 EVENTHANDLER_DEREGISTER(tcp_offload_listen_stop, listen_stop_eh); 626 EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); 627 EVENTHANDLER_DEREGISTER(route_redirect_event, route_redirect_eh); 628 629 mtx_unlock(&toedev_lock); 630 mtx_destroy(&toedev_lock); 631 632 return (0); 633} 634 635static int 636toecore_mod_handler(module_t mod, int cmd, void *arg) 637{ 638 639 if (cmd == MOD_LOAD) 640 return (toecore_load()); 641 642 if (cmd == MOD_UNLOAD) 643 return (toecore_unload()); 644 645 return (EOPNOTSUPP); 646} 647 648static moduledata_t mod_data= { 649 "toecore", 650 toecore_mod_handler, 651 0 652}; 653 654DECLARE_MODULE(toecore, mod_data, SI_SUB_EXEC, SI_ORDER_ANY); 655#endif /* defined(KLD_MODULE) || defined(INET) || defined(INET6) */ 656 657MODULE_VERSION(toecore, 1); 658 659