1/* 2 * ng_fec.c 3 */ 4 5/*- 6 * Copyright (c) 2001 Berkeley Software Design, Inc. 7 * Copyright (c) 2000, 2001 8 * Bill Paul <wpaul@osd.bsdi.com>. All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by Bill Paul. 21 * 4. Neither the name of the author nor the names of any co-contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 35 * THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * $FreeBSD$ 38 */ 39/*- 40 * Copyright (c) 1996-1999 Whistle Communications, Inc. 41 * All rights reserved. 42 * 43 * Subject to the following obligations and disclaimer of warranty, use and 44 * redistribution of this software, in source or object code forms, with or 45 * without modifications are expressly permitted by Whistle Communications; 46 * provided, however, that: 47 * 1. Any and all reproductions of the source or object code must include the 48 * copyright notice above and the following disclaimer of warranties; and 49 * 2. No rights are granted, in any manner or form, to use Whistle 50 * Communications, Inc. trademarks, including the mark "WHISTLE 51 * COMMUNICATIONS" on advertising, endorsements, or otherwise except as 52 * such appears in the above copyright notice or in the software. 53 * 54 * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND 55 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO 56 * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE, 57 * INCLUDING WITHOUT LIMITATION, ANY AND ALL IMPLIED WARRANTIES OF 58 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT. 59 * WHISTLE COMMUNICATIONS DOES NOT WARRANT, GUARANTEE, OR MAKE ANY 60 * REPRESENTATIONS REGARDING THE USE OF, OR THE RESULTS OF THE USE OF THIS 61 * SOFTWARE IN TERMS OF ITS CORRECTNESS, ACCURACY, RELIABILITY OR OTHERWISE. 62 * IN NO EVENT SHALL WHISTLE COMMUNICATIONS BE LIABLE FOR ANY DAMAGES 63 * RESULTING FROM OR ARISING OUT OF ANY USE OF THIS SOFTWARE, INCLUDING 64 * WITHOUT LIMITATION, ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 65 * PUNITIVE, OR CONSEQUENTIAL DAMAGES, PROCUREMENT OF SUBSTITUTE GOODS OR 66 * SERVICES, LOSS OF USE, DATA OR PROFITS, HOWEVER CAUSED AND UNDER ANY 67 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 68 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 69 * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY 70 * OF SUCH DAMAGE. 71 * 72 * Author: Archie Cobbs <archie@freebsd.org> 73 * 74 * $Whistle: ng_fec.c,v 1.33 1999/11/01 09:24:51 julian Exp $ 75 */ 76 77/* 78 * This module implements ethernet channel bonding using the Cisco 79 * Fast EtherChannel mechanism. Two or four ports may be combined 80 * into a single aggregate interface. 81 * 82 * Interfaces are named fec0, fec1, etc. New nodes take the 83 * first available interface name. 84 * 85 * This node also includes Berkeley packet filter support. 86 * 87 * Note that this node doesn't need to connect to any other 88 * netgraph nodes in order to do its work. 89 */ 90 91#include <sys/param.h> 92#include <sys/systm.h> 93#include <sys/errno.h> 94#include <sys/kernel.h> 95#include <sys/malloc.h> 96#include <sys/mbuf.h> 97#include <sys/errno.h> 98#include <sys/sockio.h> 99#include <sys/socket.h> 100#include <sys/syslog.h> 101#include <sys/libkern.h> 102#include <sys/queue.h> 103 104#include <net/if.h> 105#include <net/if_dl.h> 106#include <net/if_types.h> 107#include <net/if_media.h> 108#include <net/bpf.h> 109#include <net/ethernet.h> 110#include <net/route.h> 111 112#include "opt_inet.h" 113#include "opt_inet6.h" 114 115#include <netinet/in.h> 116#ifdef INET 117#include <netinet/in_systm.h> 118#include <netinet/ip.h> 119#endif 120 121#ifdef INET6 122#include <netinet/ip6.h> 123#endif 124 125#include <netgraph/ng_message.h> 126#include <netgraph/netgraph.h> 127#include <netgraph/ng_parse.h> 128#include <netgraph/ng_fec.h> 129 130/* 131 * We need a way to stash a pointer to our netgraph node in the 132 * ifnet structure so that receive handling works. As far as I can 133 * tell, although there is an AF_NETGRAPH address family, it's only 134 * used to identify sockaddr_ng structures: there is no netgraph address 135 * family domain. This means the AF_NETGRAPH entry in ifp->if_afdata 136 * should be unused, so we can use to hold our node context. 137 */ 138#define IFP2NG(ifp) ((ifp)->if_afdata[AF_NETGRAPH]) 139 140/* 141 * Current fast etherchannel implementations use either 2 or 4 142 * ports, so for now we limit the maximum bundle size to 4 interfaces. 143 */ 144#define FEC_BUNDLESIZ 4 145 146struct ng_fec_portlist { 147 struct ifnet *fec_if; 148 void (*fec_if_input) (struct ifnet *, 149 struct mbuf *); 150 int fec_idx; 151 int fec_ifstat; 152 struct ether_addr fec_mac; 153 SLIST_HEAD(__mclhd, ng_fec_mc) fec_mc_head; 154 TAILQ_ENTRY(ng_fec_portlist) fec_list; 155}; 156 157struct ng_fec_mc { 158 struct ifmultiaddr *mc_ifma; 159 SLIST_ENTRY(ng_fec_mc) mc_entries; 160}; 161 162struct ng_fec_bundle { 163 TAILQ_HEAD(,ng_fec_portlist) ng_fec_ports; 164 int fec_ifcnt; 165 int fec_btype; 166 int (*fec_if_output) (struct ifnet *, 167 struct mbuf *, 168 const struct sockaddr *, 169 struct route *); 170}; 171 172#define FEC_BTYPE_MAC 0x01 173#define FEC_BTYPE_INET 0x02 174#define FEC_BTYPE_INET6 0x03 175 176/* Node private data */ 177struct ng_fec_private { 178 struct ifnet *ifp; 179 struct ifmedia ifmedia; 180 int if_flags; 181 int if_error; /* XXX */ 182 int unit; /* Interface unit number */ 183 node_p node; /* Our netgraph node */ 184 struct ng_fec_bundle fec_bundle;/* Aggregate bundle */ 185 struct callout_handle fec_ch; /* callout handle for ticker */ 186}; 187typedef struct ng_fec_private *priv_p; 188 189/* Interface methods */ 190static void ng_fec_input(struct ifnet *, struct mbuf *); 191static void ng_fec_start(struct ifnet *ifp); 192static int ng_fec_choose_port(struct ng_fec_bundle *b, 193 struct mbuf *m, struct ifnet **ifp); 194static int ng_fec_setport(struct ifnet *ifp, u_long cmd, caddr_t data); 195static void ng_fec_init(void *arg); 196static void ng_fec_stop(struct ifnet *ifp); 197static int ng_fec_ifmedia_upd(struct ifnet *ifp); 198static void ng_fec_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); 199static int ng_fec_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); 200static int ng_fec_output(struct ifnet *ifp, struct mbuf *m0, 201 const struct sockaddr *dst, struct route *ro); 202static void ng_fec_tick(void *arg); 203static int ng_fec_addport(struct ng_fec_private *priv, char *iface); 204static int ng_fec_delport(struct ng_fec_private *priv, char *iface); 205static int ng_fec_ether_cmdmulti(struct ifnet *trifp, struct ng_fec_portlist *p, int set); 206 207#ifdef DEBUG 208static void ng_fec_print_ioctl(struct ifnet *ifp, int cmd, caddr_t data); 209#endif 210 211/* Netgraph methods */ 212static int ng_fec_mod_event(module_t, int, void *); 213static ng_constructor_t ng_fec_constructor; 214static ng_rcvmsg_t ng_fec_rcvmsg; 215static ng_shutdown_t ng_fec_shutdown; 216 217/* List of commands and how to convert arguments to/from ASCII */ 218static const struct ng_cmdlist ng_fec_cmds[] = { 219 { 220 NGM_FEC_COOKIE, 221 NGM_FEC_ADD_IFACE, 222 "add_iface", 223 &ng_parse_string_type, 224 NULL, 225 }, 226 { 227 NGM_FEC_COOKIE, 228 NGM_FEC_DEL_IFACE, 229 "del_iface", 230 &ng_parse_string_type, 231 NULL, 232 }, 233 { 234 NGM_FEC_COOKIE, 235 NGM_FEC_SET_MODE_MAC, 236 "set_mode_mac", 237 NULL, 238 NULL, 239 }, 240 { 241 NGM_FEC_COOKIE, 242 NGM_FEC_SET_MODE_INET, 243 "set_mode_inet", 244 NULL, 245 NULL, 246 }, 247 { 0 } 248}; 249 250/* Node type descriptor */ 251static struct ng_type typestruct = { 252 .version = NG_ABI_VERSION, 253 .name = NG_FEC_NODE_TYPE, 254 .mod_event = ng_fec_mod_event, 255 .constructor = ng_fec_constructor, 256 .rcvmsg = ng_fec_rcvmsg, 257 .shutdown = ng_fec_shutdown, 258 .cmdlist = ng_fec_cmds, 259}; 260NETGRAPH_INIT(fec, &typestruct); 261 262/* We keep a bitmap indicating which unit numbers are free. 263 One means the unit number is free, zero means it's taken. */ 264static int *ng_fec_units = NULL; 265static int ng_fec_units_len = 0; 266static int ng_units_in_use = 0; 267 268#define UNITS_BITSPERWORD (sizeof(*ng_fec_units) * NBBY) 269 270static struct mtx ng_fec_mtx; 271 272/* 273 * Find the first free unit number for a new interface. 274 * Increase the size of the unit bitmap as necessary. 275 */ 276static __inline int 277ng_fec_get_unit(int *unit) 278{ 279 int index, bit; 280 281 mtx_lock(&ng_fec_mtx); 282 for (index = 0; index < ng_fec_units_len 283 && ng_fec_units[index] == 0; index++); 284 if (index == ng_fec_units_len) { /* extend array */ 285 int i, *newarray, newlen; 286 287 newlen = (2 * ng_fec_units_len) + 4; 288 newarray = malloc(newlen * sizeof(*ng_fec_units), 289 M_NETGRAPH, M_NOWAIT); 290 if (newarray == NULL) { 291 mtx_unlock(&ng_fec_mtx); 292 return (ENOMEM); 293 } 294 bcopy(ng_fec_units, newarray, 295 ng_fec_units_len * sizeof(*ng_fec_units)); 296 for (i = ng_fec_units_len; i < newlen; i++) 297 newarray[i] = ~0; 298 if (ng_fec_units != NULL) 299 free(ng_fec_units, M_NETGRAPH); 300 ng_fec_units = newarray; 301 ng_fec_units_len = newlen; 302 } 303 bit = ffs(ng_fec_units[index]) - 1; 304 KASSERT(bit >= 0 && bit <= UNITS_BITSPERWORD - 1, 305 ("%s: word=%d bit=%d", __func__, ng_fec_units[index], bit)); 306 ng_fec_units[index] &= ~(1 << bit); 307 *unit = (index * UNITS_BITSPERWORD) + bit; 308 ng_units_in_use++; 309 mtx_unlock(&ng_fec_mtx); 310 return (0); 311} 312 313/* 314 * Free a no longer needed unit number. 315 */ 316static __inline void 317ng_fec_free_unit(int unit) 318{ 319 int index, bit; 320 321 index = unit / UNITS_BITSPERWORD; 322 bit = unit % UNITS_BITSPERWORD; 323 mtx_lock(&ng_fec_mtx); 324 KASSERT(index < ng_fec_units_len, 325 ("%s: unit=%d len=%d", __func__, unit, ng_fec_units_len)); 326 KASSERT((ng_fec_units[index] & (1 << bit)) == 0, 327 ("%s: unit=%d is free", __func__, unit)); 328 ng_fec_units[index] |= (1 << bit); 329 /* 330 * XXX We could think about reducing the size of ng_fec_units[] 331 * XXX here if the last portion is all ones 332 * XXX At least free it if no more units 333 * Needed if we are to eventually be able to unload. 334 */ 335 ng_units_in_use--; 336 if (ng_units_in_use == 0) { /* XXX make SMP safe */ 337 free(ng_fec_units, M_NETGRAPH); 338 ng_fec_units_len = 0; 339 ng_fec_units = NULL; 340 } 341 mtx_unlock(&ng_fec_mtx); 342} 343 344/************************************************************************ 345 INTERFACE STUFF 346 ************************************************************************/ 347 348static int 349ng_fec_addport(struct ng_fec_private *priv, char *iface) 350{ 351 struct ng_fec_bundle *b; 352 struct ifnet *ifp, *bifp; 353 struct ng_fec_portlist *p, *new; 354 355 if (priv == NULL || iface == NULL) 356 return(EINVAL); 357 358 b = &priv->fec_bundle; 359 ifp = priv->ifp; 360 361 /* Only allow reconfiguration if not running. */ 362 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 363 printf("fec%d: can't add new iface; bundle is running\n", 364 priv->unit); 365 return (EINVAL); 366 } 367 368 /* Find the interface */ 369 bifp = ifunit(iface); 370 if (bifp == NULL) { 371 printf("fec%d: tried to add iface %s, which " 372 "doesn't seem to exist\n", priv->unit, iface); 373 return(ENOENT); 374 } 375 376 /* See if we have room in the bundle */ 377 if (b->fec_ifcnt == FEC_BUNDLESIZ) { 378 printf("fec%d: can't add new iface; bundle is full\n", 379 priv->unit); 380 return(ENOSPC); 381 } 382 383 /* See if the interface is already in the bundle */ 384 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 385 if (p->fec_if == bifp) { 386 printf("fec%d: iface %s is already in this " 387 "bundle\n", priv->unit, iface); 388 return(EINVAL); 389 } 390 } 391 392 /* 393 * All interfaces must use the same output vector. Once the 394 * user attaches an interface of one type, make all subsequent 395 * interfaces have the same output vector. 396 */ 397 if (b->fec_if_output != NULL) { 398 if (b->fec_if_output != bifp->if_output) { 399 printf("fec%d: iface %s is not the same type " 400 "as the other interface(s) already in " 401 "the bundle\n", priv->unit, iface); 402 return(EINVAL); 403 } 404 } 405 406 /* Allocate new list entry. */ 407 new = malloc(sizeof(struct ng_fec_portlist), M_NETGRAPH, M_NOWAIT); 408 if (new == NULL) 409 return(ENOMEM); 410 411 IF_AFDATA_LOCK(bifp); 412 IFP2NG(bifp) = priv->node; 413 IF_AFDATA_UNLOCK(bifp); 414 415 /* 416 * If this is the first interface added to the bundle, 417 * use its MAC address for the virtual interface (and, 418 * by extension, all the other ports in the bundle). 419 */ 420 if (b->fec_ifcnt == 0) 421 if_setlladdr(ifp, IF_LLADDR(bifp), ETHER_ADDR_LEN); 422 423 b->fec_btype = FEC_BTYPE_MAC; 424 new->fec_idx = b->fec_ifcnt; 425 b->fec_ifcnt++; 426 427 /* Initialise the list of multicast addresses that we own. */ 428 SLIST_INIT(&new->fec_mc_head); 429 430 /* Save the real MAC address. */ 431 bcopy(IF_LLADDR(bifp), 432 (char *)&new->fec_mac, ETHER_ADDR_LEN); 433 434 /* Set up phony MAC address. */ 435 if_setlladdr(bifp, IF_LLADDR(ifp), ETHER_ADDR_LEN); 436 EVENTHANDLER_INVOKE(iflladdr_event, bifp); 437 438 /* Save original input vector */ 439 new->fec_if_input = bifp->if_input; 440 441 /* Override it with our own */ 442 bifp->if_input = ng_fec_input; 443 444 /* Save output vector too. */ 445 if (b->fec_if_output == NULL) 446 b->fec_if_output = bifp->if_output; 447 448 /* Add to the queue */ 449 new->fec_if = bifp; 450 new->fec_ifstat = -1; 451 TAILQ_INSERT_TAIL(&b->ng_fec_ports, new, fec_list); 452 453 /* Add multicast addresses to this port. */ 454 ng_fec_ether_cmdmulti(ifp, new, 1); 455 456 return(0); 457} 458 459static int 460ng_fec_delport(struct ng_fec_private *priv, char *iface) 461{ 462 struct ng_fec_bundle *b; 463 struct ifnet *ifp, *bifp; 464 struct ng_fec_portlist *p; 465 466 if (priv == NULL || iface == NULL) 467 return(EINVAL); 468 469 b = &priv->fec_bundle; 470 ifp = priv->ifp; 471 472 /* Only allow reconfiguration if not running. */ 473 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 474 printf("fec%d: can't remove iface; bundle is running\n", 475 priv->unit); 476 return (EINVAL); 477 } 478 479 /* Find the interface */ 480 bifp = ifunit(iface); 481 if (bifp == NULL) { 482 printf("fec%d: tried to remove iface %s, which " 483 "doesn't seem to exist\n", priv->unit, iface); 484 return(ENOENT); 485 } 486 487 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 488 if (p->fec_if == bifp) 489 break; 490 } 491 492 if (p == NULL) { 493 printf("fec%d: tried to remove iface %s which " 494 "is not in our bundle\n", priv->unit, iface); 495 return(EINVAL); 496 } 497 498 /* Stop interface */ 499 bifp->if_flags &= ~IFF_UP; 500 (*bifp->if_ioctl)(bifp, SIOCSIFFLAGS, NULL); 501 502 /* Restore MAC address. */ 503 if_setlladdr(bifp, (u_char *)&p->fec_mac, ETHER_ADDR_LEN); 504 505 /* Restore input vector */ 506 bifp->if_input = p->fec_if_input; 507 508 /* Remove our node context pointer. */ 509 IF_AFDATA_LOCK(bifp); 510 IFP2NG(bifp) = NULL; 511 IF_AFDATA_UNLOCK(bifp); 512 513 /* Delete port */ 514 TAILQ_REMOVE(&b->ng_fec_ports, p, fec_list); 515 free(p, M_NETGRAPH); 516 b->fec_ifcnt--; 517 518 if (b->fec_ifcnt == 0) 519 b->fec_if_output = NULL; 520 521 return(0); 522} 523 524static int 525ng_fec_ether_cmdmulti(struct ifnet *trifp, struct ng_fec_portlist *p, int set) 526{ 527 struct ifnet *ifp = p->fec_if; 528 struct ng_fec_mc *mc; 529 struct ifmultiaddr *ifma, *rifma = NULL; 530 struct sockaddr_dl sdl; 531 int error; 532 533 bzero((char *)&sdl, sizeof(sdl)); 534 sdl.sdl_len = sizeof(sdl); 535 sdl.sdl_family = AF_LINK; 536 sdl.sdl_type = IFT_ETHER; 537 sdl.sdl_alen = ETHER_ADDR_LEN; 538 sdl.sdl_index = ifp->if_index; 539 540 if (set) { 541 TAILQ_FOREACH(ifma, &trifp->if_multiaddrs, ifma_link) { 542 if (ifma->ifma_addr->sa_family != AF_LINK) 543 continue; 544 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 545 LLADDR(&sdl), ETHER_ADDR_LEN); 546 547 error = if_addmulti(ifp, (struct sockaddr *)&sdl, &rifma); 548 if (error) 549 return (error); 550 mc = malloc(sizeof(struct ng_fec_mc), M_DEVBUF, M_NOWAIT); 551 if (mc == NULL) 552 return (ENOMEM); 553 mc->mc_ifma = rifma; 554 SLIST_INSERT_HEAD(&p->fec_mc_head, mc, mc_entries); 555 } 556 } else { 557 while ((mc = SLIST_FIRST(&p->fec_mc_head)) != NULL) { 558 SLIST_REMOVE(&p->fec_mc_head, mc, ng_fec_mc, mc_entries); 559 if_delmulti_ifma(mc->mc_ifma); 560 free(mc, M_DEVBUF); 561 } 562 } 563 return (0); 564} 565 566static int 567ng_fec_ether_setmulti(struct ifnet *ifp) 568{ 569 struct ng_fec_private *priv; 570 struct ng_fec_bundle *b; 571 struct ng_fec_portlist *p; 572 573 priv = ifp->if_softc; 574 b = &priv->fec_bundle; 575 576 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 577 /* First, remove any existing filter entries. */ 578 ng_fec_ether_cmdmulti(ifp, p, 0); 579 /* copy all addresses from the fec interface to the port */ 580 ng_fec_ether_cmdmulti(ifp, p, 1); 581 } 582 return (0); 583} 584 585/* 586 * Pass an ioctl command down to all the underyling interfaces in a 587 * bundle. Used for setting flags. 588 */ 589 590static int 591ng_fec_setport(struct ifnet *ifp, u_long command, caddr_t data) 592{ 593 struct ng_fec_private *priv; 594 struct ng_fec_bundle *b; 595 struct ifnet *oifp; 596 struct ng_fec_portlist *p; 597 598 priv = ifp->if_softc; 599 b = &priv->fec_bundle; 600 601 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 602 oifp = p->fec_if; 603 if (oifp != NULL) 604 (*oifp->if_ioctl)(oifp, command, data); 605 } 606 607 return(0); 608} 609 610static void 611ng_fec_init(void *arg) 612{ 613 struct ng_fec_private *priv; 614 struct ng_fec_bundle *b; 615 struct ifnet *ifp, *bifp; 616 struct ng_fec_portlist *p; 617 618 priv = arg; 619 ifp = priv->ifp; 620 b = &priv->fec_bundle; 621 622 if (b->fec_ifcnt != 2 && b->fec_ifcnt != FEC_BUNDLESIZ) { 623 printf("fec%d: invalid bundle " 624 "size: %d\n", priv->unit, 625 b->fec_ifcnt); 626 return; 627 } 628 629 ng_fec_stop(ifp); 630 631 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 632 bifp = p->fec_if; 633 bifp->if_flags |= IFF_UP; 634 (*bifp->if_ioctl)(bifp, SIOCSIFFLAGS, NULL); 635 /* mark iface as up and let the monitor check it */ 636 p->fec_ifstat = -1; 637 } 638 639 ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 640 ifp->if_drv_flags |= IFF_DRV_RUNNING; 641 642 priv->fec_ch = timeout(ng_fec_tick, priv, hz); 643 644 return; 645} 646 647static void 648ng_fec_stop(struct ifnet *ifp) 649{ 650 struct ng_fec_private *priv; 651 struct ng_fec_bundle *b; 652 struct ifnet *bifp; 653 struct ng_fec_portlist *p; 654 655 priv = ifp->if_softc; 656 b = &priv->fec_bundle; 657 658 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 659 bifp = p->fec_if; 660 bifp->if_flags &= ~IFF_UP; 661 (*bifp->if_ioctl)(bifp, SIOCSIFFLAGS, NULL); 662 } 663 664 untimeout(ng_fec_tick, priv, priv->fec_ch); 665 666 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 667 668 return; 669} 670 671static void 672ng_fec_tick(void *arg) 673{ 674 struct ng_fec_private *priv; 675 struct ng_fec_bundle *b; 676 struct ifmediareq ifmr; 677 struct ifnet *ifp; 678 struct ng_fec_portlist *p; 679 int error = 0; 680 681 priv = arg; 682 b = &priv->fec_bundle; 683 684 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 685 bzero((char *)&ifmr, sizeof(ifmr)); 686 ifp = p->fec_if; 687 error = (*ifp->if_ioctl)(ifp, SIOCGIFMEDIA, (caddr_t)&ifmr); 688 if (error) { 689 printf("fec%d: failed to check status " 690 "of link %s\n", priv->unit, ifp->if_xname); 691 continue; 692 } 693 694 if (ifmr.ifm_status & IFM_AVALID) { 695 if (ifmr.ifm_status & IFM_ACTIVE) { 696 if (p->fec_ifstat == -1 || 697 p->fec_ifstat == 0) { 698 p->fec_ifstat = 1; 699 printf("fec%d: port %s in bundle " 700 "is up\n", priv->unit, 701 ifp->if_xname); 702 } 703 } else { 704 if (p->fec_ifstat == -1 || 705 p->fec_ifstat == 1) { 706 p->fec_ifstat = 0; 707 printf("fec%d: port %s in bundle " 708 "is down\n", priv->unit, 709 ifp->if_xname); 710 } 711 } 712 } 713 } 714 715 ifp = priv->ifp; 716 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 717 priv->fec_ch = timeout(ng_fec_tick, priv, hz); 718 719 return; 720} 721 722static int 723ng_fec_ifmedia_upd(struct ifnet *ifp) 724{ 725 return(0); 726} 727 728static void ng_fec_ifmedia_sts(struct ifnet *ifp, 729 struct ifmediareq *ifmr) 730{ 731 struct ng_fec_private *priv; 732 struct ng_fec_bundle *b; 733 struct ng_fec_portlist *p; 734 735 priv = ifp->if_softc; 736 b = &priv->fec_bundle; 737 738 ifmr->ifm_status = IFM_AVALID; 739 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 740 if (p->fec_ifstat == 1) { 741 ifmr->ifm_status |= IFM_ACTIVE; 742 break; 743 } 744 } 745 746 return; 747} 748 749/* 750 * Process an ioctl for the virtual interface 751 */ 752static int 753ng_fec_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 754{ 755 struct ifreq *const ifr = (struct ifreq *) data; 756 int error = 0; 757 struct ng_fec_private *priv; 758 struct ng_fec_bundle *b; 759 760 priv = ifp->if_softc; 761 b = &priv->fec_bundle; 762 763#ifdef DEBUG 764 ng_fec_print_ioctl(ifp, command, data); 765#endif 766 switch (command) { 767 768 /* These two are mostly handled at a higher layer */ 769 case SIOCSIFADDR: 770 case SIOCGIFADDR: 771 error = ether_ioctl(ifp, command, data); 772 break; 773 774 case SIOCSIFMTU: 775 if (ifr->ifr_mtu >= NG_FEC_MTU_MIN && 776 ifr->ifr_mtu <= NG_FEC_MTU_MAX) { 777 struct ng_fec_portlist *p; 778 struct ifnet *bifp; 779 780 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 781 bifp = p->fec_if; 782 error = (*bifp->if_ioctl)(bifp, SIOCSIFMTU, 783 data); 784 if (error != 0) 785 break; 786 } 787 if (error == 0) 788 ifp->if_mtu = ifr->ifr_mtu; 789 } else 790 error = EINVAL; 791 break; 792 793 /* Set flags */ 794 case SIOCSIFFLAGS: 795 /* 796 * If the interface is marked up and stopped, then start it. 797 * If it is marked down and running, then stop it. 798 */ 799 if (ifr->ifr_flags & IFF_UP) { 800 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 801 /* Sanity. */ 802 if (b->fec_ifcnt != 2 && 803 b->fec_ifcnt != FEC_BUNDLESIZ) { 804 printf("fec%d: invalid bundle " 805 "size: %d\n", priv->unit, 806 b->fec_ifcnt); 807 error = EINVAL; 808 break; 809 } 810 ng_fec_init(priv); 811 } 812 /* 813 * Bubble down changes in promisc mode to 814 * underlying interfaces. 815 */ 816 if ((ifp->if_flags & IFF_PROMISC) != 817 (priv->if_flags & IFF_PROMISC)) { 818 ng_fec_setport(ifp, command, data); 819 priv->if_flags = ifp->if_flags; 820 } 821 } else { 822 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 823 ng_fec_stop(ifp); 824 } 825 break; 826 827 case SIOCADDMULTI: 828 case SIOCDELMULTI: 829 ng_fec_ether_setmulti(ifp); 830 error = 0; 831 break; 832 case SIOCGIFMEDIA: 833 case SIOCSIFMEDIA: 834 error = ifmedia_ioctl(ifp, ifr, &priv->ifmedia, command); 835 break; 836 /* Stuff that's not supported */ 837 case SIOCSIFPHYS: 838 error = EOPNOTSUPP; 839 break; 840 841 default: 842 error = EINVAL; 843 break; 844 } 845 return (error); 846} 847 848/* 849 * This routine spies on mbufs received by underlying network device 850 * drivers. When we add an interface to our bundle, we override its 851 * if_input routine with a pointer to ng_fec_input(). This means we 852 * get to look at all the device's packets before sending them to the 853 * real ether_input() for processing by the stack. Once we verify the 854 * packet comes from an interface that's been aggregated into 855 * our bundle, we fix up the rcvif pointer and increment our 856 * packet counters so that it looks like the frames are actually 857 * coming from us. 858 */ 859static void 860ng_fec_input(struct ifnet *ifp, struct mbuf *m0) 861{ 862 struct ng_node *node; 863 struct ng_fec_private *priv; 864 struct ng_fec_bundle *b; 865 struct ifnet *bifp; 866 struct ng_fec_portlist *p; 867 868 /* Sanity check */ 869 if (ifp == NULL || m0 == NULL) 870 return; 871 872 node = IFP2NG(ifp); 873 874 /* Sanity check part II */ 875 if (node == NULL) 876 return; 877 878 priv = NG_NODE_PRIVATE(node); 879 b = &priv->fec_bundle; 880 bifp = priv->ifp; 881 882 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 883 if (p->fec_if == m0->m_pkthdr.rcvif) 884 break; 885 } 886 887 /* Wasn't meant for us; leave this frame alone. */ 888 if (p == NULL) 889 return; 890 891 /* 892 * Check for a BPF tap on the underlying interface. This 893 * is mainly a debugging aid: it allows tcpdump-ing of an 894 * individual interface in a bundle to work, which it 895 * otherwise would not. BPF tapping of our own aggregate 896 * interface will occur once we call ether_input(). 897 */ 898 BPF_MTAP(m0->m_pkthdr.rcvif, m0); 899 900 /* Convince the system that this is our frame. */ 901 m0->m_pkthdr.rcvif = bifp; 902 903 /* 904 * Count bytes on an individual interface in a bundle. 905 * The bytes will also be added to the aggregate interface 906 * once we call ether_input(). 907 */ 908 ifp->if_ibytes += m0->m_pkthdr.len; 909 910 bifp->if_ipackets++; 911 (*bifp->if_input)(bifp, m0); 912 913 return; 914} 915 916/* 917 * Take a quick peek at the packet and see if it's ok for us to use 918 * the inet or inet6 hash methods on it, if they're enabled. We do 919 * this by setting flags in the mbuf header. Once we've made up our 920 * mind what to do, we pass the frame to output vector for further 921 * processing. 922 */ 923 924static int 925ng_fec_output(struct ifnet *ifp, struct mbuf *m, 926 const struct sockaddr *dst, struct route *ro) 927{ 928 const priv_p priv = (priv_p) ifp->if_softc; 929 struct ng_fec_bundle *b; 930 int error; 931 932 /* Check interface flags */ 933 if (!((ifp->if_flags & IFF_UP) && 934 (ifp->if_drv_flags & IFF_DRV_RUNNING))) { 935 m_freem(m); 936 return (ENETDOWN); 937 } 938 939 b = &priv->fec_bundle; 940 941 switch (b->fec_btype) { 942 case FEC_BTYPE_MAC: 943 m->m_flags |= M_FEC_MAC; 944 break; 945#ifdef INET 946 case FEC_BTYPE_INET: 947 /* 948 * We can't use the INET address port selection 949 * scheme if this isn't an INET packet. 950 */ 951 if (dst->sa_family == AF_INET) 952 m->m_flags |= M_FEC_INET; 953#ifdef INET6 954 else if (dst->sa_family == AF_INET6) 955 m->m_flags |= M_FEC_INET6; 956#endif 957 else { 958#ifdef DEBUG 959 if_printf(ifp, "can't do inet aggregation of non " 960 "inet packet\n"); 961#endif 962 m->m_flags |= M_FEC_MAC; 963 } 964 break; 965#endif 966 default: 967 if_printf(ifp, "bogus hash type: %d\n", 968 b->fec_btype); 969 m_freem(m); 970 return(EINVAL); 971 break; 972 } 973 974 /* 975 * Pass the frame to the output vector for all the protocol 976 * handling. This will put the ethernet header on the packet 977 * for us. 978 */ 979 priv->if_error = 0; 980 error = (*b->fec_if_output)(ifp, m, dst, ro); 981 if (priv->if_error && !error) 982 error = priv->if_error; 983 984 return(error); 985} 986 987/* 988 * Apply a hash to the source and destination addresses in the packet 989 * in order to select an interface. Also check link status and handle 990 * dead links accordingly. 991 */ 992 993static int 994ng_fec_choose_port(struct ng_fec_bundle *b, 995 struct mbuf *m, struct ifnet **ifp) 996{ 997 struct ether_header *eh; 998 struct mbuf *m0; 999#ifdef INET 1000 struct ip *ip; 1001#ifdef INET6 1002 struct ip6_hdr *ip6; 1003#endif 1004#endif 1005 1006 struct ng_fec_portlist *p; 1007 int port = 0, mask; 1008 1009 /* 1010 * If there are only two ports, mask off all but the 1011 * last bit for XORing. If there are 4, mask off all 1012 * but the last 2 bits. 1013 */ 1014 mask = b->fec_ifcnt == 2 ? 0x1 : 0x3; 1015 eh = mtod(m, struct ether_header *); 1016#ifdef INET 1017 ip = (struct ip *)(mtod(m, char *) + 1018 sizeof(struct ether_header)); 1019#ifdef INET6 1020 ip6 = (struct ip6_hdr *)(mtod(m, char *) + 1021 sizeof(struct ether_header)); 1022#endif 1023#endif 1024 1025 /* 1026 * The fg_fec_output() routine is supposed to leave a 1027 * flag for us in the mbuf that tells us what hash to 1028 * use, but sometimes a new mbuf is prepended to the 1029 * chain, so we have to search every mbuf in the chain 1030 * to find the flags. 1031 */ 1032 m0 = m; 1033 while (m0) { 1034 if (m0->m_flags & (M_FEC_MAC|M_FEC_INET|M_FEC_INET6)) 1035 break; 1036 m0 = m0->m_next; 1037 } 1038 if (m0 == NULL) 1039 return(EINVAL); 1040 1041 switch (m0->m_flags & (M_FEC_MAC|M_FEC_INET|M_FEC_INET6)) { 1042 case M_FEC_MAC: 1043 port = (eh->ether_dhost[5] ^ 1044 eh->ether_shost[5]) & mask; 1045 break; 1046#ifdef INET 1047 case M_FEC_INET: 1048 port = (ntohl(ip->ip_dst.s_addr) ^ 1049 ntohl(ip->ip_src.s_addr)) & mask; 1050 break; 1051#ifdef INET6 1052 case M_FEC_INET6: 1053 port = (ip6->ip6_dst.s6_addr[15] ^ 1054 ip6->ip6_dst.s6_addr[15]) & mask; 1055 break; 1056#endif 1057#endif 1058 default: 1059 return(EINVAL); 1060 break; 1061 } 1062 1063 TAILQ_FOREACH(p, &b->ng_fec_ports, fec_list) { 1064 if (port == p->fec_idx) 1065 break; 1066 } 1067 1068 /* 1069 * Now that we've chosen a port, make sure it's 1070 * alive. If it's not alive, cycle through the bundle 1071 * looking for a port that is alive. If we don't find 1072 * any, return an error. 1073 */ 1074 if (p->fec_ifstat != 1) { 1075 struct ng_fec_portlist *n = NULL; 1076 1077 n = TAILQ_NEXT(p, fec_list); 1078 if (n == NULL) 1079 n = TAILQ_FIRST(&b->ng_fec_ports); 1080 while (n != p) { 1081 if (n->fec_ifstat == 1) 1082 break; 1083 n = TAILQ_NEXT(n, fec_list); 1084 if (n == NULL) 1085 n = TAILQ_FIRST(&b->ng_fec_ports); 1086 } 1087 if (n == p) 1088 return(EAGAIN); 1089 p = n; 1090 } 1091 1092 *ifp = p->fec_if; 1093 1094 return(0); 1095} 1096 1097/* 1098 * Now that the packet has been run through ether_output(), yank it 1099 * off our own send queue and stick it on the queue for the appropriate 1100 * underlying physical interface. Note that if the interface's send 1101 * queue is full, we save an error status in our private netgraph 1102 * space which will eventually be handed up to ng_fec_output(), which 1103 * will return it to the rest of the IP stack. We need to do this 1104 * in order to duplicate the effect of ether_output() returning ENOBUFS 1105 * when it detects that an interface's send queue is full. There's no 1106 * other way to signal the error status from here since the if_start() 1107 * routine is spec'ed to return void. 1108 * 1109 * Once the frame is queued, we call ether_output_frame() to initiate 1110 * transmission. 1111 */ 1112static void 1113ng_fec_start(struct ifnet *ifp) 1114{ 1115 struct ng_fec_private *priv; 1116 struct ng_fec_bundle *b; 1117 struct ifnet *oifp = NULL; 1118 struct mbuf *m0; 1119 int error; 1120 1121 priv = ifp->if_softc; 1122 b = &priv->fec_bundle; 1123 1124 IF_DEQUEUE(&ifp->if_snd, m0); 1125 if (m0 == NULL) 1126 return; 1127 1128 BPF_MTAP(ifp, m0); 1129 1130 /* Queue up packet on the proper port. */ 1131 error = ng_fec_choose_port(b, m0, &oifp); 1132 if (error) { 1133 ifp->if_ierrors++; 1134 m_freem(m0); 1135 priv->if_error = ENOBUFS; 1136 return; 1137 } 1138 ifp->if_opackets++; 1139 1140 priv->if_error = IF_HANDOFF(&oifp->if_snd, m0, oifp) ? 0 : ENOBUFS; 1141 1142 return; 1143} 1144 1145#ifdef DEBUG 1146/* 1147 * Display an ioctl to the virtual interface 1148 */ 1149 1150static void 1151ng_fec_print_ioctl(struct ifnet *ifp, int command, caddr_t data) 1152{ 1153 char *str; 1154 1155 switch (command & IOC_DIRMASK) { 1156 case IOC_VOID: 1157 str = "IO"; 1158 break; 1159 case IOC_OUT: 1160 str = "IOR"; 1161 break; 1162 case IOC_IN: 1163 str = "IOW"; 1164 break; 1165 case IOC_INOUT: 1166 str = "IORW"; 1167 break; 1168 default: 1169 str = "IO??"; 1170 } 1171 log(LOG_DEBUG, "%s: %s('%c', %d, char[%d])\n", 1172 ifp->if_xname, 1173 str, 1174 IOCGROUP(command), 1175 command & 0xff, 1176 IOCPARM_LEN(command)); 1177} 1178#endif /* DEBUG */ 1179 1180/************************************************************************ 1181 NETGRAPH NODE STUFF 1182 ************************************************************************/ 1183 1184/* 1185 * Constructor for a node 1186 */ 1187static int 1188ng_fec_constructor(node_p node) 1189{ 1190 char ifname[NG_FEC_FEC_NAME_MAX + 1]; 1191 struct ifnet *ifp; 1192 priv_p priv; 1193 const uint8_t eaddr[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; 1194 struct ng_fec_bundle *b; 1195 int error = 0; 1196 1197 /* Allocate node and interface private structures */ 1198 priv = malloc(sizeof(*priv), M_NETGRAPH, M_WAITOK | M_ZERO); 1199 1200 ifp = priv->ifp = if_alloc(IFT_ETHER); 1201 if (ifp == NULL) { 1202 free(priv, M_NETGRAPH); 1203 return (ENOSPC); 1204 } 1205 b = &priv->fec_bundle; 1206 1207 /* Link them together */ 1208 ifp->if_softc = priv; 1209 1210 /* Get an interface unit number */ 1211 if ((error = ng_fec_get_unit(&priv->unit)) != 0) { 1212 if_free(ifp); 1213 free(priv, M_NETGRAPH); 1214 return (error); 1215 } 1216 1217 /* Link together node and private info */ 1218 NG_NODE_SET_PRIVATE(node, priv); 1219 priv->node = node; 1220 1221 /* Initialize interface structure */ 1222 if_initname(ifp, NG_FEC_FEC_NAME, priv->unit); 1223 ifp->if_start = ng_fec_start; 1224 ifp->if_ioctl = ng_fec_ioctl; 1225 ifp->if_init = ng_fec_init; 1226 ifp->if_snd.ifq_maxlen = ifqmaxlen; 1227 ifp->if_mtu = NG_FEC_MTU_DEFAULT; 1228 ifp->if_flags = (IFF_SIMPLEX|IFF_BROADCAST|IFF_MULTICAST); 1229 ifp->if_addrlen = 0; /* XXX */ 1230 ifp->if_hdrlen = 0; /* XXX */ 1231 ifp->if_baudrate = 100000000; /* XXX */ 1232 TAILQ_INIT(&ifp->if_addrhead); /* XXX useless - done in if_attach */ 1233 1234 /* Give this node the same name as the interface (if possible) */ 1235 bzero(ifname, sizeof(ifname)); 1236 strlcpy(ifname, ifp->if_xname, sizeof(ifname)); 1237 if (ng_name_node(node, ifname) != 0) 1238 log(LOG_WARNING, "%s: can't acquire netgraph name\n", ifname); 1239 1240 /* Attach the interface */ 1241 ether_ifattach(ifp, eaddr); 1242 callout_handle_init(&priv->fec_ch); 1243 1244 /* Override output method with our own */ 1245 ifp->if_output = ng_fec_output; 1246 1247 TAILQ_INIT(&b->ng_fec_ports); 1248 b->fec_ifcnt = 0; 1249 1250 ifmedia_init(&priv->ifmedia, 0, 1251 ng_fec_ifmedia_upd, ng_fec_ifmedia_sts); 1252 ifmedia_add(&priv->ifmedia, IFM_ETHER|IFM_NONE, 0, NULL); 1253 ifmedia_set(&priv->ifmedia, IFM_ETHER|IFM_NONE); 1254 1255 /* Done */ 1256 return (0); 1257} 1258 1259/* 1260 * Receive a control message 1261 */ 1262static int 1263ng_fec_rcvmsg(node_p node, item_p item, hook_p lasthook) 1264{ 1265 const priv_p priv = NG_NODE_PRIVATE(node); 1266 struct ng_fec_bundle *b; 1267 struct ng_mesg *resp = NULL; 1268 struct ng_mesg *msg; 1269 char *ifname; 1270 int error = 0; 1271 1272 NGI_GET_MSG(item, msg); 1273 b = &priv->fec_bundle; 1274 1275 switch (msg->header.typecookie) { 1276 case NGM_FEC_COOKIE: 1277 switch (msg->header.cmd) { 1278 case NGM_FEC_ADD_IFACE: 1279 ifname = msg->data; 1280 error = ng_fec_addport(priv, ifname); 1281 break; 1282 case NGM_FEC_DEL_IFACE: 1283 ifname = msg->data; 1284 error = ng_fec_delport(priv, ifname); 1285 break; 1286 case NGM_FEC_SET_MODE_MAC: 1287 b->fec_btype = FEC_BTYPE_MAC; 1288 break; 1289#ifdef INET 1290 case NGM_FEC_SET_MODE_INET: 1291 b->fec_btype = FEC_BTYPE_INET; 1292 break; 1293#ifdef INET6 1294 case NGM_FEC_SET_MODE_INET6: 1295 b->fec_btype = FEC_BTYPE_INET6; 1296 break; 1297#endif 1298#endif 1299 default: 1300 error = EINVAL; 1301 break; 1302 } 1303 break; 1304 default: 1305 error = EINVAL; 1306 break; 1307 } 1308 NG_RESPOND_MSG(error, node, item, resp); 1309 NG_FREE_MSG(msg); 1310 return (error); 1311} 1312 1313/* 1314 * Shutdown and remove the node and its associated interface. 1315 */ 1316static int 1317ng_fec_shutdown(node_p node) 1318{ 1319 const priv_p priv = NG_NODE_PRIVATE(node); 1320 struct ng_fec_bundle *b; 1321 struct ng_fec_portlist *p; 1322 1323 b = &priv->fec_bundle; 1324 ng_fec_stop(priv->ifp); 1325 1326 while (!TAILQ_EMPTY(&b->ng_fec_ports)) { 1327 p = TAILQ_FIRST(&b->ng_fec_ports); 1328 ng_fec_ether_cmdmulti(priv->ifp, p, 0); 1329 ng_fec_delport(priv, p->fec_if->if_xname); 1330 } 1331 1332 ether_ifdetach(priv->ifp); 1333 if_free(priv->ifp); 1334 ifmedia_removeall(&priv->ifmedia); 1335 ng_fec_free_unit(priv->unit); 1336 free(priv, M_NETGRAPH); 1337 NG_NODE_SET_PRIVATE(node, NULL); 1338 NG_NODE_UNREF(node); 1339 return (0); 1340} 1341 1342/* 1343 * Handle loading and unloading for this node type. 1344 */ 1345static int 1346ng_fec_mod_event(module_t mod, int event, void *data) 1347{ 1348 int error = 0; 1349 1350 switch (event) { 1351 case MOD_LOAD: 1352 mtx_init(&ng_fec_mtx, "ng_fec", NULL, MTX_DEF); 1353 break; 1354 case MOD_UNLOAD: 1355 mtx_destroy(&ng_fec_mtx); 1356 break; 1357 default: 1358 error = EOPNOTSUPP; 1359 break; 1360 } 1361 return (error); 1362} 1363