1194927Sbz/*- 2194927Sbz * Copyright (c) 2008 The FreeBSD Foundation 3204805Sbz * Copyright (c) 2009-2010 Bjoern A. Zeeb <bz@FreeBSD.org> 4194927Sbz * All rights reserved. 5194927Sbz * 6194927Sbz * This software was developed by CK Software GmbH under sponsorship 7194927Sbz * from the FreeBSD Foundation. 8194927Sbz * 9194927Sbz * Redistribution and use in source and binary forms, with or without 10194927Sbz * modification, are permitted provided that the following conditions 11194927Sbz * are met: 12194927Sbz * 1. Redistributions of source code must retain the above copyright 13194927Sbz * notice, this list of conditions and the following disclaimer. 14194927Sbz * 2. Redistributions in binary form must reproduce the above copyright 15194927Sbz * notice, this list of conditions and the following disclaimer in the 16194927Sbz * documentation and/or other materials provided with the distribution. 17194927Sbz * 18194927Sbz * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19194927Sbz * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20194927Sbz * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21194927Sbz * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22194927Sbz * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23194927Sbz * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24194927Sbz * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25194927Sbz * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26194927Sbz * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27194927Sbz * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28194927Sbz * SUCH DAMAGE. 29194927Sbz */ 30194927Sbz 31194927Sbz/* 32195892Sbz * A pair of virtual back-to-back connected ethernet like interfaces 33195892Sbz * (``two interfaces with a virtual cross-over cable''). 34195892Sbz * 35194927Sbz * This is mostly intended to be used to provide connectivity between 36194927Sbz * different virtual network stack instances. 37194927Sbz */ 38194927Sbz/* 39194927Sbz * Things to re-think once we have more experience: 40195892Sbz * - ifp->if_reassign function once we can test with vimage. Depending on 41196019Srwatson * how if_vmove() is going to be improved. 42195892Sbz * - Real random etheraddrs that are checked to be uniquish; we would need 43195892Sbz * to re-do them in case we move the interface between network stacks 44195892Sbz * in a private if_reassign function. 45195892Sbz * In case we bridge to a real interface/network or between indepedent 46195892Sbz * epairs on multiple stacks/machines, we may need this. 47195892Sbz * For now let the user handle that case. 48194927Sbz */ 49194927Sbz 50194927Sbz#include <sys/cdefs.h> 51194927Sbz__FBSDID("$FreeBSD$"); 52194927Sbz 53194927Sbz#include <sys/param.h> 54194927Sbz#include <sys/kernel.h> 55194927Sbz#include <sys/mbuf.h> 56194927Sbz#include <sys/module.h> 57194927Sbz#include <sys/refcount.h> 58194927Sbz#include <sys/queue.h> 59195892Sbz#include <sys/smp.h> 60194927Sbz#include <sys/socket.h> 61194927Sbz#include <sys/sockio.h> 62194927Sbz#include <sys/sysctl.h> 63194927Sbz#include <sys/types.h> 64194927Sbz 65194927Sbz#include <net/bpf.h> 66194927Sbz#include <net/ethernet.h> 67194927Sbz#include <net/if.h> 68194927Sbz#include <net/if_clone.h> 69222246Szec#include <net/if_media.h> 70194927Sbz#include <net/if_var.h> 71194927Sbz#include <net/if_types.h> 72194927Sbz#include <net/netisr.h> 73196019Srwatson#include <net/vnet.h> 74194927Sbz 75194927SbzSYSCTL_DECL(_net_link); 76227309Sedstatic SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW, 0, "epair sysctl"); 77195892Sbz 78195892Sbz#ifdef EPAIR_DEBUG 79195892Sbzstatic int epair_debug = 0; 80215641SbzSYSCTL_INT(_net_link_epair, OID_AUTO, epair_debug, CTLFLAG_RW, 81194927Sbz &epair_debug, 0, "if_epair(4) debugging."); 82195892Sbz#define DPRINTF(fmt, arg...) \ 83195892Sbz if (epair_debug) \ 84195892Sbz printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) 85194927Sbz#else 86194927Sbz#define DPRINTF(fmt, arg...) 87194927Sbz#endif 88194927Sbz 89195892Sbzstatic void epair_nh_sintr(struct mbuf *); 90195892Sbzstatic struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); 91195892Sbzstatic void epair_nh_drainedcpu(u_int); 92195892Sbz 93195892Sbzstatic void epair_start_locked(struct ifnet *); 94222246Szecstatic int epair_media_change(struct ifnet *); 95222246Szecstatic void epair_media_status(struct ifnet *, struct ifmediareq *); 96195892Sbz 97195892Sbzstatic int epair_clone_match(struct if_clone *, const char *); 98195892Sbzstatic int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); 99195892Sbzstatic int epair_clone_destroy(struct if_clone *, struct ifnet *); 100195892Sbz 101241610Sglebiusstatic const char epairname[] = "epair"; 102241610Sglebius 103287594Shrs/* Netisr related definitions and sysctl. */ 104195892Sbzstatic struct netisr_handler epair_nh = { 105241610Sglebius .nh_name = epairname, 106195892Sbz .nh_proto = NETISR_EPAIR, 107195892Sbz .nh_policy = NETISR_POLICY_CPU, 108195892Sbz .nh_handler = epair_nh_sintr, 109195892Sbz .nh_m2cpuid = epair_nh_m2cpuid, 110195892Sbz .nh_drainedcpu = epair_nh_drainedcpu, 111195892Sbz}; 112195892Sbz 113195892Sbzstatic int 114195892Sbzsysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) 115195892Sbz{ 116195892Sbz int error, qlimit; 117195892Sbz 118195892Sbz netisr_getqlimit(&epair_nh, &qlimit); 119195892Sbz error = sysctl_handle_int(oidp, &qlimit, 0, req); 120195892Sbz if (error || !req->newptr) 121195892Sbz return (error); 122195892Sbz if (qlimit < 1) 123195892Sbz return (EINVAL); 124195892Sbz return (netisr_setqlimit(&epair_nh, qlimit)); 125195892Sbz} 126195892SbzSYSCTL_PROC(_net_link_epair, OID_AUTO, netisr_maxqlen, CTLTYPE_INT|CTLFLAG_RW, 127195892Sbz 0, 0, sysctl_epair_netisr_maxqlen, "I", 128195892Sbz "Maximum if_epair(4) netisr \"hw\" queue length"); 129195892Sbz 130194927Sbzstruct epair_softc { 131195892Sbz struct ifnet *ifp; /* This ifp. */ 132195892Sbz struct ifnet *oifp; /* other ifp of pair. */ 133222246Szec struct ifmedia media; /* Media config (fake). */ 134195892Sbz u_int refcount; /* # of mbufs in flight. */ 135195892Sbz u_int cpuid; /* CPU ID assigned upon creation. */ 136194927Sbz void (*if_qflush)(struct ifnet *); 137195892Sbz /* Original if_qflush routine. */ 138194927Sbz}; 139194927Sbz 140195892Sbz/* 141195892Sbz * Per-CPU list of ifps with data in the ifq that needs to be flushed 142195892Sbz * to the netisr ``hw'' queue before we allow any further direct queuing 143195892Sbz * to the ``hw'' queue. 144195892Sbz */ 145194927Sbzstruct epair_ifp_drain { 146194927Sbz STAILQ_ENTRY(epair_ifp_drain) ifp_next; 147194927Sbz struct ifnet *ifp; 148194927Sbz}; 149195892SbzSTAILQ_HEAD(eid_list, epair_ifp_drain); 150194927Sbz 151195892Sbz#define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ 152195892Sbz "if_epair", NULL, MTX_DEF) 153195892Sbz#define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) 154195892Sbz#define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ 155195892Sbz MA_OWNED) 156195892Sbz#define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) 157195892Sbz#define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) 158194927Sbz 159195892Sbz#ifdef INVARIANTS 160195892Sbz#define EPAIR_REFCOUNT_INIT(r, v) refcount_init((r), (v)) 161195892Sbz#define EPAIR_REFCOUNT_AQUIRE(r) refcount_acquire((r)) 162195892Sbz#define EPAIR_REFCOUNT_RELEASE(r) refcount_release((r)) 163195892Sbz#define EPAIR_REFCOUNT_ASSERT(a, p) KASSERT(a, p) 164195892Sbz#else 165195892Sbz#define EPAIR_REFCOUNT_INIT(r, v) 166195892Sbz#define EPAIR_REFCOUNT_AQUIRE(r) 167195892Sbz#define EPAIR_REFCOUNT_RELEASE(r) 168195892Sbz#define EPAIR_REFCOUNT_ASSERT(a, p) 169195892Sbz#endif 170194927Sbz 171241610Sglebiusstatic MALLOC_DEFINE(M_EPAIR, epairname, 172194927Sbz "Pair of virtual cross-over connected Ethernet-like interfaces"); 173194927Sbz 174287594Shrsstatic VNET_DEFINE(struct if_clone *, epair_cloner); 175287594Shrs#define V_epair_cloner VNET(epair_cloner) 176194927Sbz 177195892Sbz/* 178195892Sbz * DPCPU area and functions. 179195892Sbz */ 180195892Sbzstruct epair_dpcpu { 181195892Sbz struct mtx if_epair_mtx; /* Per-CPU locking. */ 182195892Sbz int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ 183195892Sbz struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with 184195892Sbz * data in the ifq. */ 185195892Sbz}; 186195892SbzDPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); 187194927Sbz 188195892Sbzstatic void 189195892Sbzepair_dpcpu_init(void) 190195892Sbz{ 191195892Sbz struct epair_dpcpu *epair_dpcpu; 192195892Sbz struct eid_list *s; 193195892Sbz u_int cpuid; 194195892Sbz 195209059Sjhb CPU_FOREACH(cpuid) { 196195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 197195892Sbz 198195892Sbz /* Initialize per-cpu lock. */ 199195892Sbz EPAIR_LOCK_INIT(epair_dpcpu); 200195892Sbz 201195892Sbz /* Driver flags are per-cpu as are our netisr "hw" queues. */ 202195892Sbz epair_dpcpu->epair_drv_flags = 0; 203195892Sbz 204195892Sbz /* 205195892Sbz * Initialize per-cpu drain list. 206195892Sbz * Manually do what STAILQ_HEAD_INITIALIZER would do. 207195892Sbz */ 208195892Sbz s = &epair_dpcpu->epair_ifp_drain_list; 209195892Sbz s->stqh_first = NULL; 210195892Sbz s->stqh_last = &s->stqh_first; 211195892Sbz } 212195892Sbz} 213195892Sbz 214195892Sbzstatic void 215195892Sbzepair_dpcpu_detach(void) 216195892Sbz{ 217195892Sbz struct epair_dpcpu *epair_dpcpu; 218195892Sbz u_int cpuid; 219195892Sbz 220209059Sjhb CPU_FOREACH(cpuid) { 221195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 222195892Sbz 223195892Sbz /* Destroy per-cpu lock. */ 224195892Sbz EPAIR_LOCK_DESTROY(epair_dpcpu); 225195892Sbz } 226195892Sbz} 227195892Sbz 228194927Sbz/* 229195892Sbz * Helper functions. 230195892Sbz */ 231195892Sbzstatic u_int 232195892Sbzcpuid_from_ifp(struct ifnet *ifp) 233195892Sbz{ 234195892Sbz struct epair_softc *sc; 235195892Sbz 236195892Sbz if (ifp == NULL) 237195892Sbz return (0); 238195892Sbz sc = ifp->if_softc; 239195892Sbz 240195892Sbz return (sc->cpuid); 241195892Sbz} 242195892Sbz 243195892Sbz/* 244194927Sbz * Netisr handler functions. 245194927Sbz */ 246194927Sbzstatic void 247195892Sbzepair_nh_sintr(struct mbuf *m) 248194927Sbz{ 249194927Sbz struct ifnet *ifp; 250194927Sbz struct epair_softc *sc; 251194927Sbz 252194927Sbz ifp = m->m_pkthdr.rcvif; 253194927Sbz (*ifp->if_input)(ifp, m); 254194927Sbz sc = ifp->if_softc; 255195892Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 256204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 257204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 258204805Sbz __func__, ifp, sc->refcount)); 259194927Sbz DPRINTF("ifp=%p refcount=%u\n", ifp, sc->refcount); 260194927Sbz} 261194927Sbz 262195892Sbzstatic struct mbuf * 263195892Sbzepair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 264195892Sbz{ 265195892Sbz 266195892Sbz *cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); 267195892Sbz 268195892Sbz return (m); 269195892Sbz} 270195892Sbz 271194927Sbzstatic void 272195892Sbzepair_nh_drainedcpu(u_int cpuid) 273194927Sbz{ 274195892Sbz struct epair_dpcpu *epair_dpcpu; 275194927Sbz struct epair_ifp_drain *elm, *tvar; 276194927Sbz struct ifnet *ifp; 277194927Sbz 278195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 279195892Sbz EPAIR_LOCK(epair_dpcpu); 280194927Sbz /* 281194927Sbz * Assume our "hw" queue and possibly ifq will be emptied 282194927Sbz * again. In case we will overflow the "hw" queue while 283194927Sbz * draining, epair_start_locked will set IFF_DRV_OACTIVE 284194927Sbz * again and we will stop and return. 285194927Sbz */ 286195892Sbz STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 287195892Sbz ifp_next, tvar) { 288194927Sbz ifp = elm->ifp; 289195892Sbz epair_dpcpu->epair_drv_flags &= ~IFF_DRV_OACTIVE; 290194927Sbz ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 291194927Sbz epair_start_locked(ifp); 292194927Sbz 293194927Sbz IFQ_LOCK(&ifp->if_snd); 294194927Sbz if (IFQ_IS_EMPTY(&ifp->if_snd)) { 295204805Sbz struct epair_softc *sc; 296204805Sbz 297195892Sbz STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list, 298195892Sbz elm, epair_ifp_drain, ifp_next); 299204805Sbz /* The cached ifp goes off the list. */ 300204805Sbz sc = ifp->if_softc; 301204805Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 302204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 303204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 304204805Sbz __func__, ifp, sc->refcount)); 305194927Sbz free(elm, M_EPAIR); 306194927Sbz } 307194927Sbz IFQ_UNLOCK(&ifp->if_snd); 308194927Sbz 309194927Sbz if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) { 310194927Sbz /* Our "hw"q overflew again. */ 311215641Sbz epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 312194927Sbz DPRINTF("hw queue length overflow at %u\n", 313195892Sbz epair_nh.nh_qlimit); 314194927Sbz break; 315194927Sbz } 316194927Sbz } 317195892Sbz EPAIR_UNLOCK(epair_dpcpu); 318194927Sbz} 319194927Sbz 320194927Sbz/* 321194927Sbz * Network interface (`if') related functions. 322194927Sbz */ 323204805Sbzstatic void 324204805Sbzepair_remove_ifp_from_draining(struct ifnet *ifp) 325204805Sbz{ 326204805Sbz struct epair_dpcpu *epair_dpcpu; 327204805Sbz struct epair_ifp_drain *elm, *tvar; 328204805Sbz u_int cpuid; 329204805Sbz 330209059Sjhb CPU_FOREACH(cpuid) { 331204805Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 332204805Sbz EPAIR_LOCK(epair_dpcpu); 333204805Sbz STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 334204805Sbz ifp_next, tvar) { 335204805Sbz if (ifp == elm->ifp) { 336204805Sbz struct epair_softc *sc; 337204805Sbz 338204805Sbz STAILQ_REMOVE( 339204805Sbz &epair_dpcpu->epair_ifp_drain_list, elm, 340204805Sbz epair_ifp_drain, ifp_next); 341204805Sbz /* The cached ifp goes off the list. */ 342204805Sbz sc = ifp->if_softc; 343204805Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 344204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 345204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 346204805Sbz __func__, ifp, sc->refcount)); 347204805Sbz free(elm, M_EPAIR); 348204805Sbz } 349204805Sbz } 350204805Sbz EPAIR_UNLOCK(epair_dpcpu); 351204805Sbz } 352204805Sbz} 353204805Sbz 354195892Sbzstatic int 355195892Sbzepair_add_ifp_for_draining(struct ifnet *ifp) 356195892Sbz{ 357195892Sbz struct epair_dpcpu *epair_dpcpu; 358204805Sbz struct epair_softc *sc; 359195892Sbz struct epair_ifp_drain *elm = NULL; 360195892Sbz 361204805Sbz sc = ifp->if_softc; 362195892Sbz epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 363204805Sbz EPAIR_LOCK_ASSERT(epair_dpcpu); 364195892Sbz STAILQ_FOREACH(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next) 365195892Sbz if (elm->ifp == ifp) 366195892Sbz break; 367201995Sbz /* If the ifp is there already, return success. */ 368195892Sbz if (elm != NULL) 369195892Sbz return (0); 370195892Sbz 371195892Sbz elm = malloc(sizeof(struct epair_ifp_drain), M_EPAIR, M_NOWAIT|M_ZERO); 372195892Sbz if (elm == NULL) 373195892Sbz return (ENOMEM); 374195892Sbz 375195892Sbz elm->ifp = ifp; 376204805Sbz /* Add a reference for the ifp pointer on the list. */ 377204805Sbz EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 378195892Sbz STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); 379195892Sbz 380195892Sbz return (0); 381195892Sbz} 382195892Sbz 383194927Sbzstatic void 384194927Sbzepair_start_locked(struct ifnet *ifp) 385194927Sbz{ 386195892Sbz struct epair_dpcpu *epair_dpcpu; 387194927Sbz struct mbuf *m; 388194927Sbz struct epair_softc *sc; 389194927Sbz struct ifnet *oifp; 390194927Sbz int error; 391194927Sbz 392194927Sbz DPRINTF("ifp=%p\n", ifp); 393195892Sbz sc = ifp->if_softc; 394195892Sbz epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 395195892Sbz EPAIR_LOCK_ASSERT(epair_dpcpu); 396194927Sbz 397194927Sbz if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 398194927Sbz return; 399194927Sbz if ((ifp->if_flags & IFF_UP) == 0) 400194927Sbz return; 401194927Sbz 402194927Sbz /* 403194927Sbz * We get patckets here from ether_output via if_handoff() 404194927Sbz * and ned to put them into the input queue of the oifp 405194927Sbz * and call oifp->if_input() via netisr/epair_sintr(). 406194927Sbz */ 407194927Sbz oifp = sc->oifp; 408194927Sbz sc = oifp->if_softc; 409194927Sbz for (;;) { 410194927Sbz IFQ_DEQUEUE(&ifp->if_snd, m); 411194927Sbz if (m == NULL) 412194927Sbz break; 413194927Sbz BPF_MTAP(ifp, m); 414194927Sbz 415194927Sbz /* 416194927Sbz * In case the outgoing interface is not usable, 417194927Sbz * drop the packet. 418194927Sbz */ 419194927Sbz if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 420194927Sbz (oifp->if_flags & IFF_UP) ==0) { 421194927Sbz ifp->if_oerrors++; 422194927Sbz m_freem(m); 423194927Sbz continue; 424194927Sbz } 425194927Sbz DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 426194927Sbz 427194927Sbz /* 428194927Sbz * Add a reference so the interface cannot go while the 429194927Sbz * packet is in transit as we rely on rcvif to stay valid. 430194927Sbz */ 431195892Sbz EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 432194927Sbz m->m_pkthdr.rcvif = oifp; 433194927Sbz CURVNET_SET_QUIET(oifp->if_vnet); 434194927Sbz error = netisr_queue(NETISR_EPAIR, m); 435194927Sbz CURVNET_RESTORE(); 436194927Sbz if (!error) { 437194927Sbz ifp->if_opackets++; 438194927Sbz /* Someone else received the packet. */ 439194927Sbz oifp->if_ipackets++; 440194927Sbz } else { 441204805Sbz /* The packet was freed already. */ 442195892Sbz epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 443194927Sbz ifp->if_drv_flags |= IFF_DRV_OACTIVE; 444204805Sbz (void) epair_add_ifp_for_draining(ifp); 445204805Sbz ifp->if_oerrors++; 446195892Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 447204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 448204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 449204805Sbz __func__, oifp, sc->refcount)); 450194927Sbz } 451194927Sbz } 452194927Sbz} 453194927Sbz 454194927Sbzstatic void 455194927Sbzepair_start(struct ifnet *ifp) 456194927Sbz{ 457195892Sbz struct epair_dpcpu *epair_dpcpu; 458194927Sbz 459195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 460195892Sbz EPAIR_LOCK(epair_dpcpu); 461194927Sbz epair_start_locked(ifp); 462195892Sbz EPAIR_UNLOCK(epair_dpcpu); 463194927Sbz} 464194927Sbz 465194927Sbzstatic int 466194927Sbzepair_transmit_locked(struct ifnet *ifp, struct mbuf *m) 467194927Sbz{ 468195892Sbz struct epair_dpcpu *epair_dpcpu; 469194927Sbz struct epair_softc *sc; 470194927Sbz struct ifnet *oifp; 471194927Sbz int error, len; 472194927Sbz short mflags; 473194927Sbz 474194927Sbz DPRINTF("ifp=%p m=%p\n", ifp, m); 475195892Sbz sc = ifp->if_softc; 476195892Sbz epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 477195892Sbz EPAIR_LOCK_ASSERT(epair_dpcpu); 478194927Sbz 479194927Sbz if (m == NULL) 480194927Sbz return (0); 481194927Sbz 482194927Sbz /* 483194927Sbz * We are not going to use the interface en/dequeue mechanism 484194927Sbz * on the TX side. We are called from ether_output_frame() 485194927Sbz * and will put the packet into the incoming queue of the 486194927Sbz * other interface of our pair via the netsir. 487194927Sbz */ 488194927Sbz if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 489194927Sbz m_freem(m); 490194927Sbz return (ENXIO); 491194927Sbz } 492194927Sbz if ((ifp->if_flags & IFF_UP) == 0) { 493194927Sbz m_freem(m); 494194927Sbz return (ENETDOWN); 495194927Sbz } 496194927Sbz 497194927Sbz BPF_MTAP(ifp, m); 498194927Sbz 499194927Sbz /* 500194927Sbz * In case the outgoing interface is not usable, 501194927Sbz * drop the packet. 502194927Sbz */ 503194927Sbz oifp = sc->oifp; 504194927Sbz if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 505194927Sbz (oifp->if_flags & IFF_UP) ==0) { 506194927Sbz ifp->if_oerrors++; 507194927Sbz m_freem(m); 508194927Sbz return (0); 509194927Sbz } 510194927Sbz len = m->m_pkthdr.len; 511194927Sbz mflags = m->m_flags; 512194927Sbz DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 513194927Sbz 514194927Sbz#ifdef ALTQ 515194927Sbz /* Support ALTQ via the clasic if_start() path. */ 516194927Sbz IF_LOCK(&ifp->if_snd); 517194927Sbz if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 518194927Sbz ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); 519194927Sbz if (error) 520194927Sbz ifp->if_snd.ifq_drops++; 521194927Sbz IF_UNLOCK(&ifp->if_snd); 522194927Sbz if (!error) { 523194927Sbz ifp->if_obytes += len; 524194927Sbz if (mflags & (M_BCAST|M_MCAST)) 525194927Sbz ifp->if_omcasts++; 526194927Sbz 527194927Sbz if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) 528194927Sbz epair_start_locked(ifp); 529194927Sbz else 530195892Sbz (void)epair_add_ifp_for_draining(ifp); 531194927Sbz } 532194927Sbz return (error); 533194927Sbz } 534194927Sbz IF_UNLOCK(&ifp->if_snd); 535194927Sbz#endif 536194927Sbz 537195892Sbz if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { 538194927Sbz /* 539194927Sbz * Our hardware queue is full, try to fall back 540194927Sbz * queuing to the ifq but do not call ifp->if_start. 541194927Sbz * Either we are lucky or the packet is gone. 542194927Sbz */ 543194927Sbz IFQ_ENQUEUE(&ifp->if_snd, m, error); 544194927Sbz if (!error) 545195892Sbz (void)epair_add_ifp_for_draining(ifp); 546194927Sbz return (error); 547194927Sbz } 548194927Sbz sc = oifp->if_softc; 549194927Sbz /* 550194927Sbz * Add a reference so the interface cannot go while the 551194927Sbz * packet is in transit as we rely on rcvif to stay valid. 552194927Sbz */ 553195892Sbz EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 554194927Sbz m->m_pkthdr.rcvif = oifp; 555194927Sbz CURVNET_SET_QUIET(oifp->if_vnet); 556194927Sbz error = netisr_queue(NETISR_EPAIR, m); 557194927Sbz CURVNET_RESTORE(); 558194927Sbz if (!error) { 559194927Sbz ifp->if_opackets++; 560194927Sbz /* 561194927Sbz * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, 562194927Sbz * but as we bypass all this we have to duplicate 563194927Sbz * the logic another time. 564194927Sbz */ 565194927Sbz ifp->if_obytes += len; 566194927Sbz if (mflags & (M_BCAST|M_MCAST)) 567194927Sbz ifp->if_omcasts++; 568194927Sbz /* Someone else received the packet. */ 569194927Sbz oifp->if_ipackets++; 570194927Sbz } else { 571194927Sbz /* The packet was freed already. */ 572195892Sbz epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 573194927Sbz ifp->if_drv_flags |= IFF_DRV_OACTIVE; 574204805Sbz ifp->if_oerrors++; 575204805Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 576204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 577204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 578204805Sbz __func__, oifp, sc->refcount)); 579194927Sbz } 580194927Sbz 581194927Sbz return (error); 582194927Sbz} 583194927Sbz 584194927Sbzstatic int 585194927Sbzepair_transmit(struct ifnet *ifp, struct mbuf *m) 586194927Sbz{ 587195892Sbz struct epair_dpcpu *epair_dpcpu; 588194927Sbz int error; 589194927Sbz 590195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 591195892Sbz EPAIR_LOCK(epair_dpcpu); 592194927Sbz error = epair_transmit_locked(ifp, m); 593195892Sbz EPAIR_UNLOCK(epair_dpcpu); 594194927Sbz return (error); 595194927Sbz} 596194927Sbz 597194927Sbzstatic void 598194927Sbzepair_qflush(struct ifnet *ifp) 599194927Sbz{ 600194927Sbz struct epair_softc *sc; 601194927Sbz 602194927Sbz sc = ifp->if_softc; 603204805Sbz KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n", 604204805Sbz __func__, ifp, sc)); 605194927Sbz /* 606204805Sbz * Remove this ifp from all backpointer lists. The interface will not 607204805Sbz * usable for flushing anyway nor should it have anything to flush 608204805Sbz * after if_qflush(). 609194927Sbz */ 610204805Sbz epair_remove_ifp_from_draining(ifp); 611204805Sbz 612194927Sbz if (sc->if_qflush) 613194927Sbz sc->if_qflush(ifp); 614194927Sbz} 615194927Sbz 616194927Sbzstatic int 617222246Szecepair_media_change(struct ifnet *ifp __unused) 618222246Szec{ 619222246Szec 620222246Szec /* Do nothing. */ 621222246Szec return (0); 622222246Szec} 623222246Szec 624222246Szecstatic void 625222246Szecepair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr) 626222246Szec{ 627222246Szec 628222246Szec imr->ifm_status = IFM_AVALID | IFM_ACTIVE; 629222246Szec imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX; 630222246Szec} 631222246Szec 632222246Szecstatic int 633194927Sbzepair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 634194927Sbz{ 635222246Szec struct epair_softc *sc; 636194927Sbz struct ifreq *ifr; 637194927Sbz int error; 638194927Sbz 639194927Sbz ifr = (struct ifreq *)data; 640194927Sbz switch (cmd) { 641194927Sbz case SIOCSIFFLAGS: 642194927Sbz case SIOCADDMULTI: 643194927Sbz case SIOCDELMULTI: 644194927Sbz error = 0; 645194927Sbz break; 646194927Sbz 647222246Szec case SIOCSIFMEDIA: 648222246Szec case SIOCGIFMEDIA: 649222246Szec sc = ifp->if_softc; 650222246Szec error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd); 651222246Szec break; 652222246Szec 653195892Sbz case SIOCSIFMTU: 654195892Sbz /* We basically allow all kinds of MTUs. */ 655195892Sbz ifp->if_mtu = ifr->ifr_mtu; 656195892Sbz error = 0; 657195892Sbz break; 658195892Sbz 659194927Sbz default: 660194927Sbz /* Let the common ethernet handler process this. */ 661194927Sbz error = ether_ioctl(ifp, cmd, data); 662194927Sbz break; 663194927Sbz } 664194927Sbz 665194927Sbz return (error); 666194927Sbz} 667194927Sbz 668194927Sbzstatic void 669194927Sbzepair_init(void *dummy __unused) 670194927Sbz{ 671194927Sbz} 672194927Sbz 673194927Sbz 674194927Sbz/* 675194927Sbz * Interface cloning functions. 676194927Sbz * We use our private ones so that we can create/destroy our secondary 677194927Sbz * device along with the primary one. 678194927Sbz */ 679194927Sbzstatic int 680194927Sbzepair_clone_match(struct if_clone *ifc, const char *name) 681194927Sbz{ 682194927Sbz const char *cp; 683194927Sbz 684194927Sbz DPRINTF("name='%s'\n", name); 685194927Sbz 686194927Sbz /* 687194927Sbz * Our base name is epair. 688194927Sbz * Our interfaces will be named epair<n>[ab]. 689194927Sbz * So accept anything of the following list: 690194927Sbz * - epair 691194927Sbz * - epair<n> 692194927Sbz * but not the epair<n>[ab] versions. 693194927Sbz */ 694241610Sglebius if (strncmp(epairname, name, sizeof(epairname)-1) != 0) 695194927Sbz return (0); 696194927Sbz 697241610Sglebius for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) { 698194927Sbz if (*cp < '0' || *cp > '9') 699194927Sbz return (0); 700194927Sbz } 701194927Sbz 702194927Sbz return (1); 703194927Sbz} 704194927Sbz 705194927Sbzstatic int 706194927Sbzepair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) 707194927Sbz{ 708194927Sbz struct epair_softc *sca, *scb; 709194927Sbz struct ifnet *ifp; 710194927Sbz char *dp; 711194927Sbz int error, unit, wildcard; 712194927Sbz uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 713194927Sbz 714194927Sbz /* 715194927Sbz * We are abusing params to create our second interface. 716241610Sglebius * Actually we already created it and called if_clone_create() 717194927Sbz * for it to do the official insertion procedure the moment we knew 718194927Sbz * it cannot fail anymore. So just do attach it here. 719194927Sbz */ 720194927Sbz if (params) { 721194927Sbz scb = (struct epair_softc *)params; 722194927Sbz ifp = scb->ifp; 723194927Sbz /* Assign a hopefully unique, locally administered etheraddr. */ 724194927Sbz eaddr[0] = 0x02; 725194927Sbz eaddr[3] = (ifp->if_index >> 8) & 0xff; 726194927Sbz eaddr[4] = ifp->if_index & 0xff; 727194927Sbz eaddr[5] = 0x0b; 728194927Sbz ether_ifattach(ifp, eaddr); 729194927Sbz /* Correctly set the name for the cloner list. */ 730194927Sbz strlcpy(name, scb->ifp->if_xname, len); 731194927Sbz return (0); 732194927Sbz } 733194927Sbz 734194927Sbz /* Try to see if a special unit was requested. */ 735194927Sbz error = ifc_name2unit(name, &unit); 736194927Sbz if (error != 0) 737194927Sbz return (error); 738194927Sbz wildcard = (unit < 0); 739194927Sbz 740194927Sbz error = ifc_alloc_unit(ifc, &unit); 741194927Sbz if (error != 0) 742194927Sbz return (error); 743194927Sbz 744194927Sbz /* 745194927Sbz * If no unit had been given, we need to adjust the ifName. 746194927Sbz * Also make sure there is space for our extra [ab] suffix. 747194927Sbz */ 748194927Sbz for (dp = name; *dp != '\0'; dp++); 749194927Sbz if (wildcard) { 750194927Sbz error = snprintf(dp, len - (dp - name), "%d", unit); 751194927Sbz if (error > len - (dp - name) - 1) { 752194927Sbz /* ifName too long. */ 753194927Sbz ifc_free_unit(ifc, unit); 754194927Sbz return (ENOSPC); 755194927Sbz } 756194927Sbz dp += error; 757194927Sbz } 758194927Sbz if (len - (dp - name) - 1 < 1) { 759194927Sbz /* No space left for our [ab] suffix. */ 760194927Sbz ifc_free_unit(ifc, unit); 761194927Sbz return (ENOSPC); 762194927Sbz } 763287594Shrs *dp = 'b'; 764194927Sbz /* Must not change dp so we can replace 'a' by 'b' later. */ 765194927Sbz *(dp+1) = '\0'; 766194927Sbz 767287594Shrs /* Check if 'a' and 'b' interfaces already exist. */ 768287594Shrs if (ifunit(name) != NULL) 769287594Shrs return (EEXIST); 770287594Shrs *dp = 'a'; 771287594Shrs if (ifunit(name) != NULL) 772287594Shrs return (EEXIST); 773287594Shrs 774194927Sbz /* Allocate memory for both [ab] interfaces */ 775194927Sbz sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 776195892Sbz EPAIR_REFCOUNT_INIT(&sca->refcount, 1); 777194927Sbz sca->ifp = if_alloc(IFT_ETHER); 778194927Sbz if (sca->ifp == NULL) { 779194927Sbz free(sca, M_EPAIR); 780194927Sbz ifc_free_unit(ifc, unit); 781194927Sbz return (ENOSPC); 782194927Sbz } 783194927Sbz 784194927Sbz scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 785195892Sbz EPAIR_REFCOUNT_INIT(&scb->refcount, 1); 786194927Sbz scb->ifp = if_alloc(IFT_ETHER); 787194927Sbz if (scb->ifp == NULL) { 788194927Sbz free(scb, M_EPAIR); 789194927Sbz if_free(sca->ifp); 790194927Sbz free(sca, M_EPAIR); 791194927Sbz ifc_free_unit(ifc, unit); 792194927Sbz return (ENOSPC); 793194927Sbz } 794194927Sbz 795194927Sbz /* 796194927Sbz * Cross-reference the interfaces so we will be able to free both. 797194927Sbz */ 798194927Sbz sca->oifp = scb->ifp; 799194927Sbz scb->oifp = sca->ifp; 800195892Sbz 801195892Sbz /* 802195892Sbz * Calculate the cpuid for netisr queueing based on the 803195892Sbz * ifIndex of the interfaces. As long as we cannot configure 804195892Sbz * this or use cpuset information easily we cannot guarantee 805195892Sbz * cache locality but we can at least allow parallelism. 806195892Sbz */ 807195892Sbz sca->cpuid = 808195892Sbz netisr_get_cpuid(sca->ifp->if_index % netisr_get_cpucount()); 809195892Sbz scb->cpuid = 810195892Sbz netisr_get_cpuid(scb->ifp->if_index % netisr_get_cpucount()); 811287594Shrs 812287594Shrs /* Initialise pseudo media types. */ 813287594Shrs ifmedia_init(&sca->media, 0, epair_media_change, epair_media_status); 814287594Shrs ifmedia_add(&sca->media, IFM_ETHER | IFM_10G_T, 0, NULL); 815287594Shrs ifmedia_set(&sca->media, IFM_ETHER | IFM_10G_T); 816287594Shrs ifmedia_init(&scb->media, 0, epair_media_change, epair_media_status); 817287594Shrs ifmedia_add(&scb->media, IFM_ETHER | IFM_10G_T, 0, NULL); 818287594Shrs ifmedia_set(&scb->media, IFM_ETHER | IFM_10G_T); 819194927Sbz 820194927Sbz /* Finish initialization of interface <n>a. */ 821194927Sbz ifp = sca->ifp; 822194927Sbz ifp->if_softc = sca; 823194927Sbz strlcpy(ifp->if_xname, name, IFNAMSIZ); 824241610Sglebius ifp->if_dname = epairname; 825194927Sbz ifp->if_dunit = unit; 826194927Sbz ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 827222247Szec ifp->if_capabilities = IFCAP_VLAN_MTU; 828222247Szec ifp->if_capenable = IFCAP_VLAN_MTU; 829194927Sbz ifp->if_start = epair_start; 830194927Sbz ifp->if_ioctl = epair_ioctl; 831194927Sbz ifp->if_init = epair_init; 832194927Sbz ifp->if_snd.ifq_maxlen = ifqmaxlen; 833194927Sbz /* Assign a hopefully unique, locally administered etheraddr. */ 834194927Sbz eaddr[0] = 0x02; 835194927Sbz eaddr[3] = (ifp->if_index >> 8) & 0xff; 836194927Sbz eaddr[4] = ifp->if_index & 0xff; 837194927Sbz eaddr[5] = 0x0a; 838194927Sbz ether_ifattach(ifp, eaddr); 839194927Sbz sca->if_qflush = ifp->if_qflush; 840194927Sbz ifp->if_qflush = epair_qflush; 841194927Sbz ifp->if_transmit = epair_transmit; 842241677Sglebius if_initbaudrate(ifp, IF_Gbps(10)); /* arbitrary maximum */ 843194927Sbz 844194927Sbz /* Swap the name and finish initialization of interface <n>b. */ 845194927Sbz *dp = 'b'; 846194927Sbz 847194927Sbz ifp = scb->ifp; 848194927Sbz ifp->if_softc = scb; 849194927Sbz strlcpy(ifp->if_xname, name, IFNAMSIZ); 850241610Sglebius ifp->if_dname = epairname; 851194927Sbz ifp->if_dunit = unit; 852194927Sbz ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 853222247Szec ifp->if_capabilities = IFCAP_VLAN_MTU; 854222247Szec ifp->if_capenable = IFCAP_VLAN_MTU; 855194927Sbz ifp->if_start = epair_start; 856194927Sbz ifp->if_ioctl = epair_ioctl; 857194927Sbz ifp->if_init = epair_init; 858194927Sbz ifp->if_snd.ifq_maxlen = ifqmaxlen; 859194927Sbz /* We need to play some tricks here for the second interface. */ 860241610Sglebius strlcpy(name, epairname, len); 861194927Sbz error = if_clone_create(name, len, (caddr_t)scb); 862194927Sbz if (error) 863241610Sglebius panic("%s: if_clone_create() for our 2nd iface failed: %d", 864194927Sbz __func__, error); 865194927Sbz scb->if_qflush = ifp->if_qflush; 866194927Sbz ifp->if_qflush = epair_qflush; 867194927Sbz ifp->if_transmit = epair_transmit; 868241677Sglebius if_initbaudrate(ifp, IF_Gbps(10)); /* arbitrary maximum */ 869194927Sbz 870194927Sbz /* 871194927Sbz * Restore name to <n>a as the ifp for this will go into the 872194927Sbz * cloner list for the initial call. 873194927Sbz */ 874194927Sbz strlcpy(name, sca->ifp->if_xname, len); 875194927Sbz DPRINTF("name='%s/%db' created sca=%p scb=%p\n", name, unit, sca, scb); 876194927Sbz 877194927Sbz /* Tell the world, that we are ready to rock. */ 878194927Sbz sca->ifp->if_drv_flags |= IFF_DRV_RUNNING; 879194927Sbz scb->ifp->if_drv_flags |= IFF_DRV_RUNNING; 880211904Sbz if_link_state_change(sca->ifp, LINK_STATE_UP); 881211904Sbz if_link_state_change(scb->ifp, LINK_STATE_UP); 882194927Sbz 883194927Sbz return (0); 884194927Sbz} 885194927Sbz 886194927Sbzstatic int 887194927Sbzepair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) 888194927Sbz{ 889194927Sbz struct ifnet *oifp; 890194927Sbz struct epair_softc *sca, *scb; 891194927Sbz int unit, error; 892194927Sbz 893194927Sbz DPRINTF("ifp=%p\n", ifp); 894194927Sbz 895194927Sbz /* 896194927Sbz * In case we called into if_clone_destroyif() ourselves 897194927Sbz * again to remove the second interface, the softc will be 898194927Sbz * NULL. In that case so not do anything but return success. 899194927Sbz */ 900194927Sbz if (ifp->if_softc == NULL) 901194927Sbz return (0); 902194927Sbz 903194927Sbz unit = ifp->if_dunit; 904194927Sbz sca = ifp->if_softc; 905194927Sbz oifp = sca->oifp; 906194927Sbz scb = oifp->if_softc; 907194927Sbz 908194927Sbz DPRINTF("ifp=%p oifp=%p\n", ifp, oifp); 909211904Sbz if_link_state_change(ifp, LINK_STATE_DOWN); 910211904Sbz if_link_state_change(oifp, LINK_STATE_DOWN); 911194927Sbz ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 912194927Sbz oifp->if_drv_flags &= ~IFF_DRV_RUNNING; 913238309Strociny 914238309Strociny /* 915238309Strociny * Get rid of our second half. As the other of the two 916238309Strociny * interfaces may reside in a different vnet, we need to 917238309Strociny * switch before freeing them. 918238309Strociny */ 919238309Strociny CURVNET_SET_QUIET(oifp->if_vnet); 920194927Sbz ether_ifdetach(oifp); 921194927Sbz /* 922194927Sbz * Wait for all packets to be dispatched to if_input. 923238309Strociny * The numbers can only go down as the interface is 924194927Sbz * detached so there is no need to use atomics. 925194927Sbz */ 926238309Strociny DPRINTF("scb refcnt=%u\n", scb->refcount); 927238309Strociny EPAIR_REFCOUNT_ASSERT(scb->refcount == 1, 928238309Strociny ("%s: ifp=%p scb->refcount!=1: %d", __func__, oifp, scb->refcount)); 929194927Sbz oifp->if_softc = NULL; 930194927Sbz error = if_clone_destroyif(ifc, oifp); 931194927Sbz if (error) 932194927Sbz panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", 933194927Sbz __func__, error); 934238309Strociny if_free(oifp); 935238309Strociny ifmedia_removeall(&scb->media); 936238309Strociny free(scb, M_EPAIR); 937238309Strociny CURVNET_RESTORE(); 938194927Sbz 939238309Strociny ether_ifdetach(ifp); 940195892Sbz /* 941238309Strociny * Wait for all packets to be dispatched to if_input. 942195892Sbz */ 943238309Strociny DPRINTF("sca refcnt=%u\n", sca->refcount); 944238309Strociny EPAIR_REFCOUNT_ASSERT(sca->refcount == 1, 945238309Strociny ("%s: ifp=%p sca->refcount!=1: %d", __func__, ifp, sca->refcount)); 946212152Sbz if_free(ifp); 947222246Szec ifmedia_removeall(&sca->media); 948194927Sbz free(sca, M_EPAIR); 949194927Sbz ifc_free_unit(ifc, unit); 950194927Sbz 951194927Sbz return (0); 952194927Sbz} 953194927Sbz 954287594Shrsstatic void 955287594Shrsvnet_epair_init(const void *unused __unused) 956287594Shrs{ 957287594Shrs 958287594Shrs V_epair_cloner = if_clone_advanced(epairname, 0, 959287594Shrs epair_clone_match, epair_clone_create, epair_clone_destroy); 960287594Shrs} 961287594ShrsVNET_SYSINIT(vnet_epair_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, 962287594Shrs vnet_epair_init, NULL); 963287594Shrs 964287594Shrsstatic void 965287594Shrsvnet_epair_uninit(const void *unused __unused) 966287594Shrs{ 967287594Shrs 968287594Shrs if_clone_detach(V_epair_cloner); 969287594Shrs} 970287594ShrsVNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, 971287594Shrs vnet_epair_uninit, NULL); 972287594Shrs 973194927Sbzstatic int 974194927Sbzepair_modevent(module_t mod, int type, void *data) 975194927Sbz{ 976195892Sbz int qlimit; 977194927Sbz 978194927Sbz switch (type) { 979194927Sbz case MOD_LOAD: 980194927Sbz /* For now limit us to one global mutex and one inq. */ 981195892Sbz epair_dpcpu_init(); 982195892Sbz epair_nh.nh_qlimit = 42 * ifqmaxlen; /* 42 shall be the number. */ 983195892Sbz if (TUNABLE_INT_FETCH("net.link.epair.netisr_maxqlen", &qlimit)) 984195892Sbz epair_nh.nh_qlimit = qlimit; 985195892Sbz netisr_register(&epair_nh); 986194927Sbz if (bootverbose) 987241610Sglebius printf("%s initialized.\n", epairname); 988194927Sbz break; 989194927Sbz case MOD_UNLOAD: 990195892Sbz netisr_unregister(&epair_nh); 991195892Sbz epair_dpcpu_detach(); 992194927Sbz if (bootverbose) 993241610Sglebius printf("%s unloaded.\n", epairname); 994194927Sbz break; 995194927Sbz default: 996194927Sbz return (EOPNOTSUPP); 997194927Sbz } 998194927Sbz return (0); 999194927Sbz} 1000194927Sbz 1001194927Sbzstatic moduledata_t epair_mod = { 1002194927Sbz "if_epair", 1003194927Sbz epair_modevent, 1004241394Skevlo 0 1005194927Sbz}; 1006194927Sbz 1007194927SbzDECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1008194927SbzMODULE_VERSION(if_epair, 1); 1009