1/* 2 * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26/* 27 * $FreeBSD$ 28 * 29 * The header contains the definitions of constants and function 30 * prototypes used only in kernelspace. 31 */ 32 33#ifndef _NET_NETMAP_KERN_H_ 34#define _NET_NETMAP_KERN_H_ 35 36#if defined(__FreeBSD__) 37 38#define likely(x) __builtin_expect(!!(x), 1) 39#define unlikely(x) __builtin_expect(!!(x), 0) 40 41#define NM_LOCK_T struct mtx 42#define NM_RWLOCK_T struct rwlock 43#define NM_SELINFO_T struct selinfo 44#define MBUF_LEN(m) ((m)->m_pkthdr.len) 45#define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) 46 47#elif defined (linux) 48 49#define NM_LOCK_T safe_spinlock_t // see bsd_glue.h 50#define NM_RWLOCK_T safe_spinlock_t // see bsd_glue.h 51#define NM_SELINFO_T wait_queue_head_t 52#define MBUF_LEN(m) ((m)->len) 53#define NM_SEND_UP(ifp, m) netif_rx(m) 54 55#ifndef DEV_NETMAP 56#define DEV_NETMAP 57#endif 58 59/* 60 * IFCAP_NETMAP goes into net_device's priv_flags (if_capenable). 61 * This was 16 bits up to linux 2.6.36, so we need a 16 bit value on older 62 * platforms and tolerate the clash with IFF_DYNAMIC and IFF_BRIDGE_PORT. 63 * For the 32-bit value, 0x100000 has no clashes until at least 3.5.1 64 */ 65#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) 66#define IFCAP_NETMAP 0x8000 67#else 68#define IFCAP_NETMAP 0x200000 69#endif 70 71#elif defined (__APPLE__) 72 73#warning apple support is incomplete. 74#define likely(x) __builtin_expect(!!(x), 1) 75#define unlikely(x) __builtin_expect(!!(x), 0) 76#define NM_LOCK_T IOLock * 77#define NM_SELINFO_T struct selinfo 78#define MBUF_LEN(m) ((m)->m_pkthdr.len) 79#define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) 80 81#else 82 83#error unsupported platform 84 85#endif /* end - platform-specific code */ 86 87#define ND(format, ...) 88#define D(format, ...) \ 89 do { \ 90 struct timeval __xxts; \ 91 microtime(&__xxts); \ 92 printf("%03d.%06d %s [%d] " format "\n", \ 93 (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 94 __FUNCTION__, __LINE__, ##__VA_ARGS__); \ 95 } while (0) 96 97/* rate limited, lps indicates how many per second */ 98#define RD(lps, format, ...) \ 99 do { \ 100 static int t0, __cnt; \ 101 if (t0 != time_second) { \ 102 t0 = time_second; \ 103 __cnt = 0; \ 104 } \ 105 if (__cnt++ < lps) \ 106 D(format, ##__VA_ARGS__); \ 107 } while (0) 108 109struct netmap_adapter; 110struct nm_bdg_fwd; 111struct nm_bridge; 112struct netmap_priv_d; 113 114/* 115 * private, kernel view of a ring. Keeps track of the status of 116 * a ring across system calls. 117 * 118 * nr_hwcur index of the next buffer to refill. 119 * It corresponds to ring->cur - ring->reserved 120 * 121 * nr_hwavail the number of slots "owned" by userspace. 122 * nr_hwavail =:= ring->avail + ring->reserved 123 * 124 * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 125 * This is so that, on a reset, buffers owned by userspace are not 126 * modified by the kernel. In particular: 127 * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with 128 * the next empty buffer as known by the hardware (next_to_check or so). 129 * TX rings: hwcur + hwofs coincides with next_to_send 130 * 131 * For received packets, slot->flags is set to nkr_slot_flags 132 * so we can provide a proper initial value (e.g. set NS_FORWARD 133 * when operating in 'transparent' mode). 134 */ 135struct netmap_kring { 136 struct netmap_ring *ring; 137 u_int nr_hwcur; 138 int nr_hwavail; 139 u_int nr_kflags; /* private driver flags */ 140#define NKR_PENDINTR 0x1 // Pending interrupt. 141 u_int nkr_num_slots; 142 143 uint16_t nkr_slot_flags; /* initial value for flags */ 144 int nkr_hwofs; /* offset between NIC and netmap ring */ 145 struct netmap_adapter *na; 146 struct nm_bdg_fwd *nkr_ft; 147 NM_SELINFO_T si; /* poll/select wait queue */ 148 NM_LOCK_T q_lock; /* used if no device lock available */ 149} __attribute__((__aligned__(64))); 150 151/* 152 * This struct extends the 'struct adapter' (or 153 * equivalent) device descriptor. It contains all fields needed to 154 * support netmap operation. 155 */ 156struct netmap_adapter { 157 /* 158 * On linux we do not have a good way to tell if an interface 159 * is netmap-capable. So we use the following trick: 160 * NA(ifp) points here, and the first entry (which hopefully 161 * always exists and is at least 32 bits) contains a magic 162 * value which we can use to detect that the interface is good. 163 */ 164 uint32_t magic; 165 uint32_t na_flags; /* future place for IFCAP_NETMAP */ 166#define NAF_SKIP_INTR 1 /* use the regular interrupt handler. 167 * useful during initialization 168 */ 169#define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ 170 int refcount; /* number of user-space descriptors using this 171 interface, which is equal to the number of 172 struct netmap_if objs in the mapped region. */ 173 /* 174 * The selwakeup in the interrupt thread can use per-ring 175 * and/or global wait queues. We track how many clients 176 * of each type we have so we can optimize the drivers, 177 * and especially avoid huge contention on the locks. 178 */ 179 int na_single; /* threads attached to a single hw queue */ 180 int na_multi; /* threads attached to multiple hw queues */ 181 182 int separate_locks; /* set if the interface suports different 183 locks for rx, tx and core. */ 184 185 u_int num_rx_rings; /* number of adapter receive rings */ 186 u_int num_tx_rings; /* number of adapter transmit rings */ 187 188 u_int num_tx_desc; /* number of descriptor in each queue */ 189 u_int num_rx_desc; 190 191 /* tx_rings and rx_rings are private but allocated 192 * as a contiguous chunk of memory. Each array has 193 * N+1 entries, for the adapter queues and for the host queue. 194 */ 195 struct netmap_kring *tx_rings; /* array of TX rings. */ 196 struct netmap_kring *rx_rings; /* array of RX rings. */ 197 198 NM_SELINFO_T tx_si, rx_si; /* global wait queues */ 199 200 /* copy of if_qflush and if_transmit pointers, to intercept 201 * packets from the network stack when netmap is active. 202 */ 203 int (*if_transmit)(struct ifnet *, struct mbuf *); 204 205 /* references to the ifnet and device routines, used by 206 * the generic netmap functions. 207 */ 208 struct ifnet *ifp; /* adapter is ifp->if_softc */ 209 210 NM_LOCK_T core_lock; /* used if no device lock available */ 211 212 int (*nm_register)(struct ifnet *, int onoff); 213 void (*nm_lock)(struct ifnet *, int what, u_int ringid); 214 int (*nm_txsync)(struct ifnet *, u_int ring, int lock); 215 int (*nm_rxsync)(struct ifnet *, u_int ring, int lock); 216 /* return configuration information */ 217 int (*nm_config)(struct ifnet *, u_int *txr, u_int *txd, 218 u_int *rxr, u_int *rxd); 219 220 /* 221 * Bridge support: 222 * 223 * bdg_port is the port number used in the bridge; 224 * na_bdg_refcount is a refcount used for bridge ports, 225 * when it goes to 0 we can detach+free this port 226 * (a bridge port is always attached if it exists; 227 * it is not always registered) 228 * na_bdg points to the bridge this NA is attached to. 229 */ 230 int bdg_port; 231 int na_bdg_refcount; 232 struct nm_bridge *na_bdg; 233 /* When we attach a physical interface to the bridge, we 234 * allow the controlling process to terminate, so we need 235 * a place to store the netmap_priv_d data structure. 236 * This is only done when physical interfaces are attached to a bridge. 237 */ 238 struct netmap_priv_d *na_kpriv; 239#ifdef linux 240 struct net_device_ops nm_ndo; 241#endif /* linux */ 242}; 243 244/* 245 * The combination of "enable" (ifp->if_capenable & IFCAP_NETMAP) 246 * and refcount gives the status of the interface, namely: 247 * 248 * enable refcount Status 249 * 250 * FALSE 0 normal operation 251 * FALSE != 0 -- (impossible) 252 * TRUE 1 netmap mode 253 * TRUE 0 being deleted. 254 */ 255 256#define NETMAP_DELETING(_na) ( ((_na)->refcount == 0) && \ 257 ( (_na)->ifp->if_capenable & IFCAP_NETMAP) ) 258 259/* 260 * parameters for (*nm_lock)(adapter, what, index) 261 */ 262enum { 263 NETMAP_NO_LOCK = 0, 264 NETMAP_CORE_LOCK, NETMAP_CORE_UNLOCK, 265 NETMAP_TX_LOCK, NETMAP_TX_UNLOCK, 266 NETMAP_RX_LOCK, NETMAP_RX_UNLOCK, 267#ifdef __FreeBSD__ 268#define NETMAP_REG_LOCK NETMAP_CORE_LOCK 269#define NETMAP_REG_UNLOCK NETMAP_CORE_UNLOCK 270#else 271 NETMAP_REG_LOCK, NETMAP_REG_UNLOCK 272#endif 273}; 274 275/* How to handle locking support in netmap_rx_irq/netmap_tx_irq */ 276#define NETMAP_LOCKED_ENTER 0x10000000 /* already locked on enter */ 277#define NETMAP_LOCKED_EXIT 0x20000000 /* keep locked on exit */ 278 279/* 280 * The following are support routines used by individual drivers to 281 * support netmap operation. 282 * 283 * netmap_attach() initializes a struct netmap_adapter, allocating the 284 * struct netmap_ring's and the struct selinfo. 285 * 286 * netmap_detach() frees the memory allocated by netmap_attach(). 287 * 288 * netmap_start() replaces the if_transmit routine of the interface, 289 * and is used to intercept packets coming from the stack. 290 * 291 * netmap_load_map/netmap_reload_map are helper routines to set/reset 292 * the dmamap for a packet buffer 293 * 294 * netmap_reset() is a helper routine to be called in the driver 295 * when reinitializing a ring. 296 */ 297int netmap_attach(struct netmap_adapter *, int); 298void netmap_detach(struct ifnet *); 299int netmap_start(struct ifnet *, struct mbuf *); 300enum txrx { NR_RX = 0, NR_TX = 1 }; 301struct netmap_slot *netmap_reset(struct netmap_adapter *na, 302 enum txrx tx, int n, u_int new_cur); 303int netmap_ring_reinit(struct netmap_kring *); 304 305/* 306 * The following bridge-related interfaces are used by other kernel modules 307 * In the version that only supports unicast or broadcast, the lookup 308 * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports, 309 * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 for unknown. 310 * XXX in practice "unknown" might be handled same as broadcast. 311 */ 312typedef u_int (*bdg_lookup_fn_t)(char *buf, u_int len, uint8_t *ring_nr, 313 struct netmap_adapter *); 314int netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func); 315u_int netmap_bdg_learning(char *, u_int, uint8_t *, struct netmap_adapter *); 316#define NM_NAME "vale" /* prefix for the bridge port name */ 317#define NM_BDG_MAXPORTS 254 /* up to 32 for bitmap, 254 ok otherwise */ 318#define NM_BDG_BROADCAST NM_BDG_MAXPORTS 319#define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1) 320 321extern u_int netmap_buf_size; 322#define NETMAP_BUF_SIZE netmap_buf_size // XXX remove 323extern int netmap_mitigate; 324extern int netmap_no_pendintr; 325extern u_int netmap_total_buffers; 326extern char *netmap_buffer_base; 327extern int netmap_verbose; // XXX debugging 328enum { /* verbose flags */ 329 NM_VERB_ON = 1, /* generic verbose */ 330 NM_VERB_HOST = 0x2, /* verbose host stack */ 331 NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */ 332 NM_VERB_TXSYNC = 0x20, 333 NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */ 334 NM_VERB_TXINTR = 0x200, 335 NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */ 336 NM_VERB_NIC_TXSYNC = 0x2000, 337}; 338 339/* 340 * NA returns a pointer to the struct netmap adapter from the ifp, 341 * WNA is used to write it. 342 * SWNA() is used for the "host stack" endpoint associated 343 * to an interface. It is allocated together with the main NA(), 344 * as an array of two objects. 345 */ 346#ifndef WNA 347#define WNA(_ifp) (_ifp)->if_pspare[0] 348#endif 349#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 350#define SWNA(_ifp) (NA(_ifp) + 1) 351 352/* 353 * Macros to determine if an interface is netmap capable or netmap enabled. 354 * See the magic field in struct netmap_adapter. 355 */ 356#ifdef __FreeBSD__ 357/* 358 * on FreeBSD just use if_capabilities and if_capenable. 359 */ 360#define NETMAP_CAPABLE(ifp) (NA(ifp) && \ 361 (ifp)->if_capabilities & IFCAP_NETMAP ) 362 363#define NETMAP_SET_CAPABLE(ifp) \ 364 (ifp)->if_capabilities |= IFCAP_NETMAP 365 366#else /* linux */ 367 368/* 369 * on linux: 370 * we check if NA(ifp) is set and its first element has a related 371 * magic value. The capenable is within the struct netmap_adapter. 372 */ 373#define NETMAP_MAGIC 0x52697a7a 374 375#define NETMAP_CAPABLE(ifp) (NA(ifp) && \ 376 ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) 377 378#define NETMAP_SET_CAPABLE(ifp) \ 379 NA(ifp)->magic = ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC 380 381#endif /* linux */ 382 383#ifdef __FreeBSD__ 384/* Callback invoked by the dma machinery after a successfull dmamap_load */ 385static void netmap_dmamap_cb(__unused void *arg, 386 __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 387{ 388} 389 390/* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 391 * XXX can we do it without a callback ? 392 */ 393static inline void 394netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 395{ 396 if (map) 397 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 398 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 399} 400 401/* update the map when a buffer changes. */ 402static inline void 403netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 404{ 405 if (map) { 406 bus_dmamap_unload(tag, map); 407 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 408 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 409 } 410} 411#else /* linux */ 412 413/* 414 * XXX How do we redefine these functions: 415 * 416 * on linux we need 417 * dma_map_single(&pdev->dev, virt_addr, len, direction) 418 * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction 419 * The len can be implicit (on netmap it is NETMAP_BUF_SIZE) 420 * unfortunately the direction is not, so we need to change 421 * something to have a cross API 422 */ 423#define netmap_load_map(_t, _m, _b) 424#define netmap_reload_map(_t, _m, _b) 425#if 0 426 struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; 427 /* set time_stamp *before* dma to help avoid a possible race */ 428 buffer_info->time_stamp = jiffies; 429 buffer_info->mapped_as_page = false; 430 buffer_info->length = len; 431 //buffer_info->next_to_watch = l; 432 /* reload dma map */ 433 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 434 NETMAP_BUF_SIZE, DMA_TO_DEVICE); 435 buffer_info->dma = dma_map_single(&adapter->pdev->dev, 436 addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); 437 438 if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 439 D("dma mapping error"); 440 /* goto dma_error; See e1000_put_txbuf() */ 441 /* XXX reset */ 442 } 443 tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX 444 445#endif 446 447/* 448 * The bus_dmamap_sync() can be one of wmb() or rmb() depending on direction. 449 */ 450#define bus_dmamap_sync(_a, _b, _c) 451 452#endif /* linux */ 453 454/* 455 * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 456 */ 457static inline int 458netmap_idx_n2k(struct netmap_kring *kr, int idx) 459{ 460 int n = kr->nkr_num_slots; 461 idx += kr->nkr_hwofs; 462 if (idx < 0) 463 return idx + n; 464 else if (idx < n) 465 return idx; 466 else 467 return idx - n; 468} 469 470 471static inline int 472netmap_idx_k2n(struct netmap_kring *kr, int idx) 473{ 474 int n = kr->nkr_num_slots; 475 idx -= kr->nkr_hwofs; 476 if (idx < 0) 477 return idx + n; 478 else if (idx < n) 479 return idx; 480 else 481 return idx - n; 482} 483 484 485/* Entries of the look-up table. */ 486struct lut_entry { 487 void *vaddr; /* virtual address. */ 488 vm_paddr_t paddr; /* physical address. */ 489}; 490 491struct netmap_obj_pool; 492extern struct lut_entry *netmap_buffer_lut; 493#define NMB_VA(i) (netmap_buffer_lut[i].vaddr) 494#define NMB_PA(i) (netmap_buffer_lut[i].paddr) 495 496/* 497 * NMB return the virtual address of a buffer (buffer 0 on bad index) 498 * PNMB also fills the physical address 499 */ 500static inline void * 501NMB(struct netmap_slot *slot) 502{ 503 uint32_t i = slot->buf_idx; 504 return (unlikely(i >= netmap_total_buffers)) ? NMB_VA(0) : NMB_VA(i); 505} 506 507static inline void * 508PNMB(struct netmap_slot *slot, uint64_t *pp) 509{ 510 uint32_t i = slot->buf_idx; 511 void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i); 512 513 *pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i); 514 return ret; 515} 516 517/* default functions to handle rx/tx interrupts */ 518int netmap_rx_irq(struct ifnet *, int, int *); 519#define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 520 521#endif /* _NET_NETMAP_KERN_H_ */ 522