1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp * 27218792Snp * $FreeBSD$ 28218792Snp * 29218792Snp */ 30218792Snp 31218792Snp#ifndef __T4_ADAPTER_H__ 32218792Snp#define __T4_ADAPTER_H__ 33218792Snp 34228561Snp#include <sys/kernel.h> 35218792Snp#include <sys/bus.h> 36218792Snp#include <sys/rman.h> 37218792Snp#include <sys/types.h> 38218792Snp#include <sys/malloc.h> 39218792Snp#include <dev/pci/pcivar.h> 40218792Snp#include <dev/pci/pcireg.h> 41218792Snp#include <machine/bus.h> 42218792Snp#include <sys/socket.h> 43218792Snp#include <sys/sysctl.h> 44218792Snp#include <net/ethernet.h> 45218792Snp#include <net/if.h> 46218792Snp#include <net/if_media.h> 47235944Sbz#include <netinet/in.h> 48218792Snp#include <netinet/tcp_lro.h> 49218792Snp 50218792Snp#include "offload.h" 51228561Snp#include "firmware/t4fw_interface.h" 52218792Snp 53218792SnpMALLOC_DECLARE(M_CXGBE); 54218792Snp#define CXGBE_UNIMPLEMENTED(s) \ 55218792Snp panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__) 56218792Snp 57218792Snp#if defined(__i386__) || defined(__amd64__) 58218792Snpstatic __inline void 59218792Snpprefetch(void *x) 60218792Snp{ 61218792Snp __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 62218792Snp} 63218792Snp#else 64218792Snp#define prefetch(x) 65218792Snp#endif 66218792Snp 67231115Snp#ifndef SYSCTL_ADD_UQUAD 68231115Snp#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD 69231115Snp#define sysctl_handle_64 sysctl_handle_quad 70231115Snp#define CTLTYPE_U64 CTLTYPE_QUAD 71231115Snp#endif 72231115Snp 73231115Snp#if (__FreeBSD_version >= 900030) || \ 74231115Snp ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000)) 75231115Snp#define SBUF_DRAIN 1 76231115Snp#endif 77231115Snp 78218792Snp#ifdef __amd64__ 79218792Snp/* XXX: need systemwide bus_space_read_8/bus_space_write_8 */ 80218792Snpstatic __inline uint64_t 81218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 82218792Snp bus_size_t offset) 83218792Snp{ 84219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 85219285Snp ("%s: can only handle mem space", __func__)); 86218792Snp 87218792Snp return (*(volatile uint64_t *)(handle + offset)); 88218792Snp} 89218792Snp 90218792Snpstatic __inline void 91218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 92218792Snp bus_size_t offset, uint64_t value) 93218792Snp{ 94219285Snp KASSERT(tag == X86_BUS_SPACE_MEM, 95219285Snp ("%s: can only handle mem space", __func__)); 96219285Snp 97218792Snp *(volatile uint64_t *)(bsh + offset) = value; 98218792Snp} 99218792Snp#else 100218792Snpstatic __inline uint64_t 101218792Snpt4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 102218792Snp bus_size_t offset) 103218792Snp{ 104218792Snp return (uint64_t)bus_space_read_4(tag, handle, offset) + 105218792Snp ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32); 106218792Snp} 107218792Snp 108218792Snpstatic __inline void 109218792Snpt4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, 110218792Snp bus_size_t offset, uint64_t value) 111218792Snp{ 112218792Snp bus_space_write_4(tag, bsh, offset, value); 113218792Snp bus_space_write_4(tag, bsh, offset + 4, value >> 32); 114218792Snp} 115218792Snp#endif 116218792Snp 117218792Snpstruct adapter; 118218792Snptypedef struct adapter adapter_t; 119218792Snp 120218792Snpenum { 121218792Snp FW_IQ_QSIZE = 256, 122218792Snp FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */ 123218792Snp 124218792Snp RX_IQ_QSIZE = 1024, 125218792Snp RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */ 126218792Snp 127228561Snp EQ_ESIZE = 64, /* All egress queues use this entry size */ 128218792Snp 129228561Snp RX_FL_ESIZE = EQ_ESIZE, /* 8 64bit addresses */ 130219392Snp#if MJUMPAGESIZE != MCLBYTES 131255050Snp FL_BUF_SIZES_MAX = 5, /* cluster, jumbop, jumbo9k, jumbo16k, extra */ 132219392Snp#else 133255050Snp FL_BUF_SIZES_MAX = 4, /* cluster, jumbo9k, jumbo16k, extra */ 134219392Snp#endif 135218792Snp 136228561Snp CTRL_EQ_QSIZE = 128, 137228561Snp 138218792Snp TX_EQ_QSIZE = 1024, 139218792Snp TX_SGL_SEGS = 36, 140218792Snp TX_WR_FLITS = SGE_MAX_WR_LEN / 8 141218792Snp}; 142218792Snp 143218792Snpenum { 144219944Snp /* adapter intr_type */ 145219944Snp INTR_INTX = (1 << 0), 146219944Snp INTR_MSI = (1 << 1), 147219944Snp INTR_MSIX = (1 << 2) 148219944Snp}; 149219944Snp 150219944Snpenum { 151245274Snp /* flags understood by begin_synchronized_op */ 152245274Snp HOLD_LOCK = (1 << 0), 153245274Snp SLEEP_OK = (1 << 1), 154245274Snp INTR_OK = (1 << 2), 155245274Snp 156245274Snp /* flags understood by end_synchronized_op */ 157245274Snp LOCK_HELD = HOLD_LOCK, 158245274Snp}; 159245274Snp 160245274Snpenum { 161218792Snp /* adapter flags */ 162218792Snp FULL_INIT_DONE = (1 << 0), 163218792Snp FW_OK = (1 << 1), 164228561Snp INTR_DIRECT = (1 << 2), /* direct interrupts for everything */ 165228561Snp MASTER_PF = (1 << 3), 166228561Snp ADAP_SYSCTL_CTX = (1 << 4), 167237263Snp TOM_INIT_DONE = (1 << 5), 168255050Snp BUF_PACKING_OK = (1 << 6), 169218792Snp 170218792Snp CXGBE_BUSY = (1 << 9), 171218792Snp 172218792Snp /* port flags */ 173218792Snp DOOMED = (1 << 0), 174228561Snp PORT_INIT_DONE = (1 << 1), 175228561Snp PORT_SYSCTL_CTX = (1 << 2), 176253691Snp HAS_TRACEQ = (1 << 3), 177218792Snp}; 178218792Snp 179245274Snp#define IS_DOOMED(pi) ((pi)->flags & DOOMED) 180245274Snp#define SET_DOOMED(pi) do {(pi)->flags |= DOOMED;} while (0) 181245274Snp#define IS_BUSY(sc) ((sc)->flags & CXGBE_BUSY) 182245274Snp#define SET_BUSY(sc) do {(sc)->flags |= CXGBE_BUSY;} while (0) 183245274Snp#define CLR_BUSY(sc) do {(sc)->flags &= ~CXGBE_BUSY;} while (0) 184218792Snp 185218792Snpstruct port_info { 186218792Snp device_t dev; 187218792Snp struct adapter *adapter; 188218792Snp 189218792Snp struct ifnet *ifp; 190218792Snp struct ifmedia media; 191218792Snp 192218792Snp struct mtx pi_lock; 193218792Snp char lockname[16]; 194218792Snp unsigned long flags; 195218792Snp int if_flags; 196218792Snp 197218792Snp uint16_t viid; 198218792Snp int16_t xact_addr_filt;/* index of exact MAC address filter */ 199218792Snp uint16_t rss_size; /* size of VI's RSS table slice */ 200218792Snp uint8_t lport; /* associated offload logical port */ 201218792Snp int8_t mdio_addr; 202218792Snp uint8_t port_type; 203218792Snp uint8_t mod_type; 204218792Snp uint8_t port_id; 205218792Snp uint8_t tx_chan; 206218792Snp 207218792Snp /* These need to be int as they are used in sysctl */ 208218792Snp int ntxq; /* # of tx queues */ 209218792Snp int first_txq; /* index of first tx queue */ 210218792Snp int nrxq; /* # of rx queues */ 211218792Snp int first_rxq; /* index of first rx queue */ 212237263Snp#ifdef TCP_OFFLOAD 213228561Snp int nofldtxq; /* # of offload tx queues */ 214228561Snp int first_ofld_txq; /* index of first offload tx queue */ 215228561Snp int nofldrxq; /* # of offload rx queues */ 216228561Snp int first_ofld_rxq; /* index of first offload rx queue */ 217228561Snp#endif 218218792Snp int tmr_idx; 219218792Snp int pktc_idx; 220218792Snp int qsize_rxq; 221218792Snp int qsize_txq; 222218792Snp 223252747Snp int linkdnrc; 224218792Snp struct link_config link_cfg; 225218792Snp struct port_stats stats; 226218792Snp 227237263Snp eventhandler_tag vlan_c; 228237263Snp 229218792Snp struct callout tick; 230228561Snp struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */ 231218792Snp 232218792Snp uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */ 233218792Snp}; 234218792Snp 235218792Snpstruct fl_sdesc { 236218792Snp bus_dmamap_t map; 237218792Snp caddr_t cl; 238255050Snp uint8_t tag_idx; /* the fl->tag entry this map comes from */ 239218792Snp#ifdef INVARIANTS 240255050Snp __be64 ba_hwtag; 241218792Snp#endif 242218792Snp}; 243218792Snp 244218792Snpstruct tx_desc { 245218792Snp __be64 flit[8]; 246218792Snp}; 247218792Snp 248218792Snpstruct tx_map { 249218792Snp struct mbuf *m; 250218792Snp bus_dmamap_t map; 251218792Snp}; 252218792Snp 253228561Snp/* DMA maps used for tx */ 254228561Snpstruct tx_maps { 255228561Snp struct tx_map *maps; 256228561Snp uint32_t map_total; /* # of DMA maps */ 257228561Snp uint32_t map_pidx; /* next map to be used */ 258228561Snp uint32_t map_cidx; /* reclaimed up to this index */ 259228561Snp uint32_t map_avail; /* # of available maps */ 260228561Snp}; 261228561Snp 262218792Snpstruct tx_sdesc { 263218792Snp uint8_t desc_used; /* # of hardware descriptors used by the WR */ 264220873Snp uint8_t credits; /* NIC txq: # of frames sent out in the WR */ 265218792Snp}; 266218792Snp 267218792Snpenum { 268218792Snp /* iq flags */ 269228561Snp IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */ 270228561Snp IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */ 271228561Snp IQ_INTR = (1 << 2), /* iq takes direct interrupt */ 272228561Snp IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */ 273220649Snp 274220649Snp /* iq state */ 275220649Snp IQS_DISABLED = 0, 276220649Snp IQS_BUSY = 1, 277220649Snp IQS_IDLE = 2, 278218792Snp}; 279218792Snp 280218792Snp/* 281218792Snp * Ingress Queue: T4 is producer, driver is consumer. 282218792Snp */ 283218792Snpstruct sge_iq { 284218792Snp bus_dma_tag_t desc_tag; 285218792Snp bus_dmamap_t desc_map; 286219290Snp bus_addr_t ba; /* bus address of descriptor ring */ 287219290Snp uint32_t flags; 288219290Snp uint16_t abs_id; /* absolute SGE id for the iq */ 289219290Snp int8_t intr_pktc_idx; /* packet count threshold index */ 290219290Snp int8_t pad0; 291219290Snp __be64 *desc; /* KVA of descriptor ring */ 292219290Snp 293228561Snp volatile int state; 294218792Snp struct adapter *adapter; 295218792Snp const __be64 *cdesc; /* current descriptor */ 296218792Snp uint8_t gen; /* generation bit */ 297218792Snp uint8_t intr_params; /* interrupt holdoff parameters */ 298228561Snp uint8_t intr_next; /* XXX: holdoff for next interrupt */ 299218792Snp uint8_t esize; /* size (bytes) of each entry in the queue */ 300218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 301218792Snp uint16_t cidx; /* consumer index */ 302228561Snp uint16_t cntxt_id; /* SGE context id for the iq */ 303228561Snp 304228561Snp STAILQ_ENTRY(sge_iq) link; 305218792Snp}; 306218792Snp 307218792Snpenum { 308228561Snp EQ_CTRL = 1, 309228561Snp EQ_ETH = 2, 310237263Snp#ifdef TCP_OFFLOAD 311228561Snp EQ_OFLD = 3, 312228561Snp#endif 313228561Snp 314218792Snp /* eq flags */ 315228561Snp EQ_TYPEMASK = 7, /* 3 lsbits hold the type */ 316228561Snp EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */ 317228561Snp EQ_DOOMED = (1 << 4), /* about to be destroyed */ 318228561Snp EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */ 319228561Snp EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */ 320218792Snp}; 321218792Snp 322248925Snp/* Listed in order of preference. Update t4_sysctls too if you change these */ 323249392Snpenum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB}; 324248925Snp 325218792Snp/* 326218792Snp * Egress Queue: driver is producer, T4 is consumer. 327218792Snp * 328218792Snp * Note: A free list is an egress queue (driver produces the buffers and T4 329218792Snp * consumes them) but it's special enough to have its own struct (see sge_fl). 330218792Snp */ 331218792Snpstruct sge_eq { 332228561Snp unsigned int flags; /* MUST be first */ 333228561Snp unsigned int cntxt_id; /* SGE context id for the eq */ 334218792Snp bus_dma_tag_t desc_tag; 335218792Snp bus_dmamap_t desc_map; 336218792Snp char lockname[16]; 337218792Snp struct mtx eq_lock; 338218792Snp 339218792Snp struct tx_desc *desc; /* KVA of descriptor ring */ 340218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 341218792Snp struct sge_qstat *spg; /* status page, for convenience */ 342248925Snp int doorbells; 343248925Snp volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */ 344248925Snp u_int udb_qid; /* relative qid within the doorbell page */ 345218792Snp uint16_t cap; /* max # of desc, for convenience */ 346218792Snp uint16_t avail; /* available descriptors, for convenience */ 347218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 348218792Snp uint16_t cidx; /* consumer idx (desc idx) */ 349218792Snp uint16_t pidx; /* producer idx (desc idx) */ 350218792Snp uint16_t pending; /* # of descriptors used since last doorbell */ 351219288Snp uint16_t iqid; /* iq that gets egr_update for the eq */ 352228561Snp uint8_t tx_chan; /* tx channel used by the eq */ 353228561Snp struct task tx_task; 354228561Snp struct callout tx_callout; 355228561Snp 356228561Snp /* stats */ 357228561Snp 358228561Snp uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for eq */ 359228561Snp uint32_t unstalled; /* recovered from stall */ 360220873Snp}; 361218792Snp 362255050Snpstruct fl_buf_info { 363255050Snp u_int size; 364255050Snp int type; 365255050Snp int hwtag:4; /* tag in low 4 bits of the pa. */ 366255050Snp uma_zone_t zone; 367255050Snp}; 368255050Snp#define FL_BUF_SIZES(sc) (sc->sge.fl_buf_sizes) 369255050Snp#define FL_BUF_SIZE(sc, x) (sc->sge.fl_buf_info[x].size) 370255050Snp#define FL_BUF_TYPE(sc, x) (sc->sge.fl_buf_info[x].type) 371255050Snp#define FL_BUF_HWTAG(sc, x) (sc->sge.fl_buf_info[x].hwtag) 372255050Snp#define FL_BUF_ZONE(sc, x) (sc->sge.fl_buf_info[x].zone) 373255050Snp 374228561Snpenum { 375228561Snp FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */ 376228561Snp FL_DOOMED = (1 << 1), /* about to be destroyed */ 377255050Snp FL_BUF_PACKING = (1 << 2), /* buffer packing enabled */ 378228561Snp}; 379228561Snp 380228561Snp#define FL_RUNNING_LOW(fl) (fl->cap - fl->needed <= fl->lowat) 381228561Snp#define FL_NOT_RUNNING_LOW(fl) (fl->cap - fl->needed >= 2 * fl->lowat) 382228561Snp 383218792Snpstruct sge_fl { 384218792Snp bus_dma_tag_t desc_tag; 385218792Snp bus_dmamap_t desc_map; 386255050Snp bus_dma_tag_t tag[FL_BUF_SIZES_MAX]; /* only first FL_BUF_SIZES(sc) are 387255050Snp valid */ 388218792Snp uint8_t tag_idx; 389218792Snp struct mtx fl_lock; 390218792Snp char lockname[16]; 391228561Snp int flags; 392218792Snp 393218792Snp __be64 *desc; /* KVA of descriptor ring, ptr to addresses */ 394218792Snp bus_addr_t ba; /* bus address of descriptor ring */ 395218792Snp struct fl_sdesc *sdesc; /* KVA of software descriptor ring */ 396218792Snp uint32_t cap; /* max # of buffers, for convenience */ 397218792Snp uint16_t qsize; /* size (# of entries) of the queue */ 398218792Snp uint16_t cntxt_id; /* SGE context id for the freelist */ 399218792Snp uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */ 400255050Snp uint32_t rx_offset; /* offset in fl buf (when buffer packing) */ 401218792Snp uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */ 402218792Snp uint32_t needed; /* # of buffers needed to fill up fl. */ 403228561Snp uint32_t lowat; /* # of buffers <= this means fl needs help */ 404218792Snp uint32_t pending; /* # of bufs allocated since last doorbell */ 405255050Snp u_int dmamap_failed; 406255050Snp struct mbuf *mstash[8]; 407228561Snp TAILQ_ENTRY(sge_fl) link; /* All starving freelists */ 408218792Snp}; 409218792Snp 410220873Snp/* txq: SGE egress queue + what's needed for Ethernet NIC */ 411218792Snpstruct sge_txq { 412218792Snp struct sge_eq eq; /* MUST be first */ 413220873Snp 414220873Snp struct ifnet *ifp; /* the interface this txq belongs to */ 415220873Snp bus_dma_tag_t tx_tag; /* tag for transmit buffers */ 416220873Snp struct buf_ring *br; /* tx buffer ring */ 417220873Snp struct tx_sdesc *sdesc; /* KVA of software descriptor ring */ 418218792Snp struct mbuf *m; /* held up due to temporary resource shortage */ 419218792Snp 420228561Snp struct tx_maps txmaps; 421219286Snp 422218792Snp /* stats for common events first */ 423218792Snp 424218792Snp uint64_t txcsum; /* # of times hardware assisted with checksum */ 425237819Snp uint64_t tso_wrs; /* # of TSO work requests */ 426218792Snp uint64_t vlan_insertion;/* # of times VLAN tag was inserted */ 427218792Snp uint64_t imm_wrs; /* # of work requests with immediate data */ 428218792Snp uint64_t sgl_wrs; /* # of work requests with direct SGL */ 429218792Snp uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */ 430218792Snp uint64_t txpkts_wrs; /* # of coalesced tx work requests */ 431218792Snp uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */ 432218792Snp 433218792Snp /* stats for not-that-common events */ 434218792Snp 435218792Snp uint32_t no_dmamap; /* no DMA map to load the mbuf */ 436218792Snp uint32_t no_desc; /* out of hardware descriptors */ 437220873Snp} __aligned(CACHE_LINE_SIZE); 438218792Snp 439218792Snp/* rxq: SGE ingress queue + SGE free list + miscellaneous items */ 440218792Snpstruct sge_rxq { 441218792Snp struct sge_iq iq; /* MUST be first */ 442228561Snp struct sge_fl fl; /* MUST follow iq */ 443218792Snp 444219290Snp struct ifnet *ifp; /* the interface this rxq belongs to */ 445237819Snp#if defined(INET) || defined(INET6) 446218792Snp struct lro_ctrl lro; /* LRO state */ 447219290Snp#endif 448218792Snp 449218792Snp /* stats for common events first */ 450218792Snp 451218792Snp uint64_t rxcsum; /* # of times hardware assisted with checksum */ 452218792Snp uint64_t vlan_extraction;/* # of times VLAN tag was extracted */ 453218792Snp 454218792Snp /* stats for not-that-common events */ 455218792Snp 456218792Snp} __aligned(CACHE_LINE_SIZE); 457218792Snp 458237263Snpstatic inline struct sge_rxq * 459237263Snpiq_to_rxq(struct sge_iq *iq) 460237263Snp{ 461237263Snp 462241733Sed return (__containerof(iq, struct sge_rxq, iq)); 463237263Snp} 464237263Snp 465237263Snp 466237263Snp#ifdef TCP_OFFLOAD 467228561Snp/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */ 468228561Snpstruct sge_ofld_rxq { 469228561Snp struct sge_iq iq; /* MUST be first */ 470228561Snp struct sge_fl fl; /* MUST follow iq */ 471228561Snp} __aligned(CACHE_LINE_SIZE); 472237263Snp 473237263Snpstatic inline struct sge_ofld_rxq * 474237263Snpiq_to_ofld_rxq(struct sge_iq *iq) 475237263Snp{ 476237263Snp 477241733Sed return (__containerof(iq, struct sge_ofld_rxq, iq)); 478237263Snp} 479228561Snp#endif 480228561Snp 481237263Snpstruct wrqe { 482237263Snp STAILQ_ENTRY(wrqe) link; 483237263Snp struct sge_wrq *wrq; 484237263Snp int wr_len; 485237263Snp uint64_t wr[] __aligned(16); 486237263Snp}; 487237263Snp 488228561Snp/* 489228561Snp * wrq: SGE egress queue that is given prebuilt work requests. Both the control 490228561Snp * and offload tx queues are of this type. 491228561Snp */ 492228561Snpstruct sge_wrq { 493220873Snp struct sge_eq eq; /* MUST be first */ 494220873Snp 495228561Snp struct adapter *adapter; 496228561Snp 497237263Snp /* List of WRs held up due to lack of tx descriptors */ 498237263Snp STAILQ_HEAD(, wrqe) wr_list; 499237263Snp 500220873Snp /* stats for common events first */ 501220873Snp 502228561Snp uint64_t tx_wrs; /* # of tx work requests */ 503220873Snp 504220873Snp /* stats for not-that-common events */ 505220873Snp 506220873Snp uint32_t no_desc; /* out of hardware descriptors */ 507220873Snp} __aligned(CACHE_LINE_SIZE); 508220873Snp 509218792Snpstruct sge { 510228561Snp int timer_val[SGE_NTIMERS]; 511228561Snp int counter_val[SGE_NCOUNTERS]; 512222701Snp int fl_starve_threshold; 513256794Snp int eq_s_qpp; 514256794Snp int iq_s_qpp; 515218792Snp 516228561Snp int nrxq; /* total # of Ethernet rx queues */ 517228561Snp int ntxq; /* total # of Ethernet tx tx queues */ 518237263Snp#ifdef TCP_OFFLOAD 519228561Snp int nofldrxq; /* total # of TOE rx queues */ 520228561Snp int nofldtxq; /* total # of TOE tx queues */ 521228561Snp#endif 522228561Snp int niq; /* total # of ingress queues */ 523228561Snp int neq; /* total # of egress queues */ 524218792Snp 525218792Snp struct sge_iq fwq; /* Firmware event queue */ 526228561Snp struct sge_wrq mgmtq; /* Management queue (control queue) */ 527228561Snp struct sge_wrq *ctrlq; /* Control queues */ 528218792Snp struct sge_txq *txq; /* NIC tx queues */ 529218792Snp struct sge_rxq *rxq; /* NIC rx queues */ 530237263Snp#ifdef TCP_OFFLOAD 531228561Snp struct sge_wrq *ofld_txq; /* TOE tx queues */ 532228561Snp struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */ 533228561Snp#endif 534218792Snp 535218792Snp uint16_t iq_start; 536218792Snp int eq_start; 537218792Snp struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */ 538218792Snp struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */ 539255050Snp 540255050Snp u_int fl_buf_sizes __aligned(CACHE_LINE_SIZE); 541255050Snp struct fl_buf_info fl_buf_info[FL_BUF_SIZES_MAX]; 542218792Snp}; 543218792Snp 544228561Snpstruct rss_header; 545228561Snptypedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, 546228561Snp struct mbuf *); 547237263Snptypedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *); 548239336Snptypedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *); 549228561Snp 550218792Snpstruct adapter { 551228561Snp SLIST_ENTRY(adapter) link; 552218792Snp device_t dev; 553218792Snp struct cdev *cdev; 554218792Snp 555218792Snp /* PCIe register resources */ 556218792Snp int regs_rid; 557218792Snp struct resource *regs_res; 558218792Snp int msix_rid; 559218792Snp struct resource *msix_res; 560218792Snp bus_space_handle_t bh; 561218792Snp bus_space_tag_t bt; 562218792Snp bus_size_t mmio_len; 563248925Snp int udbs_rid; 564248925Snp struct resource *udbs_res; 565248925Snp volatile uint8_t *udbs_base; 566218792Snp 567218792Snp unsigned int pf; 568218792Snp unsigned int mbox; 569218792Snp 570218792Snp /* Interrupt information */ 571218792Snp int intr_type; 572218792Snp int intr_count; 573218792Snp struct irq { 574218792Snp struct resource *res; 575218792Snp int rid; 576218792Snp void *tag; 577218792Snp } *irq; 578218792Snp 579218792Snp bus_dma_tag_t dmat; /* Parent DMA tag */ 580218792Snp 581218792Snp struct sge sge; 582255015Snp int lro_timeout; 583218792Snp 584228561Snp struct taskqueue *tq[NCHAN]; /* taskqueues that flush data out */ 585218792Snp struct port_info *port[MAX_NPORTS]; 586218792Snp uint8_t chan_map[NCHAN]; 587218792Snp 588237263Snp#ifdef TCP_OFFLOAD 589237263Snp void *tom_softc; /* (struct tom_data *) */ 590228561Snp struct tom_tunables tt; 591255005Snp void *iwarp_softc; /* (struct c4iw_dev *) */ 592228561Snp#endif 593222509Snp struct l2t_data *l2t; /* L2 table */ 594218792Snp struct tid_info tids; 595218792Snp 596248925Snp int doorbells; 597218792Snp int open_device_map; 598237263Snp#ifdef TCP_OFFLOAD 599228561Snp int offload_map; 600228561Snp#endif 601218792Snp int flags; 602218792Snp 603253691Snp char ifp_lockname[16]; 604253691Snp struct mtx ifp_lock; 605253691Snp struct ifnet *ifp; /* tracer ifp */ 606253691Snp struct ifmedia media; 607253691Snp int traceq; /* iq used by all tracers, -1 if none */ 608253691Snp int tracer_valid; /* bitmap of valid tracers */ 609253691Snp int tracer_enabled; /* bitmap of enabled tracers */ 610253691Snp 611218792Snp char fw_version[32]; 612245936Snp char cfg_file[32]; 613245936Snp u_int cfcsum; 614218792Snp struct adapter_params params; 615218792Snp struct t4_virt_res vres; 616218792Snp 617228561Snp uint16_t linkcaps; 618228561Snp uint16_t niccaps; 619228561Snp uint16_t toecaps; 620228561Snp uint16_t rdmacaps; 621228561Snp uint16_t iscsicaps; 622228561Snp uint16_t fcoecaps; 623220873Snp 624228561Snp struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */ 625228561Snp 626218792Snp struct mtx sc_lock; 627218792Snp char lockname[16]; 628228561Snp 629228561Snp /* Starving free lists */ 630228561Snp struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */ 631228561Snp TAILQ_HEAD(, sge_fl) sfl; 632228561Snp struct callout sfl_callout; 633228561Snp 634237263Snp an_handler_t an_handler __aligned(CACHE_LINE_SIZE); 635247291Snp fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */ 636239336Snp cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */ 637245274Snp 638245274Snp#ifdef INVARIANTS 639245274Snp const char *last_op; 640245274Snp const void *last_op_thr; 641245274Snp#endif 642218792Snp}; 643218792Snp 644218792Snp#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock) 645218792Snp#define ADAPTER_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) 646218792Snp#define ADAPTER_LOCK_ASSERT_OWNED(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) 647218792Snp#define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED) 648218792Snp 649245274Snp/* XXX: not bulletproof, but much better than nothing */ 650245274Snp#define ASSERT_SYNCHRONIZED_OP(sc) \ 651245274Snp KASSERT(IS_BUSY(sc) && \ 652245274Snp (mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \ 653245274Snp ("%s: operation not synchronized.", __func__)) 654245274Snp 655218792Snp#define PORT_LOCK(pi) mtx_lock(&(pi)->pi_lock) 656218792Snp#define PORT_UNLOCK(pi) mtx_unlock(&(pi)->pi_lock) 657218792Snp#define PORT_LOCK_ASSERT_OWNED(pi) mtx_assert(&(pi)->pi_lock, MA_OWNED) 658218792Snp#define PORT_LOCK_ASSERT_NOTOWNED(pi) mtx_assert(&(pi)->pi_lock, MA_NOTOWNED) 659218792Snp 660218792Snp#define FL_LOCK(fl) mtx_lock(&(fl)->fl_lock) 661218792Snp#define FL_TRYLOCK(fl) mtx_trylock(&(fl)->fl_lock) 662218792Snp#define FL_UNLOCK(fl) mtx_unlock(&(fl)->fl_lock) 663218792Snp#define FL_LOCK_ASSERT_OWNED(fl) mtx_assert(&(fl)->fl_lock, MA_OWNED) 664218792Snp#define FL_LOCK_ASSERT_NOTOWNED(fl) mtx_assert(&(fl)->fl_lock, MA_NOTOWNED) 665218792Snp 666218792Snp#define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl) 667218792Snp#define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl) 668218792Snp#define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl) 669218792Snp#define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl) 670218792Snp 671218792Snp#define EQ_LOCK(eq) mtx_lock(&(eq)->eq_lock) 672218792Snp#define EQ_TRYLOCK(eq) mtx_trylock(&(eq)->eq_lock) 673218792Snp#define EQ_UNLOCK(eq) mtx_unlock(&(eq)->eq_lock) 674218792Snp#define EQ_LOCK_ASSERT_OWNED(eq) mtx_assert(&(eq)->eq_lock, MA_OWNED) 675218792Snp#define EQ_LOCK_ASSERT_NOTOWNED(eq) mtx_assert(&(eq)->eq_lock, MA_NOTOWNED) 676218792Snp 677218792Snp#define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq) 678218792Snp#define TXQ_TRYLOCK(txq) EQ_TRYLOCK(&(txq)->eq) 679218792Snp#define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq) 680218792Snp#define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq) 681218792Snp#define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq) 682218792Snp 683245517Snp#define for_each_txq(pi, iter, q) \ 684245567Snp for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \ 685245567Snp iter < pi->ntxq; ++iter, ++q) 686245517Snp#define for_each_rxq(pi, iter, q) \ 687245567Snp for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \ 688245567Snp iter < pi->nrxq; ++iter, ++q) 689245517Snp#define for_each_ofld_txq(pi, iter, q) \ 690245567Snp for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \ 691245567Snp iter < pi->nofldtxq; ++iter, ++q) 692245517Snp#define for_each_ofld_rxq(pi, iter, q) \ 693245567Snp for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \ 694245567Snp iter < pi->nofldrxq; ++iter, ++q) 695218792Snp 696222510Snp/* One for errors, one for firmware events */ 697222510Snp#define T4_EXTRA_INTR 2 698218792Snp 699218792Snpstatic inline uint32_t 700218792Snpt4_read_reg(struct adapter *sc, uint32_t reg) 701218792Snp{ 702237263Snp 703218792Snp return bus_space_read_4(sc->bt, sc->bh, reg); 704218792Snp} 705218792Snp 706218792Snpstatic inline void 707218792Snpt4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val) 708218792Snp{ 709237263Snp 710218792Snp bus_space_write_4(sc->bt, sc->bh, reg, val); 711218792Snp} 712218792Snp 713218792Snpstatic inline uint64_t 714218792Snpt4_read_reg64(struct adapter *sc, uint32_t reg) 715218792Snp{ 716237263Snp 717218792Snp return t4_bus_space_read_8(sc->bt, sc->bh, reg); 718218792Snp} 719218792Snp 720218792Snpstatic inline void 721218792Snpt4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val) 722218792Snp{ 723237263Snp 724218792Snp t4_bus_space_write_8(sc->bt, sc->bh, reg, val); 725218792Snp} 726218792Snp 727218792Snpstatic inline void 728218792Snpt4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val) 729218792Snp{ 730237263Snp 731218792Snp *val = pci_read_config(sc->dev, reg, 1); 732218792Snp} 733218792Snp 734218792Snpstatic inline void 735218792Snpt4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val) 736218792Snp{ 737237263Snp 738218792Snp pci_write_config(sc->dev, reg, val, 1); 739218792Snp} 740218792Snp 741218792Snpstatic inline void 742218792Snpt4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val) 743218792Snp{ 744237263Snp 745218792Snp *val = pci_read_config(sc->dev, reg, 2); 746218792Snp} 747218792Snp 748218792Snpstatic inline void 749218792Snpt4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val) 750218792Snp{ 751237263Snp 752218792Snp pci_write_config(sc->dev, reg, val, 2); 753218792Snp} 754218792Snp 755218792Snpstatic inline void 756218792Snpt4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val) 757218792Snp{ 758237263Snp 759218792Snp *val = pci_read_config(sc->dev, reg, 4); 760218792Snp} 761218792Snp 762218792Snpstatic inline void 763218792Snpt4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val) 764218792Snp{ 765237263Snp 766218792Snp pci_write_config(sc->dev, reg, val, 4); 767218792Snp} 768218792Snp 769218792Snpstatic inline struct port_info * 770218792Snpadap2pinfo(struct adapter *sc, int idx) 771218792Snp{ 772237263Snp 773218792Snp return (sc->port[idx]); 774218792Snp} 775218792Snp 776218792Snpstatic inline void 777218792Snpt4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[]) 778218792Snp{ 779237263Snp 780218792Snp bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN); 781218792Snp} 782218792Snp 783248925Snpstatic inline bool 784248925Snpis_10G_port(const struct port_info *pi) 785218792Snp{ 786237263Snp 787218792Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0); 788218792Snp} 789218792Snp 790250092Snpstatic inline bool 791250092Snpis_40G_port(const struct port_info *pi) 792250092Snp{ 793250092Snp 794250092Snp return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0); 795250092Snp} 796250092Snp 797248925Snpstatic inline int 798248925Snptx_resume_threshold(struct sge_eq *eq) 799228561Snp{ 800237263Snp 801228561Snp return (eq->qsize / 4); 802228561Snp} 803228561Snp 804219286Snp/* t4_main.c */ 805228561Snpvoid t4_tx_task(void *, int); 806228561Snpvoid t4_tx_callout(void *); 807218792Snpint t4_os_find_pci_capability(struct adapter *, int); 808218792Snpint t4_os_pci_save_state(struct adapter *); 809218792Snpint t4_os_pci_restore_state(struct adapter *); 810218792Snpvoid t4_os_portmod_changed(const struct adapter *, int); 811252747Snpvoid t4_os_link_changed(struct adapter *, int, int, int); 812228561Snpvoid t4_iterate(void (*)(struct adapter *, void *), void *); 813228561Snpint t4_register_cpl_handler(struct adapter *, int, cpl_handler_t); 814237263Snpint t4_register_an_handler(struct adapter *, an_handler_t); 815239336Snpint t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t); 816239338Snpint t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 817245274Snpint begin_synchronized_op(struct adapter *, struct port_info *, int, char *); 818245274Snpvoid end_synchronized_op(struct adapter *, int); 819218792Snp 820219286Snp/* t4_sge.c */ 821219392Snpvoid t4_sge_modload(void); 822248925Snpvoid t4_init_sge_cpl_handlers(struct adapter *); 823248925Snpvoid t4_tweak_chip_settings(struct adapter *); 824248925Snpint t4_read_chip_settings(struct adapter *); 825218792Snpint t4_create_dma_tag(struct adapter *); 826253829Snpvoid t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *, 827253829Snp struct sysctl_oid_list *); 828218792Snpint t4_destroy_dma_tag(struct adapter *); 829220873Snpint t4_setup_adapter_queues(struct adapter *); 830220873Snpint t4_teardown_adapter_queues(struct adapter *); 831228561Snpint t4_setup_port_queues(struct port_info *); 832228561Snpint t4_teardown_port_queues(struct port_info *); 833228561Snpint t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int); 834228561Snpvoid t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t); 835218792Snpvoid t4_intr_all(void *); 836222510Snpvoid t4_intr(void *); 837218792Snpvoid t4_intr_err(void *); 838218792Snpvoid t4_intr_evt(void *); 839237263Snpvoid t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *); 840218792Snpint t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *); 841218792Snpvoid t4_update_fl_bufsize(struct ifnet *); 842228561Snpint can_resume_tx(struct sge_eq *); 843218792Snp 844253691Snp/* t4_tracer.c */ 845253691Snpstruct t4_tracer; 846253691Snpvoid t4_tracer_modload(void); 847253691Snpvoid t4_tracer_modunload(void); 848253691Snpvoid t4_tracer_port_detach(struct adapter *); 849253691Snpint t4_get_tracer(struct adapter *, struct t4_tracer *); 850253691Snpint t4_set_tracer(struct adapter *, struct t4_tracer *); 851253691Snpint t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 852253691Snpint t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *); 853253691Snp 854237263Snpstatic inline struct wrqe * 855237263Snpalloc_wrqe(int wr_len, struct sge_wrq *wrq) 856228561Snp{ 857237263Snp int len = offsetof(struct wrqe, wr) + wr_len; 858237263Snp struct wrqe *wr; 859228561Snp 860237263Snp wr = malloc(len, M_CXGBE, M_NOWAIT); 861237263Snp if (__predict_false(wr == NULL)) 862237263Snp return (NULL); 863237263Snp wr->wr_len = wr_len; 864237263Snp wr->wrq = wrq; 865237263Snp return (wr); 866237263Snp} 867237263Snp 868237263Snpstatic inline void * 869237263Snpwrtod(struct wrqe *wr) 870237263Snp{ 871237263Snp return (&wr->wr[0]); 872237263Snp} 873237263Snp 874237263Snpstatic inline void 875237263Snpfree_wrqe(struct wrqe *wr) 876237263Snp{ 877237263Snp free(wr, M_CXGBE); 878237263Snp} 879237263Snp 880237263Snpstatic inline void 881237263Snpt4_wrq_tx(struct adapter *sc, struct wrqe *wr) 882237263Snp{ 883237263Snp struct sge_wrq *wrq = wr->wrq; 884237263Snp 885228561Snp TXQ_LOCK(wrq); 886237263Snp t4_wrq_tx_locked(sc, wrq, wr); 887228561Snp TXQ_UNLOCK(wrq); 888228561Snp} 889228561Snp 890218792Snp#endif 891