t4_main.c revision 281955
1303980Sngie/*- 2303980Sngie * Copyright (c) 2011 Chelsio Communications, Inc. 3303980Sngie * All rights reserved. 4303980Sngie * Written by: Navdeep Parhar <np@FreeBSD.org> 5303980Sngie * 6303980Sngie * Redistribution and use in source and binary forms, with or without 7303980Sngie * modification, are permitted provided that the following conditions 8303980Sngie * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 281955 2015-04-24 23:26:44Z hiren $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct sx t4_list_lock; 164SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct sx t4_uld_list_lock; 167SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200static int t4_rsrv_noflowq = 0; 201TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203#ifdef TCP_OFFLOAD 204#define NOFLDTXQ_10G 8 205static int t4_nofldtxq10g = -1; 206TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208#define NOFLDRXQ_10G 2 209static int t4_nofldrxq10g = -1; 210TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212#define NOFLDTXQ_1G 2 213static int t4_nofldtxq1g = -1; 214TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216#define NOFLDRXQ_1G 1 217static int t4_nofldrxq1g = -1; 218TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219#endif 220 221#ifdef DEV_NETMAP 222#define NNMTXQ_10G 2 223static int t4_nnmtxq10g = -1; 224TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 225 226#define NNMRXQ_10G 2 227static int t4_nnmrxq10g = -1; 228TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 229 230#define NNMTXQ_1G 1 231static int t4_nnmtxq1g = -1; 232TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 233 234#define NNMRXQ_1G 1 235static int t4_nnmrxq1g = -1; 236TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 237#endif 238 239/* 240 * Holdoff parameters for 10G and 1G ports. 241 */ 242#define TMR_IDX_10G 1 243static int t4_tmr_idx_10g = TMR_IDX_10G; 244TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 245 246#define PKTC_IDX_10G (-1) 247static int t4_pktc_idx_10g = PKTC_IDX_10G; 248TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 249 250#define TMR_IDX_1G 1 251static int t4_tmr_idx_1g = TMR_IDX_1G; 252TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 253 254#define PKTC_IDX_1G (-1) 255static int t4_pktc_idx_1g = PKTC_IDX_1G; 256TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 257 258/* 259 * Size (# of entries) of each tx and rx queue. 260 */ 261static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 262TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 263 264static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 266 267/* 268 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 269 */ 270static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 271TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 272 273/* 274 * Configuration file. 275 */ 276#define DEFAULT_CF "default" 277#define FLASH_CF "flash" 278#define UWIRE_CF "uwire" 279#define FPGA_CF "fpga" 280static char t4_cfg_file[32] = DEFAULT_CF; 281TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 282 283/* 284 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 285 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 286 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 287 * mark or when signalled to do so, 0 to never emit PAUSE. 288 */ 289static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 290TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 291 292/* 293 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 294 * encouraged respectively). 295 */ 296static unsigned int t4_fw_install = 1; 297TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 298 299/* 300 * ASIC features that will be used. Disable the ones you don't want so that the 301 * chip resources aren't wasted on features that will not be used. 302 */ 303static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 304TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 305 306static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 307TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 308 309static int t4_toecaps_allowed = -1; 310TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 311 312static int t4_rdmacaps_allowed = 0; 313TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 314 315static int t4_iscsicaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 317 318static int t4_fcoecaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 320 321static int t5_write_combine = 0; 322TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 323 324struct intrs_and_queues { 325 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 326 uint16_t nirq; /* Total # of vectors */ 327 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 328 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 329 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 330 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 331 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 332 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 333 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 334#ifdef TCP_OFFLOAD 335 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 336 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 337 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 338 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 339#endif 340#ifdef DEV_NETMAP 341 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 342 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 343 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 344 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 345#endif 346}; 347 348struct filter_entry { 349 uint32_t valid:1; /* filter allocated and valid */ 350 uint32_t locked:1; /* filter is administratively locked */ 351 uint32_t pending:1; /* filter action is pending firmware reply */ 352 uint32_t smtidx:8; /* Source MAC Table index for smac */ 353 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 354 355 struct t4_filter_specification fs; 356}; 357 358static int map_bars_0_and_4(struct adapter *); 359static int map_bar_2(struct adapter *); 360static void setup_memwin(struct adapter *); 361static int validate_mem_range(struct adapter *, uint32_t, int); 362static int fwmtype_to_hwmtype(int); 363static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 364 uint32_t *); 365static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 366static uint32_t position_memwin(struct adapter *, int, uint32_t); 367static int cfg_itype_and_nqueues(struct adapter *, int, int, 368 struct intrs_and_queues *); 369static int prep_firmware(struct adapter *); 370static int partition_resources(struct adapter *, const struct firmware *, 371 const char *); 372static int get_params__pre_init(struct adapter *); 373static int get_params__post_init(struct adapter *); 374static int set_params__post_init(struct adapter *); 375static void t4_set_desc(struct adapter *); 376static void build_medialist(struct port_info *, struct ifmedia *); 377static int cxgbe_init_synchronized(struct port_info *); 378static int cxgbe_uninit_synchronized(struct port_info *); 379static int setup_intr_handlers(struct adapter *); 380static void quiesce_eq(struct adapter *, struct sge_eq *); 381static void quiesce_iq(struct adapter *, struct sge_iq *); 382static void quiesce_fl(struct adapter *, struct sge_fl *); 383static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 384 driver_intr_t *, void *, char *); 385static int t4_free_irq(struct adapter *, struct irq *); 386static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 387 unsigned int); 388static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 389static void cxgbe_tick(void *); 390static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 391static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 392 struct mbuf *); 393static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 394static int fw_msg_not_handled(struct adapter *, const __be64 *); 395static int t4_sysctls(struct adapter *); 396static int cxgbe_sysctls(struct port_info *); 397static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 398static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 399static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 400static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 401static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 402static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 403static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 404static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 405static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 406static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 407static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 408#ifdef SBUF_DRAIN 409static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 410static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 411static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 412static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 413static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 414static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 415static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 416static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 417static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 418static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 419static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 420static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 422static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 423static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 424static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 425static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 427static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 428static int sysctl_tids(SYSCTL_HANDLER_ARGS); 429static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 430static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 431static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 432static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 433static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 434#endif 435static inline void txq_start(struct ifnet *, struct sge_txq *); 436static uint32_t fconf_to_mode(uint32_t); 437static uint32_t mode_to_fconf(uint32_t); 438static uint32_t fspec_to_fconf(struct t4_filter_specification *); 439static int get_filter_mode(struct adapter *, uint32_t *); 440static int set_filter_mode(struct adapter *, uint32_t); 441static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 442static int get_filter(struct adapter *, struct t4_filter *); 443static int set_filter(struct adapter *, struct t4_filter *); 444static int del_filter(struct adapter *, struct t4_filter *); 445static void clear_filter(struct filter_entry *); 446static int set_filter_wr(struct adapter *, int); 447static int del_filter_wr(struct adapter *, int); 448static int get_sge_context(struct adapter *, struct t4_sge_context *); 449static int load_fw(struct adapter *, struct t4_data *); 450static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 451static int read_i2c(struct adapter *, struct t4_i2c_data *); 452static int set_sched_class(struct adapter *, struct t4_sched_params *); 453static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 454#ifdef TCP_OFFLOAD 455static int toe_capability(struct port_info *, int); 456#endif 457static int mod_event(module_t, int, void *); 458 459struct { 460 uint16_t device; 461 char *desc; 462} t4_pciids[] = { 463 {0xa000, "Chelsio Terminator 4 FPGA"}, 464 {0x4400, "Chelsio T440-dbg"}, 465 {0x4401, "Chelsio T420-CR"}, 466 {0x4402, "Chelsio T422-CR"}, 467 {0x4403, "Chelsio T440-CR"}, 468 {0x4404, "Chelsio T420-BCH"}, 469 {0x4405, "Chelsio T440-BCH"}, 470 {0x4406, "Chelsio T440-CH"}, 471 {0x4407, "Chelsio T420-SO"}, 472 {0x4408, "Chelsio T420-CX"}, 473 {0x4409, "Chelsio T420-BT"}, 474 {0x440a, "Chelsio T404-BT"}, 475 {0x440e, "Chelsio T440-LP-CR"}, 476}, t5_pciids[] = { 477 {0xb000, "Chelsio Terminator 5 FPGA"}, 478 {0x5400, "Chelsio T580-dbg"}, 479 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 480 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 481 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 482 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 483 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 484 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 485 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 486 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 487 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 488 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 489 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 490 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 491 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 492#ifdef notyet 493 {0x5404, "Chelsio T520-BCH"}, 494 {0x5405, "Chelsio T540-BCH"}, 495 {0x5406, "Chelsio T540-CH"}, 496 {0x5408, "Chelsio T520-CX"}, 497 {0x540b, "Chelsio B520-SR"}, 498 {0x540c, "Chelsio B504-BT"}, 499 {0x540f, "Chelsio Amsterdam"}, 500 {0x5413, "Chelsio T580-CHR"}, 501#endif 502}; 503 504#ifdef TCP_OFFLOAD 505/* 506 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 507 * exactly the same for both rxq and ofld_rxq. 508 */ 509CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 510CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 511#endif 512 513/* No easy way to include t4_msg.h before adapter.h so we check this way */ 514CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 515CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 516 517CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 518 519static int 520t4_probe(device_t dev) 521{ 522 int i; 523 uint16_t v = pci_get_vendor(dev); 524 uint16_t d = pci_get_device(dev); 525 uint8_t f = pci_get_function(dev); 526 527 if (v != PCI_VENDOR_ID_CHELSIO) 528 return (ENXIO); 529 530 /* Attach only to PF0 of the FPGA */ 531 if (d == 0xa000 && f != 0) 532 return (ENXIO); 533 534 for (i = 0; i < nitems(t4_pciids); i++) { 535 if (d == t4_pciids[i].device) { 536 device_set_desc(dev, t4_pciids[i].desc); 537 return (BUS_PROBE_DEFAULT); 538 } 539 } 540 541 return (ENXIO); 542} 543 544static int 545t5_probe(device_t dev) 546{ 547 int i; 548 uint16_t v = pci_get_vendor(dev); 549 uint16_t d = pci_get_device(dev); 550 uint8_t f = pci_get_function(dev); 551 552 if (v != PCI_VENDOR_ID_CHELSIO) 553 return (ENXIO); 554 555 /* Attach only to PF0 of the FPGA */ 556 if (d == 0xb000 && f != 0) 557 return (ENXIO); 558 559 for (i = 0; i < nitems(t5_pciids); i++) { 560 if (d == t5_pciids[i].device) { 561 device_set_desc(dev, t5_pciids[i].desc); 562 return (BUS_PROBE_DEFAULT); 563 } 564 } 565 566 return (ENXIO); 567} 568 569static int 570t4_attach(device_t dev) 571{ 572 struct adapter *sc; 573 int rc = 0, i, n10g, n1g, rqidx, tqidx; 574 struct intrs_and_queues iaq; 575 struct sge *s; 576#ifdef TCP_OFFLOAD 577 int ofld_rqidx, ofld_tqidx; 578#endif 579#ifdef DEV_NETMAP 580 int nm_rqidx, nm_tqidx; 581#endif 582 583 sc = device_get_softc(dev); 584 sc->dev = dev; 585 586 pci_enable_busmaster(dev); 587 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 588 uint32_t v; 589 590 pci_set_max_read_req(dev, 4096); 591 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 592 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 593 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 594 595 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 596 } 597 598 sc->traceq = -1; 599 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 600 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 601 device_get_nameunit(dev)); 602 603 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 604 device_get_nameunit(dev)); 605 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 606 sx_xlock(&t4_list_lock); 607 SLIST_INSERT_HEAD(&t4_list, sc, link); 608 sx_xunlock(&t4_list_lock); 609 610 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 611 TAILQ_INIT(&sc->sfl); 612 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 613 614 rc = map_bars_0_and_4(sc); 615 if (rc != 0) 616 goto done; /* error message displayed already */ 617 618 /* 619 * This is the real PF# to which we're attaching. Works from within PCI 620 * passthrough environments too, where pci_get_function() could return a 621 * different PF# depending on the passthrough configuration. We need to 622 * use the real PF# in all our communication with the firmware. 623 */ 624 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 625 sc->mbox = sc->pf; 626 627 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 628 sc->an_handler = an_not_handled; 629 for (i = 0; i < nitems(sc->cpl_handler); i++) 630 sc->cpl_handler[i] = cpl_not_handled; 631 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 632 sc->fw_msg_handler[i] = fw_msg_not_handled; 633 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 634 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 635 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 636 t4_init_sge_cpl_handlers(sc); 637 638 /* Prepare the adapter for operation */ 639 rc = -t4_prep_adapter(sc); 640 if (rc != 0) { 641 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 642 goto done; 643 } 644 645 /* 646 * Do this really early, with the memory windows set up even before the 647 * character device. The userland tool's register i/o and mem read 648 * will work even in "recovery mode". 649 */ 650 setup_memwin(sc); 651 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 652 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 653 device_get_nameunit(dev)); 654 if (sc->cdev == NULL) 655 device_printf(dev, "failed to create nexus char device.\n"); 656 else 657 sc->cdev->si_drv1 = sc; 658 659 /* Go no further if recovery mode has been requested. */ 660 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 661 device_printf(dev, "recovery mode.\n"); 662 goto done; 663 } 664 665 /* Prepare the firmware for operation */ 666 rc = prep_firmware(sc); 667 if (rc != 0) 668 goto done; /* error message displayed already */ 669 670 rc = get_params__post_init(sc); 671 if (rc != 0) 672 goto done; /* error message displayed already */ 673 674 rc = set_params__post_init(sc); 675 if (rc != 0) 676 goto done; /* error message displayed already */ 677 678 rc = map_bar_2(sc); 679 if (rc != 0) 680 goto done; /* error message displayed already */ 681 682 rc = t4_create_dma_tag(sc); 683 if (rc != 0) 684 goto done; /* error message displayed already */ 685 686 /* 687 * First pass over all the ports - allocate VIs and initialize some 688 * basic parameters like mac address, port type, etc. We also figure 689 * out whether a port is 10G or 1G and use that information when 690 * calculating how many interrupts to attempt to allocate. 691 */ 692 n10g = n1g = 0; 693 for_each_port(sc, i) { 694 struct port_info *pi; 695 696 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 697 sc->port[i] = pi; 698 699 /* These must be set before t4_port_init */ 700 pi->adapter = sc; 701 pi->port_id = i; 702 703 /* Allocate the vi and initialize parameters like mac addr */ 704 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 705 if (rc != 0) { 706 device_printf(dev, "unable to initialize port %d: %d\n", 707 i, rc); 708 free(pi, M_CXGBE); 709 sc->port[i] = NULL; 710 goto done; 711 } 712 713 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 714 pi->link_cfg.requested_fc |= t4_pause_settings; 715 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 716 pi->link_cfg.fc |= t4_pause_settings; 717 718 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 719 if (rc != 0) { 720 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 721 free(pi, M_CXGBE); 722 sc->port[i] = NULL; 723 goto done; 724 } 725 726 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 727 device_get_nameunit(dev), i); 728 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 729 sc->chan_map[pi->tx_chan] = i; 730 731 if (is_10G_port(pi) || is_40G_port(pi)) { 732 n10g++; 733 pi->tmr_idx = t4_tmr_idx_10g; 734 pi->pktc_idx = t4_pktc_idx_10g; 735 } else { 736 n1g++; 737 pi->tmr_idx = t4_tmr_idx_1g; 738 pi->pktc_idx = t4_pktc_idx_1g; 739 } 740 741 pi->xact_addr_filt = -1; 742 pi->linkdnrc = -1; 743 744 pi->qsize_rxq = t4_qsize_rxq; 745 pi->qsize_txq = t4_qsize_txq; 746 747 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 748 if (pi->dev == NULL) { 749 device_printf(dev, 750 "failed to add device for port %d.\n", i); 751 rc = ENXIO; 752 goto done; 753 } 754 device_set_softc(pi->dev, pi); 755 } 756 757 /* 758 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 759 */ 760 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 761 if (rc != 0) 762 goto done; /* error message displayed already */ 763 764 sc->intr_type = iaq.intr_type; 765 sc->intr_count = iaq.nirq; 766 767 s = &sc->sge; 768 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 769 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 770 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 771 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 772 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 773#ifdef TCP_OFFLOAD 774 if (is_offload(sc)) { 775 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 776 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 777 s->neq += s->nofldtxq + s->nofldrxq; 778 s->niq += s->nofldrxq; 779 780 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 781 M_CXGBE, M_ZERO | M_WAITOK); 782 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 783 M_CXGBE, M_ZERO | M_WAITOK); 784 } 785#endif 786#ifdef DEV_NETMAP 787 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 788 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 789 s->neq += s->nnmtxq + s->nnmrxq; 790 s->niq += s->nnmrxq; 791 792 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 793 M_CXGBE, M_ZERO | M_WAITOK); 794 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796#endif 797 798 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 799 M_ZERO | M_WAITOK); 800 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 801 M_ZERO | M_WAITOK); 802 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 803 M_ZERO | M_WAITOK); 804 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 805 M_ZERO | M_WAITOK); 806 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 807 M_ZERO | M_WAITOK); 808 809 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 t4_init_l2t(sc, M_WAITOK); 813 814 /* 815 * Second pass over the ports. This time we know the number of rx and 816 * tx queues that each port should get. 817 */ 818 rqidx = tqidx = 0; 819#ifdef TCP_OFFLOAD 820 ofld_rqidx = ofld_tqidx = 0; 821#endif 822#ifdef DEV_NETMAP 823 nm_rqidx = nm_tqidx = 0; 824#endif 825 for_each_port(sc, i) { 826 struct port_info *pi = sc->port[i]; 827 828 if (pi == NULL) 829 continue; 830 831 pi->first_rxq = rqidx; 832 pi->first_txq = tqidx; 833 if (is_10G_port(pi) || is_40G_port(pi)) { 834 pi->flags |= iaq.intr_flags_10g; 835 pi->nrxq = iaq.nrxq10g; 836 pi->ntxq = iaq.ntxq10g; 837 } else { 838 pi->flags |= iaq.intr_flags_1g; 839 pi->nrxq = iaq.nrxq1g; 840 pi->ntxq = iaq.ntxq1g; 841 } 842 843 if (pi->ntxq > 1) 844 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 845 else 846 pi->rsrv_noflowq = 0; 847 848 rqidx += pi->nrxq; 849 tqidx += pi->ntxq; 850#ifdef TCP_OFFLOAD 851 if (is_offload(sc)) { 852 pi->first_ofld_rxq = ofld_rqidx; 853 pi->first_ofld_txq = ofld_tqidx; 854 if (is_10G_port(pi) || is_40G_port(pi)) { 855 pi->nofldrxq = iaq.nofldrxq10g; 856 pi->nofldtxq = iaq.nofldtxq10g; 857 } else { 858 pi->nofldrxq = iaq.nofldrxq1g; 859 pi->nofldtxq = iaq.nofldtxq1g; 860 } 861 ofld_rqidx += pi->nofldrxq; 862 ofld_tqidx += pi->nofldtxq; 863 } 864#endif 865#ifdef DEV_NETMAP 866 pi->first_nm_rxq = nm_rqidx; 867 pi->first_nm_txq = nm_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nnmrxq = iaq.nnmrxq10g; 870 pi->nnmtxq = iaq.nnmtxq10g; 871 } else { 872 pi->nnmrxq = iaq.nnmrxq1g; 873 pi->nnmtxq = iaq.nnmtxq1g; 874 } 875 nm_rqidx += pi->nnmrxq; 876 nm_tqidx += pi->nnmtxq; 877#endif 878 } 879 880 rc = setup_intr_handlers(sc); 881 if (rc != 0) { 882 device_printf(dev, 883 "failed to setup interrupt handlers: %d\n", rc); 884 goto done; 885 } 886 887 rc = bus_generic_attach(dev); 888 if (rc != 0) { 889 device_printf(dev, 890 "failed to attach all child ports: %d\n", rc); 891 goto done; 892 } 893 894 device_printf(dev, 895 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 896 sc->params.pci.width, sc->params.nports, sc->intr_count, 897 sc->intr_type == INTR_MSIX ? "MSI-X" : 898 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 899 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 900 901 t4_set_desc(sc); 902 903done: 904 if (rc != 0 && sc->cdev) { 905 /* cdev was created and so cxgbetool works; recover that way. */ 906 device_printf(dev, 907 "error during attach, adapter is now in recovery mode.\n"); 908 rc = 0; 909 } 910 911 if (rc != 0) 912 t4_detach(dev); 913 else 914 t4_sysctls(sc); 915 916 return (rc); 917} 918 919/* 920 * Idempotent 921 */ 922static int 923t4_detach(device_t dev) 924{ 925 struct adapter *sc; 926 struct port_info *pi; 927 int i, rc; 928 929 sc = device_get_softc(dev); 930 931 if (sc->flags & FULL_INIT_DONE) 932 t4_intr_disable(sc); 933 934 if (sc->cdev) { 935 destroy_dev(sc->cdev); 936 sc->cdev = NULL; 937 } 938 939 rc = bus_generic_detach(dev); 940 if (rc) { 941 device_printf(dev, 942 "failed to detach child devices: %d\n", rc); 943 return (rc); 944 } 945 946 for (i = 0; i < sc->intr_count; i++) 947 t4_free_irq(sc, &sc->irq[i]); 948 949 for (i = 0; i < MAX_NPORTS; i++) { 950 pi = sc->port[i]; 951 if (pi) { 952 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 953 if (pi->dev) 954 device_delete_child(dev, pi->dev); 955 956 mtx_destroy(&pi->pi_lock); 957 free(pi, M_CXGBE); 958 } 959 } 960 961 if (sc->flags & FULL_INIT_DONE) 962 adapter_full_uninit(sc); 963 964 if (sc->flags & FW_OK) 965 t4_fw_bye(sc, sc->mbox); 966 967 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 968 pci_release_msi(dev); 969 970 if (sc->regs_res) 971 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 972 sc->regs_res); 973 974 if (sc->udbs_res) 975 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 976 sc->udbs_res); 977 978 if (sc->msix_res) 979 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 980 sc->msix_res); 981 982 if (sc->l2t) 983 t4_free_l2t(sc->l2t); 984 985#ifdef TCP_OFFLOAD 986 free(sc->sge.ofld_rxq, M_CXGBE); 987 free(sc->sge.ofld_txq, M_CXGBE); 988#endif 989#ifdef DEV_NETMAP 990 free(sc->sge.nm_rxq, M_CXGBE); 991 free(sc->sge.nm_txq, M_CXGBE); 992#endif 993 free(sc->irq, M_CXGBE); 994 free(sc->sge.rxq, M_CXGBE); 995 free(sc->sge.txq, M_CXGBE); 996 free(sc->sge.ctrlq, M_CXGBE); 997 free(sc->sge.iqmap, M_CXGBE); 998 free(sc->sge.eqmap, M_CXGBE); 999 free(sc->tids.ftid_tab, M_CXGBE); 1000 t4_destroy_dma_tag(sc); 1001 if (mtx_initialized(&sc->sc_lock)) { 1002 sx_xlock(&t4_list_lock); 1003 SLIST_REMOVE(&t4_list, sc, adapter, link); 1004 sx_xunlock(&t4_list_lock); 1005 mtx_destroy(&sc->sc_lock); 1006 } 1007 1008 if (mtx_initialized(&sc->tids.ftid_lock)) 1009 mtx_destroy(&sc->tids.ftid_lock); 1010 if (mtx_initialized(&sc->sfl_lock)) 1011 mtx_destroy(&sc->sfl_lock); 1012 if (mtx_initialized(&sc->ifp_lock)) 1013 mtx_destroy(&sc->ifp_lock); 1014 1015 bzero(sc, sizeof(*sc)); 1016 1017 return (0); 1018} 1019 1020static int 1021cxgbe_probe(device_t dev) 1022{ 1023 char buf[128]; 1024 struct port_info *pi = device_get_softc(dev); 1025 1026 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1027 device_set_desc_copy(dev, buf); 1028 1029 return (BUS_PROBE_DEFAULT); 1030} 1031 1032#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1033 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1034 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1035#define T4_CAP_ENABLE (T4_CAP) 1036 1037static int 1038cxgbe_attach(device_t dev) 1039{ 1040 struct port_info *pi = device_get_softc(dev); 1041 struct ifnet *ifp; 1042 char *s; 1043 int n, o; 1044 1045 /* Allocate an ifnet and set it up */ 1046 ifp = if_alloc(IFT_ETHER); 1047 if (ifp == NULL) { 1048 device_printf(dev, "Cannot allocate ifnet\n"); 1049 return (ENOMEM); 1050 } 1051 pi->ifp = ifp; 1052 ifp->if_softc = pi; 1053 1054 callout_init(&pi->tick, CALLOUT_MPSAFE); 1055 1056 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1057 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1058 1059 ifp->if_init = cxgbe_init; 1060 ifp->if_ioctl = cxgbe_ioctl; 1061 ifp->if_transmit = cxgbe_transmit; 1062 ifp->if_qflush = cxgbe_qflush; 1063 1064 ifp->if_capabilities = T4_CAP; 1065#ifdef TCP_OFFLOAD 1066 if (is_offload(pi->adapter)) 1067 ifp->if_capabilities |= IFCAP_TOE; 1068#endif 1069 ifp->if_capenable = T4_CAP_ENABLE; 1070 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1071 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1072 1073 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1074 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1075 ifp->if_hw_tsomaxsegsize = 65536; 1076 1077 /* Initialize ifmedia for this port */ 1078 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1079 cxgbe_media_status); 1080 build_medialist(pi, &pi->media); 1081 1082 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1083 EVENTHANDLER_PRI_ANY); 1084 1085 ether_ifattach(ifp, pi->hw_addr); 1086 1087 n = 128; 1088 s = malloc(n, M_CXGBE, M_WAITOK); 1089 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1090 MPASS(n > o); 1091#ifdef TCP_OFFLOAD 1092 if (is_offload(pi->adapter)) { 1093 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1094 pi->nofldtxq, pi->nofldrxq); 1095 MPASS(n > o); 1096 } 1097#endif 1098#ifdef DEV_NETMAP 1099 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1100 pi->nnmrxq); 1101 MPASS(n > o); 1102#endif 1103 device_printf(dev, "%s\n", s); 1104 free(s, M_CXGBE); 1105 1106#ifdef DEV_NETMAP 1107 /* nm_media handled here to keep implementation private to this file */ 1108 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1109 cxgbe_media_status); 1110 build_medialist(pi, &pi->nm_media); 1111 create_netmap_ifnet(pi); /* logs errors it something fails */ 1112#endif 1113 cxgbe_sysctls(pi); 1114 1115 return (0); 1116} 1117 1118static int 1119cxgbe_detach(device_t dev) 1120{ 1121 struct port_info *pi = device_get_softc(dev); 1122 struct adapter *sc = pi->adapter; 1123 struct ifnet *ifp = pi->ifp; 1124 1125 /* Tell if_ioctl and if_init that the port is going away */ 1126 ADAPTER_LOCK(sc); 1127 SET_DOOMED(pi); 1128 wakeup(&sc->flags); 1129 while (IS_BUSY(sc)) 1130 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1131 SET_BUSY(sc); 1132#ifdef INVARIANTS 1133 sc->last_op = "t4detach"; 1134 sc->last_op_thr = curthread; 1135#endif 1136 ADAPTER_UNLOCK(sc); 1137 1138 if (pi->flags & HAS_TRACEQ) { 1139 sc->traceq = -1; /* cloner should not create ifnet */ 1140 t4_tracer_port_detach(sc); 1141 } 1142 1143 if (pi->vlan_c) 1144 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1145 1146 PORT_LOCK(pi); 1147 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1148 callout_stop(&pi->tick); 1149 PORT_UNLOCK(pi); 1150 callout_drain(&pi->tick); 1151 1152 /* Let detach proceed even if these fail. */ 1153 cxgbe_uninit_synchronized(pi); 1154 port_full_uninit(pi); 1155 1156 ifmedia_removeall(&pi->media); 1157 ether_ifdetach(pi->ifp); 1158 if_free(pi->ifp); 1159 1160#ifdef DEV_NETMAP 1161 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1162 destroy_netmap_ifnet(pi); 1163#endif 1164 1165 ADAPTER_LOCK(sc); 1166 CLR_BUSY(sc); 1167 wakeup(&sc->flags); 1168 ADAPTER_UNLOCK(sc); 1169 1170 return (0); 1171} 1172 1173static void 1174cxgbe_init(void *arg) 1175{ 1176 struct port_info *pi = arg; 1177 struct adapter *sc = pi->adapter; 1178 1179 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1180 return; 1181 cxgbe_init_synchronized(pi); 1182 end_synchronized_op(sc, 0); 1183} 1184 1185static int 1186cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1187{ 1188 int rc = 0, mtu, flags, can_sleep; 1189 struct port_info *pi = ifp->if_softc; 1190 struct adapter *sc = pi->adapter; 1191 struct ifreq *ifr = (struct ifreq *)data; 1192 uint32_t mask; 1193 1194 switch (cmd) { 1195 case SIOCSIFMTU: 1196 mtu = ifr->ifr_mtu; 1197 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1198 return (EINVAL); 1199 1200 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1201 if (rc) 1202 return (rc); 1203 ifp->if_mtu = mtu; 1204 if (pi->flags & PORT_INIT_DONE) { 1205 t4_update_fl_bufsize(ifp); 1206 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1207 rc = update_mac_settings(ifp, XGMAC_MTU); 1208 } 1209 end_synchronized_op(sc, 0); 1210 break; 1211 1212 case SIOCSIFFLAGS: 1213 can_sleep = 0; 1214redo_sifflags: 1215 rc = begin_synchronized_op(sc, pi, 1216 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1217 if (rc) 1218 return (rc); 1219 1220 if (ifp->if_flags & IFF_UP) { 1221 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1222 flags = pi->if_flags; 1223 if ((ifp->if_flags ^ flags) & 1224 (IFF_PROMISC | IFF_ALLMULTI)) { 1225 if (can_sleep == 1) { 1226 end_synchronized_op(sc, 0); 1227 can_sleep = 0; 1228 goto redo_sifflags; 1229 } 1230 rc = update_mac_settings(ifp, 1231 XGMAC_PROMISC | XGMAC_ALLMULTI); 1232 } 1233 } else { 1234 if (can_sleep == 0) { 1235 end_synchronized_op(sc, LOCK_HELD); 1236 can_sleep = 1; 1237 goto redo_sifflags; 1238 } 1239 rc = cxgbe_init_synchronized(pi); 1240 } 1241 pi->if_flags = ifp->if_flags; 1242 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1243 if (can_sleep == 0) { 1244 end_synchronized_op(sc, LOCK_HELD); 1245 can_sleep = 1; 1246 goto redo_sifflags; 1247 } 1248 rc = cxgbe_uninit_synchronized(pi); 1249 } 1250 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1251 break; 1252 1253 case SIOCADDMULTI: 1254 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1255 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1256 if (rc) 1257 return (rc); 1258 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1259 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1260 end_synchronized_op(sc, LOCK_HELD); 1261 break; 1262 1263 case SIOCSIFCAP: 1264 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1265 if (rc) 1266 return (rc); 1267 1268 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1269 if (mask & IFCAP_TXCSUM) { 1270 ifp->if_capenable ^= IFCAP_TXCSUM; 1271 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1272 1273 if (IFCAP_TSO4 & ifp->if_capenable && 1274 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1275 ifp->if_capenable &= ~IFCAP_TSO4; 1276 if_printf(ifp, 1277 "tso4 disabled due to -txcsum.\n"); 1278 } 1279 } 1280 if (mask & IFCAP_TXCSUM_IPV6) { 1281 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1282 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1283 1284 if (IFCAP_TSO6 & ifp->if_capenable && 1285 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1286 ifp->if_capenable &= ~IFCAP_TSO6; 1287 if_printf(ifp, 1288 "tso6 disabled due to -txcsum6.\n"); 1289 } 1290 } 1291 if (mask & IFCAP_RXCSUM) 1292 ifp->if_capenable ^= IFCAP_RXCSUM; 1293 if (mask & IFCAP_RXCSUM_IPV6) 1294 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1295 1296 /* 1297 * Note that we leave CSUM_TSO alone (it is always set). The 1298 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1299 * sending a TSO request our way, so it's sufficient to toggle 1300 * IFCAP_TSOx only. 1301 */ 1302 if (mask & IFCAP_TSO4) { 1303 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1304 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1305 if_printf(ifp, "enable txcsum first.\n"); 1306 rc = EAGAIN; 1307 goto fail; 1308 } 1309 ifp->if_capenable ^= IFCAP_TSO4; 1310 } 1311 if (mask & IFCAP_TSO6) { 1312 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1313 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1314 if_printf(ifp, "enable txcsum6 first.\n"); 1315 rc = EAGAIN; 1316 goto fail; 1317 } 1318 ifp->if_capenable ^= IFCAP_TSO6; 1319 } 1320 if (mask & IFCAP_LRO) { 1321#if defined(INET) || defined(INET6) 1322 int i; 1323 struct sge_rxq *rxq; 1324 1325 ifp->if_capenable ^= IFCAP_LRO; 1326 for_each_rxq(pi, i, rxq) { 1327 if (ifp->if_capenable & IFCAP_LRO) 1328 rxq->iq.flags |= IQ_LRO_ENABLED; 1329 else 1330 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1331 } 1332#endif 1333 } 1334#ifdef TCP_OFFLOAD 1335 if (mask & IFCAP_TOE) { 1336 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1337 1338 rc = toe_capability(pi, enable); 1339 if (rc != 0) 1340 goto fail; 1341 1342 ifp->if_capenable ^= mask; 1343 } 1344#endif 1345 if (mask & IFCAP_VLAN_HWTAGGING) { 1346 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1347 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1348 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1349 } 1350 if (mask & IFCAP_VLAN_MTU) { 1351 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1352 1353 /* Need to find out how to disable auto-mtu-inflation */ 1354 } 1355 if (mask & IFCAP_VLAN_HWTSO) 1356 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1357 if (mask & IFCAP_VLAN_HWCSUM) 1358 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1359 1360#ifdef VLAN_CAPABILITIES 1361 VLAN_CAPABILITIES(ifp); 1362#endif 1363fail: 1364 end_synchronized_op(sc, 0); 1365 break; 1366 1367 case SIOCSIFMEDIA: 1368 case SIOCGIFMEDIA: 1369 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1370 break; 1371 1372 default: 1373 rc = ether_ioctl(ifp, cmd, data); 1374 } 1375 1376 return (rc); 1377} 1378 1379static int 1380cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1381{ 1382 struct port_info *pi = ifp->if_softc; 1383 struct adapter *sc = pi->adapter; 1384 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1385 struct buf_ring *br; 1386 int rc; 1387 1388 M_ASSERTPKTHDR(m); 1389 1390 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1391 m_freem(m); 1392 return (ENETDOWN); 1393 } 1394 1395 /* check if flowid is set */ 1396 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1397 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1398 + pi->rsrv_noflowq); 1399 br = txq->br; 1400 1401 if (TXQ_TRYLOCK(txq) == 0) { 1402 struct sge_eq *eq = &txq->eq; 1403 1404 /* 1405 * It is possible that t4_eth_tx finishes up and releases the 1406 * lock between the TRYLOCK above and the drbr_enqueue here. We 1407 * need to make sure that this mbuf doesn't just sit there in 1408 * the drbr. 1409 */ 1410 1411 rc = drbr_enqueue(ifp, br, m); 1412 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1413 !(eq->flags & EQ_DOOMED)) 1414 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1415 return (rc); 1416 } 1417 1418 /* 1419 * txq->m is the mbuf that is held up due to a temporary shortage of 1420 * resources and it should be put on the wire first. Then what's in 1421 * drbr and finally the mbuf that was just passed in to us. 1422 * 1423 * Return code should indicate the fate of the mbuf that was passed in 1424 * this time. 1425 */ 1426 1427 TXQ_LOCK_ASSERT_OWNED(txq); 1428 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1429 1430 /* Queued for transmission. */ 1431 1432 rc = drbr_enqueue(ifp, br, m); 1433 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1434 (void) t4_eth_tx(ifp, txq, m); 1435 TXQ_UNLOCK(txq); 1436 return (rc); 1437 } 1438 1439 /* Direct transmission. */ 1440 rc = t4_eth_tx(ifp, txq, m); 1441 if (rc != 0 && txq->m) 1442 rc = 0; /* held, will be transmitted soon (hopefully) */ 1443 1444 TXQ_UNLOCK(txq); 1445 return (rc); 1446} 1447 1448static void 1449cxgbe_qflush(struct ifnet *ifp) 1450{ 1451 struct port_info *pi = ifp->if_softc; 1452 struct sge_txq *txq; 1453 int i; 1454 struct mbuf *m; 1455 1456 /* queues do not exist if !PORT_INIT_DONE. */ 1457 if (pi->flags & PORT_INIT_DONE) { 1458 for_each_txq(pi, i, txq) { 1459 TXQ_LOCK(txq); 1460 m_freem(txq->m); 1461 txq->m = NULL; 1462 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1463 m_freem(m); 1464 TXQ_UNLOCK(txq); 1465 } 1466 } 1467 if_qflush(ifp); 1468} 1469 1470static int 1471cxgbe_media_change(struct ifnet *ifp) 1472{ 1473 struct port_info *pi = ifp->if_softc; 1474 1475 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1476 1477 return (EOPNOTSUPP); 1478} 1479 1480static void 1481cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1482{ 1483 struct port_info *pi = ifp->if_softc; 1484 struct ifmedia *media = NULL; 1485 struct ifmedia_entry *cur; 1486 int speed = pi->link_cfg.speed; 1487#ifdef INVARIANTS 1488 int data = (pi->port_type << 8) | pi->mod_type; 1489#endif 1490 1491 if (ifp == pi->ifp) 1492 media = &pi->media; 1493#ifdef DEV_NETMAP 1494 else if (ifp == pi->nm_ifp) 1495 media = &pi->nm_media; 1496#endif 1497 MPASS(media != NULL); 1498 1499 cur = media->ifm_cur; 1500 MPASS(cur->ifm_data == data); 1501 1502 ifmr->ifm_status = IFM_AVALID; 1503 if (!pi->link_cfg.link_ok) 1504 return; 1505 1506 ifmr->ifm_status |= IFM_ACTIVE; 1507 1508 /* active and current will differ iff current media is autoselect. */ 1509 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1510 return; 1511 1512 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1513 if (speed == SPEED_10000) 1514 ifmr->ifm_active |= IFM_10G_T; 1515 else if (speed == SPEED_1000) 1516 ifmr->ifm_active |= IFM_1000_T; 1517 else if (speed == SPEED_100) 1518 ifmr->ifm_active |= IFM_100_TX; 1519 else if (speed == SPEED_10) 1520 ifmr->ifm_active |= IFM_10_T; 1521 else 1522 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1523 speed)); 1524} 1525 1526void 1527t4_fatal_err(struct adapter *sc) 1528{ 1529 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1530 t4_intr_disable(sc); 1531 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1532 device_get_nameunit(sc->dev)); 1533} 1534 1535static int 1536map_bars_0_and_4(struct adapter *sc) 1537{ 1538 sc->regs_rid = PCIR_BAR(0); 1539 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1540 &sc->regs_rid, RF_ACTIVE); 1541 if (sc->regs_res == NULL) { 1542 device_printf(sc->dev, "cannot map registers.\n"); 1543 return (ENXIO); 1544 } 1545 sc->bt = rman_get_bustag(sc->regs_res); 1546 sc->bh = rman_get_bushandle(sc->regs_res); 1547 sc->mmio_len = rman_get_size(sc->regs_res); 1548 setbit(&sc->doorbells, DOORBELL_KDB); 1549 1550 sc->msix_rid = PCIR_BAR(4); 1551 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1552 &sc->msix_rid, RF_ACTIVE); 1553 if (sc->msix_res == NULL) { 1554 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1555 return (ENXIO); 1556 } 1557 1558 return (0); 1559} 1560 1561static int 1562map_bar_2(struct adapter *sc) 1563{ 1564 1565 /* 1566 * T4: only iWARP driver uses the userspace doorbells. There is no need 1567 * to map it if RDMA is disabled. 1568 */ 1569 if (is_t4(sc) && sc->rdmacaps == 0) 1570 return (0); 1571 1572 sc->udbs_rid = PCIR_BAR(2); 1573 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1574 &sc->udbs_rid, RF_ACTIVE); 1575 if (sc->udbs_res == NULL) { 1576 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1577 return (ENXIO); 1578 } 1579 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1580 1581 if (is_t5(sc)) { 1582 setbit(&sc->doorbells, DOORBELL_UDB); 1583#if defined(__i386__) || defined(__amd64__) 1584 if (t5_write_combine) { 1585 int rc; 1586 1587 /* 1588 * Enable write combining on BAR2. This is the 1589 * userspace doorbell BAR and is split into 128B 1590 * (UDBS_SEG_SIZE) doorbell regions, each associated 1591 * with an egress queue. The first 64B has the doorbell 1592 * and the second 64B can be used to submit a tx work 1593 * request with an implicit doorbell. 1594 */ 1595 1596 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1597 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1598 if (rc == 0) { 1599 clrbit(&sc->doorbells, DOORBELL_UDB); 1600 setbit(&sc->doorbells, DOORBELL_WCWR); 1601 setbit(&sc->doorbells, DOORBELL_UDBWC); 1602 } else { 1603 device_printf(sc->dev, 1604 "couldn't enable write combining: %d\n", 1605 rc); 1606 } 1607 1608 t4_write_reg(sc, A_SGE_STAT_CFG, 1609 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1610 } 1611#endif 1612 } 1613 1614 return (0); 1615} 1616 1617static const struct memwin t4_memwin[] = { 1618 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1619 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1620 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1621}; 1622 1623static const struct memwin t5_memwin[] = { 1624 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1625 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1626 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1627}; 1628 1629static void 1630setup_memwin(struct adapter *sc) 1631{ 1632 const struct memwin *mw; 1633 int i, n; 1634 uint32_t bar0; 1635 1636 if (is_t4(sc)) { 1637 /* 1638 * Read low 32b of bar0 indirectly via the hardware backdoor 1639 * mechanism. Works from within PCI passthrough environments 1640 * too, where rman_get_start() can return a different value. We 1641 * need to program the T4 memory window decoders with the actual 1642 * addresses that will be coming across the PCIe link. 1643 */ 1644 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1645 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1646 1647 mw = &t4_memwin[0]; 1648 n = nitems(t4_memwin); 1649 } else { 1650 /* T5 uses the relative offset inside the PCIe BAR */ 1651 bar0 = 0; 1652 1653 mw = &t5_memwin[0]; 1654 n = nitems(t5_memwin); 1655 } 1656 1657 for (i = 0; i < n; i++, mw++) { 1658 t4_write_reg(sc, 1659 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1660 (mw->base + bar0) | V_BIR(0) | 1661 V_WINDOW(ilog2(mw->aperture) - 10)); 1662 } 1663 1664 /* flush */ 1665 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1666} 1667 1668/* 1669 * Verify that the memory range specified by the addr/len pair is valid and lies 1670 * entirely within a single region (EDCx or MCx). 1671 */ 1672static int 1673validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1674{ 1675 uint32_t em, addr_len, maddr, mlen; 1676 1677 /* Memory can only be accessed in naturally aligned 4 byte units */ 1678 if (addr & 3 || len & 3 || len == 0) 1679 return (EINVAL); 1680 1681 /* Enabled memories */ 1682 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1683 if (em & F_EDRAM0_ENABLE) { 1684 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1685 maddr = G_EDRAM0_BASE(addr_len) << 20; 1686 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1687 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1688 addr + len <= maddr + mlen) 1689 return (0); 1690 } 1691 if (em & F_EDRAM1_ENABLE) { 1692 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1693 maddr = G_EDRAM1_BASE(addr_len) << 20; 1694 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1695 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1696 addr + len <= maddr + mlen) 1697 return (0); 1698 } 1699 if (em & F_EXT_MEM_ENABLE) { 1700 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1701 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1702 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1703 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1704 addr + len <= maddr + mlen) 1705 return (0); 1706 } 1707 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1708 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1709 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1710 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1711 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1712 addr + len <= maddr + mlen) 1713 return (0); 1714 } 1715 1716 return (EFAULT); 1717} 1718 1719static int 1720fwmtype_to_hwmtype(int mtype) 1721{ 1722 1723 switch (mtype) { 1724 case FW_MEMTYPE_EDC0: 1725 return (MEM_EDC0); 1726 case FW_MEMTYPE_EDC1: 1727 return (MEM_EDC1); 1728 case FW_MEMTYPE_EXTMEM: 1729 return (MEM_MC0); 1730 case FW_MEMTYPE_EXTMEM1: 1731 return (MEM_MC1); 1732 default: 1733 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1734 } 1735} 1736 1737/* 1738 * Verify that the memory range specified by the memtype/offset/len pair is 1739 * valid and lies entirely within the memtype specified. The global address of 1740 * the start of the range is returned in addr. 1741 */ 1742static int 1743validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1744 uint32_t *addr) 1745{ 1746 uint32_t em, addr_len, maddr, mlen; 1747 1748 /* Memory can only be accessed in naturally aligned 4 byte units */ 1749 if (off & 3 || len & 3 || len == 0) 1750 return (EINVAL); 1751 1752 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1753 switch (fwmtype_to_hwmtype(mtype)) { 1754 case MEM_EDC0: 1755 if (!(em & F_EDRAM0_ENABLE)) 1756 return (EINVAL); 1757 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1758 maddr = G_EDRAM0_BASE(addr_len) << 20; 1759 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1760 break; 1761 case MEM_EDC1: 1762 if (!(em & F_EDRAM1_ENABLE)) 1763 return (EINVAL); 1764 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1765 maddr = G_EDRAM1_BASE(addr_len) << 20; 1766 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1767 break; 1768 case MEM_MC: 1769 if (!(em & F_EXT_MEM_ENABLE)) 1770 return (EINVAL); 1771 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1772 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1773 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1774 break; 1775 case MEM_MC1: 1776 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1777 return (EINVAL); 1778 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1779 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1780 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1781 break; 1782 default: 1783 return (EINVAL); 1784 } 1785 1786 if (mlen > 0 && off < mlen && off + len <= mlen) { 1787 *addr = maddr + off; /* global address */ 1788 return (0); 1789 } 1790 1791 return (EFAULT); 1792} 1793 1794static void 1795memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1796{ 1797 const struct memwin *mw; 1798 1799 if (is_t4(sc)) { 1800 KASSERT(win >= 0 && win < nitems(t4_memwin), 1801 ("%s: incorrect memwin# (%d)", __func__, win)); 1802 mw = &t4_memwin[win]; 1803 } else { 1804 KASSERT(win >= 0 && win < nitems(t5_memwin), 1805 ("%s: incorrect memwin# (%d)", __func__, win)); 1806 mw = &t5_memwin[win]; 1807 } 1808 1809 if (base != NULL) 1810 *base = mw->base; 1811 if (aperture != NULL) 1812 *aperture = mw->aperture; 1813} 1814 1815/* 1816 * Positions the memory window such that it can be used to access the specified 1817 * address in the chip's address space. The return value is the offset of addr 1818 * from the start of the window. 1819 */ 1820static uint32_t 1821position_memwin(struct adapter *sc, int n, uint32_t addr) 1822{ 1823 uint32_t start, pf; 1824 uint32_t reg; 1825 1826 KASSERT(n >= 0 && n <= 3, 1827 ("%s: invalid window %d.", __func__, n)); 1828 KASSERT((addr & 3) == 0, 1829 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1830 1831 if (is_t4(sc)) { 1832 pf = 0; 1833 start = addr & ~0xf; /* start must be 16B aligned */ 1834 } else { 1835 pf = V_PFNUM(sc->pf); 1836 start = addr & ~0x7f; /* start must be 128B aligned */ 1837 } 1838 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1839 1840 t4_write_reg(sc, reg, start | pf); 1841 t4_read_reg(sc, reg); 1842 1843 return (addr - start); 1844} 1845 1846static int 1847cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1848 struct intrs_and_queues *iaq) 1849{ 1850 int rc, itype, navail, nrxq10g, nrxq1g, n; 1851 int nofldrxq10g = 0, nofldrxq1g = 0; 1852 int nnmrxq10g = 0, nnmrxq1g = 0; 1853 1854 bzero(iaq, sizeof(*iaq)); 1855 1856 iaq->ntxq10g = t4_ntxq10g; 1857 iaq->ntxq1g = t4_ntxq1g; 1858 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1859 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1860 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1861#ifdef TCP_OFFLOAD 1862 if (is_offload(sc)) { 1863 iaq->nofldtxq10g = t4_nofldtxq10g; 1864 iaq->nofldtxq1g = t4_nofldtxq1g; 1865 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1866 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1867 } 1868#endif 1869#ifdef DEV_NETMAP 1870 iaq->nnmtxq10g = t4_nnmtxq10g; 1871 iaq->nnmtxq1g = t4_nnmtxq1g; 1872 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1873 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1874#endif 1875 1876 for (itype = INTR_MSIX; itype; itype >>= 1) { 1877 1878 if ((itype & t4_intr_types) == 0) 1879 continue; /* not allowed */ 1880 1881 if (itype == INTR_MSIX) 1882 navail = pci_msix_count(sc->dev); 1883 else if (itype == INTR_MSI) 1884 navail = pci_msi_count(sc->dev); 1885 else 1886 navail = 1; 1887restart: 1888 if (navail == 0) 1889 continue; 1890 1891 iaq->intr_type = itype; 1892 iaq->intr_flags_10g = 0; 1893 iaq->intr_flags_1g = 0; 1894 1895 /* 1896 * Best option: an interrupt vector for errors, one for the 1897 * firmware event queue, and one for every rxq (NIC, TOE, and 1898 * netmap). 1899 */ 1900 iaq->nirq = T4_EXTRA_INTR; 1901 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1902 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1903 if (iaq->nirq <= navail && 1904 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1905 iaq->intr_flags_10g = INTR_ALL; 1906 iaq->intr_flags_1g = INTR_ALL; 1907 goto allocate; 1908 } 1909 1910 /* 1911 * Second best option: a vector for errors, one for the firmware 1912 * event queue, and vectors for either all the NIC rx queues or 1913 * all the TOE rx queues. The queues that don't get vectors 1914 * will forward their interrupts to those that do. 1915 * 1916 * Note: netmap rx queues cannot be created early and so they 1917 * can't be setup to receive forwarded interrupts for others. 1918 */ 1919 iaq->nirq = T4_EXTRA_INTR; 1920 if (nrxq10g >= nofldrxq10g) { 1921 iaq->intr_flags_10g = INTR_RXQ; 1922 iaq->nirq += n10g * nrxq10g; 1923#ifdef DEV_NETMAP 1924 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1925#endif 1926 } else { 1927 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1928 iaq->nirq += n10g * nofldrxq10g; 1929#ifdef DEV_NETMAP 1930 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1931#endif 1932 } 1933 if (nrxq1g >= nofldrxq1g) { 1934 iaq->intr_flags_1g = INTR_RXQ; 1935 iaq->nirq += n1g * nrxq1g; 1936#ifdef DEV_NETMAP 1937 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1938#endif 1939 } else { 1940 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1941 iaq->nirq += n1g * nofldrxq1g; 1942#ifdef DEV_NETMAP 1943 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1944#endif 1945 } 1946 if (iaq->nirq <= navail && 1947 (itype != INTR_MSI || powerof2(iaq->nirq))) 1948 goto allocate; 1949 1950 /* 1951 * Next best option: an interrupt vector for errors, one for the 1952 * firmware event queue, and at least one per port. At this 1953 * point we know we'll have to downsize nrxq and/or nofldrxq 1954 * and/or nnmrxq to fit what's available to us. 1955 */ 1956 iaq->nirq = T4_EXTRA_INTR; 1957 iaq->nirq += n10g + n1g; 1958 if (iaq->nirq <= navail) { 1959 int leftover = navail - iaq->nirq; 1960 1961 if (n10g > 0) { 1962 int target = max(nrxq10g, nofldrxq10g); 1963 1964 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1965 INTR_RXQ : INTR_OFLD_RXQ; 1966 1967 n = 1; 1968 while (n < target && leftover >= n10g) { 1969 leftover -= n10g; 1970 iaq->nirq += n10g; 1971 n++; 1972 } 1973 iaq->nrxq10g = min(n, nrxq10g); 1974#ifdef TCP_OFFLOAD 1975 iaq->nofldrxq10g = min(n, nofldrxq10g); 1976#endif 1977#ifdef DEV_NETMAP 1978 iaq->nnmrxq10g = min(n, nnmrxq10g); 1979#endif 1980 } 1981 1982 if (n1g > 0) { 1983 int target = max(nrxq1g, nofldrxq1g); 1984 1985 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1986 INTR_RXQ : INTR_OFLD_RXQ; 1987 1988 n = 1; 1989 while (n < target && leftover >= n1g) { 1990 leftover -= n1g; 1991 iaq->nirq += n1g; 1992 n++; 1993 } 1994 iaq->nrxq1g = min(n, nrxq1g); 1995#ifdef TCP_OFFLOAD 1996 iaq->nofldrxq1g = min(n, nofldrxq1g); 1997#endif 1998#ifdef DEV_NETMAP 1999 iaq->nnmrxq1g = min(n, nnmrxq1g); 2000#endif 2001 } 2002 2003 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2004 goto allocate; 2005 } 2006 2007 /* 2008 * Least desirable option: one interrupt vector for everything. 2009 */ 2010 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2011 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2012#ifdef TCP_OFFLOAD 2013 if (is_offload(sc)) 2014 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2015#endif 2016#ifdef DEV_NETMAP 2017 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2018#endif 2019 2020allocate: 2021 navail = iaq->nirq; 2022 rc = 0; 2023 if (itype == INTR_MSIX) 2024 rc = pci_alloc_msix(sc->dev, &navail); 2025 else if (itype == INTR_MSI) 2026 rc = pci_alloc_msi(sc->dev, &navail); 2027 2028 if (rc == 0) { 2029 if (navail == iaq->nirq) 2030 return (0); 2031 2032 /* 2033 * Didn't get the number requested. Use whatever number 2034 * the kernel is willing to allocate (it's in navail). 2035 */ 2036 device_printf(sc->dev, "fewer vectors than requested, " 2037 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2038 itype, iaq->nirq, navail); 2039 pci_release_msi(sc->dev); 2040 goto restart; 2041 } 2042 2043 device_printf(sc->dev, 2044 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2045 itype, rc, iaq->nirq, navail); 2046 } 2047 2048 device_printf(sc->dev, 2049 "failed to find a usable interrupt type. " 2050 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2051 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2052 2053 return (ENXIO); 2054} 2055 2056#define FW_VERSION(chip) ( \ 2057 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2058 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2059 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2060 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2061#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2062 2063struct fw_info { 2064 uint8_t chip; 2065 char *kld_name; 2066 char *fw_mod_name; 2067 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2068} fw_info[] = { 2069 { 2070 .chip = CHELSIO_T4, 2071 .kld_name = "t4fw_cfg", 2072 .fw_mod_name = "t4fw", 2073 .fw_hdr = { 2074 .chip = FW_HDR_CHIP_T4, 2075 .fw_ver = htobe32_const(FW_VERSION(T4)), 2076 .intfver_nic = FW_INTFVER(T4, NIC), 2077 .intfver_vnic = FW_INTFVER(T4, VNIC), 2078 .intfver_ofld = FW_INTFVER(T4, OFLD), 2079 .intfver_ri = FW_INTFVER(T4, RI), 2080 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2081 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2082 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2083 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2084 }, 2085 }, { 2086 .chip = CHELSIO_T5, 2087 .kld_name = "t5fw_cfg", 2088 .fw_mod_name = "t5fw", 2089 .fw_hdr = { 2090 .chip = FW_HDR_CHIP_T5, 2091 .fw_ver = htobe32_const(FW_VERSION(T5)), 2092 .intfver_nic = FW_INTFVER(T5, NIC), 2093 .intfver_vnic = FW_INTFVER(T5, VNIC), 2094 .intfver_ofld = FW_INTFVER(T5, OFLD), 2095 .intfver_ri = FW_INTFVER(T5, RI), 2096 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2097 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2098 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2099 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2100 }, 2101 } 2102}; 2103 2104static struct fw_info * 2105find_fw_info(int chip) 2106{ 2107 int i; 2108 2109 for (i = 0; i < nitems(fw_info); i++) { 2110 if (fw_info[i].chip == chip) 2111 return (&fw_info[i]); 2112 } 2113 return (NULL); 2114} 2115 2116/* 2117 * Is the given firmware API compatible with the one the driver was compiled 2118 * with? 2119 */ 2120static int 2121fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2122{ 2123 2124 /* short circuit if it's the exact same firmware version */ 2125 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2126 return (1); 2127 2128 /* 2129 * XXX: Is this too conservative? Perhaps I should limit this to the 2130 * features that are supported in the driver. 2131 */ 2132#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2133 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2134 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2135 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2136 return (1); 2137#undef SAME_INTF 2138 2139 return (0); 2140} 2141 2142/* 2143 * The firmware in the KLD is usable, but should it be installed? This routine 2144 * explains itself in detail if it indicates the KLD firmware should be 2145 * installed. 2146 */ 2147static int 2148should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2149{ 2150 const char *reason; 2151 2152 if (!card_fw_usable) { 2153 reason = "incompatible or unusable"; 2154 goto install; 2155 } 2156 2157 if (k > c) { 2158 reason = "older than the version bundled with this driver"; 2159 goto install; 2160 } 2161 2162 if (t4_fw_install == 2 && k != c) { 2163 reason = "different than the version bundled with this driver"; 2164 goto install; 2165 } 2166 2167 return (0); 2168 2169install: 2170 if (t4_fw_install == 0) { 2171 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2172 "but the driver is prohibited from installing a different " 2173 "firmware on the card.\n", 2174 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2175 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2176 2177 return (0); 2178 } 2179 2180 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2181 "installing firmware %u.%u.%u.%u on card.\n", 2182 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2183 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2184 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2185 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2186 2187 return (1); 2188} 2189/* 2190 * Establish contact with the firmware and determine if we are the master driver 2191 * or not, and whether we are responsible for chip initialization. 2192 */ 2193static int 2194prep_firmware(struct adapter *sc) 2195{ 2196 const struct firmware *fw = NULL, *default_cfg; 2197 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2198 enum dev_state state; 2199 struct fw_info *fw_info; 2200 struct fw_hdr *card_fw; /* fw on the card */ 2201 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2202 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2203 against */ 2204 2205 /* Contact firmware. */ 2206 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2207 if (rc < 0 || state == DEV_STATE_ERR) { 2208 rc = -rc; 2209 device_printf(sc->dev, 2210 "failed to connect to the firmware: %d, %d.\n", rc, state); 2211 return (rc); 2212 } 2213 pf = rc; 2214 if (pf == sc->mbox) 2215 sc->flags |= MASTER_PF; 2216 else if (state == DEV_STATE_UNINIT) { 2217 /* 2218 * We didn't get to be the master so we definitely won't be 2219 * configuring the chip. It's a bug if someone else hasn't 2220 * configured it already. 2221 */ 2222 device_printf(sc->dev, "couldn't be master(%d), " 2223 "device not already initialized either(%d).\n", rc, state); 2224 return (EDOOFUS); 2225 } 2226 2227 /* This is the firmware whose headers the driver was compiled against */ 2228 fw_info = find_fw_info(chip_id(sc)); 2229 if (fw_info == NULL) { 2230 device_printf(sc->dev, 2231 "unable to look up firmware information for chip %d.\n", 2232 chip_id(sc)); 2233 return (EINVAL); 2234 } 2235 drv_fw = &fw_info->fw_hdr; 2236 2237 /* 2238 * The firmware KLD contains many modules. The KLD name is also the 2239 * name of the module that contains the default config file. 2240 */ 2241 default_cfg = firmware_get(fw_info->kld_name); 2242 2243 /* Read the header of the firmware on the card */ 2244 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2245 rc = -t4_read_flash(sc, FLASH_FW_START, 2246 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2247 if (rc == 0) 2248 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2249 else { 2250 device_printf(sc->dev, 2251 "Unable to read card's firmware header: %d\n", rc); 2252 card_fw_usable = 0; 2253 } 2254 2255 /* This is the firmware in the KLD */ 2256 fw = firmware_get(fw_info->fw_mod_name); 2257 if (fw != NULL) { 2258 kld_fw = (const void *)fw->data; 2259 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2260 } else { 2261 kld_fw = NULL; 2262 kld_fw_usable = 0; 2263 } 2264 2265 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2266 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2267 /* 2268 * Common case: the firmware on the card is an exact match and 2269 * the KLD is an exact match too, or the KLD is 2270 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2271 * here -- use cxgbetool loadfw if you want to reinstall the 2272 * same firmware as the one on the card. 2273 */ 2274 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2275 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2276 be32toh(card_fw->fw_ver))) { 2277 2278 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2279 if (rc != 0) { 2280 device_printf(sc->dev, 2281 "failed to install firmware: %d\n", rc); 2282 goto done; 2283 } 2284 2285 /* Installed successfully, update the cached header too. */ 2286 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2287 card_fw_usable = 1; 2288 need_fw_reset = 0; /* already reset as part of load_fw */ 2289 } 2290 2291 if (!card_fw_usable) { 2292 uint32_t d, c, k; 2293 2294 d = ntohl(drv_fw->fw_ver); 2295 c = ntohl(card_fw->fw_ver); 2296 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2297 2298 device_printf(sc->dev, "Cannot find a usable firmware: " 2299 "fw_install %d, chip state %d, " 2300 "driver compiled with %d.%d.%d.%d, " 2301 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2302 t4_fw_install, state, 2303 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2304 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2305 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2306 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2307 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2308 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2309 rc = EINVAL; 2310 goto done; 2311 } 2312 2313 /* We're using whatever's on the card and it's known to be good. */ 2314 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2315 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2316 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2317 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2318 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2319 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2320 t4_get_tp_version(sc, &sc->params.tp_vers); 2321 2322 /* Reset device */ 2323 if (need_fw_reset && 2324 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2325 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2326 if (rc != ETIMEDOUT && rc != EIO) 2327 t4_fw_bye(sc, sc->mbox); 2328 goto done; 2329 } 2330 sc->flags |= FW_OK; 2331 2332 rc = get_params__pre_init(sc); 2333 if (rc != 0) 2334 goto done; /* error message displayed already */ 2335 2336 /* Partition adapter resources as specified in the config file. */ 2337 if (state == DEV_STATE_UNINIT) { 2338 2339 KASSERT(sc->flags & MASTER_PF, 2340 ("%s: trying to change chip settings when not master.", 2341 __func__)); 2342 2343 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2344 if (rc != 0) 2345 goto done; /* error message displayed already */ 2346 2347 t4_tweak_chip_settings(sc); 2348 2349 /* get basic stuff going */ 2350 rc = -t4_fw_initialize(sc, sc->mbox); 2351 if (rc != 0) { 2352 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2353 goto done; 2354 } 2355 } else { 2356 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2357 sc->cfcsum = 0; 2358 } 2359 2360done: 2361 free(card_fw, M_CXGBE); 2362 if (fw != NULL) 2363 firmware_put(fw, FIRMWARE_UNLOAD); 2364 if (default_cfg != NULL) 2365 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2366 2367 return (rc); 2368} 2369 2370#define FW_PARAM_DEV(param) \ 2371 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2372 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2373#define FW_PARAM_PFVF(param) \ 2374 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2375 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2376 2377/* 2378 * Partition chip resources for use between various PFs, VFs, etc. 2379 */ 2380static int 2381partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2382 const char *name_prefix) 2383{ 2384 const struct firmware *cfg = NULL; 2385 int rc = 0; 2386 struct fw_caps_config_cmd caps; 2387 uint32_t mtype, moff, finicsum, cfcsum; 2388 2389 /* 2390 * Figure out what configuration file to use. Pick the default config 2391 * file for the card if the user hasn't specified one explicitly. 2392 */ 2393 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2394 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2395 /* Card specific overrides go here. */ 2396 if (pci_get_device(sc->dev) == 0x440a) 2397 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2398 if (is_fpga(sc)) 2399 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2400 } 2401 2402 /* 2403 * We need to load another module if the profile is anything except 2404 * "default" or "flash". 2405 */ 2406 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2407 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2408 char s[32]; 2409 2410 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2411 cfg = firmware_get(s); 2412 if (cfg == NULL) { 2413 if (default_cfg != NULL) { 2414 device_printf(sc->dev, 2415 "unable to load module \"%s\" for " 2416 "configuration profile \"%s\", will use " 2417 "the default config file instead.\n", 2418 s, sc->cfg_file); 2419 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2420 "%s", DEFAULT_CF); 2421 } else { 2422 device_printf(sc->dev, 2423 "unable to load module \"%s\" for " 2424 "configuration profile \"%s\", will use " 2425 "the config file on the card's flash " 2426 "instead.\n", s, sc->cfg_file); 2427 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2428 "%s", FLASH_CF); 2429 } 2430 } 2431 } 2432 2433 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2434 default_cfg == NULL) { 2435 device_printf(sc->dev, 2436 "default config file not available, will use the config " 2437 "file on the card's flash instead.\n"); 2438 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2439 } 2440 2441 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2442 u_int cflen, i, n; 2443 const uint32_t *cfdata; 2444 uint32_t param, val, addr, off, mw_base, mw_aperture; 2445 2446 KASSERT(cfg != NULL || default_cfg != NULL, 2447 ("%s: no config to upload", __func__)); 2448 2449 /* 2450 * Ask the firmware where it wants us to upload the config file. 2451 */ 2452 param = FW_PARAM_DEV(CF); 2453 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2454 if (rc != 0) { 2455 /* No support for config file? Shouldn't happen. */ 2456 device_printf(sc->dev, 2457 "failed to query config file location: %d.\n", rc); 2458 goto done; 2459 } 2460 mtype = G_FW_PARAMS_PARAM_Y(val); 2461 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2462 2463 /* 2464 * XXX: sheer laziness. We deliberately added 4 bytes of 2465 * useless stuffing/comments at the end of the config file so 2466 * it's ok to simply throw away the last remaining bytes when 2467 * the config file is not an exact multiple of 4. This also 2468 * helps with the validate_mt_off_len check. 2469 */ 2470 if (cfg != NULL) { 2471 cflen = cfg->datasize & ~3; 2472 cfdata = cfg->data; 2473 } else { 2474 cflen = default_cfg->datasize & ~3; 2475 cfdata = default_cfg->data; 2476 } 2477 2478 if (cflen > FLASH_CFG_MAX_SIZE) { 2479 device_printf(sc->dev, 2480 "config file too long (%d, max allowed is %d). " 2481 "Will try to use the config on the card, if any.\n", 2482 cflen, FLASH_CFG_MAX_SIZE); 2483 goto use_config_on_flash; 2484 } 2485 2486 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2487 if (rc != 0) { 2488 device_printf(sc->dev, 2489 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2490 "Will try to use the config on the card, if any.\n", 2491 __func__, mtype, moff, cflen, rc); 2492 goto use_config_on_flash; 2493 } 2494 2495 memwin_info(sc, 2, &mw_base, &mw_aperture); 2496 while (cflen) { 2497 off = position_memwin(sc, 2, addr); 2498 n = min(cflen, mw_aperture - off); 2499 for (i = 0; i < n; i += 4) 2500 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2501 cflen -= n; 2502 addr += n; 2503 } 2504 } else { 2505use_config_on_flash: 2506 mtype = FW_MEMTYPE_FLASH; 2507 moff = t4_flash_cfg_addr(sc); 2508 } 2509 2510 bzero(&caps, sizeof(caps)); 2511 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2512 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2513 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2514 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2515 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2516 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2517 if (rc != 0) { 2518 device_printf(sc->dev, 2519 "failed to pre-process config file: %d " 2520 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2521 goto done; 2522 } 2523 2524 finicsum = be32toh(caps.finicsum); 2525 cfcsum = be32toh(caps.cfcsum); 2526 if (finicsum != cfcsum) { 2527 device_printf(sc->dev, 2528 "WARNING: config file checksum mismatch: %08x %08x\n", 2529 finicsum, cfcsum); 2530 } 2531 sc->cfcsum = cfcsum; 2532 2533#define LIMIT_CAPS(x) do { \ 2534 caps.x &= htobe16(t4_##x##_allowed); \ 2535} while (0) 2536 2537 /* 2538 * Let the firmware know what features will (not) be used so it can tune 2539 * things accordingly. 2540 */ 2541 LIMIT_CAPS(linkcaps); 2542 LIMIT_CAPS(niccaps); 2543 LIMIT_CAPS(toecaps); 2544 LIMIT_CAPS(rdmacaps); 2545 LIMIT_CAPS(iscsicaps); 2546 LIMIT_CAPS(fcoecaps); 2547#undef LIMIT_CAPS 2548 2549 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2550 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2551 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2552 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2553 if (rc != 0) { 2554 device_printf(sc->dev, 2555 "failed to process config file: %d.\n", rc); 2556 } 2557done: 2558 if (cfg != NULL) 2559 firmware_put(cfg, FIRMWARE_UNLOAD); 2560 return (rc); 2561} 2562 2563/* 2564 * Retrieve parameters that are needed (or nice to have) very early. 2565 */ 2566static int 2567get_params__pre_init(struct adapter *sc) 2568{ 2569 int rc; 2570 uint32_t param[2], val[2]; 2571 struct fw_devlog_cmd cmd; 2572 struct devlog_params *dlog = &sc->params.devlog; 2573 2574 param[0] = FW_PARAM_DEV(PORTVEC); 2575 param[1] = FW_PARAM_DEV(CCLK); 2576 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2577 if (rc != 0) { 2578 device_printf(sc->dev, 2579 "failed to query parameters (pre_init): %d.\n", rc); 2580 return (rc); 2581 } 2582 2583 sc->params.portvec = val[0]; 2584 sc->params.nports = bitcount32(val[0]); 2585 sc->params.vpd.cclk = val[1]; 2586 2587 /* Read device log parameters. */ 2588 bzero(&cmd, sizeof(cmd)); 2589 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2590 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2591 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2592 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2593 if (rc != 0) { 2594 device_printf(sc->dev, 2595 "failed to get devlog parameters: %d.\n", rc); 2596 bzero(dlog, sizeof (*dlog)); 2597 rc = 0; /* devlog isn't critical for device operation */ 2598 } else { 2599 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2600 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2601 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2602 dlog->size = be32toh(cmd.memsize_devlog); 2603 } 2604 2605 return (rc); 2606} 2607 2608/* 2609 * Retrieve various parameters that are of interest to the driver. The device 2610 * has been initialized by the firmware at this point. 2611 */ 2612static int 2613get_params__post_init(struct adapter *sc) 2614{ 2615 int rc; 2616 uint32_t param[7], val[7]; 2617 struct fw_caps_config_cmd caps; 2618 2619 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2620 param[1] = FW_PARAM_PFVF(EQ_START); 2621 param[2] = FW_PARAM_PFVF(FILTER_START); 2622 param[3] = FW_PARAM_PFVF(FILTER_END); 2623 param[4] = FW_PARAM_PFVF(L2T_START); 2624 param[5] = FW_PARAM_PFVF(L2T_END); 2625 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2626 if (rc != 0) { 2627 device_printf(sc->dev, 2628 "failed to query parameters (post_init): %d.\n", rc); 2629 return (rc); 2630 } 2631 2632 sc->sge.iq_start = val[0]; 2633 sc->sge.eq_start = val[1]; 2634 sc->tids.ftid_base = val[2]; 2635 sc->tids.nftids = val[3] - val[2] + 1; 2636 sc->params.ftid_min = val[2]; 2637 sc->params.ftid_max = val[3]; 2638 sc->vres.l2t.start = val[4]; 2639 sc->vres.l2t.size = val[5] - val[4] + 1; 2640 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2641 ("%s: L2 table size (%u) larger than expected (%u)", 2642 __func__, sc->vres.l2t.size, L2T_SIZE)); 2643 2644 /* get capabilites */ 2645 bzero(&caps, sizeof(caps)); 2646 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2647 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2648 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2649 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2650 if (rc != 0) { 2651 device_printf(sc->dev, 2652 "failed to get card capabilities: %d.\n", rc); 2653 return (rc); 2654 } 2655 2656#define READ_CAPS(x) do { \ 2657 sc->x = htobe16(caps.x); \ 2658} while (0) 2659 READ_CAPS(linkcaps); 2660 READ_CAPS(niccaps); 2661 READ_CAPS(toecaps); 2662 READ_CAPS(rdmacaps); 2663 READ_CAPS(iscsicaps); 2664 READ_CAPS(fcoecaps); 2665 2666 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2667 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2668 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2669 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2670 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2671 if (rc != 0) { 2672 device_printf(sc->dev, 2673 "failed to query NIC parameters: %d.\n", rc); 2674 return (rc); 2675 } 2676 sc->tids.etid_base = val[0]; 2677 sc->params.etid_min = val[0]; 2678 sc->tids.netids = val[1] - val[0] + 1; 2679 sc->params.netids = sc->tids.netids; 2680 sc->params.eo_wr_cred = val[2]; 2681 sc->params.ethoffload = 1; 2682 } 2683 2684 if (sc->toecaps) { 2685 /* query offload-related parameters */ 2686 param[0] = FW_PARAM_DEV(NTID); 2687 param[1] = FW_PARAM_PFVF(SERVER_START); 2688 param[2] = FW_PARAM_PFVF(SERVER_END); 2689 param[3] = FW_PARAM_PFVF(TDDP_START); 2690 param[4] = FW_PARAM_PFVF(TDDP_END); 2691 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2692 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2693 if (rc != 0) { 2694 device_printf(sc->dev, 2695 "failed to query TOE parameters: %d.\n", rc); 2696 return (rc); 2697 } 2698 sc->tids.ntids = val[0]; 2699 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2700 sc->tids.stid_base = val[1]; 2701 sc->tids.nstids = val[2] - val[1] + 1; 2702 sc->vres.ddp.start = val[3]; 2703 sc->vres.ddp.size = val[4] - val[3] + 1; 2704 sc->params.ofldq_wr_cred = val[5]; 2705 sc->params.offload = 1; 2706 } 2707 if (sc->rdmacaps) { 2708 param[0] = FW_PARAM_PFVF(STAG_START); 2709 param[1] = FW_PARAM_PFVF(STAG_END); 2710 param[2] = FW_PARAM_PFVF(RQ_START); 2711 param[3] = FW_PARAM_PFVF(RQ_END); 2712 param[4] = FW_PARAM_PFVF(PBL_START); 2713 param[5] = FW_PARAM_PFVF(PBL_END); 2714 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2715 if (rc != 0) { 2716 device_printf(sc->dev, 2717 "failed to query RDMA parameters(1): %d.\n", rc); 2718 return (rc); 2719 } 2720 sc->vres.stag.start = val[0]; 2721 sc->vres.stag.size = val[1] - val[0] + 1; 2722 sc->vres.rq.start = val[2]; 2723 sc->vres.rq.size = val[3] - val[2] + 1; 2724 sc->vres.pbl.start = val[4]; 2725 sc->vres.pbl.size = val[5] - val[4] + 1; 2726 2727 param[0] = FW_PARAM_PFVF(SQRQ_START); 2728 param[1] = FW_PARAM_PFVF(SQRQ_END); 2729 param[2] = FW_PARAM_PFVF(CQ_START); 2730 param[3] = FW_PARAM_PFVF(CQ_END); 2731 param[4] = FW_PARAM_PFVF(OCQ_START); 2732 param[5] = FW_PARAM_PFVF(OCQ_END); 2733 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2734 if (rc != 0) { 2735 device_printf(sc->dev, 2736 "failed to query RDMA parameters(2): %d.\n", rc); 2737 return (rc); 2738 } 2739 sc->vres.qp.start = val[0]; 2740 sc->vres.qp.size = val[1] - val[0] + 1; 2741 sc->vres.cq.start = val[2]; 2742 sc->vres.cq.size = val[3] - val[2] + 1; 2743 sc->vres.ocq.start = val[4]; 2744 sc->vres.ocq.size = val[5] - val[4] + 1; 2745 } 2746 if (sc->iscsicaps) { 2747 param[0] = FW_PARAM_PFVF(ISCSI_START); 2748 param[1] = FW_PARAM_PFVF(ISCSI_END); 2749 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2750 if (rc != 0) { 2751 device_printf(sc->dev, 2752 "failed to query iSCSI parameters: %d.\n", rc); 2753 return (rc); 2754 } 2755 sc->vres.iscsi.start = val[0]; 2756 sc->vres.iscsi.size = val[1] - val[0] + 1; 2757 } 2758 2759 /* 2760 * We've got the params we wanted to query via the firmware. Now grab 2761 * some others directly from the chip. 2762 */ 2763 rc = t4_read_chip_settings(sc); 2764 2765 return (rc); 2766} 2767 2768static int 2769set_params__post_init(struct adapter *sc) 2770{ 2771 uint32_t param, val; 2772 2773 /* ask for encapsulated CPLs */ 2774 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2775 val = 1; 2776 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2777 2778 return (0); 2779} 2780 2781#undef FW_PARAM_PFVF 2782#undef FW_PARAM_DEV 2783 2784static void 2785t4_set_desc(struct adapter *sc) 2786{ 2787 char buf[128]; 2788 struct adapter_params *p = &sc->params; 2789 2790 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2791 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2792 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2793 2794 device_set_desc_copy(sc->dev, buf); 2795} 2796 2797static void 2798build_medialist(struct port_info *pi, struct ifmedia *media) 2799{ 2800 int data, m; 2801 2802 PORT_LOCK(pi); 2803 2804 ifmedia_removeall(media); 2805 2806 m = IFM_ETHER | IFM_FDX; 2807 data = (pi->port_type << 8) | pi->mod_type; 2808 2809 switch(pi->port_type) { 2810 case FW_PORT_TYPE_BT_XFI: 2811 case FW_PORT_TYPE_BT_XAUI: 2812 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2813 /* fall through */ 2814 2815 case FW_PORT_TYPE_BT_SGMII: 2816 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2817 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2818 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2819 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2820 break; 2821 2822 case FW_PORT_TYPE_CX4: 2823 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2824 ifmedia_set(media, m | IFM_10G_CX4); 2825 break; 2826 2827 case FW_PORT_TYPE_QSFP_10G: 2828 case FW_PORT_TYPE_SFP: 2829 case FW_PORT_TYPE_FIBER_XFI: 2830 case FW_PORT_TYPE_FIBER_XAUI: 2831 switch (pi->mod_type) { 2832 2833 case FW_PORT_MOD_TYPE_LR: 2834 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2835 ifmedia_set(media, m | IFM_10G_LR); 2836 break; 2837 2838 case FW_PORT_MOD_TYPE_SR: 2839 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2840 ifmedia_set(media, m | IFM_10G_SR); 2841 break; 2842 2843 case FW_PORT_MOD_TYPE_LRM: 2844 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2845 ifmedia_set(media, m | IFM_10G_LRM); 2846 break; 2847 2848 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2849 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2850 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2851 ifmedia_set(media, m | IFM_10G_TWINAX); 2852 break; 2853 2854 case FW_PORT_MOD_TYPE_NONE: 2855 m &= ~IFM_FDX; 2856 ifmedia_add(media, m | IFM_NONE, data, NULL); 2857 ifmedia_set(media, m | IFM_NONE); 2858 break; 2859 2860 case FW_PORT_MOD_TYPE_NA: 2861 case FW_PORT_MOD_TYPE_ER: 2862 default: 2863 device_printf(pi->dev, 2864 "unknown port_type (%d), mod_type (%d)\n", 2865 pi->port_type, pi->mod_type); 2866 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2867 ifmedia_set(media, m | IFM_UNKNOWN); 2868 break; 2869 } 2870 break; 2871 2872 case FW_PORT_TYPE_QSFP: 2873 switch (pi->mod_type) { 2874 2875 case FW_PORT_MOD_TYPE_LR: 2876 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2877 ifmedia_set(media, m | IFM_40G_LR4); 2878 break; 2879 2880 case FW_PORT_MOD_TYPE_SR: 2881 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2882 ifmedia_set(media, m | IFM_40G_SR4); 2883 break; 2884 2885 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2886 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2887 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2888 ifmedia_set(media, m | IFM_40G_CR4); 2889 break; 2890 2891 case FW_PORT_MOD_TYPE_NONE: 2892 m &= ~IFM_FDX; 2893 ifmedia_add(media, m | IFM_NONE, data, NULL); 2894 ifmedia_set(media, m | IFM_NONE); 2895 break; 2896 2897 default: 2898 device_printf(pi->dev, 2899 "unknown port_type (%d), mod_type (%d)\n", 2900 pi->port_type, pi->mod_type); 2901 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2902 ifmedia_set(media, m | IFM_UNKNOWN); 2903 break; 2904 } 2905 break; 2906 2907 default: 2908 device_printf(pi->dev, 2909 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2910 pi->mod_type); 2911 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2912 ifmedia_set(media, m | IFM_UNKNOWN); 2913 break; 2914 } 2915 2916 PORT_UNLOCK(pi); 2917} 2918 2919#define FW_MAC_EXACT_CHUNK 7 2920 2921/* 2922 * Program the port's XGMAC based on parameters in ifnet. The caller also 2923 * indicates which parameters should be programmed (the rest are left alone). 2924 */ 2925int 2926update_mac_settings(struct ifnet *ifp, int flags) 2927{ 2928 int rc = 0; 2929 struct port_info *pi = ifp->if_softc; 2930 struct adapter *sc = pi->adapter; 2931 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2932 uint16_t viid = 0xffff; 2933 int16_t *xact_addr_filt = NULL; 2934 2935 ASSERT_SYNCHRONIZED_OP(sc); 2936 KASSERT(flags, ("%s: not told what to update.", __func__)); 2937 2938 if (ifp == pi->ifp) { 2939 viid = pi->viid; 2940 xact_addr_filt = &pi->xact_addr_filt; 2941 } 2942#ifdef DEV_NETMAP 2943 else if (ifp == pi->nm_ifp) { 2944 viid = pi->nm_viid; 2945 xact_addr_filt = &pi->nm_xact_addr_filt; 2946 } 2947#endif 2948 if (flags & XGMAC_MTU) 2949 mtu = ifp->if_mtu; 2950 2951 if (flags & XGMAC_PROMISC) 2952 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2953 2954 if (flags & XGMAC_ALLMULTI) 2955 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2956 2957 if (flags & XGMAC_VLANEX) 2958 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2959 2960 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2961 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2962 1, vlanex, false); 2963 if (rc) { 2964 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2965 rc); 2966 return (rc); 2967 } 2968 } 2969 2970 if (flags & XGMAC_UCADDR) { 2971 uint8_t ucaddr[ETHER_ADDR_LEN]; 2972 2973 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2974 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2975 true, true); 2976 if (rc < 0) { 2977 rc = -rc; 2978 if_printf(ifp, "change_mac failed: %d\n", rc); 2979 return (rc); 2980 } else { 2981 *xact_addr_filt = rc; 2982 rc = 0; 2983 } 2984 } 2985 2986 if (flags & XGMAC_MCADDRS) { 2987 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2988 int del = 1; 2989 uint64_t hash = 0; 2990 struct ifmultiaddr *ifma; 2991 int i = 0, j; 2992 2993 if_maddr_rlock(ifp); 2994 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2995 if (ifma->ifma_addr->sa_family != AF_LINK) 2996 continue; 2997 mcaddr[i++] = 2998 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2999 3000 if (i == FW_MAC_EXACT_CHUNK) { 3001 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3002 i, mcaddr, NULL, &hash, 0); 3003 if (rc < 0) { 3004 rc = -rc; 3005 for (j = 0; j < i; j++) { 3006 if_printf(ifp, 3007 "failed to add mc address" 3008 " %02x:%02x:%02x:" 3009 "%02x:%02x:%02x rc=%d\n", 3010 mcaddr[j][0], mcaddr[j][1], 3011 mcaddr[j][2], mcaddr[j][3], 3012 mcaddr[j][4], mcaddr[j][5], 3013 rc); 3014 } 3015 goto mcfail; 3016 } 3017 del = 0; 3018 i = 0; 3019 } 3020 } 3021 if (i > 0) { 3022 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3023 mcaddr, NULL, &hash, 0); 3024 if (rc < 0) { 3025 rc = -rc; 3026 for (j = 0; j < i; j++) { 3027 if_printf(ifp, 3028 "failed to add mc address" 3029 " %02x:%02x:%02x:" 3030 "%02x:%02x:%02x rc=%d\n", 3031 mcaddr[j][0], mcaddr[j][1], 3032 mcaddr[j][2], mcaddr[j][3], 3033 mcaddr[j][4], mcaddr[j][5], 3034 rc); 3035 } 3036 goto mcfail; 3037 } 3038 } 3039 3040 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3041 if (rc != 0) 3042 if_printf(ifp, "failed to set mc address hash: %d", rc); 3043mcfail: 3044 if_maddr_runlock(ifp); 3045 } 3046 3047 return (rc); 3048} 3049 3050/* 3051 * {begin|end}_synchronized_op must be called from the same thread. 3052 */ 3053int 3054begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3055 char *wmesg) 3056{ 3057 int rc, pri; 3058 3059#ifdef WITNESS 3060 /* the caller thinks it's ok to sleep, but is it really? */ 3061 if (flags & SLEEP_OK) 3062 pause("t4slptst", 1); 3063#endif 3064 3065 if (INTR_OK) 3066 pri = PCATCH; 3067 else 3068 pri = 0; 3069 3070 ADAPTER_LOCK(sc); 3071 for (;;) { 3072 3073 if (pi && IS_DOOMED(pi)) { 3074 rc = ENXIO; 3075 goto done; 3076 } 3077 3078 if (!IS_BUSY(sc)) { 3079 rc = 0; 3080 break; 3081 } 3082 3083 if (!(flags & SLEEP_OK)) { 3084 rc = EBUSY; 3085 goto done; 3086 } 3087 3088 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3089 rc = EINTR; 3090 goto done; 3091 } 3092 } 3093 3094 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3095 SET_BUSY(sc); 3096#ifdef INVARIANTS 3097 sc->last_op = wmesg; 3098 sc->last_op_thr = curthread; 3099#endif 3100 3101done: 3102 if (!(flags & HOLD_LOCK) || rc) 3103 ADAPTER_UNLOCK(sc); 3104 3105 return (rc); 3106} 3107 3108/* 3109 * {begin|end}_synchronized_op must be called from the same thread. 3110 */ 3111void 3112end_synchronized_op(struct adapter *sc, int flags) 3113{ 3114 3115 if (flags & LOCK_HELD) 3116 ADAPTER_LOCK_ASSERT_OWNED(sc); 3117 else 3118 ADAPTER_LOCK(sc); 3119 3120 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3121 CLR_BUSY(sc); 3122 wakeup(&sc->flags); 3123 ADAPTER_UNLOCK(sc); 3124} 3125 3126static int 3127cxgbe_init_synchronized(struct port_info *pi) 3128{ 3129 struct adapter *sc = pi->adapter; 3130 struct ifnet *ifp = pi->ifp; 3131 int rc = 0; 3132 3133 ASSERT_SYNCHRONIZED_OP(sc); 3134 3135 if (isset(&sc->open_device_map, pi->port_id)) { 3136 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3137 ("mismatch between open_device_map and if_drv_flags")); 3138 return (0); /* already running */ 3139 } 3140 3141 if (!(sc->flags & FULL_INIT_DONE) && 3142 ((rc = adapter_full_init(sc)) != 0)) 3143 return (rc); /* error message displayed already */ 3144 3145 if (!(pi->flags & PORT_INIT_DONE) && 3146 ((rc = port_full_init(pi)) != 0)) 3147 return (rc); /* error message displayed already */ 3148 3149 rc = update_mac_settings(ifp, XGMAC_ALL); 3150 if (rc) 3151 goto done; /* error message displayed already */ 3152 3153 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3154 if (rc != 0) { 3155 if_printf(ifp, "enable_vi failed: %d\n", rc); 3156 goto done; 3157 } 3158 3159 /* 3160 * The first iq of the first port to come up is used for tracing. 3161 */ 3162 if (sc->traceq < 0) { 3163 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3164 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3165 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3166 V_QUEUENUMBER(sc->traceq)); 3167 pi->flags |= HAS_TRACEQ; 3168 } 3169 3170 /* all ok */ 3171 setbit(&sc->open_device_map, pi->port_id); 3172 PORT_LOCK(pi); 3173 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3174 PORT_UNLOCK(pi); 3175 3176 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3177done: 3178 if (rc != 0) 3179 cxgbe_uninit_synchronized(pi); 3180 3181 return (rc); 3182} 3183 3184/* 3185 * Idempotent. 3186 */ 3187static int 3188cxgbe_uninit_synchronized(struct port_info *pi) 3189{ 3190 struct adapter *sc = pi->adapter; 3191 struct ifnet *ifp = pi->ifp; 3192 int rc; 3193 3194 ASSERT_SYNCHRONIZED_OP(sc); 3195 3196 /* 3197 * Disable the VI so that all its data in either direction is discarded 3198 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3199 * tick) intact as the TP can deliver negative advice or data that it's 3200 * holding in its RAM (for an offloaded connection) even after the VI is 3201 * disabled. 3202 */ 3203 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3204 if (rc) { 3205 if_printf(ifp, "disable_vi failed: %d\n", rc); 3206 return (rc); 3207 } 3208 3209 clrbit(&sc->open_device_map, pi->port_id); 3210 PORT_LOCK(pi); 3211 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3212 PORT_UNLOCK(pi); 3213 3214 pi->link_cfg.link_ok = 0; 3215 pi->link_cfg.speed = 0; 3216 pi->linkdnrc = -1; 3217 t4_os_link_changed(sc, pi->port_id, 0, -1); 3218 3219 return (0); 3220} 3221 3222/* 3223 * It is ok for this function to fail midway and return right away. t4_detach 3224 * will walk the entire sc->irq list and clean up whatever is valid. 3225 */ 3226static int 3227setup_intr_handlers(struct adapter *sc) 3228{ 3229 int rc, rid, p, q; 3230 char s[8]; 3231 struct irq *irq; 3232 struct port_info *pi; 3233 struct sge_rxq *rxq; 3234#ifdef TCP_OFFLOAD 3235 struct sge_ofld_rxq *ofld_rxq; 3236#endif 3237#ifdef DEV_NETMAP 3238 struct sge_nm_rxq *nm_rxq; 3239#endif 3240 3241 /* 3242 * Setup interrupts. 3243 */ 3244 irq = &sc->irq[0]; 3245 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3246 if (sc->intr_count == 1) 3247 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3248 3249 /* Multiple interrupts. */ 3250 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3251 ("%s: too few intr.", __func__)); 3252 3253 /* The first one is always error intr */ 3254 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3255 if (rc != 0) 3256 return (rc); 3257 irq++; 3258 rid++; 3259 3260 /* The second one is always the firmware event queue */ 3261 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3262 if (rc != 0) 3263 return (rc); 3264 irq++; 3265 rid++; 3266 3267 for_each_port(sc, p) { 3268 pi = sc->port[p]; 3269 3270 if (pi->flags & INTR_RXQ) { 3271 for_each_rxq(pi, q, rxq) { 3272 snprintf(s, sizeof(s), "%d.%d", p, q); 3273 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3274 s); 3275 if (rc != 0) 3276 return (rc); 3277 irq++; 3278 rid++; 3279 } 3280 } 3281#ifdef TCP_OFFLOAD 3282 if (pi->flags & INTR_OFLD_RXQ) { 3283 for_each_ofld_rxq(pi, q, ofld_rxq) { 3284 snprintf(s, sizeof(s), "%d,%d", p, q); 3285 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3286 ofld_rxq, s); 3287 if (rc != 0) 3288 return (rc); 3289 irq++; 3290 rid++; 3291 } 3292 } 3293#endif 3294#ifdef DEV_NETMAP 3295 if (pi->flags & INTR_NM_RXQ) { 3296 for_each_nm_rxq(pi, q, nm_rxq) { 3297 snprintf(s, sizeof(s), "%d-%d", p, q); 3298 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3299 nm_rxq, s); 3300 if (rc != 0) 3301 return (rc); 3302 irq++; 3303 rid++; 3304 } 3305 } 3306#endif 3307 } 3308 MPASS(irq == &sc->irq[sc->intr_count]); 3309 3310 return (0); 3311} 3312 3313int 3314adapter_full_init(struct adapter *sc) 3315{ 3316 int rc, i; 3317 3318 ASSERT_SYNCHRONIZED_OP(sc); 3319 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3320 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3321 ("%s: FULL_INIT_DONE already", __func__)); 3322 3323 /* 3324 * queues that belong to the adapter (not any particular port). 3325 */ 3326 rc = t4_setup_adapter_queues(sc); 3327 if (rc != 0) 3328 goto done; 3329 3330 for (i = 0; i < nitems(sc->tq); i++) { 3331 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3332 taskqueue_thread_enqueue, &sc->tq[i]); 3333 if (sc->tq[i] == NULL) { 3334 device_printf(sc->dev, 3335 "failed to allocate task queue %d\n", i); 3336 rc = ENOMEM; 3337 goto done; 3338 } 3339 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3340 device_get_nameunit(sc->dev), i); 3341 } 3342 3343 t4_intr_enable(sc); 3344 sc->flags |= FULL_INIT_DONE; 3345done: 3346 if (rc != 0) 3347 adapter_full_uninit(sc); 3348 3349 return (rc); 3350} 3351 3352int 3353adapter_full_uninit(struct adapter *sc) 3354{ 3355 int i; 3356 3357 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3358 3359 t4_teardown_adapter_queues(sc); 3360 3361 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3362 taskqueue_free(sc->tq[i]); 3363 sc->tq[i] = NULL; 3364 } 3365 3366 sc->flags &= ~FULL_INIT_DONE; 3367 3368 return (0); 3369} 3370 3371int 3372port_full_init(struct port_info *pi) 3373{ 3374 struct adapter *sc = pi->adapter; 3375 struct ifnet *ifp = pi->ifp; 3376 uint16_t *rss; 3377 struct sge_rxq *rxq; 3378 int rc, i, j; 3379 3380 ASSERT_SYNCHRONIZED_OP(sc); 3381 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3382 ("%s: PORT_INIT_DONE already", __func__)); 3383 3384 sysctl_ctx_init(&pi->ctx); 3385 pi->flags |= PORT_SYSCTL_CTX; 3386 3387 /* 3388 * Allocate tx/rx/fl queues for this port. 3389 */ 3390 rc = t4_setup_port_queues(pi); 3391 if (rc != 0) 3392 goto done; /* error message displayed already */ 3393 3394 /* 3395 * Setup RSS for this port. Save a copy of the RSS table for later use. 3396 */ 3397 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3398 for (i = 0; i < pi->rss_size;) { 3399 for_each_rxq(pi, j, rxq) { 3400 rss[i++] = rxq->iq.abs_id; 3401 if (i == pi->rss_size) 3402 break; 3403 } 3404 } 3405 3406 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3407 pi->rss_size); 3408 if (rc != 0) { 3409 if_printf(ifp, "rss_config failed: %d\n", rc); 3410 goto done; 3411 } 3412 3413 pi->rss = rss; 3414 pi->flags |= PORT_INIT_DONE; 3415done: 3416 if (rc != 0) 3417 port_full_uninit(pi); 3418 3419 return (rc); 3420} 3421 3422/* 3423 * Idempotent. 3424 */ 3425int 3426port_full_uninit(struct port_info *pi) 3427{ 3428 struct adapter *sc = pi->adapter; 3429 int i; 3430 struct sge_rxq *rxq; 3431 struct sge_txq *txq; 3432#ifdef TCP_OFFLOAD 3433 struct sge_ofld_rxq *ofld_rxq; 3434 struct sge_wrq *ofld_txq; 3435#endif 3436 3437 if (pi->flags & PORT_INIT_DONE) { 3438 3439 /* Need to quiesce queues. XXX: ctrl queues? */ 3440 3441 for_each_txq(pi, i, txq) { 3442 quiesce_eq(sc, &txq->eq); 3443 } 3444 3445#ifdef TCP_OFFLOAD 3446 for_each_ofld_txq(pi, i, ofld_txq) { 3447 quiesce_eq(sc, &ofld_txq->eq); 3448 } 3449#endif 3450 3451 for_each_rxq(pi, i, rxq) { 3452 quiesce_iq(sc, &rxq->iq); 3453 quiesce_fl(sc, &rxq->fl); 3454 } 3455 3456#ifdef TCP_OFFLOAD 3457 for_each_ofld_rxq(pi, i, ofld_rxq) { 3458 quiesce_iq(sc, &ofld_rxq->iq); 3459 quiesce_fl(sc, &ofld_rxq->fl); 3460 } 3461#endif 3462 free(pi->rss, M_CXGBE); 3463 } 3464 3465 t4_teardown_port_queues(pi); 3466 pi->flags &= ~PORT_INIT_DONE; 3467 3468 return (0); 3469} 3470 3471static void 3472quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3473{ 3474 EQ_LOCK(eq); 3475 eq->flags |= EQ_DOOMED; 3476 3477 /* 3478 * Wait for the response to a credit flush if one's 3479 * pending. 3480 */ 3481 while (eq->flags & EQ_CRFLUSHED) 3482 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3483 EQ_UNLOCK(eq); 3484 3485 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3486 pause("callout", 10); /* Still iffy */ 3487 3488 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3489} 3490 3491static void 3492quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3493{ 3494 (void) sc; /* unused */ 3495 3496 /* Synchronize with the interrupt handler */ 3497 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3498 pause("iqfree", 1); 3499} 3500 3501static void 3502quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3503{ 3504 mtx_lock(&sc->sfl_lock); 3505 FL_LOCK(fl); 3506 fl->flags |= FL_DOOMED; 3507 FL_UNLOCK(fl); 3508 mtx_unlock(&sc->sfl_lock); 3509 3510 callout_drain(&sc->sfl_callout); 3511 KASSERT((fl->flags & FL_STARVING) == 0, 3512 ("%s: still starving", __func__)); 3513} 3514 3515static int 3516t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3517 driver_intr_t *handler, void *arg, char *name) 3518{ 3519 int rc; 3520 3521 irq->rid = rid; 3522 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3523 RF_SHAREABLE | RF_ACTIVE); 3524 if (irq->res == NULL) { 3525 device_printf(sc->dev, 3526 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3527 return (ENOMEM); 3528 } 3529 3530 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3531 NULL, handler, arg, &irq->tag); 3532 if (rc != 0) { 3533 device_printf(sc->dev, 3534 "failed to setup interrupt for rid %d, name %s: %d\n", 3535 rid, name, rc); 3536 } else if (name) 3537 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3538 3539 return (rc); 3540} 3541 3542static int 3543t4_free_irq(struct adapter *sc, struct irq *irq) 3544{ 3545 if (irq->tag) 3546 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3547 if (irq->res) 3548 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3549 3550 bzero(irq, sizeof(*irq)); 3551 3552 return (0); 3553} 3554 3555static void 3556reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3557 unsigned int end) 3558{ 3559 uint32_t *p = (uint32_t *)(buf + start); 3560 3561 for ( ; start <= end; start += sizeof(uint32_t)) 3562 *p++ = t4_read_reg(sc, start); 3563} 3564 3565static void 3566t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3567{ 3568 int i, n; 3569 const unsigned int *reg_ranges; 3570 static const unsigned int t4_reg_ranges[] = { 3571 0x1008, 0x1108, 3572 0x1180, 0x11b4, 3573 0x11fc, 0x123c, 3574 0x1300, 0x173c, 3575 0x1800, 0x18fc, 3576 0x3000, 0x30d8, 3577 0x30e0, 0x5924, 3578 0x5960, 0x59d4, 3579 0x5a00, 0x5af8, 3580 0x6000, 0x6098, 3581 0x6100, 0x6150, 3582 0x6200, 0x6208, 3583 0x6240, 0x6248, 3584 0x6280, 0x6338, 3585 0x6370, 0x638c, 3586 0x6400, 0x643c, 3587 0x6500, 0x6524, 3588 0x6a00, 0x6a38, 3589 0x6a60, 0x6a78, 3590 0x6b00, 0x6b84, 3591 0x6bf0, 0x6c84, 3592 0x6cf0, 0x6d84, 3593 0x6df0, 0x6e84, 3594 0x6ef0, 0x6f84, 3595 0x6ff0, 0x7084, 3596 0x70f0, 0x7184, 3597 0x71f0, 0x7284, 3598 0x72f0, 0x7384, 3599 0x73f0, 0x7450, 3600 0x7500, 0x7530, 3601 0x7600, 0x761c, 3602 0x7680, 0x76cc, 3603 0x7700, 0x7798, 3604 0x77c0, 0x77fc, 3605 0x7900, 0x79fc, 3606 0x7b00, 0x7c38, 3607 0x7d00, 0x7efc, 3608 0x8dc0, 0x8e1c, 3609 0x8e30, 0x8e78, 3610 0x8ea0, 0x8f6c, 3611 0x8fc0, 0x9074, 3612 0x90fc, 0x90fc, 3613 0x9400, 0x9458, 3614 0x9600, 0x96bc, 3615 0x9800, 0x9808, 3616 0x9820, 0x983c, 3617 0x9850, 0x9864, 3618 0x9c00, 0x9c6c, 3619 0x9c80, 0x9cec, 3620 0x9d00, 0x9d6c, 3621 0x9d80, 0x9dec, 3622 0x9e00, 0x9e6c, 3623 0x9e80, 0x9eec, 3624 0x9f00, 0x9f6c, 3625 0x9f80, 0x9fec, 3626 0xd004, 0xd03c, 3627 0xdfc0, 0xdfe0, 3628 0xe000, 0xea7c, 3629 0xf000, 0x11110, 3630 0x11118, 0x11190, 3631 0x19040, 0x1906c, 3632 0x19078, 0x19080, 3633 0x1908c, 0x19124, 3634 0x19150, 0x191b0, 3635 0x191d0, 0x191e8, 3636 0x19238, 0x1924c, 3637 0x193f8, 0x19474, 3638 0x19490, 0x194f8, 3639 0x19800, 0x19f30, 3640 0x1a000, 0x1a06c, 3641 0x1a0b0, 0x1a120, 3642 0x1a128, 0x1a138, 3643 0x1a190, 0x1a1c4, 3644 0x1a1fc, 0x1a1fc, 3645 0x1e040, 0x1e04c, 3646 0x1e284, 0x1e28c, 3647 0x1e2c0, 0x1e2c0, 3648 0x1e2e0, 0x1e2e0, 3649 0x1e300, 0x1e384, 3650 0x1e3c0, 0x1e3c8, 3651 0x1e440, 0x1e44c, 3652 0x1e684, 0x1e68c, 3653 0x1e6c0, 0x1e6c0, 3654 0x1e6e0, 0x1e6e0, 3655 0x1e700, 0x1e784, 3656 0x1e7c0, 0x1e7c8, 3657 0x1e840, 0x1e84c, 3658 0x1ea84, 0x1ea8c, 3659 0x1eac0, 0x1eac0, 3660 0x1eae0, 0x1eae0, 3661 0x1eb00, 0x1eb84, 3662 0x1ebc0, 0x1ebc8, 3663 0x1ec40, 0x1ec4c, 3664 0x1ee84, 0x1ee8c, 3665 0x1eec0, 0x1eec0, 3666 0x1eee0, 0x1eee0, 3667 0x1ef00, 0x1ef84, 3668 0x1efc0, 0x1efc8, 3669 0x1f040, 0x1f04c, 3670 0x1f284, 0x1f28c, 3671 0x1f2c0, 0x1f2c0, 3672 0x1f2e0, 0x1f2e0, 3673 0x1f300, 0x1f384, 3674 0x1f3c0, 0x1f3c8, 3675 0x1f440, 0x1f44c, 3676 0x1f684, 0x1f68c, 3677 0x1f6c0, 0x1f6c0, 3678 0x1f6e0, 0x1f6e0, 3679 0x1f700, 0x1f784, 3680 0x1f7c0, 0x1f7c8, 3681 0x1f840, 0x1f84c, 3682 0x1fa84, 0x1fa8c, 3683 0x1fac0, 0x1fac0, 3684 0x1fae0, 0x1fae0, 3685 0x1fb00, 0x1fb84, 3686 0x1fbc0, 0x1fbc8, 3687 0x1fc40, 0x1fc4c, 3688 0x1fe84, 0x1fe8c, 3689 0x1fec0, 0x1fec0, 3690 0x1fee0, 0x1fee0, 3691 0x1ff00, 0x1ff84, 3692 0x1ffc0, 0x1ffc8, 3693 0x20000, 0x2002c, 3694 0x20100, 0x2013c, 3695 0x20190, 0x201c8, 3696 0x20200, 0x20318, 3697 0x20400, 0x20528, 3698 0x20540, 0x20614, 3699 0x21000, 0x21040, 3700 0x2104c, 0x21060, 3701 0x210c0, 0x210ec, 3702 0x21200, 0x21268, 3703 0x21270, 0x21284, 3704 0x212fc, 0x21388, 3705 0x21400, 0x21404, 3706 0x21500, 0x21518, 3707 0x2152c, 0x2153c, 3708 0x21550, 0x21554, 3709 0x21600, 0x21600, 3710 0x21608, 0x21628, 3711 0x21630, 0x2163c, 3712 0x21700, 0x2171c, 3713 0x21780, 0x2178c, 3714 0x21800, 0x21c38, 3715 0x21c80, 0x21d7c, 3716 0x21e00, 0x21e04, 3717 0x22000, 0x2202c, 3718 0x22100, 0x2213c, 3719 0x22190, 0x221c8, 3720 0x22200, 0x22318, 3721 0x22400, 0x22528, 3722 0x22540, 0x22614, 3723 0x23000, 0x23040, 3724 0x2304c, 0x23060, 3725 0x230c0, 0x230ec, 3726 0x23200, 0x23268, 3727 0x23270, 0x23284, 3728 0x232fc, 0x23388, 3729 0x23400, 0x23404, 3730 0x23500, 0x23518, 3731 0x2352c, 0x2353c, 3732 0x23550, 0x23554, 3733 0x23600, 0x23600, 3734 0x23608, 0x23628, 3735 0x23630, 0x2363c, 3736 0x23700, 0x2371c, 3737 0x23780, 0x2378c, 3738 0x23800, 0x23c38, 3739 0x23c80, 0x23d7c, 3740 0x23e00, 0x23e04, 3741 0x24000, 0x2402c, 3742 0x24100, 0x2413c, 3743 0x24190, 0x241c8, 3744 0x24200, 0x24318, 3745 0x24400, 0x24528, 3746 0x24540, 0x24614, 3747 0x25000, 0x25040, 3748 0x2504c, 0x25060, 3749 0x250c0, 0x250ec, 3750 0x25200, 0x25268, 3751 0x25270, 0x25284, 3752 0x252fc, 0x25388, 3753 0x25400, 0x25404, 3754 0x25500, 0x25518, 3755 0x2552c, 0x2553c, 3756 0x25550, 0x25554, 3757 0x25600, 0x25600, 3758 0x25608, 0x25628, 3759 0x25630, 0x2563c, 3760 0x25700, 0x2571c, 3761 0x25780, 0x2578c, 3762 0x25800, 0x25c38, 3763 0x25c80, 0x25d7c, 3764 0x25e00, 0x25e04, 3765 0x26000, 0x2602c, 3766 0x26100, 0x2613c, 3767 0x26190, 0x261c8, 3768 0x26200, 0x26318, 3769 0x26400, 0x26528, 3770 0x26540, 0x26614, 3771 0x27000, 0x27040, 3772 0x2704c, 0x27060, 3773 0x270c0, 0x270ec, 3774 0x27200, 0x27268, 3775 0x27270, 0x27284, 3776 0x272fc, 0x27388, 3777 0x27400, 0x27404, 3778 0x27500, 0x27518, 3779 0x2752c, 0x2753c, 3780 0x27550, 0x27554, 3781 0x27600, 0x27600, 3782 0x27608, 0x27628, 3783 0x27630, 0x2763c, 3784 0x27700, 0x2771c, 3785 0x27780, 0x2778c, 3786 0x27800, 0x27c38, 3787 0x27c80, 0x27d7c, 3788 0x27e00, 0x27e04 3789 }; 3790 static const unsigned int t5_reg_ranges[] = { 3791 0x1008, 0x1148, 3792 0x1180, 0x11b4, 3793 0x11fc, 0x123c, 3794 0x1280, 0x173c, 3795 0x1800, 0x18fc, 3796 0x3000, 0x3028, 3797 0x3060, 0x30d8, 3798 0x30e0, 0x30fc, 3799 0x3140, 0x357c, 3800 0x35a8, 0x35cc, 3801 0x35ec, 0x35ec, 3802 0x3600, 0x5624, 3803 0x56cc, 0x575c, 3804 0x580c, 0x5814, 3805 0x5890, 0x58bc, 3806 0x5940, 0x59dc, 3807 0x59fc, 0x5a18, 3808 0x5a60, 0x5a9c, 3809 0x5b94, 0x5bfc, 3810 0x6000, 0x6040, 3811 0x6058, 0x614c, 3812 0x7700, 0x7798, 3813 0x77c0, 0x78fc, 3814 0x7b00, 0x7c54, 3815 0x7d00, 0x7efc, 3816 0x8dc0, 0x8de0, 3817 0x8df8, 0x8e84, 3818 0x8ea0, 0x8f84, 3819 0x8fc0, 0x90f8, 3820 0x9400, 0x9470, 3821 0x9600, 0x96f4, 3822 0x9800, 0x9808, 3823 0x9820, 0x983c, 3824 0x9850, 0x9864, 3825 0x9c00, 0x9c6c, 3826 0x9c80, 0x9cec, 3827 0x9d00, 0x9d6c, 3828 0x9d80, 0x9dec, 3829 0x9e00, 0x9e6c, 3830 0x9e80, 0x9eec, 3831 0x9f00, 0x9f6c, 3832 0x9f80, 0xa020, 3833 0xd004, 0xd03c, 3834 0xdfc0, 0xdfe0, 3835 0xe000, 0x11088, 3836 0x1109c, 0x11110, 3837 0x11118, 0x1117c, 3838 0x11190, 0x11204, 3839 0x19040, 0x1906c, 3840 0x19078, 0x19080, 3841 0x1908c, 0x19124, 3842 0x19150, 0x191b0, 3843 0x191d0, 0x191e8, 3844 0x19238, 0x19290, 3845 0x193f8, 0x19474, 3846 0x19490, 0x194cc, 3847 0x194f0, 0x194f8, 3848 0x19c00, 0x19c60, 3849 0x19c94, 0x19e10, 3850 0x19e50, 0x19f34, 3851 0x19f40, 0x19f50, 3852 0x19f90, 0x19fe4, 3853 0x1a000, 0x1a06c, 3854 0x1a0b0, 0x1a120, 3855 0x1a128, 0x1a138, 3856 0x1a190, 0x1a1c4, 3857 0x1a1fc, 0x1a1fc, 3858 0x1e008, 0x1e00c, 3859 0x1e040, 0x1e04c, 3860 0x1e284, 0x1e290, 3861 0x1e2c0, 0x1e2c0, 3862 0x1e2e0, 0x1e2e0, 3863 0x1e300, 0x1e384, 3864 0x1e3c0, 0x1e3c8, 3865 0x1e408, 0x1e40c, 3866 0x1e440, 0x1e44c, 3867 0x1e684, 0x1e690, 3868 0x1e6c0, 0x1e6c0, 3869 0x1e6e0, 0x1e6e0, 3870 0x1e700, 0x1e784, 3871 0x1e7c0, 0x1e7c8, 3872 0x1e808, 0x1e80c, 3873 0x1e840, 0x1e84c, 3874 0x1ea84, 0x1ea90, 3875 0x1eac0, 0x1eac0, 3876 0x1eae0, 0x1eae0, 3877 0x1eb00, 0x1eb84, 3878 0x1ebc0, 0x1ebc8, 3879 0x1ec08, 0x1ec0c, 3880 0x1ec40, 0x1ec4c, 3881 0x1ee84, 0x1ee90, 3882 0x1eec0, 0x1eec0, 3883 0x1eee0, 0x1eee0, 3884 0x1ef00, 0x1ef84, 3885 0x1efc0, 0x1efc8, 3886 0x1f008, 0x1f00c, 3887 0x1f040, 0x1f04c, 3888 0x1f284, 0x1f290, 3889 0x1f2c0, 0x1f2c0, 3890 0x1f2e0, 0x1f2e0, 3891 0x1f300, 0x1f384, 3892 0x1f3c0, 0x1f3c8, 3893 0x1f408, 0x1f40c, 3894 0x1f440, 0x1f44c, 3895 0x1f684, 0x1f690, 3896 0x1f6c0, 0x1f6c0, 3897 0x1f6e0, 0x1f6e0, 3898 0x1f700, 0x1f784, 3899 0x1f7c0, 0x1f7c8, 3900 0x1f808, 0x1f80c, 3901 0x1f840, 0x1f84c, 3902 0x1fa84, 0x1fa90, 3903 0x1fac0, 0x1fac0, 3904 0x1fae0, 0x1fae0, 3905 0x1fb00, 0x1fb84, 3906 0x1fbc0, 0x1fbc8, 3907 0x1fc08, 0x1fc0c, 3908 0x1fc40, 0x1fc4c, 3909 0x1fe84, 0x1fe90, 3910 0x1fec0, 0x1fec0, 3911 0x1fee0, 0x1fee0, 3912 0x1ff00, 0x1ff84, 3913 0x1ffc0, 0x1ffc8, 3914 0x30000, 0x30030, 3915 0x30100, 0x30144, 3916 0x30190, 0x301d0, 3917 0x30200, 0x30318, 3918 0x30400, 0x3052c, 3919 0x30540, 0x3061c, 3920 0x30800, 0x30834, 3921 0x308c0, 0x30908, 3922 0x30910, 0x309ac, 3923 0x30a00, 0x30a2c, 3924 0x30a44, 0x30a50, 3925 0x30a74, 0x30c24, 3926 0x30d00, 0x30d00, 3927 0x30d08, 0x30d14, 3928 0x30d1c, 0x30d20, 3929 0x30d3c, 0x30d50, 3930 0x31200, 0x3120c, 3931 0x31220, 0x31220, 3932 0x31240, 0x31240, 3933 0x31600, 0x3160c, 3934 0x31a00, 0x31a1c, 3935 0x31e00, 0x31e20, 3936 0x31e38, 0x31e3c, 3937 0x31e80, 0x31e80, 3938 0x31e88, 0x31ea8, 3939 0x31eb0, 0x31eb4, 3940 0x31ec8, 0x31ed4, 3941 0x31fb8, 0x32004, 3942 0x32200, 0x32200, 3943 0x32208, 0x32240, 3944 0x32248, 0x32280, 3945 0x32288, 0x322c0, 3946 0x322c8, 0x322fc, 3947 0x32600, 0x32630, 3948 0x32a00, 0x32abc, 3949 0x32b00, 0x32b70, 3950 0x33000, 0x33048, 3951 0x33060, 0x3309c, 3952 0x330f0, 0x33148, 3953 0x33160, 0x3319c, 3954 0x331f0, 0x332e4, 3955 0x332f8, 0x333e4, 3956 0x333f8, 0x33448, 3957 0x33460, 0x3349c, 3958 0x334f0, 0x33548, 3959 0x33560, 0x3359c, 3960 0x335f0, 0x336e4, 3961 0x336f8, 0x337e4, 3962 0x337f8, 0x337fc, 3963 0x33814, 0x33814, 3964 0x3382c, 0x3382c, 3965 0x33880, 0x3388c, 3966 0x338e8, 0x338ec, 3967 0x33900, 0x33948, 3968 0x33960, 0x3399c, 3969 0x339f0, 0x33ae4, 3970 0x33af8, 0x33b10, 3971 0x33b28, 0x33b28, 3972 0x33b3c, 0x33b50, 3973 0x33bf0, 0x33c10, 3974 0x33c28, 0x33c28, 3975 0x33c3c, 0x33c50, 3976 0x33cf0, 0x33cfc, 3977 0x34000, 0x34030, 3978 0x34100, 0x34144, 3979 0x34190, 0x341d0, 3980 0x34200, 0x34318, 3981 0x34400, 0x3452c, 3982 0x34540, 0x3461c, 3983 0x34800, 0x34834, 3984 0x348c0, 0x34908, 3985 0x34910, 0x349ac, 3986 0x34a00, 0x34a2c, 3987 0x34a44, 0x34a50, 3988 0x34a74, 0x34c24, 3989 0x34d00, 0x34d00, 3990 0x34d08, 0x34d14, 3991 0x34d1c, 0x34d20, 3992 0x34d3c, 0x34d50, 3993 0x35200, 0x3520c, 3994 0x35220, 0x35220, 3995 0x35240, 0x35240, 3996 0x35600, 0x3560c, 3997 0x35a00, 0x35a1c, 3998 0x35e00, 0x35e20, 3999 0x35e38, 0x35e3c, 4000 0x35e80, 0x35e80, 4001 0x35e88, 0x35ea8, 4002 0x35eb0, 0x35eb4, 4003 0x35ec8, 0x35ed4, 4004 0x35fb8, 0x36004, 4005 0x36200, 0x36200, 4006 0x36208, 0x36240, 4007 0x36248, 0x36280, 4008 0x36288, 0x362c0, 4009 0x362c8, 0x362fc, 4010 0x36600, 0x36630, 4011 0x36a00, 0x36abc, 4012 0x36b00, 0x36b70, 4013 0x37000, 0x37048, 4014 0x37060, 0x3709c, 4015 0x370f0, 0x37148, 4016 0x37160, 0x3719c, 4017 0x371f0, 0x372e4, 4018 0x372f8, 0x373e4, 4019 0x373f8, 0x37448, 4020 0x37460, 0x3749c, 4021 0x374f0, 0x37548, 4022 0x37560, 0x3759c, 4023 0x375f0, 0x376e4, 4024 0x376f8, 0x377e4, 4025 0x377f8, 0x377fc, 4026 0x37814, 0x37814, 4027 0x3782c, 0x3782c, 4028 0x37880, 0x3788c, 4029 0x378e8, 0x378ec, 4030 0x37900, 0x37948, 4031 0x37960, 0x3799c, 4032 0x379f0, 0x37ae4, 4033 0x37af8, 0x37b10, 4034 0x37b28, 0x37b28, 4035 0x37b3c, 0x37b50, 4036 0x37bf0, 0x37c10, 4037 0x37c28, 0x37c28, 4038 0x37c3c, 0x37c50, 4039 0x37cf0, 0x37cfc, 4040 0x38000, 0x38030, 4041 0x38100, 0x38144, 4042 0x38190, 0x381d0, 4043 0x38200, 0x38318, 4044 0x38400, 0x3852c, 4045 0x38540, 0x3861c, 4046 0x38800, 0x38834, 4047 0x388c0, 0x38908, 4048 0x38910, 0x389ac, 4049 0x38a00, 0x38a2c, 4050 0x38a44, 0x38a50, 4051 0x38a74, 0x38c24, 4052 0x38d00, 0x38d00, 4053 0x38d08, 0x38d14, 4054 0x38d1c, 0x38d20, 4055 0x38d3c, 0x38d50, 4056 0x39200, 0x3920c, 4057 0x39220, 0x39220, 4058 0x39240, 0x39240, 4059 0x39600, 0x3960c, 4060 0x39a00, 0x39a1c, 4061 0x39e00, 0x39e20, 4062 0x39e38, 0x39e3c, 4063 0x39e80, 0x39e80, 4064 0x39e88, 0x39ea8, 4065 0x39eb0, 0x39eb4, 4066 0x39ec8, 0x39ed4, 4067 0x39fb8, 0x3a004, 4068 0x3a200, 0x3a200, 4069 0x3a208, 0x3a240, 4070 0x3a248, 0x3a280, 4071 0x3a288, 0x3a2c0, 4072 0x3a2c8, 0x3a2fc, 4073 0x3a600, 0x3a630, 4074 0x3aa00, 0x3aabc, 4075 0x3ab00, 0x3ab70, 4076 0x3b000, 0x3b048, 4077 0x3b060, 0x3b09c, 4078 0x3b0f0, 0x3b148, 4079 0x3b160, 0x3b19c, 4080 0x3b1f0, 0x3b2e4, 4081 0x3b2f8, 0x3b3e4, 4082 0x3b3f8, 0x3b448, 4083 0x3b460, 0x3b49c, 4084 0x3b4f0, 0x3b548, 4085 0x3b560, 0x3b59c, 4086 0x3b5f0, 0x3b6e4, 4087 0x3b6f8, 0x3b7e4, 4088 0x3b7f8, 0x3b7fc, 4089 0x3b814, 0x3b814, 4090 0x3b82c, 0x3b82c, 4091 0x3b880, 0x3b88c, 4092 0x3b8e8, 0x3b8ec, 4093 0x3b900, 0x3b948, 4094 0x3b960, 0x3b99c, 4095 0x3b9f0, 0x3bae4, 4096 0x3baf8, 0x3bb10, 4097 0x3bb28, 0x3bb28, 4098 0x3bb3c, 0x3bb50, 4099 0x3bbf0, 0x3bc10, 4100 0x3bc28, 0x3bc28, 4101 0x3bc3c, 0x3bc50, 4102 0x3bcf0, 0x3bcfc, 4103 0x3c000, 0x3c030, 4104 0x3c100, 0x3c144, 4105 0x3c190, 0x3c1d0, 4106 0x3c200, 0x3c318, 4107 0x3c400, 0x3c52c, 4108 0x3c540, 0x3c61c, 4109 0x3c800, 0x3c834, 4110 0x3c8c0, 0x3c908, 4111 0x3c910, 0x3c9ac, 4112 0x3ca00, 0x3ca2c, 4113 0x3ca44, 0x3ca50, 4114 0x3ca74, 0x3cc24, 4115 0x3cd00, 0x3cd00, 4116 0x3cd08, 0x3cd14, 4117 0x3cd1c, 0x3cd20, 4118 0x3cd3c, 0x3cd50, 4119 0x3d200, 0x3d20c, 4120 0x3d220, 0x3d220, 4121 0x3d240, 0x3d240, 4122 0x3d600, 0x3d60c, 4123 0x3da00, 0x3da1c, 4124 0x3de00, 0x3de20, 4125 0x3de38, 0x3de3c, 4126 0x3de80, 0x3de80, 4127 0x3de88, 0x3dea8, 4128 0x3deb0, 0x3deb4, 4129 0x3dec8, 0x3ded4, 4130 0x3dfb8, 0x3e004, 4131 0x3e200, 0x3e200, 4132 0x3e208, 0x3e240, 4133 0x3e248, 0x3e280, 4134 0x3e288, 0x3e2c0, 4135 0x3e2c8, 0x3e2fc, 4136 0x3e600, 0x3e630, 4137 0x3ea00, 0x3eabc, 4138 0x3eb00, 0x3eb70, 4139 0x3f000, 0x3f048, 4140 0x3f060, 0x3f09c, 4141 0x3f0f0, 0x3f148, 4142 0x3f160, 0x3f19c, 4143 0x3f1f0, 0x3f2e4, 4144 0x3f2f8, 0x3f3e4, 4145 0x3f3f8, 0x3f448, 4146 0x3f460, 0x3f49c, 4147 0x3f4f0, 0x3f548, 4148 0x3f560, 0x3f59c, 4149 0x3f5f0, 0x3f6e4, 4150 0x3f6f8, 0x3f7e4, 4151 0x3f7f8, 0x3f7fc, 4152 0x3f814, 0x3f814, 4153 0x3f82c, 0x3f82c, 4154 0x3f880, 0x3f88c, 4155 0x3f8e8, 0x3f8ec, 4156 0x3f900, 0x3f948, 4157 0x3f960, 0x3f99c, 4158 0x3f9f0, 0x3fae4, 4159 0x3faf8, 0x3fb10, 4160 0x3fb28, 0x3fb28, 4161 0x3fb3c, 0x3fb50, 4162 0x3fbf0, 0x3fc10, 4163 0x3fc28, 0x3fc28, 4164 0x3fc3c, 0x3fc50, 4165 0x3fcf0, 0x3fcfc, 4166 0x40000, 0x4000c, 4167 0x40040, 0x40068, 4168 0x4007c, 0x40144, 4169 0x40180, 0x4018c, 4170 0x40200, 0x40298, 4171 0x402ac, 0x4033c, 4172 0x403f8, 0x403fc, 4173 0x41304, 0x413c4, 4174 0x41400, 0x4141c, 4175 0x41480, 0x414d0, 4176 0x44000, 0x44078, 4177 0x440c0, 0x44278, 4178 0x442c0, 0x44478, 4179 0x444c0, 0x44678, 4180 0x446c0, 0x44878, 4181 0x448c0, 0x449fc, 4182 0x45000, 0x45068, 4183 0x45080, 0x45084, 4184 0x450a0, 0x450b0, 4185 0x45200, 0x45268, 4186 0x45280, 0x45284, 4187 0x452a0, 0x452b0, 4188 0x460c0, 0x460e4, 4189 0x47000, 0x4708c, 4190 0x47200, 0x47250, 4191 0x47400, 0x47420, 4192 0x47600, 0x47618, 4193 0x47800, 0x47814, 4194 0x48000, 0x4800c, 4195 0x48040, 0x48068, 4196 0x4807c, 0x48144, 4197 0x48180, 0x4818c, 4198 0x48200, 0x48298, 4199 0x482ac, 0x4833c, 4200 0x483f8, 0x483fc, 4201 0x49304, 0x493c4, 4202 0x49400, 0x4941c, 4203 0x49480, 0x494d0, 4204 0x4c000, 0x4c078, 4205 0x4c0c0, 0x4c278, 4206 0x4c2c0, 0x4c478, 4207 0x4c4c0, 0x4c678, 4208 0x4c6c0, 0x4c878, 4209 0x4c8c0, 0x4c9fc, 4210 0x4d000, 0x4d068, 4211 0x4d080, 0x4d084, 4212 0x4d0a0, 0x4d0b0, 4213 0x4d200, 0x4d268, 4214 0x4d280, 0x4d284, 4215 0x4d2a0, 0x4d2b0, 4216 0x4e0c0, 0x4e0e4, 4217 0x4f000, 0x4f08c, 4218 0x4f200, 0x4f250, 4219 0x4f400, 0x4f420, 4220 0x4f600, 0x4f618, 4221 0x4f800, 0x4f814, 4222 0x50000, 0x500cc, 4223 0x50400, 0x50400, 4224 0x50800, 0x508cc, 4225 0x50c00, 0x50c00, 4226 0x51000, 0x5101c, 4227 0x51300, 0x51308, 4228 }; 4229 4230 if (is_t4(sc)) { 4231 reg_ranges = &t4_reg_ranges[0]; 4232 n = nitems(t4_reg_ranges); 4233 } else { 4234 reg_ranges = &t5_reg_ranges[0]; 4235 n = nitems(t5_reg_ranges); 4236 } 4237 4238 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4239 for (i = 0; i < n; i += 2) 4240 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4241} 4242 4243static void 4244cxgbe_tick(void *arg) 4245{ 4246 struct port_info *pi = arg; 4247 struct adapter *sc = pi->adapter; 4248 struct ifnet *ifp = pi->ifp; 4249 struct sge_txq *txq; 4250 int i, drops; 4251 struct port_stats *s = &pi->stats; 4252 4253 PORT_LOCK(pi); 4254 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4255 PORT_UNLOCK(pi); 4256 return; /* without scheduling another callout */ 4257 } 4258 4259 t4_get_port_stats(sc, pi->tx_chan, s); 4260 4261 ifp->if_opackets = s->tx_frames - s->tx_pause; 4262 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4263 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4264 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4265 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4266 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4267 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4268 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4269 s->rx_trunc3; 4270 for (i = 0; i < 4; i++) { 4271 if (pi->rx_chan_map & (1 << i)) { 4272 uint32_t v; 4273 4274 /* 4275 * XXX: indirect reads from the same ADDR/DATA pair can 4276 * race with each other. 4277 */ 4278 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4279 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4280 ifp->if_iqdrops += v; 4281 } 4282 } 4283 4284 drops = s->tx_drop; 4285 for_each_txq(pi, i, txq) 4286 drops += txq->br->br_drops; 4287 ifp->if_snd.ifq_drops = drops; 4288 4289 ifp->if_oerrors = s->tx_error_frames; 4290 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4291 s->rx_fcs_err + s->rx_len_err; 4292 4293 callout_schedule(&pi->tick, hz); 4294 PORT_UNLOCK(pi); 4295} 4296 4297static void 4298cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4299{ 4300 struct ifnet *vlan; 4301 4302 if (arg != ifp || ifp->if_type != IFT_ETHER) 4303 return; 4304 4305 vlan = VLAN_DEVAT(ifp, vid); 4306 VLAN_SETCOOKIE(vlan, ifp); 4307} 4308 4309static int 4310cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4311{ 4312 4313#ifdef INVARIANTS 4314 panic("%s: opcode 0x%02x on iq %p with payload %p", 4315 __func__, rss->opcode, iq, m); 4316#else 4317 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4318 __func__, rss->opcode, iq, m); 4319 m_freem(m); 4320#endif 4321 return (EDOOFUS); 4322} 4323 4324int 4325t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4326{ 4327 uintptr_t *loc, new; 4328 4329 if (opcode >= nitems(sc->cpl_handler)) 4330 return (EINVAL); 4331 4332 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4333 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4334 atomic_store_rel_ptr(loc, new); 4335 4336 return (0); 4337} 4338 4339static int 4340an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4341{ 4342 4343#ifdef INVARIANTS 4344 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4345#else 4346 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4347 __func__, iq, ctrl); 4348#endif 4349 return (EDOOFUS); 4350} 4351 4352int 4353t4_register_an_handler(struct adapter *sc, an_handler_t h) 4354{ 4355 uintptr_t *loc, new; 4356 4357 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4358 loc = (uintptr_t *) &sc->an_handler; 4359 atomic_store_rel_ptr(loc, new); 4360 4361 return (0); 4362} 4363 4364static int 4365fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4366{ 4367 const struct cpl_fw6_msg *cpl = 4368 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4369 4370#ifdef INVARIANTS 4371 panic("%s: fw_msg type %d", __func__, cpl->type); 4372#else 4373 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4374#endif 4375 return (EDOOFUS); 4376} 4377 4378int 4379t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4380{ 4381 uintptr_t *loc, new; 4382 4383 if (type >= nitems(sc->fw_msg_handler)) 4384 return (EINVAL); 4385 4386 /* 4387 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4388 * handler dispatch table. Reject any attempt to install a handler for 4389 * this subtype. 4390 */ 4391 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4392 return (EINVAL); 4393 4394 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4395 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4396 atomic_store_rel_ptr(loc, new); 4397 4398 return (0); 4399} 4400 4401static int 4402t4_sysctls(struct adapter *sc) 4403{ 4404 struct sysctl_ctx_list *ctx; 4405 struct sysctl_oid *oid; 4406 struct sysctl_oid_list *children, *c0; 4407 static char *caps[] = { 4408 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4409 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4410 "\6HASHFILTER\7ETHOFLD", 4411 "\20\1TOE", /* caps[2] toecaps */ 4412 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4413 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4414 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4415 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4416 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4417 "\4PO_INITIAOR\5PO_TARGET" 4418 }; 4419 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4420 4421 ctx = device_get_sysctl_ctx(sc->dev); 4422 4423 /* 4424 * dev.t4nex.X. 4425 */ 4426 oid = device_get_sysctl_tree(sc->dev); 4427 c0 = children = SYSCTL_CHILDREN(oid); 4428 4429 sc->sc_do_rxcopy = 1; 4430 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4431 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4432 4433 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4434 sc->params.nports, "# of ports"); 4435 4436 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4437 NULL, chip_rev(sc), "chip hardware revision"); 4438 4439 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4440 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4441 4442 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4443 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4444 4445 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4446 sc->cfcsum, "config file checksum"); 4447 4448 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4449 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4450 sysctl_bitfield, "A", "available doorbells"); 4451 4452 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4453 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4454 sysctl_bitfield, "A", "available link capabilities"); 4455 4456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4457 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4458 sysctl_bitfield, "A", "available NIC capabilities"); 4459 4460 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4461 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4462 sysctl_bitfield, "A", "available TCP offload capabilities"); 4463 4464 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4465 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4466 sysctl_bitfield, "A", "available RDMA capabilities"); 4467 4468 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4469 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4470 sysctl_bitfield, "A", "available iSCSI capabilities"); 4471 4472 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4473 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4474 sysctl_bitfield, "A", "available FCoE capabilities"); 4475 4476 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4477 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4478 4479 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4480 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4481 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4482 "interrupt holdoff timer values (us)"); 4483 4484 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4485 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4486 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4487 "interrupt holdoff packet counter values"); 4488 4489 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4490 NULL, sc->tids.nftids, "number of filters"); 4491 4492 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4493 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4494 "chip temperature (in Celsius)"); 4495 4496 t4_sge_sysctls(sc, ctx, children); 4497 4498 sc->lro_timeout = 100; 4499 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4500 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4501 4502#ifdef SBUF_DRAIN 4503 /* 4504 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4505 */ 4506 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4507 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4508 "logs and miscellaneous information"); 4509 children = SYSCTL_CHILDREN(oid); 4510 4511 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4512 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4513 sysctl_cctrl, "A", "congestion control"); 4514 4515 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4516 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4517 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4518 4519 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4520 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4521 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4522 4523 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4524 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4525 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4526 4527 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4528 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4529 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4530 4531 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4532 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4533 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4534 4535 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4536 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4537 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4538 4539 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4540 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4541 sysctl_cim_la, "A", "CIM logic analyzer"); 4542 4543 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4544 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4545 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4546 4547 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4548 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4549 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4550 4551 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4552 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4553 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4554 4555 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4556 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4557 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4558 4559 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4560 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4561 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4562 4563 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4564 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4565 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4566 4567 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4568 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4569 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4570 4571 if (is_t5(sc)) { 4572 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4573 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4574 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4575 4576 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4577 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4578 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4579 } 4580 4581 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4582 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4583 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4584 4585 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4586 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4587 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4588 4589 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4590 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4591 sysctl_cpl_stats, "A", "CPL statistics"); 4592 4593 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4594 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4595 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4596 4597 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4598 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4599 sysctl_devlog, "A", "firmware's device log"); 4600 4601 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4602 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4603 sysctl_fcoe_stats, "A", "FCoE statistics"); 4604 4605 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4606 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4607 sysctl_hw_sched, "A", "hardware scheduler "); 4608 4609 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4610 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4611 sysctl_l2t, "A", "hardware L2 table"); 4612 4613 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4614 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4615 sysctl_lb_stats, "A", "loopback statistics"); 4616 4617 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4618 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4619 sysctl_meminfo, "A", "memory regions"); 4620 4621 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4622 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4623 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4624 4625 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4626 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4627 sysctl_path_mtus, "A", "path MTUs"); 4628 4629 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4630 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4631 sysctl_pm_stats, "A", "PM statistics"); 4632 4633 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4634 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4635 sysctl_rdma_stats, "A", "RDMA statistics"); 4636 4637 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4638 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4639 sysctl_tcp_stats, "A", "TCP statistics"); 4640 4641 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4642 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4643 sysctl_tids, "A", "TID information"); 4644 4645 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4646 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4647 sysctl_tp_err_stats, "A", "TP error statistics"); 4648 4649 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4650 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4651 sysctl_tp_la, "A", "TP logic analyzer"); 4652 4653 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4654 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4655 sysctl_tx_rate, "A", "Tx rate"); 4656 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4659 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4660 4661 if (is_t5(sc)) { 4662 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4663 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4664 sysctl_wcwr_stats, "A", "write combined work requests"); 4665 } 4666#endif 4667 4668#ifdef TCP_OFFLOAD 4669 if (is_offload(sc)) { 4670 /* 4671 * dev.t4nex.X.toe. 4672 */ 4673 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4674 NULL, "TOE parameters"); 4675 children = SYSCTL_CHILDREN(oid); 4676 4677 sc->tt.sndbuf = 256 * 1024; 4678 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4679 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4680 4681 sc->tt.ddp = 0; 4682 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4683 &sc->tt.ddp, 0, "DDP allowed"); 4684 4685 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4686 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4687 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4688 4689 sc->tt.ddp_thres = 4690 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4691 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4692 &sc->tt.ddp_thres, 0, "DDP threshold"); 4693 4694 sc->tt.rx_coalesce = 1; 4695 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4696 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4697 4698 sc->tt.tx_align = 1; 4699 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4700 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4701 } 4702#endif 4703 4704 4705 return (0); 4706} 4707 4708static int 4709cxgbe_sysctls(struct port_info *pi) 4710{ 4711 struct sysctl_ctx_list *ctx; 4712 struct sysctl_oid *oid; 4713 struct sysctl_oid_list *children; 4714 struct adapter *sc = pi->adapter; 4715 4716 ctx = device_get_sysctl_ctx(pi->dev); 4717 4718 /* 4719 * dev.cxgbe.X. 4720 */ 4721 oid = device_get_sysctl_tree(pi->dev); 4722 children = SYSCTL_CHILDREN(oid); 4723 4724 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4725 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4726 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4727 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4728 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4729 "PHY temperature (in Celsius)"); 4730 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4731 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4732 "PHY firmware version"); 4733 } 4734 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4735 &pi->nrxq, 0, "# of rx queues"); 4736 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4737 &pi->ntxq, 0, "# of tx queues"); 4738 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4739 &pi->first_rxq, 0, "index of first rx queue"); 4740 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4741 &pi->first_txq, 0, "index of first tx queue"); 4742 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4743 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4744 "Reserve queue 0 for non-flowid packets"); 4745 4746#ifdef TCP_OFFLOAD 4747 if (is_offload(sc)) { 4748 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4749 &pi->nofldrxq, 0, 4750 "# of rx queues for offloaded TCP connections"); 4751 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4752 &pi->nofldtxq, 0, 4753 "# of tx queues for offloaded TCP connections"); 4754 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4755 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4756 "index of first TOE rx queue"); 4757 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4758 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4759 "index of first TOE tx queue"); 4760 } 4761#endif 4762#ifdef DEV_NETMAP 4763 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4764 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4765 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4766 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4767 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4768 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4769 "index of first netmap rx queue"); 4770 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4771 CTLFLAG_RD, &pi->first_nm_txq, 0, 4772 "index of first netmap tx queue"); 4773#endif 4774 4775 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4776 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4777 "holdoff timer index"); 4778 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4779 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4780 "holdoff packet counter index"); 4781 4782 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4783 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4784 "rx queue size"); 4785 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4786 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4787 "tx queue size"); 4788 4789 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4790 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4791 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4792 4793 /* 4794 * dev.cxgbe.X.stats. 4795 */ 4796 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4797 NULL, "port statistics"); 4798 children = SYSCTL_CHILDREN(oid); 4799 4800#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4801 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4802 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4803 sysctl_handle_t4_reg64, "QU", desc) 4804 4805 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4806 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4807 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4808 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4809 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4810 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4811 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4812 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4813 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4814 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4815 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4816 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4817 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4818 "# of tx frames in this range", 4819 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4820 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4821 "# of tx frames in this range", 4822 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4823 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4824 "# of tx frames in this range", 4825 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4826 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4827 "# of tx frames in this range", 4828 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4829 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4830 "# of tx frames in this range", 4831 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4832 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4833 "# of tx frames in this range", 4834 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4835 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4836 "# of tx frames in this range", 4837 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4838 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4839 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4840 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4841 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4842 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4843 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4844 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4845 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4846 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4847 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4848 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4849 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4850 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4851 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4852 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4853 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4854 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4855 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4856 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4857 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4858 4859 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4860 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4861 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4862 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4863 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4864 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4865 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4866 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4867 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4868 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4869 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4870 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4871 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4872 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4873 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4874 "# of frames received with bad FCS", 4875 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4876 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4877 "# of frames received with length error", 4878 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4879 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4880 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4881 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4882 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4883 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4884 "# of rx frames in this range", 4885 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4886 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4887 "# of rx frames in this range", 4888 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4889 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4890 "# of rx frames in this range", 4891 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4892 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4893 "# of rx frames in this range", 4894 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4895 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4896 "# of rx frames in this range", 4897 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4898 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4899 "# of rx frames in this range", 4900 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4901 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4902 "# of rx frames in this range", 4903 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4904 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4905 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4906 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4907 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4908 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4909 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4910 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4911 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4912 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4913 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4914 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4915 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4916 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4917 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4918 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4919 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4920 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4922 4923#undef SYSCTL_ADD_T4_REG64 4924 4925#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4926 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4927 &pi->stats.name, desc) 4928 4929 /* We get these from port_stats and they may be stale by upto 1s */ 4930 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4931 "# drops due to buffer-group 0 overflows"); 4932 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4933 "# drops due to buffer-group 1 overflows"); 4934 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4935 "# drops due to buffer-group 2 overflows"); 4936 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4937 "# drops due to buffer-group 3 overflows"); 4938 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4939 "# of buffer-group 0 truncated packets"); 4940 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4941 "# of buffer-group 1 truncated packets"); 4942 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4943 "# of buffer-group 2 truncated packets"); 4944 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4945 "# of buffer-group 3 truncated packets"); 4946 4947#undef SYSCTL_ADD_T4_PORTSTAT 4948 4949 return (0); 4950} 4951 4952static int 4953sysctl_int_array(SYSCTL_HANDLER_ARGS) 4954{ 4955 int rc, *i, space = 0; 4956 struct sbuf sb; 4957 4958 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4959 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 4960 if (space) 4961 sbuf_printf(&sb, " "); 4962 sbuf_printf(&sb, "%d", *i); 4963 space = 1; 4964 } 4965 sbuf_finish(&sb); 4966 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4967 sbuf_delete(&sb); 4968 return (rc); 4969} 4970 4971static int 4972sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4973{ 4974 int rc; 4975 struct sbuf *sb; 4976 4977 rc = sysctl_wire_old_buffer(req, 0); 4978 if (rc != 0) 4979 return(rc); 4980 4981 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4982 if (sb == NULL) 4983 return (ENOMEM); 4984 4985 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4986 rc = sbuf_finish(sb); 4987 sbuf_delete(sb); 4988 4989 return (rc); 4990} 4991 4992static int 4993sysctl_btphy(SYSCTL_HANDLER_ARGS) 4994{ 4995 struct port_info *pi = arg1; 4996 int op = arg2; 4997 struct adapter *sc = pi->adapter; 4998 u_int v; 4999 int rc; 5000 5001 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5002 if (rc) 5003 return (rc); 5004 /* XXX: magic numbers */ 5005 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5006 &v); 5007 end_synchronized_op(sc, 0); 5008 if (rc) 5009 return (rc); 5010 if (op == 0) 5011 v /= 256; 5012 5013 rc = sysctl_handle_int(oidp, &v, 0, req); 5014 return (rc); 5015} 5016 5017static int 5018sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5019{ 5020 struct port_info *pi = arg1; 5021 int rc, val; 5022 5023 val = pi->rsrv_noflowq; 5024 rc = sysctl_handle_int(oidp, &val, 0, req); 5025 if (rc != 0 || req->newptr == NULL) 5026 return (rc); 5027 5028 if ((val >= 1) && (pi->ntxq > 1)) 5029 pi->rsrv_noflowq = 1; 5030 else 5031 pi->rsrv_noflowq = 0; 5032 5033 return (rc); 5034} 5035 5036static int 5037sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5038{ 5039 struct port_info *pi = arg1; 5040 struct adapter *sc = pi->adapter; 5041 int idx, rc, i; 5042 struct sge_rxq *rxq; 5043#ifdef TCP_OFFLOAD 5044 struct sge_ofld_rxq *ofld_rxq; 5045#endif 5046 uint8_t v; 5047 5048 idx = pi->tmr_idx; 5049 5050 rc = sysctl_handle_int(oidp, &idx, 0, req); 5051 if (rc != 0 || req->newptr == NULL) 5052 return (rc); 5053 5054 if (idx < 0 || idx >= SGE_NTIMERS) 5055 return (EINVAL); 5056 5057 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5058 "t4tmr"); 5059 if (rc) 5060 return (rc); 5061 5062 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5063 for_each_rxq(pi, i, rxq) { 5064#ifdef atomic_store_rel_8 5065 atomic_store_rel_8(&rxq->iq.intr_params, v); 5066#else 5067 rxq->iq.intr_params = v; 5068#endif 5069 } 5070#ifdef TCP_OFFLOAD 5071 for_each_ofld_rxq(pi, i, ofld_rxq) { 5072#ifdef atomic_store_rel_8 5073 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5074#else 5075 ofld_rxq->iq.intr_params = v; 5076#endif 5077 } 5078#endif 5079 pi->tmr_idx = idx; 5080 5081 end_synchronized_op(sc, LOCK_HELD); 5082 return (0); 5083} 5084 5085static int 5086sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5087{ 5088 struct port_info *pi = arg1; 5089 struct adapter *sc = pi->adapter; 5090 int idx, rc; 5091 5092 idx = pi->pktc_idx; 5093 5094 rc = sysctl_handle_int(oidp, &idx, 0, req); 5095 if (rc != 0 || req->newptr == NULL) 5096 return (rc); 5097 5098 if (idx < -1 || idx >= SGE_NCOUNTERS) 5099 return (EINVAL); 5100 5101 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5102 "t4pktc"); 5103 if (rc) 5104 return (rc); 5105 5106 if (pi->flags & PORT_INIT_DONE) 5107 rc = EBUSY; /* cannot be changed once the queues are created */ 5108 else 5109 pi->pktc_idx = idx; 5110 5111 end_synchronized_op(sc, LOCK_HELD); 5112 return (rc); 5113} 5114 5115static int 5116sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5117{ 5118 struct port_info *pi = arg1; 5119 struct adapter *sc = pi->adapter; 5120 int qsize, rc; 5121 5122 qsize = pi->qsize_rxq; 5123 5124 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5125 if (rc != 0 || req->newptr == NULL) 5126 return (rc); 5127 5128 if (qsize < 128 || (qsize & 7)) 5129 return (EINVAL); 5130 5131 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5132 "t4rxqs"); 5133 if (rc) 5134 return (rc); 5135 5136 if (pi->flags & PORT_INIT_DONE) 5137 rc = EBUSY; /* cannot be changed once the queues are created */ 5138 else 5139 pi->qsize_rxq = qsize; 5140 5141 end_synchronized_op(sc, LOCK_HELD); 5142 return (rc); 5143} 5144 5145static int 5146sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5147{ 5148 struct port_info *pi = arg1; 5149 struct adapter *sc = pi->adapter; 5150 int qsize, rc; 5151 5152 qsize = pi->qsize_txq; 5153 5154 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5155 if (rc != 0 || req->newptr == NULL) 5156 return (rc); 5157 5158 /* bufring size must be powerof2 */ 5159 if (qsize < 128 || !powerof2(qsize)) 5160 return (EINVAL); 5161 5162 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5163 "t4txqs"); 5164 if (rc) 5165 return (rc); 5166 5167 if (pi->flags & PORT_INIT_DONE) 5168 rc = EBUSY; /* cannot be changed once the queues are created */ 5169 else 5170 pi->qsize_txq = qsize; 5171 5172 end_synchronized_op(sc, LOCK_HELD); 5173 return (rc); 5174} 5175 5176static int 5177sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5178{ 5179 struct port_info *pi = arg1; 5180 struct adapter *sc = pi->adapter; 5181 struct link_config *lc = &pi->link_cfg; 5182 int rc; 5183 5184 if (req->newptr == NULL) { 5185 struct sbuf *sb; 5186 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5187 5188 rc = sysctl_wire_old_buffer(req, 0); 5189 if (rc != 0) 5190 return(rc); 5191 5192 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5193 if (sb == NULL) 5194 return (ENOMEM); 5195 5196 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5197 rc = sbuf_finish(sb); 5198 sbuf_delete(sb); 5199 } else { 5200 char s[2]; 5201 int n; 5202 5203 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5204 s[1] = 0; 5205 5206 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5207 if (rc != 0) 5208 return(rc); 5209 5210 if (s[1] != 0) 5211 return (EINVAL); 5212 if (s[0] < '0' || s[0] > '9') 5213 return (EINVAL); /* not a number */ 5214 n = s[0] - '0'; 5215 if (n & ~(PAUSE_TX | PAUSE_RX)) 5216 return (EINVAL); /* some other bit is set too */ 5217 5218 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5219 if (rc) 5220 return (rc); 5221 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5222 int link_ok = lc->link_ok; 5223 5224 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5225 lc->requested_fc |= n; 5226 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5227 lc->link_ok = link_ok; /* restore */ 5228 } 5229 end_synchronized_op(sc, 0); 5230 } 5231 5232 return (rc); 5233} 5234 5235static int 5236sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5237{ 5238 struct adapter *sc = arg1; 5239 int reg = arg2; 5240 uint64_t val; 5241 5242 val = t4_read_reg64(sc, reg); 5243 5244 return (sysctl_handle_64(oidp, &val, 0, req)); 5245} 5246 5247static int 5248sysctl_temperature(SYSCTL_HANDLER_ARGS) 5249{ 5250 struct adapter *sc = arg1; 5251 int rc, t; 5252 uint32_t param, val; 5253 5254 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5255 if (rc) 5256 return (rc); 5257 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5258 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5259 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5260 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5261 end_synchronized_op(sc, 0); 5262 if (rc) 5263 return (rc); 5264 5265 /* unknown is returned as 0 but we display -1 in that case */ 5266 t = val == 0 ? -1 : val; 5267 5268 rc = sysctl_handle_int(oidp, &t, 0, req); 5269 return (rc); 5270} 5271 5272#ifdef SBUF_DRAIN 5273static int 5274sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5275{ 5276 struct adapter *sc = arg1; 5277 struct sbuf *sb; 5278 int rc, i; 5279 uint16_t incr[NMTUS][NCCTRL_WIN]; 5280 static const char *dec_fac[] = { 5281 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5282 "0.9375" 5283 }; 5284 5285 rc = sysctl_wire_old_buffer(req, 0); 5286 if (rc != 0) 5287 return (rc); 5288 5289 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5290 if (sb == NULL) 5291 return (ENOMEM); 5292 5293 t4_read_cong_tbl(sc, incr); 5294 5295 for (i = 0; i < NCCTRL_WIN; ++i) { 5296 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5297 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5298 incr[5][i], incr[6][i], incr[7][i]); 5299 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5300 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5301 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5302 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5303 } 5304 5305 rc = sbuf_finish(sb); 5306 sbuf_delete(sb); 5307 5308 return (rc); 5309} 5310 5311static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5312 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5313 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5314 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5315}; 5316 5317static int 5318sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5319{ 5320 struct adapter *sc = arg1; 5321 struct sbuf *sb; 5322 int rc, i, n, qid = arg2; 5323 uint32_t *buf, *p; 5324 char *qtype; 5325 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5326 5327 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5328 ("%s: bad qid %d\n", __func__, qid)); 5329 5330 if (qid < CIM_NUM_IBQ) { 5331 /* inbound queue */ 5332 qtype = "IBQ"; 5333 n = 4 * CIM_IBQ_SIZE; 5334 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5335 rc = t4_read_cim_ibq(sc, qid, buf, n); 5336 } else { 5337 /* outbound queue */ 5338 qtype = "OBQ"; 5339 qid -= CIM_NUM_IBQ; 5340 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5341 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5342 rc = t4_read_cim_obq(sc, qid, buf, n); 5343 } 5344 5345 if (rc < 0) { 5346 rc = -rc; 5347 goto done; 5348 } 5349 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5350 5351 rc = sysctl_wire_old_buffer(req, 0); 5352 if (rc != 0) 5353 goto done; 5354 5355 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5356 if (sb == NULL) { 5357 rc = ENOMEM; 5358 goto done; 5359 } 5360 5361 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5362 for (i = 0, p = buf; i < n; i += 16, p += 4) 5363 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5364 p[2], p[3]); 5365 5366 rc = sbuf_finish(sb); 5367 sbuf_delete(sb); 5368done: 5369 free(buf, M_CXGBE); 5370 return (rc); 5371} 5372 5373static int 5374sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5375{ 5376 struct adapter *sc = arg1; 5377 u_int cfg; 5378 struct sbuf *sb; 5379 uint32_t *buf, *p; 5380 int rc; 5381 5382 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5383 if (rc != 0) 5384 return (rc); 5385 5386 rc = sysctl_wire_old_buffer(req, 0); 5387 if (rc != 0) 5388 return (rc); 5389 5390 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5391 if (sb == NULL) 5392 return (ENOMEM); 5393 5394 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5395 M_ZERO | M_WAITOK); 5396 5397 rc = -t4_cim_read_la(sc, buf, NULL); 5398 if (rc != 0) 5399 goto done; 5400 5401 sbuf_printf(sb, "Status Data PC%s", 5402 cfg & F_UPDBGLACAPTPCONLY ? "" : 5403 " LS0Stat LS0Addr LS0Data"); 5404 5405 KASSERT((sc->params.cim_la_size & 7) == 0, 5406 ("%s: p will walk off the end of buf", __func__)); 5407 5408 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5409 if (cfg & F_UPDBGLACAPTPCONLY) { 5410 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5411 p[6], p[7]); 5412 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5413 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5414 p[4] & 0xff, p[5] >> 8); 5415 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5416 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5417 p[1] & 0xf, p[2] >> 4); 5418 } else { 5419 sbuf_printf(sb, 5420 "\n %02x %x%07x %x%07x %08x %08x " 5421 "%08x%08x%08x%08x", 5422 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5423 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5424 p[6], p[7]); 5425 } 5426 } 5427 5428 rc = sbuf_finish(sb); 5429 sbuf_delete(sb); 5430done: 5431 free(buf, M_CXGBE); 5432 return (rc); 5433} 5434 5435static int 5436sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5437{ 5438 struct adapter *sc = arg1; 5439 u_int i; 5440 struct sbuf *sb; 5441 uint32_t *buf, *p; 5442 int rc; 5443 5444 rc = sysctl_wire_old_buffer(req, 0); 5445 if (rc != 0) 5446 return (rc); 5447 5448 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5449 if (sb == NULL) 5450 return (ENOMEM); 5451 5452 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5453 M_ZERO | M_WAITOK); 5454 5455 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5456 p = buf; 5457 5458 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5459 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5460 p[1], p[0]); 5461 } 5462 5463 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5464 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5465 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5466 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5467 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5468 (p[1] >> 2) | ((p[2] & 3) << 30), 5469 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5470 p[0] & 1); 5471 } 5472 5473 rc = sbuf_finish(sb); 5474 sbuf_delete(sb); 5475 free(buf, M_CXGBE); 5476 return (rc); 5477} 5478 5479static int 5480sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5481{ 5482 struct adapter *sc = arg1; 5483 u_int i; 5484 struct sbuf *sb; 5485 uint32_t *buf, *p; 5486 int rc; 5487 5488 rc = sysctl_wire_old_buffer(req, 0); 5489 if (rc != 0) 5490 return (rc); 5491 5492 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5493 if (sb == NULL) 5494 return (ENOMEM); 5495 5496 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5497 M_ZERO | M_WAITOK); 5498 5499 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5500 p = buf; 5501 5502 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5503 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5504 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5505 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5506 p[4], p[3], p[2], p[1], p[0]); 5507 } 5508 5509 sbuf_printf(sb, "\n\nCntl ID Data"); 5510 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5511 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5512 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5513 } 5514 5515 rc = sbuf_finish(sb); 5516 sbuf_delete(sb); 5517 free(buf, M_CXGBE); 5518 return (rc); 5519} 5520 5521static int 5522sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5523{ 5524 struct adapter *sc = arg1; 5525 struct sbuf *sb; 5526 int rc, i; 5527 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5528 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5529 uint16_t thres[CIM_NUM_IBQ]; 5530 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5531 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5532 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5533 5534 if (is_t4(sc)) { 5535 cim_num_obq = CIM_NUM_OBQ; 5536 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5537 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5538 } else { 5539 cim_num_obq = CIM_NUM_OBQ_T5; 5540 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5541 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5542 } 5543 nq = CIM_NUM_IBQ + cim_num_obq; 5544 5545 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5546 if (rc == 0) 5547 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5548 if (rc != 0) 5549 return (rc); 5550 5551 t4_read_cimq_cfg(sc, base, size, thres); 5552 5553 rc = sysctl_wire_old_buffer(req, 0); 5554 if (rc != 0) 5555 return (rc); 5556 5557 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5558 if (sb == NULL) 5559 return (ENOMEM); 5560 5561 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5562 5563 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5564 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5565 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5566 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5567 G_QUEREMFLITS(p[2]) * 16); 5568 for ( ; i < nq; i++, p += 4, wr += 2) 5569 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5570 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5571 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5572 G_QUEREMFLITS(p[2]) * 16); 5573 5574 rc = sbuf_finish(sb); 5575 sbuf_delete(sb); 5576 5577 return (rc); 5578} 5579 5580static int 5581sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5582{ 5583 struct adapter *sc = arg1; 5584 struct sbuf *sb; 5585 int rc; 5586 struct tp_cpl_stats stats; 5587 5588 rc = sysctl_wire_old_buffer(req, 0); 5589 if (rc != 0) 5590 return (rc); 5591 5592 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5593 if (sb == NULL) 5594 return (ENOMEM); 5595 5596 t4_tp_get_cpl_stats(sc, &stats); 5597 5598 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5599 "channel 3\n"); 5600 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5601 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5602 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5603 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5604 5605 rc = sbuf_finish(sb); 5606 sbuf_delete(sb); 5607 5608 return (rc); 5609} 5610 5611static int 5612sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5613{ 5614 struct adapter *sc = arg1; 5615 struct sbuf *sb; 5616 int rc; 5617 struct tp_usm_stats stats; 5618 5619 rc = sysctl_wire_old_buffer(req, 0); 5620 if (rc != 0) 5621 return(rc); 5622 5623 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5624 if (sb == NULL) 5625 return (ENOMEM); 5626 5627 t4_get_usm_stats(sc, &stats); 5628 5629 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5630 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5631 sbuf_printf(sb, "Drops: %u", stats.drops); 5632 5633 rc = sbuf_finish(sb); 5634 sbuf_delete(sb); 5635 5636 return (rc); 5637} 5638 5639const char *devlog_level_strings[] = { 5640 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5641 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5642 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5643 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5644 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5645 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5646}; 5647 5648const char *devlog_facility_strings[] = { 5649 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5650 [FW_DEVLOG_FACILITY_CF] = "CF", 5651 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5652 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5653 [FW_DEVLOG_FACILITY_RES] = "RES", 5654 [FW_DEVLOG_FACILITY_HW] = "HW", 5655 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5656 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5657 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5658 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5659 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5660 [FW_DEVLOG_FACILITY_VI] = "VI", 5661 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5662 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5663 [FW_DEVLOG_FACILITY_TM] = "TM", 5664 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5665 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5666 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5667 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5668 [FW_DEVLOG_FACILITY_RI] = "RI", 5669 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5670 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5671 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5672 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5673}; 5674 5675static int 5676sysctl_devlog(SYSCTL_HANDLER_ARGS) 5677{ 5678 struct adapter *sc = arg1; 5679 struct devlog_params *dparams = &sc->params.devlog; 5680 struct fw_devlog_e *buf, *e; 5681 int i, j, rc, nentries, first = 0, m; 5682 struct sbuf *sb; 5683 uint64_t ftstamp = UINT64_MAX; 5684 5685 if (dparams->start == 0) { 5686 dparams->memtype = FW_MEMTYPE_EDC0; 5687 dparams->start = 0x84000; 5688 dparams->size = 32768; 5689 } 5690 5691 nentries = dparams->size / sizeof(struct fw_devlog_e); 5692 5693 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5694 if (buf == NULL) 5695 return (ENOMEM); 5696 5697 m = fwmtype_to_hwmtype(dparams->memtype); 5698 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5699 if (rc != 0) 5700 goto done; 5701 5702 for (i = 0; i < nentries; i++) { 5703 e = &buf[i]; 5704 5705 if (e->timestamp == 0) 5706 break; /* end */ 5707 5708 e->timestamp = be64toh(e->timestamp); 5709 e->seqno = be32toh(e->seqno); 5710 for (j = 0; j < 8; j++) 5711 e->params[j] = be32toh(e->params[j]); 5712 5713 if (e->timestamp < ftstamp) { 5714 ftstamp = e->timestamp; 5715 first = i; 5716 } 5717 } 5718 5719 if (buf[first].timestamp == 0) 5720 goto done; /* nothing in the log */ 5721 5722 rc = sysctl_wire_old_buffer(req, 0); 5723 if (rc != 0) 5724 goto done; 5725 5726 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5727 if (sb == NULL) { 5728 rc = ENOMEM; 5729 goto done; 5730 } 5731 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5732 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5733 5734 i = first; 5735 do { 5736 e = &buf[i]; 5737 if (e->timestamp == 0) 5738 break; /* end */ 5739 5740 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5741 e->seqno, e->timestamp, 5742 (e->level < nitems(devlog_level_strings) ? 5743 devlog_level_strings[e->level] : "UNKNOWN"), 5744 (e->facility < nitems(devlog_facility_strings) ? 5745 devlog_facility_strings[e->facility] : "UNKNOWN")); 5746 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5747 e->params[2], e->params[3], e->params[4], 5748 e->params[5], e->params[6], e->params[7]); 5749 5750 if (++i == nentries) 5751 i = 0; 5752 } while (i != first); 5753 5754 rc = sbuf_finish(sb); 5755 sbuf_delete(sb); 5756done: 5757 free(buf, M_CXGBE); 5758 return (rc); 5759} 5760 5761static int 5762sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5763{ 5764 struct adapter *sc = arg1; 5765 struct sbuf *sb; 5766 int rc; 5767 struct tp_fcoe_stats stats[4]; 5768 5769 rc = sysctl_wire_old_buffer(req, 0); 5770 if (rc != 0) 5771 return (rc); 5772 5773 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5774 if (sb == NULL) 5775 return (ENOMEM); 5776 5777 t4_get_fcoe_stats(sc, 0, &stats[0]); 5778 t4_get_fcoe_stats(sc, 1, &stats[1]); 5779 t4_get_fcoe_stats(sc, 2, &stats[2]); 5780 t4_get_fcoe_stats(sc, 3, &stats[3]); 5781 5782 sbuf_printf(sb, " channel 0 channel 1 " 5783 "channel 2 channel 3\n"); 5784 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5785 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5786 stats[3].octetsDDP); 5787 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5788 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5789 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5790 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5791 stats[3].framesDrop); 5792 5793 rc = sbuf_finish(sb); 5794 sbuf_delete(sb); 5795 5796 return (rc); 5797} 5798 5799static int 5800sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5801{ 5802 struct adapter *sc = arg1; 5803 struct sbuf *sb; 5804 int rc, i; 5805 unsigned int map, kbps, ipg, mode; 5806 unsigned int pace_tab[NTX_SCHED]; 5807 5808 rc = sysctl_wire_old_buffer(req, 0); 5809 if (rc != 0) 5810 return (rc); 5811 5812 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5813 if (sb == NULL) 5814 return (ENOMEM); 5815 5816 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5817 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5818 t4_read_pace_tbl(sc, pace_tab); 5819 5820 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5821 "Class IPG (0.1 ns) Flow IPG (us)"); 5822 5823 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5824 t4_get_tx_sched(sc, i, &kbps, &ipg); 5825 sbuf_printf(sb, "\n %u %-5s %u ", i, 5826 (mode & (1 << i)) ? "flow" : "class", map & 3); 5827 if (kbps) 5828 sbuf_printf(sb, "%9u ", kbps); 5829 else 5830 sbuf_printf(sb, " disabled "); 5831 5832 if (ipg) 5833 sbuf_printf(sb, "%13u ", ipg); 5834 else 5835 sbuf_printf(sb, " disabled "); 5836 5837 if (pace_tab[i]) 5838 sbuf_printf(sb, "%10u", pace_tab[i]); 5839 else 5840 sbuf_printf(sb, " disabled"); 5841 } 5842 5843 rc = sbuf_finish(sb); 5844 sbuf_delete(sb); 5845 5846 return (rc); 5847} 5848 5849static int 5850sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5851{ 5852 struct adapter *sc = arg1; 5853 struct sbuf *sb; 5854 int rc, i, j; 5855 uint64_t *p0, *p1; 5856 struct lb_port_stats s[2]; 5857 static const char *stat_name[] = { 5858 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5859 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5860 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5861 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5862 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5863 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5864 "BG2FramesTrunc:", "BG3FramesTrunc:" 5865 }; 5866 5867 rc = sysctl_wire_old_buffer(req, 0); 5868 if (rc != 0) 5869 return (rc); 5870 5871 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5872 if (sb == NULL) 5873 return (ENOMEM); 5874 5875 memset(s, 0, sizeof(s)); 5876 5877 for (i = 0; i < 4; i += 2) { 5878 t4_get_lb_stats(sc, i, &s[0]); 5879 t4_get_lb_stats(sc, i + 1, &s[1]); 5880 5881 p0 = &s[0].octets; 5882 p1 = &s[1].octets; 5883 sbuf_printf(sb, "%s Loopback %u" 5884 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5885 5886 for (j = 0; j < nitems(stat_name); j++) 5887 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5888 *p0++, *p1++); 5889 } 5890 5891 rc = sbuf_finish(sb); 5892 sbuf_delete(sb); 5893 5894 return (rc); 5895} 5896 5897static int 5898sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5899{ 5900 int rc = 0; 5901 struct port_info *pi = arg1; 5902 struct sbuf *sb; 5903 static const char *linkdnreasons[] = { 5904 "non-specific", "remote fault", "autoneg failed", "reserved3", 5905 "PHY overheated", "unknown", "rx los", "reserved7" 5906 }; 5907 5908 rc = sysctl_wire_old_buffer(req, 0); 5909 if (rc != 0) 5910 return(rc); 5911 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5912 if (sb == NULL) 5913 return (ENOMEM); 5914 5915 if (pi->linkdnrc < 0) 5916 sbuf_printf(sb, "n/a"); 5917 else if (pi->linkdnrc < nitems(linkdnreasons)) 5918 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5919 else 5920 sbuf_printf(sb, "%d", pi->linkdnrc); 5921 5922 rc = sbuf_finish(sb); 5923 sbuf_delete(sb); 5924 5925 return (rc); 5926} 5927 5928struct mem_desc { 5929 unsigned int base; 5930 unsigned int limit; 5931 unsigned int idx; 5932}; 5933 5934static int 5935mem_desc_cmp(const void *a, const void *b) 5936{ 5937 return ((const struct mem_desc *)a)->base - 5938 ((const struct mem_desc *)b)->base; 5939} 5940 5941static void 5942mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5943 unsigned int to) 5944{ 5945 unsigned int size; 5946 5947 size = to - from + 1; 5948 if (size == 0) 5949 return; 5950 5951 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5952 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5953} 5954 5955static int 5956sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5957{ 5958 struct adapter *sc = arg1; 5959 struct sbuf *sb; 5960 int rc, i, n; 5961 uint32_t lo, hi, used, alloc; 5962 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5963 static const char *region[] = { 5964 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5965 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5966 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5967 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5968 "RQUDP region:", "PBL region:", "TXPBL region:", 5969 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5970 "On-chip queues:" 5971 }; 5972 struct mem_desc avail[4]; 5973 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5974 struct mem_desc *md = mem; 5975 5976 rc = sysctl_wire_old_buffer(req, 0); 5977 if (rc != 0) 5978 return (rc); 5979 5980 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5981 if (sb == NULL) 5982 return (ENOMEM); 5983 5984 for (i = 0; i < nitems(mem); i++) { 5985 mem[i].limit = 0; 5986 mem[i].idx = i; 5987 } 5988 5989 /* Find and sort the populated memory ranges */ 5990 i = 0; 5991 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5992 if (lo & F_EDRAM0_ENABLE) { 5993 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5994 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5995 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5996 avail[i].idx = 0; 5997 i++; 5998 } 5999 if (lo & F_EDRAM1_ENABLE) { 6000 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6001 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6002 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6003 avail[i].idx = 1; 6004 i++; 6005 } 6006 if (lo & F_EXT_MEM_ENABLE) { 6007 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6008 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6009 avail[i].limit = avail[i].base + 6010 (G_EXT_MEM_SIZE(hi) << 20); 6011 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6012 i++; 6013 } 6014 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6015 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6016 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6017 avail[i].limit = avail[i].base + 6018 (G_EXT_MEM1_SIZE(hi) << 20); 6019 avail[i].idx = 4; 6020 i++; 6021 } 6022 if (!i) /* no memory available */ 6023 return 0; 6024 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6025 6026 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6027 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6028 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6029 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6030 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6031 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6032 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6033 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6034 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6035 6036 /* the next few have explicit upper bounds */ 6037 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6038 md->limit = md->base - 1 + 6039 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6040 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6041 md++; 6042 6043 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6044 md->limit = md->base - 1 + 6045 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6046 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6047 md++; 6048 6049 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6050 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6051 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6052 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6053 } else { 6054 md->base = 0; 6055 md->idx = nitems(region); /* hide it */ 6056 } 6057 md++; 6058 6059#define ulp_region(reg) \ 6060 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6061 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6062 6063 ulp_region(RX_ISCSI); 6064 ulp_region(RX_TDDP); 6065 ulp_region(TX_TPT); 6066 ulp_region(RX_STAG); 6067 ulp_region(RX_RQ); 6068 ulp_region(RX_RQUDP); 6069 ulp_region(RX_PBL); 6070 ulp_region(TX_PBL); 6071#undef ulp_region 6072 6073 md->base = 0; 6074 md->idx = nitems(region); 6075 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6076 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6077 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6078 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6079 } 6080 md++; 6081 6082 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6083 md->limit = md->base + sc->tids.ntids - 1; 6084 md++; 6085 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6086 md->limit = md->base + sc->tids.ntids - 1; 6087 md++; 6088 6089 md->base = sc->vres.ocq.start; 6090 if (sc->vres.ocq.size) 6091 md->limit = md->base + sc->vres.ocq.size - 1; 6092 else 6093 md->idx = nitems(region); /* hide it */ 6094 md++; 6095 6096 /* add any address-space holes, there can be up to 3 */ 6097 for (n = 0; n < i - 1; n++) 6098 if (avail[n].limit < avail[n + 1].base) 6099 (md++)->base = avail[n].limit; 6100 if (avail[n].limit) 6101 (md++)->base = avail[n].limit; 6102 6103 n = md - mem; 6104 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6105 6106 for (lo = 0; lo < i; lo++) 6107 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6108 avail[lo].limit - 1); 6109 6110 sbuf_printf(sb, "\n"); 6111 for (i = 0; i < n; i++) { 6112 if (mem[i].idx >= nitems(region)) 6113 continue; /* skip holes */ 6114 if (!mem[i].limit) 6115 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6116 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6117 mem[i].limit); 6118 } 6119 6120 sbuf_printf(sb, "\n"); 6121 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6122 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6123 mem_region_show(sb, "uP RAM:", lo, hi); 6124 6125 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6126 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6127 mem_region_show(sb, "uP Extmem2:", lo, hi); 6128 6129 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6130 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6131 G_PMRXMAXPAGE(lo), 6132 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6133 (lo & F_PMRXNUMCHN) ? 2 : 1); 6134 6135 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6136 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6137 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6138 G_PMTXMAXPAGE(lo), 6139 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6140 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6141 sbuf_printf(sb, "%u p-structs\n", 6142 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6143 6144 for (i = 0; i < 4; i++) { 6145 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6146 if (is_t4(sc)) { 6147 used = G_USED(lo); 6148 alloc = G_ALLOC(lo); 6149 } else { 6150 used = G_T5_USED(lo); 6151 alloc = G_T5_ALLOC(lo); 6152 } 6153 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6154 i, used, alloc); 6155 } 6156 for (i = 0; i < 4; i++) { 6157 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6158 if (is_t4(sc)) { 6159 used = G_USED(lo); 6160 alloc = G_ALLOC(lo); 6161 } else { 6162 used = G_T5_USED(lo); 6163 alloc = G_T5_ALLOC(lo); 6164 } 6165 sbuf_printf(sb, 6166 "\nLoopback %d using %u pages out of %u allocated", 6167 i, used, alloc); 6168 } 6169 6170 rc = sbuf_finish(sb); 6171 sbuf_delete(sb); 6172 6173 return (rc); 6174} 6175 6176static inline void 6177tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6178{ 6179 *mask = x | y; 6180 y = htobe64(y); 6181 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6182} 6183 6184static int 6185sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6186{ 6187 struct adapter *sc = arg1; 6188 struct sbuf *sb; 6189 int rc, i, n; 6190 6191 rc = sysctl_wire_old_buffer(req, 0); 6192 if (rc != 0) 6193 return (rc); 6194 6195 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6196 if (sb == NULL) 6197 return (ENOMEM); 6198 6199 sbuf_printf(sb, 6200 "Idx Ethernet address Mask Vld Ports PF" 6201 " VF Replication P0 P1 P2 P3 ML"); 6202 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6203 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6204 for (i = 0; i < n; i++) { 6205 uint64_t tcamx, tcamy, mask; 6206 uint32_t cls_lo, cls_hi; 6207 uint8_t addr[ETHER_ADDR_LEN]; 6208 6209 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6210 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6211 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6212 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6213 6214 if (tcamx & tcamy) 6215 continue; 6216 6217 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6218 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6219 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6220 addr[3], addr[4], addr[5], (uintmax_t)mask, 6221 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6222 G_PORTMAP(cls_hi), G_PF(cls_lo), 6223 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6224 6225 if (cls_lo & F_REPLICATE) { 6226 struct fw_ldst_cmd ldst_cmd; 6227 6228 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6229 ldst_cmd.op_to_addrspace = 6230 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6231 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6232 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6233 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6234 ldst_cmd.u.mps.fid_ctl = 6235 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6236 V_FW_LDST_CMD_CTL(i)); 6237 6238 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6239 "t4mps"); 6240 if (rc) 6241 break; 6242 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6243 sizeof(ldst_cmd), &ldst_cmd); 6244 end_synchronized_op(sc, 0); 6245 6246 if (rc != 0) { 6247 sbuf_printf(sb, 6248 " ------------ error %3u ------------", rc); 6249 rc = 0; 6250 } else { 6251 sbuf_printf(sb, " %08x %08x %08x %08x", 6252 be32toh(ldst_cmd.u.mps.rplc127_96), 6253 be32toh(ldst_cmd.u.mps.rplc95_64), 6254 be32toh(ldst_cmd.u.mps.rplc63_32), 6255 be32toh(ldst_cmd.u.mps.rplc31_0)); 6256 } 6257 } else 6258 sbuf_printf(sb, "%36s", ""); 6259 6260 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6261 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6262 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6263 } 6264 6265 if (rc) 6266 (void) sbuf_finish(sb); 6267 else 6268 rc = sbuf_finish(sb); 6269 sbuf_delete(sb); 6270 6271 return (rc); 6272} 6273 6274static int 6275sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6276{ 6277 struct adapter *sc = arg1; 6278 struct sbuf *sb; 6279 int rc; 6280 uint16_t mtus[NMTUS]; 6281 6282 rc = sysctl_wire_old_buffer(req, 0); 6283 if (rc != 0) 6284 return (rc); 6285 6286 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6287 if (sb == NULL) 6288 return (ENOMEM); 6289 6290 t4_read_mtu_tbl(sc, mtus, NULL); 6291 6292 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6293 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6294 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6295 mtus[14], mtus[15]); 6296 6297 rc = sbuf_finish(sb); 6298 sbuf_delete(sb); 6299 6300 return (rc); 6301} 6302 6303static int 6304sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6305{ 6306 struct adapter *sc = arg1; 6307 struct sbuf *sb; 6308 int rc, i; 6309 uint32_t cnt[PM_NSTATS]; 6310 uint64_t cyc[PM_NSTATS]; 6311 static const char *rx_stats[] = { 6312 "Read:", "Write bypass:", "Write mem:", "Flush:" 6313 }; 6314 static const char *tx_stats[] = { 6315 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6316 }; 6317 6318 rc = sysctl_wire_old_buffer(req, 0); 6319 if (rc != 0) 6320 return (rc); 6321 6322 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6323 if (sb == NULL) 6324 return (ENOMEM); 6325 6326 t4_pmtx_get_stats(sc, cnt, cyc); 6327 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6328 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6329 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6330 cyc[i]); 6331 6332 t4_pmrx_get_stats(sc, cnt, cyc); 6333 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6334 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6335 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6336 cyc[i]); 6337 6338 rc = sbuf_finish(sb); 6339 sbuf_delete(sb); 6340 6341 return (rc); 6342} 6343 6344static int 6345sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6346{ 6347 struct adapter *sc = arg1; 6348 struct sbuf *sb; 6349 int rc; 6350 struct tp_rdma_stats stats; 6351 6352 rc = sysctl_wire_old_buffer(req, 0); 6353 if (rc != 0) 6354 return (rc); 6355 6356 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6357 if (sb == NULL) 6358 return (ENOMEM); 6359 6360 t4_tp_get_rdma_stats(sc, &stats); 6361 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6362 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6363 6364 rc = sbuf_finish(sb); 6365 sbuf_delete(sb); 6366 6367 return (rc); 6368} 6369 6370static int 6371sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6372{ 6373 struct adapter *sc = arg1; 6374 struct sbuf *sb; 6375 int rc; 6376 struct tp_tcp_stats v4, v6; 6377 6378 rc = sysctl_wire_old_buffer(req, 0); 6379 if (rc != 0) 6380 return (rc); 6381 6382 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6383 if (sb == NULL) 6384 return (ENOMEM); 6385 6386 t4_tp_get_tcp_stats(sc, &v4, &v6); 6387 sbuf_printf(sb, 6388 " IP IPv6\n"); 6389 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6390 v4.tcpOutRsts, v6.tcpOutRsts); 6391 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6392 v4.tcpInSegs, v6.tcpInSegs); 6393 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6394 v4.tcpOutSegs, v6.tcpOutSegs); 6395 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6396 v4.tcpRetransSegs, v6.tcpRetransSegs); 6397 6398 rc = sbuf_finish(sb); 6399 sbuf_delete(sb); 6400 6401 return (rc); 6402} 6403 6404static int 6405sysctl_tids(SYSCTL_HANDLER_ARGS) 6406{ 6407 struct adapter *sc = arg1; 6408 struct sbuf *sb; 6409 int rc; 6410 struct tid_info *t = &sc->tids; 6411 6412 rc = sysctl_wire_old_buffer(req, 0); 6413 if (rc != 0) 6414 return (rc); 6415 6416 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6417 if (sb == NULL) 6418 return (ENOMEM); 6419 6420 if (t->natids) { 6421 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6422 t->atids_in_use); 6423 } 6424 6425 if (t->ntids) { 6426 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6427 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6428 6429 if (b) { 6430 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6431 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6432 t->ntids - 1); 6433 } else { 6434 sbuf_printf(sb, "TID range: %u-%u", 6435 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6436 t->ntids - 1); 6437 } 6438 } else 6439 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6440 sbuf_printf(sb, ", in use: %u\n", 6441 atomic_load_acq_int(&t->tids_in_use)); 6442 } 6443 6444 if (t->nstids) { 6445 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6446 t->stid_base + t->nstids - 1, t->stids_in_use); 6447 } 6448 6449 if (t->nftids) { 6450 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6451 t->ftid_base + t->nftids - 1); 6452 } 6453 6454 if (t->netids) { 6455 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6456 t->etid_base + t->netids - 1); 6457 } 6458 6459 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6460 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6461 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6462 6463 rc = sbuf_finish(sb); 6464 sbuf_delete(sb); 6465 6466 return (rc); 6467} 6468 6469static int 6470sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6471{ 6472 struct adapter *sc = arg1; 6473 struct sbuf *sb; 6474 int rc; 6475 struct tp_err_stats stats; 6476 6477 rc = sysctl_wire_old_buffer(req, 0); 6478 if (rc != 0) 6479 return (rc); 6480 6481 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6482 if (sb == NULL) 6483 return (ENOMEM); 6484 6485 t4_tp_get_err_stats(sc, &stats); 6486 6487 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6488 "channel 3\n"); 6489 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6490 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6491 stats.macInErrs[3]); 6492 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6493 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6494 stats.hdrInErrs[3]); 6495 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6496 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6497 stats.tcpInErrs[3]); 6498 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6499 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6500 stats.tcp6InErrs[3]); 6501 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6502 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6503 stats.tnlCongDrops[3]); 6504 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6505 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6506 stats.tnlTxDrops[3]); 6507 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6508 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6509 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6510 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6511 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6512 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6513 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6514 stats.ofldNoNeigh, stats.ofldCongDefer); 6515 6516 rc = sbuf_finish(sb); 6517 sbuf_delete(sb); 6518 6519 return (rc); 6520} 6521 6522struct field_desc { 6523 const char *name; 6524 u_int start; 6525 u_int width; 6526}; 6527 6528static void 6529field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6530{ 6531 char buf[32]; 6532 int line_size = 0; 6533 6534 while (f->name) { 6535 uint64_t mask = (1ULL << f->width) - 1; 6536 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6537 ((uintmax_t)v >> f->start) & mask); 6538 6539 if (line_size + len >= 79) { 6540 line_size = 8; 6541 sbuf_printf(sb, "\n "); 6542 } 6543 sbuf_printf(sb, "%s ", buf); 6544 line_size += len + 1; 6545 f++; 6546 } 6547 sbuf_printf(sb, "\n"); 6548} 6549 6550static struct field_desc tp_la0[] = { 6551 { "RcfOpCodeOut", 60, 4 }, 6552 { "State", 56, 4 }, 6553 { "WcfState", 52, 4 }, 6554 { "RcfOpcSrcOut", 50, 2 }, 6555 { "CRxError", 49, 1 }, 6556 { "ERxError", 48, 1 }, 6557 { "SanityFailed", 47, 1 }, 6558 { "SpuriousMsg", 46, 1 }, 6559 { "FlushInputMsg", 45, 1 }, 6560 { "FlushInputCpl", 44, 1 }, 6561 { "RssUpBit", 43, 1 }, 6562 { "RssFilterHit", 42, 1 }, 6563 { "Tid", 32, 10 }, 6564 { "InitTcb", 31, 1 }, 6565 { "LineNumber", 24, 7 }, 6566 { "Emsg", 23, 1 }, 6567 { "EdataOut", 22, 1 }, 6568 { "Cmsg", 21, 1 }, 6569 { "CdataOut", 20, 1 }, 6570 { "EreadPdu", 19, 1 }, 6571 { "CreadPdu", 18, 1 }, 6572 { "TunnelPkt", 17, 1 }, 6573 { "RcfPeerFin", 16, 1 }, 6574 { "RcfReasonOut", 12, 4 }, 6575 { "TxCchannel", 10, 2 }, 6576 { "RcfTxChannel", 8, 2 }, 6577 { "RxEchannel", 6, 2 }, 6578 { "RcfRxChannel", 5, 1 }, 6579 { "RcfDataOutSrdy", 4, 1 }, 6580 { "RxDvld", 3, 1 }, 6581 { "RxOoDvld", 2, 1 }, 6582 { "RxCongestion", 1, 1 }, 6583 { "TxCongestion", 0, 1 }, 6584 { NULL } 6585}; 6586 6587static struct field_desc tp_la1[] = { 6588 { "CplCmdIn", 56, 8 }, 6589 { "CplCmdOut", 48, 8 }, 6590 { "ESynOut", 47, 1 }, 6591 { "EAckOut", 46, 1 }, 6592 { "EFinOut", 45, 1 }, 6593 { "ERstOut", 44, 1 }, 6594 { "SynIn", 43, 1 }, 6595 { "AckIn", 42, 1 }, 6596 { "FinIn", 41, 1 }, 6597 { "RstIn", 40, 1 }, 6598 { "DataIn", 39, 1 }, 6599 { "DataInVld", 38, 1 }, 6600 { "PadIn", 37, 1 }, 6601 { "RxBufEmpty", 36, 1 }, 6602 { "RxDdp", 35, 1 }, 6603 { "RxFbCongestion", 34, 1 }, 6604 { "TxFbCongestion", 33, 1 }, 6605 { "TxPktSumSrdy", 32, 1 }, 6606 { "RcfUlpType", 28, 4 }, 6607 { "Eread", 27, 1 }, 6608 { "Ebypass", 26, 1 }, 6609 { "Esave", 25, 1 }, 6610 { "Static0", 24, 1 }, 6611 { "Cread", 23, 1 }, 6612 { "Cbypass", 22, 1 }, 6613 { "Csave", 21, 1 }, 6614 { "CPktOut", 20, 1 }, 6615 { "RxPagePoolFull", 18, 2 }, 6616 { "RxLpbkPkt", 17, 1 }, 6617 { "TxLpbkPkt", 16, 1 }, 6618 { "RxVfValid", 15, 1 }, 6619 { "SynLearned", 14, 1 }, 6620 { "SetDelEntry", 13, 1 }, 6621 { "SetInvEntry", 12, 1 }, 6622 { "CpcmdDvld", 11, 1 }, 6623 { "CpcmdSave", 10, 1 }, 6624 { "RxPstructsFull", 8, 2 }, 6625 { "EpcmdDvld", 7, 1 }, 6626 { "EpcmdFlush", 6, 1 }, 6627 { "EpcmdTrimPrefix", 5, 1 }, 6628 { "EpcmdTrimPostfix", 4, 1 }, 6629 { "ERssIp4Pkt", 3, 1 }, 6630 { "ERssIp6Pkt", 2, 1 }, 6631 { "ERssTcpUdpPkt", 1, 1 }, 6632 { "ERssFceFipPkt", 0, 1 }, 6633 { NULL } 6634}; 6635 6636static struct field_desc tp_la2[] = { 6637 { "CplCmdIn", 56, 8 }, 6638 { "MpsVfVld", 55, 1 }, 6639 { "MpsPf", 52, 3 }, 6640 { "MpsVf", 44, 8 }, 6641 { "SynIn", 43, 1 }, 6642 { "AckIn", 42, 1 }, 6643 { "FinIn", 41, 1 }, 6644 { "RstIn", 40, 1 }, 6645 { "DataIn", 39, 1 }, 6646 { "DataInVld", 38, 1 }, 6647 { "PadIn", 37, 1 }, 6648 { "RxBufEmpty", 36, 1 }, 6649 { "RxDdp", 35, 1 }, 6650 { "RxFbCongestion", 34, 1 }, 6651 { "TxFbCongestion", 33, 1 }, 6652 { "TxPktSumSrdy", 32, 1 }, 6653 { "RcfUlpType", 28, 4 }, 6654 { "Eread", 27, 1 }, 6655 { "Ebypass", 26, 1 }, 6656 { "Esave", 25, 1 }, 6657 { "Static0", 24, 1 }, 6658 { "Cread", 23, 1 }, 6659 { "Cbypass", 22, 1 }, 6660 { "Csave", 21, 1 }, 6661 { "CPktOut", 20, 1 }, 6662 { "RxPagePoolFull", 18, 2 }, 6663 { "RxLpbkPkt", 17, 1 }, 6664 { "TxLpbkPkt", 16, 1 }, 6665 { "RxVfValid", 15, 1 }, 6666 { "SynLearned", 14, 1 }, 6667 { "SetDelEntry", 13, 1 }, 6668 { "SetInvEntry", 12, 1 }, 6669 { "CpcmdDvld", 11, 1 }, 6670 { "CpcmdSave", 10, 1 }, 6671 { "RxPstructsFull", 8, 2 }, 6672 { "EpcmdDvld", 7, 1 }, 6673 { "EpcmdFlush", 6, 1 }, 6674 { "EpcmdTrimPrefix", 5, 1 }, 6675 { "EpcmdTrimPostfix", 4, 1 }, 6676 { "ERssIp4Pkt", 3, 1 }, 6677 { "ERssIp6Pkt", 2, 1 }, 6678 { "ERssTcpUdpPkt", 1, 1 }, 6679 { "ERssFceFipPkt", 0, 1 }, 6680 { NULL } 6681}; 6682 6683static void 6684tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6685{ 6686 6687 field_desc_show(sb, *p, tp_la0); 6688} 6689 6690static void 6691tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6692{ 6693 6694 if (idx) 6695 sbuf_printf(sb, "\n"); 6696 field_desc_show(sb, p[0], tp_la0); 6697 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6698 field_desc_show(sb, p[1], tp_la0); 6699} 6700 6701static void 6702tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6703{ 6704 6705 if (idx) 6706 sbuf_printf(sb, "\n"); 6707 field_desc_show(sb, p[0], tp_la0); 6708 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6709 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6710} 6711 6712static int 6713sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6714{ 6715 struct adapter *sc = arg1; 6716 struct sbuf *sb; 6717 uint64_t *buf, *p; 6718 int rc; 6719 u_int i, inc; 6720 void (*show_func)(struct sbuf *, uint64_t *, int); 6721 6722 rc = sysctl_wire_old_buffer(req, 0); 6723 if (rc != 0) 6724 return (rc); 6725 6726 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6727 if (sb == NULL) 6728 return (ENOMEM); 6729 6730 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6731 6732 t4_tp_read_la(sc, buf, NULL); 6733 p = buf; 6734 6735 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6736 case 2: 6737 inc = 2; 6738 show_func = tp_la_show2; 6739 break; 6740 case 3: 6741 inc = 2; 6742 show_func = tp_la_show3; 6743 break; 6744 default: 6745 inc = 1; 6746 show_func = tp_la_show; 6747 } 6748 6749 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6750 (*show_func)(sb, p, i); 6751 6752 rc = sbuf_finish(sb); 6753 sbuf_delete(sb); 6754 free(buf, M_CXGBE); 6755 return (rc); 6756} 6757 6758static int 6759sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6760{ 6761 struct adapter *sc = arg1; 6762 struct sbuf *sb; 6763 int rc; 6764 u64 nrate[NCHAN], orate[NCHAN]; 6765 6766 rc = sysctl_wire_old_buffer(req, 0); 6767 if (rc != 0) 6768 return (rc); 6769 6770 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6771 if (sb == NULL) 6772 return (ENOMEM); 6773 6774 t4_get_chan_txrate(sc, nrate, orate); 6775 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6776 "channel 3\n"); 6777 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6778 nrate[0], nrate[1], nrate[2], nrate[3]); 6779 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6780 orate[0], orate[1], orate[2], orate[3]); 6781 6782 rc = sbuf_finish(sb); 6783 sbuf_delete(sb); 6784 6785 return (rc); 6786} 6787 6788static int 6789sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6790{ 6791 struct adapter *sc = arg1; 6792 struct sbuf *sb; 6793 uint32_t *buf, *p; 6794 int rc, i; 6795 6796 rc = sysctl_wire_old_buffer(req, 0); 6797 if (rc != 0) 6798 return (rc); 6799 6800 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6801 if (sb == NULL) 6802 return (ENOMEM); 6803 6804 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6805 M_ZERO | M_WAITOK); 6806 6807 t4_ulprx_read_la(sc, buf); 6808 p = buf; 6809 6810 sbuf_printf(sb, " Pcmd Type Message" 6811 " Data"); 6812 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6813 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6814 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6815 } 6816 6817 rc = sbuf_finish(sb); 6818 sbuf_delete(sb); 6819 free(buf, M_CXGBE); 6820 return (rc); 6821} 6822 6823static int 6824sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6825{ 6826 struct adapter *sc = arg1; 6827 struct sbuf *sb; 6828 int rc, v; 6829 6830 rc = sysctl_wire_old_buffer(req, 0); 6831 if (rc != 0) 6832 return (rc); 6833 6834 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6835 if (sb == NULL) 6836 return (ENOMEM); 6837 6838 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6839 if (G_STATSOURCE_T5(v) == 7) { 6840 if (G_STATMODE(v) == 0) { 6841 sbuf_printf(sb, "total %d, incomplete %d", 6842 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6843 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6844 } else if (G_STATMODE(v) == 1) { 6845 sbuf_printf(sb, "total %d, data overflow %d", 6846 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6847 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6848 } 6849 } 6850 rc = sbuf_finish(sb); 6851 sbuf_delete(sb); 6852 6853 return (rc); 6854} 6855#endif 6856 6857static inline void 6858txq_start(struct ifnet *ifp, struct sge_txq *txq) 6859{ 6860 struct buf_ring *br; 6861 struct mbuf *m; 6862 6863 TXQ_LOCK_ASSERT_OWNED(txq); 6864 6865 br = txq->br; 6866 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6867 if (m) 6868 t4_eth_tx(ifp, txq, m); 6869} 6870 6871void 6872t4_tx_callout(void *arg) 6873{ 6874 struct sge_eq *eq = arg; 6875 struct adapter *sc; 6876 6877 if (EQ_TRYLOCK(eq) == 0) 6878 goto reschedule; 6879 6880 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6881 EQ_UNLOCK(eq); 6882reschedule: 6883 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6884 callout_schedule(&eq->tx_callout, 1); 6885 return; 6886 } 6887 6888 EQ_LOCK_ASSERT_OWNED(eq); 6889 6890 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6891 6892 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6893 struct sge_txq *txq = arg; 6894 struct port_info *pi = txq->ifp->if_softc; 6895 6896 sc = pi->adapter; 6897 } else { 6898 struct sge_wrq *wrq = arg; 6899 6900 sc = wrq->adapter; 6901 } 6902 6903 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6904 } 6905 6906 EQ_UNLOCK(eq); 6907} 6908 6909void 6910t4_tx_task(void *arg, int count) 6911{ 6912 struct sge_eq *eq = arg; 6913 6914 EQ_LOCK(eq); 6915 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6916 struct sge_txq *txq = arg; 6917 txq_start(txq->ifp, txq); 6918 } else { 6919 struct sge_wrq *wrq = arg; 6920 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6921 } 6922 EQ_UNLOCK(eq); 6923} 6924 6925static uint32_t 6926fconf_to_mode(uint32_t fconf) 6927{ 6928 uint32_t mode; 6929 6930 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6931 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6932 6933 if (fconf & F_FRAGMENTATION) 6934 mode |= T4_FILTER_IP_FRAGMENT; 6935 6936 if (fconf & F_MPSHITTYPE) 6937 mode |= T4_FILTER_MPS_HIT_TYPE; 6938 6939 if (fconf & F_MACMATCH) 6940 mode |= T4_FILTER_MAC_IDX; 6941 6942 if (fconf & F_ETHERTYPE) 6943 mode |= T4_FILTER_ETH_TYPE; 6944 6945 if (fconf & F_PROTOCOL) 6946 mode |= T4_FILTER_IP_PROTO; 6947 6948 if (fconf & F_TOS) 6949 mode |= T4_FILTER_IP_TOS; 6950 6951 if (fconf & F_VLAN) 6952 mode |= T4_FILTER_VLAN; 6953 6954 if (fconf & F_VNIC_ID) 6955 mode |= T4_FILTER_VNIC; 6956 6957 if (fconf & F_PORT) 6958 mode |= T4_FILTER_PORT; 6959 6960 if (fconf & F_FCOE) 6961 mode |= T4_FILTER_FCoE; 6962 6963 return (mode); 6964} 6965 6966static uint32_t 6967mode_to_fconf(uint32_t mode) 6968{ 6969 uint32_t fconf = 0; 6970 6971 if (mode & T4_FILTER_IP_FRAGMENT) 6972 fconf |= F_FRAGMENTATION; 6973 6974 if (mode & T4_FILTER_MPS_HIT_TYPE) 6975 fconf |= F_MPSHITTYPE; 6976 6977 if (mode & T4_FILTER_MAC_IDX) 6978 fconf |= F_MACMATCH; 6979 6980 if (mode & T4_FILTER_ETH_TYPE) 6981 fconf |= F_ETHERTYPE; 6982 6983 if (mode & T4_FILTER_IP_PROTO) 6984 fconf |= F_PROTOCOL; 6985 6986 if (mode & T4_FILTER_IP_TOS) 6987 fconf |= F_TOS; 6988 6989 if (mode & T4_FILTER_VLAN) 6990 fconf |= F_VLAN; 6991 6992 if (mode & T4_FILTER_VNIC) 6993 fconf |= F_VNIC_ID; 6994 6995 if (mode & T4_FILTER_PORT) 6996 fconf |= F_PORT; 6997 6998 if (mode & T4_FILTER_FCoE) 6999 fconf |= F_FCOE; 7000 7001 return (fconf); 7002} 7003 7004static uint32_t 7005fspec_to_fconf(struct t4_filter_specification *fs) 7006{ 7007 uint32_t fconf = 0; 7008 7009 if (fs->val.frag || fs->mask.frag) 7010 fconf |= F_FRAGMENTATION; 7011 7012 if (fs->val.matchtype || fs->mask.matchtype) 7013 fconf |= F_MPSHITTYPE; 7014 7015 if (fs->val.macidx || fs->mask.macidx) 7016 fconf |= F_MACMATCH; 7017 7018 if (fs->val.ethtype || fs->mask.ethtype) 7019 fconf |= F_ETHERTYPE; 7020 7021 if (fs->val.proto || fs->mask.proto) 7022 fconf |= F_PROTOCOL; 7023 7024 if (fs->val.tos || fs->mask.tos) 7025 fconf |= F_TOS; 7026 7027 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7028 fconf |= F_VLAN; 7029 7030 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7031 fconf |= F_VNIC_ID; 7032 7033 if (fs->val.iport || fs->mask.iport) 7034 fconf |= F_PORT; 7035 7036 if (fs->val.fcoe || fs->mask.fcoe) 7037 fconf |= F_FCOE; 7038 7039 return (fconf); 7040} 7041 7042static int 7043get_filter_mode(struct adapter *sc, uint32_t *mode) 7044{ 7045 int rc; 7046 uint32_t fconf; 7047 7048 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7049 "t4getfm"); 7050 if (rc) 7051 return (rc); 7052 7053 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7054 A_TP_VLAN_PRI_MAP); 7055 7056 if (sc->params.tp.vlan_pri_map != fconf) { 7057 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7058 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7059 fconf); 7060 } 7061 7062 *mode = fconf_to_mode(fconf); 7063 7064 end_synchronized_op(sc, LOCK_HELD); 7065 return (0); 7066} 7067 7068static int 7069set_filter_mode(struct adapter *sc, uint32_t mode) 7070{ 7071 uint32_t fconf; 7072 int rc; 7073 7074 fconf = mode_to_fconf(mode); 7075 7076 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7077 "t4setfm"); 7078 if (rc) 7079 return (rc); 7080 7081 if (sc->tids.ftids_in_use > 0) { 7082 rc = EBUSY; 7083 goto done; 7084 } 7085 7086#ifdef TCP_OFFLOAD 7087 if (sc->offload_map) { 7088 rc = EBUSY; 7089 goto done; 7090 } 7091#endif 7092 7093 rc = -t4_set_filter_mode(sc, fconf); 7094done: 7095 end_synchronized_op(sc, LOCK_HELD); 7096 return (rc); 7097} 7098 7099static inline uint64_t 7100get_filter_hits(struct adapter *sc, uint32_t fid) 7101{ 7102 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7103 uint64_t hits; 7104 7105 memwin_info(sc, 0, &mw_base, NULL); 7106 off = position_memwin(sc, 0, 7107 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7108 if (is_t4(sc)) { 7109 hits = t4_read_reg64(sc, mw_base + off + 16); 7110 hits = be64toh(hits); 7111 } else { 7112 hits = t4_read_reg(sc, mw_base + off + 24); 7113 hits = be32toh(hits); 7114 } 7115 7116 return (hits); 7117} 7118 7119static int 7120get_filter(struct adapter *sc, struct t4_filter *t) 7121{ 7122 int i, rc, nfilters = sc->tids.nftids; 7123 struct filter_entry *f; 7124 7125 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7126 "t4getf"); 7127 if (rc) 7128 return (rc); 7129 7130 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7131 t->idx >= nfilters) { 7132 t->idx = 0xffffffff; 7133 goto done; 7134 } 7135 7136 f = &sc->tids.ftid_tab[t->idx]; 7137 for (i = t->idx; i < nfilters; i++, f++) { 7138 if (f->valid) { 7139 t->idx = i; 7140 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7141 t->smtidx = f->smtidx; 7142 if (f->fs.hitcnts) 7143 t->hits = get_filter_hits(sc, t->idx); 7144 else 7145 t->hits = UINT64_MAX; 7146 t->fs = f->fs; 7147 7148 goto done; 7149 } 7150 } 7151 7152 t->idx = 0xffffffff; 7153done: 7154 end_synchronized_op(sc, LOCK_HELD); 7155 return (0); 7156} 7157 7158static int 7159set_filter(struct adapter *sc, struct t4_filter *t) 7160{ 7161 unsigned int nfilters, nports; 7162 struct filter_entry *f; 7163 int i, rc; 7164 7165 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7166 if (rc) 7167 return (rc); 7168 7169 nfilters = sc->tids.nftids; 7170 nports = sc->params.nports; 7171 7172 if (nfilters == 0) { 7173 rc = ENOTSUP; 7174 goto done; 7175 } 7176 7177 if (!(sc->flags & FULL_INIT_DONE)) { 7178 rc = EAGAIN; 7179 goto done; 7180 } 7181 7182 if (t->idx >= nfilters) { 7183 rc = EINVAL; 7184 goto done; 7185 } 7186 7187 /* Validate against the global filter mode */ 7188 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7189 sc->params.tp.vlan_pri_map) { 7190 rc = E2BIG; 7191 goto done; 7192 } 7193 7194 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7195 rc = EINVAL; 7196 goto done; 7197 } 7198 7199 if (t->fs.val.iport >= nports) { 7200 rc = EINVAL; 7201 goto done; 7202 } 7203 7204 /* Can't specify an iq if not steering to it */ 7205 if (!t->fs.dirsteer && t->fs.iq) { 7206 rc = EINVAL; 7207 goto done; 7208 } 7209 7210 /* IPv6 filter idx must be 4 aligned */ 7211 if (t->fs.type == 1 && 7212 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7213 rc = EINVAL; 7214 goto done; 7215 } 7216 7217 if (sc->tids.ftid_tab == NULL) { 7218 KASSERT(sc->tids.ftids_in_use == 0, 7219 ("%s: no memory allocated but filters_in_use > 0", 7220 __func__)); 7221 7222 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7223 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7224 if (sc->tids.ftid_tab == NULL) { 7225 rc = ENOMEM; 7226 goto done; 7227 } 7228 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7229 } 7230 7231 for (i = 0; i < 4; i++) { 7232 f = &sc->tids.ftid_tab[t->idx + i]; 7233 7234 if (f->pending || f->valid) { 7235 rc = EBUSY; 7236 goto done; 7237 } 7238 if (f->locked) { 7239 rc = EPERM; 7240 goto done; 7241 } 7242 7243 if (t->fs.type == 0) 7244 break; 7245 } 7246 7247 f = &sc->tids.ftid_tab[t->idx]; 7248 f->fs = t->fs; 7249 7250 rc = set_filter_wr(sc, t->idx); 7251done: 7252 end_synchronized_op(sc, 0); 7253 7254 if (rc == 0) { 7255 mtx_lock(&sc->tids.ftid_lock); 7256 for (;;) { 7257 if (f->pending == 0) { 7258 rc = f->valid ? 0 : EIO; 7259 break; 7260 } 7261 7262 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7263 PCATCH, "t4setfw", 0)) { 7264 rc = EINPROGRESS; 7265 break; 7266 } 7267 } 7268 mtx_unlock(&sc->tids.ftid_lock); 7269 } 7270 return (rc); 7271} 7272 7273static int 7274del_filter(struct adapter *sc, struct t4_filter *t) 7275{ 7276 unsigned int nfilters; 7277 struct filter_entry *f; 7278 int rc; 7279 7280 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7281 if (rc) 7282 return (rc); 7283 7284 nfilters = sc->tids.nftids; 7285 7286 if (nfilters == 0) { 7287 rc = ENOTSUP; 7288 goto done; 7289 } 7290 7291 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7292 t->idx >= nfilters) { 7293 rc = EINVAL; 7294 goto done; 7295 } 7296 7297 if (!(sc->flags & FULL_INIT_DONE)) { 7298 rc = EAGAIN; 7299 goto done; 7300 } 7301 7302 f = &sc->tids.ftid_tab[t->idx]; 7303 7304 if (f->pending) { 7305 rc = EBUSY; 7306 goto done; 7307 } 7308 if (f->locked) { 7309 rc = EPERM; 7310 goto done; 7311 } 7312 7313 if (f->valid) { 7314 t->fs = f->fs; /* extra info for the caller */ 7315 rc = del_filter_wr(sc, t->idx); 7316 } 7317 7318done: 7319 end_synchronized_op(sc, 0); 7320 7321 if (rc == 0) { 7322 mtx_lock(&sc->tids.ftid_lock); 7323 for (;;) { 7324 if (f->pending == 0) { 7325 rc = f->valid ? EIO : 0; 7326 break; 7327 } 7328 7329 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7330 PCATCH, "t4delfw", 0)) { 7331 rc = EINPROGRESS; 7332 break; 7333 } 7334 } 7335 mtx_unlock(&sc->tids.ftid_lock); 7336 } 7337 7338 return (rc); 7339} 7340 7341static void 7342clear_filter(struct filter_entry *f) 7343{ 7344 if (f->l2t) 7345 t4_l2t_release(f->l2t); 7346 7347 bzero(f, sizeof (*f)); 7348} 7349 7350static int 7351set_filter_wr(struct adapter *sc, int fidx) 7352{ 7353 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7354 struct wrqe *wr; 7355 struct fw_filter_wr *fwr; 7356 unsigned int ftid; 7357 7358 ASSERT_SYNCHRONIZED_OP(sc); 7359 7360 if (f->fs.newdmac || f->fs.newvlan) { 7361 /* This filter needs an L2T entry; allocate one. */ 7362 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7363 if (f->l2t == NULL) 7364 return (EAGAIN); 7365 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7366 f->fs.dmac)) { 7367 t4_l2t_release(f->l2t); 7368 f->l2t = NULL; 7369 return (ENOMEM); 7370 } 7371 } 7372 7373 ftid = sc->tids.ftid_base + fidx; 7374 7375 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7376 if (wr == NULL) 7377 return (ENOMEM); 7378 7379 fwr = wrtod(wr); 7380 bzero(fwr, sizeof (*fwr)); 7381 7382 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7383 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7384 fwr->tid_to_iq = 7385 htobe32(V_FW_FILTER_WR_TID(ftid) | 7386 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7387 V_FW_FILTER_WR_NOREPLY(0) | 7388 V_FW_FILTER_WR_IQ(f->fs.iq)); 7389 fwr->del_filter_to_l2tix = 7390 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7391 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7392 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7393 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7394 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7395 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7396 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7397 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7398 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7399 f->fs.newvlan == VLAN_REWRITE) | 7400 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7401 f->fs.newvlan == VLAN_REWRITE) | 7402 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7403 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7404 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7405 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7406 fwr->ethtype = htobe16(f->fs.val.ethtype); 7407 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7408 fwr->frag_to_ovlan_vldm = 7409 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7410 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7411 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7412 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7413 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7414 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7415 fwr->smac_sel = 0; 7416 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7417 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7418 fwr->maci_to_matchtypem = 7419 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7420 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7421 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7422 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7423 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7424 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7425 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7426 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7427 fwr->ptcl = f->fs.val.proto; 7428 fwr->ptclm = f->fs.mask.proto; 7429 fwr->ttyp = f->fs.val.tos; 7430 fwr->ttypm = f->fs.mask.tos; 7431 fwr->ivlan = htobe16(f->fs.val.vlan); 7432 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7433 fwr->ovlan = htobe16(f->fs.val.vnic); 7434 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7435 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7436 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7437 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7438 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7439 fwr->lp = htobe16(f->fs.val.dport); 7440 fwr->lpm = htobe16(f->fs.mask.dport); 7441 fwr->fp = htobe16(f->fs.val.sport); 7442 fwr->fpm = htobe16(f->fs.mask.sport); 7443 if (f->fs.newsmac) 7444 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7445 7446 f->pending = 1; 7447 sc->tids.ftids_in_use++; 7448 7449 t4_wrq_tx(sc, wr); 7450 return (0); 7451} 7452 7453static int 7454del_filter_wr(struct adapter *sc, int fidx) 7455{ 7456 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7457 struct wrqe *wr; 7458 struct fw_filter_wr *fwr; 7459 unsigned int ftid; 7460 7461 ftid = sc->tids.ftid_base + fidx; 7462 7463 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7464 if (wr == NULL) 7465 return (ENOMEM); 7466 fwr = wrtod(wr); 7467 bzero(fwr, sizeof (*fwr)); 7468 7469 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7470 7471 f->pending = 1; 7472 t4_wrq_tx(sc, wr); 7473 return (0); 7474} 7475 7476int 7477t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7478{ 7479 struct adapter *sc = iq->adapter; 7480 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7481 unsigned int idx = GET_TID(rpl); 7482 unsigned int rc; 7483 struct filter_entry *f; 7484 7485 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7486 rss->opcode)); 7487 7488 if (is_ftid(sc, idx)) { 7489 7490 idx -= sc->tids.ftid_base; 7491 f = &sc->tids.ftid_tab[idx]; 7492 rc = G_COOKIE(rpl->cookie); 7493 7494 mtx_lock(&sc->tids.ftid_lock); 7495 if (rc == FW_FILTER_WR_FLT_ADDED) { 7496 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7497 __func__, idx)); 7498 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7499 f->pending = 0; /* asynchronous setup completed */ 7500 f->valid = 1; 7501 } else { 7502 if (rc != FW_FILTER_WR_FLT_DELETED) { 7503 /* Add or delete failed, display an error */ 7504 log(LOG_ERR, 7505 "filter %u setup failed with error %u\n", 7506 idx, rc); 7507 } 7508 7509 clear_filter(f); 7510 sc->tids.ftids_in_use--; 7511 } 7512 wakeup(&sc->tids.ftid_tab); 7513 mtx_unlock(&sc->tids.ftid_lock); 7514 } 7515 7516 return (0); 7517} 7518 7519static int 7520get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7521{ 7522 int rc; 7523 7524 if (cntxt->cid > M_CTXTQID) 7525 return (EINVAL); 7526 7527 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7528 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7529 return (EINVAL); 7530 7531 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7532 if (rc) 7533 return (rc); 7534 7535 if (sc->flags & FW_OK) { 7536 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7537 &cntxt->data[0]); 7538 if (rc == 0) 7539 goto done; 7540 } 7541 7542 /* 7543 * Read via firmware failed or wasn't even attempted. Read directly via 7544 * the backdoor. 7545 */ 7546 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7547done: 7548 end_synchronized_op(sc, 0); 7549 return (rc); 7550} 7551 7552static int 7553load_fw(struct adapter *sc, struct t4_data *fw) 7554{ 7555 int rc; 7556 uint8_t *fw_data; 7557 7558 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7559 if (rc) 7560 return (rc); 7561 7562 if (sc->flags & FULL_INIT_DONE) { 7563 rc = EBUSY; 7564 goto done; 7565 } 7566 7567 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7568 if (fw_data == NULL) { 7569 rc = ENOMEM; 7570 goto done; 7571 } 7572 7573 rc = copyin(fw->data, fw_data, fw->len); 7574 if (rc == 0) 7575 rc = -t4_load_fw(sc, fw_data, fw->len); 7576 7577 free(fw_data, M_CXGBE); 7578done: 7579 end_synchronized_op(sc, 0); 7580 return (rc); 7581} 7582 7583static int 7584read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7585{ 7586 uint32_t addr, off, remaining, i, n; 7587 uint32_t *buf, *b; 7588 uint32_t mw_base, mw_aperture; 7589 int rc; 7590 uint8_t *dst; 7591 7592 rc = validate_mem_range(sc, mr->addr, mr->len); 7593 if (rc != 0) 7594 return (rc); 7595 7596 memwin_info(sc, win, &mw_base, &mw_aperture); 7597 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7598 addr = mr->addr; 7599 remaining = mr->len; 7600 dst = (void *)mr->data; 7601 7602 while (remaining) { 7603 off = position_memwin(sc, win, addr); 7604 7605 /* number of bytes that we'll copy in the inner loop */ 7606 n = min(remaining, mw_aperture - off); 7607 for (i = 0; i < n; i += 4) 7608 *b++ = t4_read_reg(sc, mw_base + off + i); 7609 7610 rc = copyout(buf, dst, n); 7611 if (rc != 0) 7612 break; 7613 7614 b = buf; 7615 dst += n; 7616 remaining -= n; 7617 addr += n; 7618 } 7619 7620 free(buf, M_CXGBE); 7621 return (rc); 7622} 7623 7624static int 7625read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7626{ 7627 int rc; 7628 7629 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7630 return (EINVAL); 7631 7632 if (i2cd->len > sizeof(i2cd->data)) 7633 return (EFBIG); 7634 7635 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7636 if (rc) 7637 return (rc); 7638 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7639 i2cd->offset, i2cd->len, &i2cd->data[0]); 7640 end_synchronized_op(sc, 0); 7641 7642 return (rc); 7643} 7644 7645static int 7646in_range(int val, int lo, int hi) 7647{ 7648 7649 return (val < 0 || (val <= hi && val >= lo)); 7650} 7651 7652static int 7653set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7654{ 7655 int fw_subcmd, fw_type, rc; 7656 7657 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7658 if (rc) 7659 return (rc); 7660 7661 if (!(sc->flags & FULL_INIT_DONE)) { 7662 rc = EAGAIN; 7663 goto done; 7664 } 7665 7666 /* 7667 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7668 * sub-command and type are in common locations.) 7669 */ 7670 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7671 fw_subcmd = FW_SCHED_SC_CONFIG; 7672 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7673 fw_subcmd = FW_SCHED_SC_PARAMS; 7674 else { 7675 rc = EINVAL; 7676 goto done; 7677 } 7678 if (p->type == SCHED_CLASS_TYPE_PACKET) 7679 fw_type = FW_SCHED_TYPE_PKTSCHED; 7680 else { 7681 rc = EINVAL; 7682 goto done; 7683 } 7684 7685 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7686 /* Vet our parameters ..*/ 7687 if (p->u.config.minmax < 0) { 7688 rc = EINVAL; 7689 goto done; 7690 } 7691 7692 /* And pass the request to the firmware ...*/ 7693 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7694 goto done; 7695 } 7696 7697 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7698 int fw_level; 7699 int fw_mode; 7700 int fw_rateunit; 7701 int fw_ratemode; 7702 7703 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7704 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7705 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7706 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7707 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7708 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7709 else { 7710 rc = EINVAL; 7711 goto done; 7712 } 7713 7714 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7715 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7716 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7717 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7718 else { 7719 rc = EINVAL; 7720 goto done; 7721 } 7722 7723 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7724 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7725 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7726 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7727 else { 7728 rc = EINVAL; 7729 goto done; 7730 } 7731 7732 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7733 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7734 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7735 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7736 else { 7737 rc = EINVAL; 7738 goto done; 7739 } 7740 7741 /* Vet our parameters ... */ 7742 if (!in_range(p->u.params.channel, 0, 3) || 7743 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7744 !in_range(p->u.params.minrate, 0, 10000000) || 7745 !in_range(p->u.params.maxrate, 0, 10000000) || 7746 !in_range(p->u.params.weight, 0, 100)) { 7747 rc = ERANGE; 7748 goto done; 7749 } 7750 7751 /* 7752 * Translate any unset parameters into the firmware's 7753 * nomenclature and/or fail the call if the parameters 7754 * are required ... 7755 */ 7756 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7757 p->u.params.channel < 0 || p->u.params.cl < 0) { 7758 rc = EINVAL; 7759 goto done; 7760 } 7761 if (p->u.params.minrate < 0) 7762 p->u.params.minrate = 0; 7763 if (p->u.params.maxrate < 0) { 7764 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7765 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7766 rc = EINVAL; 7767 goto done; 7768 } else 7769 p->u.params.maxrate = 0; 7770 } 7771 if (p->u.params.weight < 0) { 7772 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7773 rc = EINVAL; 7774 goto done; 7775 } else 7776 p->u.params.weight = 0; 7777 } 7778 if (p->u.params.pktsize < 0) { 7779 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7780 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7781 rc = EINVAL; 7782 goto done; 7783 } else 7784 p->u.params.pktsize = 0; 7785 } 7786 7787 /* See what the firmware thinks of the request ... */ 7788 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7789 fw_rateunit, fw_ratemode, p->u.params.channel, 7790 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7791 p->u.params.weight, p->u.params.pktsize, 1); 7792 goto done; 7793 } 7794 7795 rc = EINVAL; 7796done: 7797 end_synchronized_op(sc, 0); 7798 return (rc); 7799} 7800 7801static int 7802set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7803{ 7804 struct port_info *pi = NULL; 7805 struct sge_txq *txq; 7806 uint32_t fw_mnem, fw_queue, fw_class; 7807 int i, rc; 7808 7809 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7810 if (rc) 7811 return (rc); 7812 7813 if (!(sc->flags & FULL_INIT_DONE)) { 7814 rc = EAGAIN; 7815 goto done; 7816 } 7817 7818 if (p->port >= sc->params.nports) { 7819 rc = EINVAL; 7820 goto done; 7821 } 7822 7823 pi = sc->port[p->port]; 7824 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7825 rc = EINVAL; 7826 goto done; 7827 } 7828 7829 /* 7830 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7831 * Scheduling Class in this case). 7832 */ 7833 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7834 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7835 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7836 7837 /* 7838 * If op.queue is non-negative, then we're only changing the scheduling 7839 * on a single specified TX queue. 7840 */ 7841 if (p->queue >= 0) { 7842 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7843 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7844 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7845 &fw_class); 7846 goto done; 7847 } 7848 7849 /* 7850 * Change the scheduling on all the TX queues for the 7851 * interface. 7852 */ 7853 for_each_txq(pi, i, txq) { 7854 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7855 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7856 &fw_class); 7857 if (rc) 7858 goto done; 7859 } 7860 7861 rc = 0; 7862done: 7863 end_synchronized_op(sc, 0); 7864 return (rc); 7865} 7866 7867int 7868t4_os_find_pci_capability(struct adapter *sc, int cap) 7869{ 7870 int i; 7871 7872 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7873} 7874 7875int 7876t4_os_pci_save_state(struct adapter *sc) 7877{ 7878 device_t dev; 7879 struct pci_devinfo *dinfo; 7880 7881 dev = sc->dev; 7882 dinfo = device_get_ivars(dev); 7883 7884 pci_cfg_save(dev, dinfo, 0); 7885 return (0); 7886} 7887 7888int 7889t4_os_pci_restore_state(struct adapter *sc) 7890{ 7891 device_t dev; 7892 struct pci_devinfo *dinfo; 7893 7894 dev = sc->dev; 7895 dinfo = device_get_ivars(dev); 7896 7897 pci_cfg_restore(dev, dinfo); 7898 return (0); 7899} 7900 7901void 7902t4_os_portmod_changed(const struct adapter *sc, int idx) 7903{ 7904 struct port_info *pi = sc->port[idx]; 7905 static const char *mod_str[] = { 7906 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7907 }; 7908 7909 build_medialist(pi, &pi->media); 7910#ifdef DEV_NETMAP 7911 build_medialist(pi, &pi->nm_media); 7912#endif 7913 7914 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7915 if_printf(pi->ifp, "transceiver unplugged.\n"); 7916 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7917 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7918 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7919 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7920 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7921 if_printf(pi->ifp, "%s transceiver inserted.\n", 7922 mod_str[pi->mod_type]); 7923 } else { 7924 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7925 pi->mod_type); 7926 } 7927} 7928 7929void 7930t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7931{ 7932 struct port_info *pi = sc->port[idx]; 7933 struct ifnet *ifp = pi->ifp; 7934 7935 if (link_stat) { 7936 pi->linkdnrc = -1; 7937 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7938 if_link_state_change(ifp, LINK_STATE_UP); 7939 } else { 7940 if (reason >= 0) 7941 pi->linkdnrc = reason; 7942 if_link_state_change(ifp, LINK_STATE_DOWN); 7943 } 7944} 7945 7946void 7947t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7948{ 7949 struct adapter *sc; 7950 7951 sx_slock(&t4_list_lock); 7952 SLIST_FOREACH(sc, &t4_list, link) { 7953 /* 7954 * func should not make any assumptions about what state sc is 7955 * in - the only guarantee is that sc->sc_lock is a valid lock. 7956 */ 7957 func(sc, arg); 7958 } 7959 sx_sunlock(&t4_list_lock); 7960} 7961 7962static int 7963t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7964{ 7965 return (0); 7966} 7967 7968static int 7969t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7970{ 7971 return (0); 7972} 7973 7974static int 7975t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7976 struct thread *td) 7977{ 7978 int rc; 7979 struct adapter *sc = dev->si_drv1; 7980 7981 rc = priv_check(td, PRIV_DRIVER); 7982 if (rc != 0) 7983 return (rc); 7984 7985 switch (cmd) { 7986 case CHELSIO_T4_GETREG: { 7987 struct t4_reg *edata = (struct t4_reg *)data; 7988 7989 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7990 return (EFAULT); 7991 7992 if (edata->size == 4) 7993 edata->val = t4_read_reg(sc, edata->addr); 7994 else if (edata->size == 8) 7995 edata->val = t4_read_reg64(sc, edata->addr); 7996 else 7997 return (EINVAL); 7998 7999 break; 8000 } 8001 case CHELSIO_T4_SETREG: { 8002 struct t4_reg *edata = (struct t4_reg *)data; 8003 8004 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8005 return (EFAULT); 8006 8007 if (edata->size == 4) { 8008 if (edata->val & 0xffffffff00000000) 8009 return (EINVAL); 8010 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8011 } else if (edata->size == 8) 8012 t4_write_reg64(sc, edata->addr, edata->val); 8013 else 8014 return (EINVAL); 8015 break; 8016 } 8017 case CHELSIO_T4_REGDUMP: { 8018 struct t4_regdump *regs = (struct t4_regdump *)data; 8019 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8020 uint8_t *buf; 8021 8022 if (regs->len < reglen) { 8023 regs->len = reglen; /* hint to the caller */ 8024 return (ENOBUFS); 8025 } 8026 8027 regs->len = reglen; 8028 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8029 t4_get_regs(sc, regs, buf); 8030 rc = copyout(buf, regs->data, reglen); 8031 free(buf, M_CXGBE); 8032 break; 8033 } 8034 case CHELSIO_T4_GET_FILTER_MODE: 8035 rc = get_filter_mode(sc, (uint32_t *)data); 8036 break; 8037 case CHELSIO_T4_SET_FILTER_MODE: 8038 rc = set_filter_mode(sc, *(uint32_t *)data); 8039 break; 8040 case CHELSIO_T4_GET_FILTER: 8041 rc = get_filter(sc, (struct t4_filter *)data); 8042 break; 8043 case CHELSIO_T4_SET_FILTER: 8044 rc = set_filter(sc, (struct t4_filter *)data); 8045 break; 8046 case CHELSIO_T4_DEL_FILTER: 8047 rc = del_filter(sc, (struct t4_filter *)data); 8048 break; 8049 case CHELSIO_T4_GET_SGE_CONTEXT: 8050 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8051 break; 8052 case CHELSIO_T4_LOAD_FW: 8053 rc = load_fw(sc, (struct t4_data *)data); 8054 break; 8055 case CHELSIO_T4_GET_MEM: 8056 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8057 break; 8058 case CHELSIO_T4_GET_I2C: 8059 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8060 break; 8061 case CHELSIO_T4_CLEAR_STATS: { 8062 int i; 8063 u_int port_id = *(uint32_t *)data; 8064 struct port_info *pi; 8065 8066 if (port_id >= sc->params.nports) 8067 return (EINVAL); 8068 pi = sc->port[port_id]; 8069 8070 /* MAC stats */ 8071 t4_clr_port_stats(sc, pi->tx_chan); 8072 8073 if (pi->flags & PORT_INIT_DONE) { 8074 struct sge_rxq *rxq; 8075 struct sge_txq *txq; 8076 struct sge_wrq *wrq; 8077 8078 for_each_rxq(pi, i, rxq) { 8079#if defined(INET) || defined(INET6) 8080 rxq->lro.lro_queued = 0; 8081 rxq->lro.lro_flushed = 0; 8082#endif 8083 rxq->rxcsum = 0; 8084 rxq->vlan_extraction = 0; 8085 } 8086 8087 for_each_txq(pi, i, txq) { 8088 txq->txcsum = 0; 8089 txq->tso_wrs = 0; 8090 txq->vlan_insertion = 0; 8091 txq->imm_wrs = 0; 8092 txq->sgl_wrs = 0; 8093 txq->txpkt_wrs = 0; 8094 txq->txpkts_wrs = 0; 8095 txq->txpkts_pkts = 0; 8096 txq->br->br_drops = 0; 8097 txq->no_dmamap = 0; 8098 txq->no_desc = 0; 8099 } 8100 8101#ifdef TCP_OFFLOAD 8102 /* nothing to clear for each ofld_rxq */ 8103 8104 for_each_ofld_txq(pi, i, wrq) { 8105 wrq->tx_wrs = 0; 8106 wrq->no_desc = 0; 8107 } 8108#endif 8109 wrq = &sc->sge.ctrlq[pi->port_id]; 8110 wrq->tx_wrs = 0; 8111 wrq->no_desc = 0; 8112 } 8113 break; 8114 } 8115 case CHELSIO_T4_SCHED_CLASS: 8116 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8117 break; 8118 case CHELSIO_T4_SCHED_QUEUE: 8119 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8120 break; 8121 case CHELSIO_T4_GET_TRACER: 8122 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8123 break; 8124 case CHELSIO_T4_SET_TRACER: 8125 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8126 break; 8127 default: 8128 rc = EINVAL; 8129 } 8130 8131 return (rc); 8132} 8133 8134#ifdef TCP_OFFLOAD 8135void 8136t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8137 const unsigned int *pgsz_order) 8138{ 8139 struct port_info *pi = ifp->if_softc; 8140 struct adapter *sc = pi->adapter; 8141 8142 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8143 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8144 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8145 V_HPZ3(pgsz_order[3])); 8146} 8147 8148static int 8149toe_capability(struct port_info *pi, int enable) 8150{ 8151 int rc; 8152 struct adapter *sc = pi->adapter; 8153 8154 ASSERT_SYNCHRONIZED_OP(sc); 8155 8156 if (!is_offload(sc)) 8157 return (ENODEV); 8158 8159 if (enable) { 8160 /* 8161 * We need the port's queues around so that we're able to send 8162 * and receive CPLs to/from the TOE even if the ifnet for this 8163 * port has never been UP'd administratively. 8164 */ 8165 if (!(pi->flags & PORT_INIT_DONE)) { 8166 rc = cxgbe_init_synchronized(pi); 8167 if (rc) 8168 return (rc); 8169 } 8170 8171 if (isset(&sc->offload_map, pi->port_id)) 8172 return (0); 8173 8174 if (!(sc->flags & TOM_INIT_DONE)) { 8175 rc = t4_activate_uld(sc, ULD_TOM); 8176 if (rc == EAGAIN) { 8177 log(LOG_WARNING, 8178 "You must kldload t4_tom.ko before trying " 8179 "to enable TOE on a cxgbe interface.\n"); 8180 } 8181 if (rc != 0) 8182 return (rc); 8183 KASSERT(sc->tom_softc != NULL, 8184 ("%s: TOM activated but softc NULL", __func__)); 8185 KASSERT(sc->flags & TOM_INIT_DONE, 8186 ("%s: TOM activated but flag not set", __func__)); 8187 } 8188 8189 setbit(&sc->offload_map, pi->port_id); 8190 } else { 8191 if (!isset(&sc->offload_map, pi->port_id)) 8192 return (0); 8193 8194 KASSERT(sc->flags & TOM_INIT_DONE, 8195 ("%s: TOM never initialized?", __func__)); 8196 clrbit(&sc->offload_map, pi->port_id); 8197 } 8198 8199 return (0); 8200} 8201 8202/* 8203 * Add an upper layer driver to the global list. 8204 */ 8205int 8206t4_register_uld(struct uld_info *ui) 8207{ 8208 int rc = 0; 8209 struct uld_info *u; 8210 8211 sx_xlock(&t4_uld_list_lock); 8212 SLIST_FOREACH(u, &t4_uld_list, link) { 8213 if (u->uld_id == ui->uld_id) { 8214 rc = EEXIST; 8215 goto done; 8216 } 8217 } 8218 8219 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8220 ui->refcount = 0; 8221done: 8222 sx_xunlock(&t4_uld_list_lock); 8223 return (rc); 8224} 8225 8226int 8227t4_unregister_uld(struct uld_info *ui) 8228{ 8229 int rc = EINVAL; 8230 struct uld_info *u; 8231 8232 sx_xlock(&t4_uld_list_lock); 8233 8234 SLIST_FOREACH(u, &t4_uld_list, link) { 8235 if (u == ui) { 8236 if (ui->refcount > 0) { 8237 rc = EBUSY; 8238 goto done; 8239 } 8240 8241 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8242 rc = 0; 8243 goto done; 8244 } 8245 } 8246done: 8247 sx_xunlock(&t4_uld_list_lock); 8248 return (rc); 8249} 8250 8251int 8252t4_activate_uld(struct adapter *sc, int id) 8253{ 8254 int rc = EAGAIN; 8255 struct uld_info *ui; 8256 8257 ASSERT_SYNCHRONIZED_OP(sc); 8258 8259 sx_slock(&t4_uld_list_lock); 8260 8261 SLIST_FOREACH(ui, &t4_uld_list, link) { 8262 if (ui->uld_id == id) { 8263 rc = ui->activate(sc); 8264 if (rc == 0) 8265 ui->refcount++; 8266 goto done; 8267 } 8268 } 8269done: 8270 sx_sunlock(&t4_uld_list_lock); 8271 8272 return (rc); 8273} 8274 8275int 8276t4_deactivate_uld(struct adapter *sc, int id) 8277{ 8278 int rc = EINVAL; 8279 struct uld_info *ui; 8280 8281 ASSERT_SYNCHRONIZED_OP(sc); 8282 8283 sx_slock(&t4_uld_list_lock); 8284 8285 SLIST_FOREACH(ui, &t4_uld_list, link) { 8286 if (ui->uld_id == id) { 8287 rc = ui->deactivate(sc); 8288 if (rc == 0) 8289 ui->refcount--; 8290 goto done; 8291 } 8292 } 8293done: 8294 sx_sunlock(&t4_uld_list_lock); 8295 8296 return (rc); 8297} 8298#endif 8299 8300/* 8301 * Come up with reasonable defaults for some of the tunables, provided they're 8302 * not set by the user (in which case we'll use the values as is). 8303 */ 8304static void 8305tweak_tunables(void) 8306{ 8307 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8308 8309 if (t4_ntxq10g < 1) 8310 t4_ntxq10g = min(nc, NTXQ_10G); 8311 8312 if (t4_ntxq1g < 1) 8313 t4_ntxq1g = min(nc, NTXQ_1G); 8314 8315 if (t4_nrxq10g < 1) 8316 t4_nrxq10g = min(nc, NRXQ_10G); 8317 8318 if (t4_nrxq1g < 1) 8319 t4_nrxq1g = min(nc, NRXQ_1G); 8320 8321#ifdef TCP_OFFLOAD 8322 if (t4_nofldtxq10g < 1) 8323 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8324 8325 if (t4_nofldtxq1g < 1) 8326 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8327 8328 if (t4_nofldrxq10g < 1) 8329 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8330 8331 if (t4_nofldrxq1g < 1) 8332 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8333 8334 if (t4_toecaps_allowed == -1) 8335 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8336#else 8337 if (t4_toecaps_allowed == -1) 8338 t4_toecaps_allowed = 0; 8339#endif 8340 8341#ifdef DEV_NETMAP 8342 if (t4_nnmtxq10g < 1) 8343 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8344 8345 if (t4_nnmtxq1g < 1) 8346 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8347 8348 if (t4_nnmrxq10g < 1) 8349 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8350 8351 if (t4_nnmrxq1g < 1) 8352 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8353#endif 8354 8355 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8356 t4_tmr_idx_10g = TMR_IDX_10G; 8357 8358 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8359 t4_pktc_idx_10g = PKTC_IDX_10G; 8360 8361 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8362 t4_tmr_idx_1g = TMR_IDX_1G; 8363 8364 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8365 t4_pktc_idx_1g = PKTC_IDX_1G; 8366 8367 if (t4_qsize_txq < 128) 8368 t4_qsize_txq = 128; 8369 8370 if (t4_qsize_rxq < 128) 8371 t4_qsize_rxq = 128; 8372 while (t4_qsize_rxq & 7) 8373 t4_qsize_rxq++; 8374 8375 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8376} 8377 8378static struct sx mlu; /* mod load unload */ 8379SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8380 8381static int 8382mod_event(module_t mod, int cmd, void *arg) 8383{ 8384 int rc = 0; 8385 static int loaded = 0; 8386 8387 switch (cmd) { 8388 case MOD_LOAD: 8389 sx_xlock(&mlu); 8390 if (loaded++ == 0) { 8391 t4_sge_modload(); 8392 sx_init(&t4_list_lock, "T4/T5 adapters"); 8393 SLIST_INIT(&t4_list); 8394#ifdef TCP_OFFLOAD 8395 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8396 SLIST_INIT(&t4_uld_list); 8397#endif 8398 t4_tracer_modload(); 8399 tweak_tunables(); 8400 } 8401 sx_xunlock(&mlu); 8402 break; 8403 8404 case MOD_UNLOAD: 8405 sx_xlock(&mlu); 8406 if (--loaded == 0) { 8407 int tries; 8408 8409 sx_slock(&t4_list_lock); 8410 if (!SLIST_EMPTY(&t4_list)) { 8411 rc = EBUSY; 8412 sx_sunlock(&t4_list_lock); 8413 goto done_unload; 8414 } 8415#ifdef TCP_OFFLOAD 8416 sx_slock(&t4_uld_list_lock); 8417 if (!SLIST_EMPTY(&t4_uld_list)) { 8418 rc = EBUSY; 8419 sx_sunlock(&t4_uld_list_lock); 8420 sx_sunlock(&t4_list_lock); 8421 goto done_unload; 8422 } 8423#endif 8424 tries = 0; 8425 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8426 uprintf("%ju clusters with custom free routine " 8427 "still is use.\n", t4_sge_extfree_refs()); 8428 pause("t4unload", 2 * hz); 8429 } 8430#ifdef TCP_OFFLOAD 8431 sx_sunlock(&t4_uld_list_lock); 8432#endif 8433 sx_sunlock(&t4_list_lock); 8434 8435 if (t4_sge_extfree_refs() == 0) { 8436 t4_tracer_modunload(); 8437#ifdef TCP_OFFLOAD 8438 sx_destroy(&t4_uld_list_lock); 8439#endif 8440 sx_destroy(&t4_list_lock); 8441 t4_sge_modunload(); 8442 loaded = 0; 8443 } else { 8444 rc = EBUSY; 8445 loaded++; /* undo earlier decrement */ 8446 } 8447 } 8448done_unload: 8449 sx_xunlock(&mlu); 8450 break; 8451 } 8452 8453 return (rc); 8454} 8455 8456static devclass_t t4_devclass, t5_devclass; 8457static devclass_t cxgbe_devclass, cxl_devclass; 8458 8459DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8460MODULE_VERSION(t4nex, 1); 8461MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8462 8463DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8464MODULE_VERSION(t5nex, 1); 8465MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8466 8467DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8468MODULE_VERSION(cxgbe, 1); 8469 8470DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8471MODULE_VERSION(cxl, 1); 8472