t4_main.c revision 284052
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 284052 2015-06-06 09:28:40Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/systm.h> 40#include <sys/counter.h> 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> 49#include <sys/firmware.h> 50#include <sys/sbuf.h> 51#include <sys/smp.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/sysctl.h> 55#include <net/ethernet.h> 56#include <net/if.h> 57#include <net/if_types.h> 58#include <net/if_dl.h> 59#include <net/if_vlan_var.h> 60#if defined(__i386__) || defined(__amd64__) 61#include <vm/vm.h> 62#include <vm/pmap.h> 63#endif 64 65#include "common/common.h" 66#include "common/t4_msg.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "t4_ioctl.h" 70#include "t4_l2t.h" 71#include "t4_mp_ring.h" 72 73/* T4 bus driver interface */ 74static int t4_probe(device_t); 75static int t4_attach(device_t); 76static int t4_detach(device_t); 77static device_method_t t4_methods[] = { 78 DEVMETHOD(device_probe, t4_probe), 79 DEVMETHOD(device_attach, t4_attach), 80 DEVMETHOD(device_detach, t4_detach), 81 82 DEVMETHOD_END 83}; 84static driver_t t4_driver = { 85 "t4nex", 86 t4_methods, 87 sizeof(struct adapter) 88}; 89 90 91/* T4 port (cxgbe) interface */ 92static int cxgbe_probe(device_t); 93static int cxgbe_attach(device_t); 94static int cxgbe_detach(device_t); 95static device_method_t cxgbe_methods[] = { 96 DEVMETHOD(device_probe, cxgbe_probe), 97 DEVMETHOD(device_attach, cxgbe_attach), 98 DEVMETHOD(device_detach, cxgbe_detach), 99 { 0, 0 } 100}; 101static driver_t cxgbe_driver = { 102 "cxgbe", 103 cxgbe_methods, 104 sizeof(struct port_info) 105}; 106 107static d_ioctl_t t4_ioctl; 108static d_open_t t4_open; 109static d_close_t t4_close; 110 111static struct cdevsw t4_cdevsw = { 112 .d_version = D_VERSION, 113 .d_flags = 0, 114 .d_open = t4_open, 115 .d_close = t4_close, 116 .d_ioctl = t4_ioctl, 117 .d_name = "t4nex", 118}; 119 120/* T5 bus driver interface */ 121static int t5_probe(device_t); 122static device_method_t t5_methods[] = { 123 DEVMETHOD(device_probe, t5_probe), 124 DEVMETHOD(device_attach, t4_attach), 125 DEVMETHOD(device_detach, t4_detach), 126 127 DEVMETHOD_END 128}; 129static driver_t t5_driver = { 130 "t5nex", 131 t5_methods, 132 sizeof(struct adapter) 133}; 134 135 136/* T5 port (cxl) interface */ 137static driver_t cxl_driver = { 138 "cxl", 139 cxgbe_methods, 140 sizeof(struct port_info) 141}; 142 143static struct cdevsw t5_cdevsw = { 144 .d_version = D_VERSION, 145 .d_flags = 0, 146 .d_open = t4_open, 147 .d_close = t4_close, 148 .d_ioctl = t4_ioctl, 149 .d_name = "t5nex", 150}; 151 152/* ifnet + media interface */ 153static void cxgbe_init(void *); 154static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 155static int cxgbe_transmit(struct ifnet *, struct mbuf *); 156static void cxgbe_qflush(struct ifnet *); 157static int cxgbe_media_change(struct ifnet *); 158static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 159 160MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 161 162/* 163 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 164 * then ADAPTER_LOCK, then t4_uld_list_lock. 165 */ 166static struct sx t4_list_lock; 167SLIST_HEAD(, adapter) t4_list; 168#ifdef TCP_OFFLOAD 169static struct sx t4_uld_list_lock; 170SLIST_HEAD(, uld_info) t4_uld_list; 171#endif 172 173/* 174 * Tunables. See tweak_tunables() too. 175 * 176 * Each tunable is set to a default value here if it's known at compile-time. 177 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 178 * provide a reasonable default when the driver is loaded. 179 * 180 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 181 * T5 are under hw.cxl. 182 */ 183 184/* 185 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 186 */ 187#define NTXQ_10G 16 188static int t4_ntxq10g = -1; 189TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 190 191#define NRXQ_10G 8 192static int t4_nrxq10g = -1; 193TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 194 195#define NTXQ_1G 4 196static int t4_ntxq1g = -1; 197TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 198 199#define NRXQ_1G 2 200static int t4_nrxq1g = -1; 201TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 202 203static int t4_rsrv_noflowq = 0; 204TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 205 206#ifdef TCP_OFFLOAD 207#define NOFLDTXQ_10G 8 208static int t4_nofldtxq10g = -1; 209TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 210 211#define NOFLDRXQ_10G 2 212static int t4_nofldrxq10g = -1; 213TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 214 215#define NOFLDTXQ_1G 2 216static int t4_nofldtxq1g = -1; 217TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 218 219#define NOFLDRXQ_1G 1 220static int t4_nofldrxq1g = -1; 221TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 222#endif 223 224#ifdef DEV_NETMAP 225#define NNMTXQ_10G 2 226static int t4_nnmtxq10g = -1; 227TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 228 229#define NNMRXQ_10G 2 230static int t4_nnmrxq10g = -1; 231TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 232 233#define NNMTXQ_1G 1 234static int t4_nnmtxq1g = -1; 235TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 236 237#define NNMRXQ_1G 1 238static int t4_nnmrxq1g = -1; 239TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 240#endif 241 242/* 243 * Holdoff parameters for 10G and 1G ports. 244 */ 245#define TMR_IDX_10G 1 246static int t4_tmr_idx_10g = TMR_IDX_10G; 247TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 248 249#define PKTC_IDX_10G (-1) 250static int t4_pktc_idx_10g = PKTC_IDX_10G; 251TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 252 253#define TMR_IDX_1G 1 254static int t4_tmr_idx_1g = TMR_IDX_1G; 255TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 256 257#define PKTC_IDX_1G (-1) 258static int t4_pktc_idx_1g = PKTC_IDX_1G; 259TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 260 261/* 262 * Size (# of entries) of each tx and rx queue. 263 */ 264static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 266 267static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 268TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 269 270/* 271 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 272 */ 273static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 274TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 275 276/* 277 * Configuration file. 278 */ 279#define DEFAULT_CF "default" 280#define FLASH_CF "flash" 281#define UWIRE_CF "uwire" 282#define FPGA_CF "fpga" 283static char t4_cfg_file[32] = DEFAULT_CF; 284TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 285 286/* 287 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 288 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 289 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 290 * mark or when signalled to do so, 0 to never emit PAUSE. 291 */ 292static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 293TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 294 295/* 296 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 297 * encouraged respectively). 298 */ 299static unsigned int t4_fw_install = 1; 300TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 301 302/* 303 * ASIC features that will be used. Disable the ones you don't want so that the 304 * chip resources aren't wasted on features that will not be used. 305 */ 306static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 307TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 308 309static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 310TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 311 312static int t4_toecaps_allowed = -1; 313TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 314 315static int t4_rdmacaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 317 318static int t4_iscsicaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 320 321static int t4_fcoecaps_allowed = 0; 322TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 323 324static int t5_write_combine = 0; 325TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 326 327struct intrs_and_queues { 328 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 329 uint16_t nirq; /* Total # of vectors */ 330 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 331 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 332 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 333 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 334 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 335 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 336 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 337#ifdef TCP_OFFLOAD 338 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 339 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 340 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 341 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 342#endif 343#ifdef DEV_NETMAP 344 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 345 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 346 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 347 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 348#endif 349}; 350 351struct filter_entry { 352 uint32_t valid:1; /* filter allocated and valid */ 353 uint32_t locked:1; /* filter is administratively locked */ 354 uint32_t pending:1; /* filter action is pending firmware reply */ 355 uint32_t smtidx:8; /* Source MAC Table index for smac */ 356 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 357 358 struct t4_filter_specification fs; 359}; 360 361static int map_bars_0_and_4(struct adapter *); 362static int map_bar_2(struct adapter *); 363static void setup_memwin(struct adapter *); 364static int validate_mem_range(struct adapter *, uint32_t, int); 365static int fwmtype_to_hwmtype(int); 366static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 367 uint32_t *); 368static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 369static uint32_t position_memwin(struct adapter *, int, uint32_t); 370static int cfg_itype_and_nqueues(struct adapter *, int, int, 371 struct intrs_and_queues *); 372static int prep_firmware(struct adapter *); 373static int partition_resources(struct adapter *, const struct firmware *, 374 const char *); 375static int get_params__pre_init(struct adapter *); 376static int get_params__post_init(struct adapter *); 377static int set_params__post_init(struct adapter *); 378static void t4_set_desc(struct adapter *); 379static void build_medialist(struct port_info *, struct ifmedia *); 380static int cxgbe_init_synchronized(struct port_info *); 381static int cxgbe_uninit_synchronized(struct port_info *); 382static int setup_intr_handlers(struct adapter *); 383static void quiesce_txq(struct adapter *, struct sge_txq *); 384static void quiesce_wrq(struct adapter *, struct sge_wrq *); 385static void quiesce_iq(struct adapter *, struct sge_iq *); 386static void quiesce_fl(struct adapter *, struct sge_fl *); 387static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 388 driver_intr_t *, void *, char *); 389static int t4_free_irq(struct adapter *, struct irq *); 390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 391 unsigned int); 392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 393static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 394static void cxgbe_tick(void *); 395static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 396static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 397 struct mbuf *); 398static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 399static int fw_msg_not_handled(struct adapter *, const __be64 *); 400static int t4_sysctls(struct adapter *); 401static int cxgbe_sysctls(struct port_info *); 402static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 403static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 404static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 405static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 406static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 407static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 408static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 409static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 410static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 411static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 412static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 413#ifdef SBUF_DRAIN 414static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 415static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 416static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 417static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 418static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 419static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 420static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 422static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 423static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 424static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 425static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 427static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 428static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 429static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 430static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 431static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 433static int sysctl_tids(SYSCTL_HANDLER_ARGS); 434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 439#endif 440static uint32_t fconf_to_mode(uint32_t); 441static uint32_t mode_to_fconf(uint32_t); 442static uint32_t fspec_to_fconf(struct t4_filter_specification *); 443static int get_filter_mode(struct adapter *, uint32_t *); 444static int set_filter_mode(struct adapter *, uint32_t); 445static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 446static int get_filter(struct adapter *, struct t4_filter *); 447static int set_filter(struct adapter *, struct t4_filter *); 448static int del_filter(struct adapter *, struct t4_filter *); 449static void clear_filter(struct filter_entry *); 450static int set_filter_wr(struct adapter *, int); 451static int del_filter_wr(struct adapter *, int); 452static int get_sge_context(struct adapter *, struct t4_sge_context *); 453static int load_fw(struct adapter *, struct t4_data *); 454static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 455static int read_i2c(struct adapter *, struct t4_i2c_data *); 456static int set_sched_class(struct adapter *, struct t4_sched_params *); 457static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 458#ifdef TCP_OFFLOAD 459static int toe_capability(struct port_info *, int); 460#endif 461static int mod_event(module_t, int, void *); 462 463struct { 464 uint16_t device; 465 char *desc; 466} t4_pciids[] = { 467 {0xa000, "Chelsio Terminator 4 FPGA"}, 468 {0x4400, "Chelsio T440-dbg"}, 469 {0x4401, "Chelsio T420-CR"}, 470 {0x4402, "Chelsio T422-CR"}, 471 {0x4403, "Chelsio T440-CR"}, 472 {0x4404, "Chelsio T420-BCH"}, 473 {0x4405, "Chelsio T440-BCH"}, 474 {0x4406, "Chelsio T440-CH"}, 475 {0x4407, "Chelsio T420-SO"}, 476 {0x4408, "Chelsio T420-CX"}, 477 {0x4409, "Chelsio T420-BT"}, 478 {0x440a, "Chelsio T404-BT"}, 479 {0x440e, "Chelsio T440-LP-CR"}, 480}, t5_pciids[] = { 481 {0xb000, "Chelsio Terminator 5 FPGA"}, 482 {0x5400, "Chelsio T580-dbg"}, 483 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 484 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 485 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 486 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 487 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 488 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 489 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 490 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 491 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 492 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 493 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 494 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 495 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 496#ifdef notyet 497 {0x5404, "Chelsio T520-BCH"}, 498 {0x5405, "Chelsio T540-BCH"}, 499 {0x5406, "Chelsio T540-CH"}, 500 {0x5408, "Chelsio T520-CX"}, 501 {0x540b, "Chelsio B520-SR"}, 502 {0x540c, "Chelsio B504-BT"}, 503 {0x540f, "Chelsio Amsterdam"}, 504 {0x5413, "Chelsio T580-CHR"}, 505#endif 506}; 507 508#ifdef TCP_OFFLOAD 509/* 510 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 511 * exactly the same for both rxq and ofld_rxq. 512 */ 513CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 514CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 515#endif 516 517/* No easy way to include t4_msg.h before adapter.h so we check this way */ 518CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 519CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 520 521CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 522 523static int 524t4_probe(device_t dev) 525{ 526 int i; 527 uint16_t v = pci_get_vendor(dev); 528 uint16_t d = pci_get_device(dev); 529 uint8_t f = pci_get_function(dev); 530 531 if (v != PCI_VENDOR_ID_CHELSIO) 532 return (ENXIO); 533 534 /* Attach only to PF0 of the FPGA */ 535 if (d == 0xa000 && f != 0) 536 return (ENXIO); 537 538 for (i = 0; i < nitems(t4_pciids); i++) { 539 if (d == t4_pciids[i].device) { 540 device_set_desc(dev, t4_pciids[i].desc); 541 return (BUS_PROBE_DEFAULT); 542 } 543 } 544 545 return (ENXIO); 546} 547 548static int 549t5_probe(device_t dev) 550{ 551 int i; 552 uint16_t v = pci_get_vendor(dev); 553 uint16_t d = pci_get_device(dev); 554 uint8_t f = pci_get_function(dev); 555 556 if (v != PCI_VENDOR_ID_CHELSIO) 557 return (ENXIO); 558 559 /* Attach only to PF0 of the FPGA */ 560 if (d == 0xb000 && f != 0) 561 return (ENXIO); 562 563 for (i = 0; i < nitems(t5_pciids); i++) { 564 if (d == t5_pciids[i].device) { 565 device_set_desc(dev, t5_pciids[i].desc); 566 return (BUS_PROBE_DEFAULT); 567 } 568 } 569 570 return (ENXIO); 571} 572 573static int 574t4_attach(device_t dev) 575{ 576 struct adapter *sc; 577 int rc = 0, i, n10g, n1g, rqidx, tqidx; 578 struct intrs_and_queues iaq; 579 struct sge *s; 580#ifdef TCP_OFFLOAD 581 int ofld_rqidx, ofld_tqidx; 582#endif 583#ifdef DEV_NETMAP 584 int nm_rqidx, nm_tqidx; 585#endif 586 587 sc = device_get_softc(dev); 588 sc->dev = dev; 589 590 pci_enable_busmaster(dev); 591 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 592 uint32_t v; 593 594 pci_set_max_read_req(dev, 4096); 595 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 596 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 597 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 598 599 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 600 } 601 602 sc->traceq = -1; 603 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 604 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 605 device_get_nameunit(dev)); 606 607 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 608 device_get_nameunit(dev)); 609 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 610 sx_xlock(&t4_list_lock); 611 SLIST_INSERT_HEAD(&t4_list, sc, link); 612 sx_xunlock(&t4_list_lock); 613 614 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 615 TAILQ_INIT(&sc->sfl); 616 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 617 618 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 619 620 rc = map_bars_0_and_4(sc); 621 if (rc != 0) 622 goto done; /* error message displayed already */ 623 624 /* 625 * This is the real PF# to which we're attaching. Works from within PCI 626 * passthrough environments too, where pci_get_function() could return a 627 * different PF# depending on the passthrough configuration. We need to 628 * use the real PF# in all our communication with the firmware. 629 */ 630 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 631 sc->mbox = sc->pf; 632 633 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 634 sc->an_handler = an_not_handled; 635 for (i = 0; i < nitems(sc->cpl_handler); i++) 636 sc->cpl_handler[i] = cpl_not_handled; 637 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 638 sc->fw_msg_handler[i] = fw_msg_not_handled; 639 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 640 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 642 t4_init_sge_cpl_handlers(sc); 643 644 /* Prepare the adapter for operation */ 645 rc = -t4_prep_adapter(sc); 646 if (rc != 0) { 647 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 648 goto done; 649 } 650 651 /* 652 * Do this really early, with the memory windows set up even before the 653 * character device. The userland tool's register i/o and mem read 654 * will work even in "recovery mode". 655 */ 656 setup_memwin(sc); 657 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 658 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 659 device_get_nameunit(dev)); 660 if (sc->cdev == NULL) 661 device_printf(dev, "failed to create nexus char device.\n"); 662 else 663 sc->cdev->si_drv1 = sc; 664 665 /* Go no further if recovery mode has been requested. */ 666 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 667 device_printf(dev, "recovery mode.\n"); 668 goto done; 669 } 670 671#if defined(__i386__) 672 if ((cpu_feature & CPUID_CX8) == 0) { 673 device_printf(dev, "64 bit atomics not available.\n"); 674 rc = ENOTSUP; 675 goto done; 676 } 677#endif 678 679 /* Prepare the firmware for operation */ 680 rc = prep_firmware(sc); 681 if (rc != 0) 682 goto done; /* error message displayed already */ 683 684 rc = get_params__post_init(sc); 685 if (rc != 0) 686 goto done; /* error message displayed already */ 687 688 rc = set_params__post_init(sc); 689 if (rc != 0) 690 goto done; /* error message displayed already */ 691 692 rc = map_bar_2(sc); 693 if (rc != 0) 694 goto done; /* error message displayed already */ 695 696 rc = t4_create_dma_tag(sc); 697 if (rc != 0) 698 goto done; /* error message displayed already */ 699 700 /* 701 * First pass over all the ports - allocate VIs and initialize some 702 * basic parameters like mac address, port type, etc. We also figure 703 * out whether a port is 10G or 1G and use that information when 704 * calculating how many interrupts to attempt to allocate. 705 */ 706 n10g = n1g = 0; 707 for_each_port(sc, i) { 708 struct port_info *pi; 709 710 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 711 sc->port[i] = pi; 712 713 /* These must be set before t4_port_init */ 714 pi->adapter = sc; 715 pi->port_id = i; 716 717 /* Allocate the vi and initialize parameters like mac addr */ 718 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 719 if (rc != 0) { 720 device_printf(dev, "unable to initialize port %d: %d\n", 721 i, rc); 722 free(pi, M_CXGBE); 723 sc->port[i] = NULL; 724 goto done; 725 } 726 727 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 728 pi->link_cfg.requested_fc |= t4_pause_settings; 729 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 730 pi->link_cfg.fc |= t4_pause_settings; 731 732 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 733 if (rc != 0) { 734 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 735 free(pi, M_CXGBE); 736 sc->port[i] = NULL; 737 goto done; 738 } 739 740 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 741 device_get_nameunit(dev), i); 742 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 743 sc->chan_map[pi->tx_chan] = i; 744 745 if (is_10G_port(pi) || is_40G_port(pi)) { 746 n10g++; 747 pi->tmr_idx = t4_tmr_idx_10g; 748 pi->pktc_idx = t4_pktc_idx_10g; 749 } else { 750 n1g++; 751 pi->tmr_idx = t4_tmr_idx_1g; 752 pi->pktc_idx = t4_pktc_idx_1g; 753 } 754 755 pi->xact_addr_filt = -1; 756 pi->linkdnrc = -1; 757 758 pi->qsize_rxq = t4_qsize_rxq; 759 pi->qsize_txq = t4_qsize_txq; 760 761 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 762 if (pi->dev == NULL) { 763 device_printf(dev, 764 "failed to add device for port %d.\n", i); 765 rc = ENXIO; 766 goto done; 767 } 768 device_set_softc(pi->dev, pi); 769 } 770 771 /* 772 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 773 */ 774 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 775 if (rc != 0) 776 goto done; /* error message displayed already */ 777 778 sc->intr_type = iaq.intr_type; 779 sc->intr_count = iaq.nirq; 780 781 s = &sc->sge; 782 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 783 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 784 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 785 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 786 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 787#ifdef TCP_OFFLOAD 788 if (is_offload(sc)) { 789 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 790 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 791 s->neq += s->nofldtxq + s->nofldrxq; 792 s->niq += s->nofldrxq; 793 794 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 797 M_CXGBE, M_ZERO | M_WAITOK); 798 } 799#endif 800#ifdef DEV_NETMAP 801 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 802 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 803 s->neq += s->nnmtxq + s->nnmrxq; 804 s->niq += s->nnmrxq; 805 806 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 807 M_CXGBE, M_ZERO | M_WAITOK); 808 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 809 M_CXGBE, M_ZERO | M_WAITOK); 810#endif 811 812 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 813 M_ZERO | M_WAITOK); 814 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 815 M_ZERO | M_WAITOK); 816 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 817 M_ZERO | M_WAITOK); 818 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 819 M_ZERO | M_WAITOK); 820 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 821 M_ZERO | M_WAITOK); 822 823 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 824 M_ZERO | M_WAITOK); 825 826 t4_init_l2t(sc, M_WAITOK); 827 828 /* 829 * Second pass over the ports. This time we know the number of rx and 830 * tx queues that each port should get. 831 */ 832 rqidx = tqidx = 0; 833#ifdef TCP_OFFLOAD 834 ofld_rqidx = ofld_tqidx = 0; 835#endif 836#ifdef DEV_NETMAP 837 nm_rqidx = nm_tqidx = 0; 838#endif 839 for_each_port(sc, i) { 840 struct port_info *pi = sc->port[i]; 841 842 if (pi == NULL) 843 continue; 844 845 pi->first_rxq = rqidx; 846 pi->first_txq = tqidx; 847 if (is_10G_port(pi) || is_40G_port(pi)) { 848 pi->flags |= iaq.intr_flags_10g; 849 pi->nrxq = iaq.nrxq10g; 850 pi->ntxq = iaq.ntxq10g; 851 } else { 852 pi->flags |= iaq.intr_flags_1g; 853 pi->nrxq = iaq.nrxq1g; 854 pi->ntxq = iaq.ntxq1g; 855 } 856 857 if (pi->ntxq > 1) 858 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 859 else 860 pi->rsrv_noflowq = 0; 861 862 rqidx += pi->nrxq; 863 tqidx += pi->ntxq; 864#ifdef TCP_OFFLOAD 865 if (is_offload(sc)) { 866 pi->first_ofld_rxq = ofld_rqidx; 867 pi->first_ofld_txq = ofld_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nofldrxq = iaq.nofldrxq10g; 870 pi->nofldtxq = iaq.nofldtxq10g; 871 } else { 872 pi->nofldrxq = iaq.nofldrxq1g; 873 pi->nofldtxq = iaq.nofldtxq1g; 874 } 875 ofld_rqidx += pi->nofldrxq; 876 ofld_tqidx += pi->nofldtxq; 877 } 878#endif 879#ifdef DEV_NETMAP 880 pi->first_nm_rxq = nm_rqidx; 881 pi->first_nm_txq = nm_tqidx; 882 if (is_10G_port(pi) || is_40G_port(pi)) { 883 pi->nnmrxq = iaq.nnmrxq10g; 884 pi->nnmtxq = iaq.nnmtxq10g; 885 } else { 886 pi->nnmrxq = iaq.nnmrxq1g; 887 pi->nnmtxq = iaq.nnmtxq1g; 888 } 889 nm_rqidx += pi->nnmrxq; 890 nm_tqidx += pi->nnmtxq; 891#endif 892 } 893 894 rc = setup_intr_handlers(sc); 895 if (rc != 0) { 896 device_printf(dev, 897 "failed to setup interrupt handlers: %d\n", rc); 898 goto done; 899 } 900 901 rc = bus_generic_attach(dev); 902 if (rc != 0) { 903 device_printf(dev, 904 "failed to attach all child ports: %d\n", rc); 905 goto done; 906 } 907 908 device_printf(dev, 909 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 910 sc->params.pci.width, sc->params.nports, sc->intr_count, 911 sc->intr_type == INTR_MSIX ? "MSI-X" : 912 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 913 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 914 915 t4_set_desc(sc); 916 917done: 918 if (rc != 0 && sc->cdev) { 919 /* cdev was created and so cxgbetool works; recover that way. */ 920 device_printf(dev, 921 "error during attach, adapter is now in recovery mode.\n"); 922 rc = 0; 923 } 924 925 if (rc != 0) 926 t4_detach(dev); 927 else 928 t4_sysctls(sc); 929 930 return (rc); 931} 932 933/* 934 * Idempotent 935 */ 936static int 937t4_detach(device_t dev) 938{ 939 struct adapter *sc; 940 struct port_info *pi; 941 int i, rc; 942 943 sc = device_get_softc(dev); 944 945 if (sc->flags & FULL_INIT_DONE) 946 t4_intr_disable(sc); 947 948 if (sc->cdev) { 949 destroy_dev(sc->cdev); 950 sc->cdev = NULL; 951 } 952 953 rc = bus_generic_detach(dev); 954 if (rc) { 955 device_printf(dev, 956 "failed to detach child devices: %d\n", rc); 957 return (rc); 958 } 959 960 for (i = 0; i < sc->intr_count; i++) 961 t4_free_irq(sc, &sc->irq[i]); 962 963 for (i = 0; i < MAX_NPORTS; i++) { 964 pi = sc->port[i]; 965 if (pi) { 966 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 967 if (pi->dev) 968 device_delete_child(dev, pi->dev); 969 970 mtx_destroy(&pi->pi_lock); 971 free(pi, M_CXGBE); 972 } 973 } 974 975 if (sc->flags & FULL_INIT_DONE) 976 adapter_full_uninit(sc); 977 978 if (sc->flags & FW_OK) 979 t4_fw_bye(sc, sc->mbox); 980 981 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 982 pci_release_msi(dev); 983 984 if (sc->regs_res) 985 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 986 sc->regs_res); 987 988 if (sc->udbs_res) 989 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 990 sc->udbs_res); 991 992 if (sc->msix_res) 993 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 994 sc->msix_res); 995 996 if (sc->l2t) 997 t4_free_l2t(sc->l2t); 998 999#ifdef TCP_OFFLOAD 1000 free(sc->sge.ofld_rxq, M_CXGBE); 1001 free(sc->sge.ofld_txq, M_CXGBE); 1002#endif 1003#ifdef DEV_NETMAP 1004 free(sc->sge.nm_rxq, M_CXGBE); 1005 free(sc->sge.nm_txq, M_CXGBE); 1006#endif 1007 free(sc->irq, M_CXGBE); 1008 free(sc->sge.rxq, M_CXGBE); 1009 free(sc->sge.txq, M_CXGBE); 1010 free(sc->sge.ctrlq, M_CXGBE); 1011 free(sc->sge.iqmap, M_CXGBE); 1012 free(sc->sge.eqmap, M_CXGBE); 1013 free(sc->tids.ftid_tab, M_CXGBE); 1014 t4_destroy_dma_tag(sc); 1015 if (mtx_initialized(&sc->sc_lock)) { 1016 sx_xlock(&t4_list_lock); 1017 SLIST_REMOVE(&t4_list, sc, adapter, link); 1018 sx_xunlock(&t4_list_lock); 1019 mtx_destroy(&sc->sc_lock); 1020 } 1021 1022 if (mtx_initialized(&sc->tids.ftid_lock)) 1023 mtx_destroy(&sc->tids.ftid_lock); 1024 if (mtx_initialized(&sc->sfl_lock)) 1025 mtx_destroy(&sc->sfl_lock); 1026 if (mtx_initialized(&sc->ifp_lock)) 1027 mtx_destroy(&sc->ifp_lock); 1028 if (mtx_initialized(&sc->regwin_lock)) 1029 mtx_destroy(&sc->regwin_lock); 1030 1031 bzero(sc, sizeof(*sc)); 1032 1033 return (0); 1034} 1035 1036static int 1037cxgbe_probe(device_t dev) 1038{ 1039 char buf[128]; 1040 struct port_info *pi = device_get_softc(dev); 1041 1042 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1043 device_set_desc_copy(dev, buf); 1044 1045 return (BUS_PROBE_DEFAULT); 1046} 1047 1048#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1049 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1050 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1051#define T4_CAP_ENABLE (T4_CAP) 1052 1053static int 1054cxgbe_attach(device_t dev) 1055{ 1056 struct port_info *pi = device_get_softc(dev); 1057 struct ifnet *ifp; 1058 char *s; 1059 int n, o; 1060 1061 /* Allocate an ifnet and set it up */ 1062 ifp = if_alloc(IFT_ETHER); 1063 if (ifp == NULL) { 1064 device_printf(dev, "Cannot allocate ifnet\n"); 1065 return (ENOMEM); 1066 } 1067 pi->ifp = ifp; 1068 ifp->if_softc = pi; 1069 1070 callout_init(&pi->tick, CALLOUT_MPSAFE); 1071 1072 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1073 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1074 1075 ifp->if_init = cxgbe_init; 1076 ifp->if_ioctl = cxgbe_ioctl; 1077 ifp->if_transmit = cxgbe_transmit; 1078 ifp->if_qflush = cxgbe_qflush; 1079 1080 ifp->if_capabilities = T4_CAP; 1081#ifdef TCP_OFFLOAD 1082 if (is_offload(pi->adapter)) 1083 ifp->if_capabilities |= IFCAP_TOE; 1084#endif 1085 ifp->if_capenable = T4_CAP_ENABLE; 1086 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1087 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1088 1089 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1090 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1091 ifp->if_hw_tsomaxsegsize = 65536; 1092 1093 /* Initialize ifmedia for this port */ 1094 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1095 cxgbe_media_status); 1096 build_medialist(pi, &pi->media); 1097 1098 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1099 EVENTHANDLER_PRI_ANY); 1100 1101 ether_ifattach(ifp, pi->hw_addr); 1102 1103 n = 128; 1104 s = malloc(n, M_CXGBE, M_WAITOK); 1105 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1106 MPASS(n > o); 1107#ifdef TCP_OFFLOAD 1108 if (is_offload(pi->adapter)) { 1109 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1110 pi->nofldtxq, pi->nofldrxq); 1111 MPASS(n > o); 1112 } 1113#endif 1114#ifdef DEV_NETMAP 1115 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1116 pi->nnmrxq); 1117 MPASS(n > o); 1118#endif 1119 device_printf(dev, "%s\n", s); 1120 free(s, M_CXGBE); 1121 1122#ifdef DEV_NETMAP 1123 /* nm_media handled here to keep implementation private to this file */ 1124 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1125 cxgbe_media_status); 1126 build_medialist(pi, &pi->nm_media); 1127 create_netmap_ifnet(pi); /* logs errors it something fails */ 1128#endif 1129 cxgbe_sysctls(pi); 1130 1131 return (0); 1132} 1133 1134static int 1135cxgbe_detach(device_t dev) 1136{ 1137 struct port_info *pi = device_get_softc(dev); 1138 struct adapter *sc = pi->adapter; 1139 struct ifnet *ifp = pi->ifp; 1140 1141 /* Tell if_ioctl and if_init that the port is going away */ 1142 ADAPTER_LOCK(sc); 1143 SET_DOOMED(pi); 1144 wakeup(&sc->flags); 1145 while (IS_BUSY(sc)) 1146 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1147 SET_BUSY(sc); 1148#ifdef INVARIANTS 1149 sc->last_op = "t4detach"; 1150 sc->last_op_thr = curthread; 1151#endif 1152 ADAPTER_UNLOCK(sc); 1153 1154 if (pi->flags & HAS_TRACEQ) { 1155 sc->traceq = -1; /* cloner should not create ifnet */ 1156 t4_tracer_port_detach(sc); 1157 } 1158 1159 if (pi->vlan_c) 1160 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1161 1162 PORT_LOCK(pi); 1163 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1164 callout_stop(&pi->tick); 1165 PORT_UNLOCK(pi); 1166 callout_drain(&pi->tick); 1167 1168 /* Let detach proceed even if these fail. */ 1169 cxgbe_uninit_synchronized(pi); 1170 port_full_uninit(pi); 1171 1172 ifmedia_removeall(&pi->media); 1173 ether_ifdetach(pi->ifp); 1174 if_free(pi->ifp); 1175 1176#ifdef DEV_NETMAP 1177 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1178 destroy_netmap_ifnet(pi); 1179#endif 1180 1181 ADAPTER_LOCK(sc); 1182 CLR_BUSY(sc); 1183 wakeup(&sc->flags); 1184 ADAPTER_UNLOCK(sc); 1185 1186 return (0); 1187} 1188 1189static void 1190cxgbe_init(void *arg) 1191{ 1192 struct port_info *pi = arg; 1193 struct adapter *sc = pi->adapter; 1194 1195 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1196 return; 1197 cxgbe_init_synchronized(pi); 1198 end_synchronized_op(sc, 0); 1199} 1200 1201static int 1202cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1203{ 1204 int rc = 0, mtu, flags, can_sleep; 1205 struct port_info *pi = ifp->if_softc; 1206 struct adapter *sc = pi->adapter; 1207 struct ifreq *ifr = (struct ifreq *)data; 1208 uint32_t mask; 1209 1210 switch (cmd) { 1211 case SIOCSIFMTU: 1212 mtu = ifr->ifr_mtu; 1213 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1214 return (EINVAL); 1215 1216 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1217 if (rc) 1218 return (rc); 1219 ifp->if_mtu = mtu; 1220 if (pi->flags & PORT_INIT_DONE) { 1221 t4_update_fl_bufsize(ifp); 1222 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1223 rc = update_mac_settings(ifp, XGMAC_MTU); 1224 } 1225 end_synchronized_op(sc, 0); 1226 break; 1227 1228 case SIOCSIFFLAGS: 1229 can_sleep = 0; 1230redo_sifflags: 1231 rc = begin_synchronized_op(sc, pi, 1232 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1233 if (rc) 1234 return (rc); 1235 1236 if (ifp->if_flags & IFF_UP) { 1237 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1238 flags = pi->if_flags; 1239 if ((ifp->if_flags ^ flags) & 1240 (IFF_PROMISC | IFF_ALLMULTI)) { 1241 if (can_sleep == 1) { 1242 end_synchronized_op(sc, 0); 1243 can_sleep = 0; 1244 goto redo_sifflags; 1245 } 1246 rc = update_mac_settings(ifp, 1247 XGMAC_PROMISC | XGMAC_ALLMULTI); 1248 } 1249 } else { 1250 if (can_sleep == 0) { 1251 end_synchronized_op(sc, LOCK_HELD); 1252 can_sleep = 1; 1253 goto redo_sifflags; 1254 } 1255 rc = cxgbe_init_synchronized(pi); 1256 } 1257 pi->if_flags = ifp->if_flags; 1258 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1259 if (can_sleep == 0) { 1260 end_synchronized_op(sc, LOCK_HELD); 1261 can_sleep = 1; 1262 goto redo_sifflags; 1263 } 1264 rc = cxgbe_uninit_synchronized(pi); 1265 } 1266 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1267 break; 1268 1269 case SIOCADDMULTI: 1270 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1271 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1272 if (rc) 1273 return (rc); 1274 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1275 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1276 end_synchronized_op(sc, LOCK_HELD); 1277 break; 1278 1279 case SIOCSIFCAP: 1280 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1281 if (rc) 1282 return (rc); 1283 1284 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1285 if (mask & IFCAP_TXCSUM) { 1286 ifp->if_capenable ^= IFCAP_TXCSUM; 1287 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1288 1289 if (IFCAP_TSO4 & ifp->if_capenable && 1290 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1291 ifp->if_capenable &= ~IFCAP_TSO4; 1292 if_printf(ifp, 1293 "tso4 disabled due to -txcsum.\n"); 1294 } 1295 } 1296 if (mask & IFCAP_TXCSUM_IPV6) { 1297 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1298 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1299 1300 if (IFCAP_TSO6 & ifp->if_capenable && 1301 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1302 ifp->if_capenable &= ~IFCAP_TSO6; 1303 if_printf(ifp, 1304 "tso6 disabled due to -txcsum6.\n"); 1305 } 1306 } 1307 if (mask & IFCAP_RXCSUM) 1308 ifp->if_capenable ^= IFCAP_RXCSUM; 1309 if (mask & IFCAP_RXCSUM_IPV6) 1310 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1311 1312 /* 1313 * Note that we leave CSUM_TSO alone (it is always set). The 1314 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1315 * sending a TSO request our way, so it's sufficient to toggle 1316 * IFCAP_TSOx only. 1317 */ 1318 if (mask & IFCAP_TSO4) { 1319 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1320 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1321 if_printf(ifp, "enable txcsum first.\n"); 1322 rc = EAGAIN; 1323 goto fail; 1324 } 1325 ifp->if_capenable ^= IFCAP_TSO4; 1326 } 1327 if (mask & IFCAP_TSO6) { 1328 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1329 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1330 if_printf(ifp, "enable txcsum6 first.\n"); 1331 rc = EAGAIN; 1332 goto fail; 1333 } 1334 ifp->if_capenable ^= IFCAP_TSO6; 1335 } 1336 if (mask & IFCAP_LRO) { 1337#if defined(INET) || defined(INET6) 1338 int i; 1339 struct sge_rxq *rxq; 1340 1341 ifp->if_capenable ^= IFCAP_LRO; 1342 for_each_rxq(pi, i, rxq) { 1343 if (ifp->if_capenable & IFCAP_LRO) 1344 rxq->iq.flags |= IQ_LRO_ENABLED; 1345 else 1346 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1347 } 1348#endif 1349 } 1350#ifdef TCP_OFFLOAD 1351 if (mask & IFCAP_TOE) { 1352 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1353 1354 rc = toe_capability(pi, enable); 1355 if (rc != 0) 1356 goto fail; 1357 1358 ifp->if_capenable ^= mask; 1359 } 1360#endif 1361 if (mask & IFCAP_VLAN_HWTAGGING) { 1362 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1363 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1364 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1365 } 1366 if (mask & IFCAP_VLAN_MTU) { 1367 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1368 1369 /* Need to find out how to disable auto-mtu-inflation */ 1370 } 1371 if (mask & IFCAP_VLAN_HWTSO) 1372 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1373 if (mask & IFCAP_VLAN_HWCSUM) 1374 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1375 1376#ifdef VLAN_CAPABILITIES 1377 VLAN_CAPABILITIES(ifp); 1378#endif 1379fail: 1380 end_synchronized_op(sc, 0); 1381 break; 1382 1383 case SIOCSIFMEDIA: 1384 case SIOCGIFMEDIA: 1385 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1386 break; 1387 1388 default: 1389 rc = ether_ioctl(ifp, cmd, data); 1390 } 1391 1392 return (rc); 1393} 1394 1395static int 1396cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1397{ 1398 struct port_info *pi = ifp->if_softc; 1399 struct adapter *sc = pi->adapter; 1400 struct sge_txq *txq; 1401 void *items[1]; 1402 int rc; 1403 1404 M_ASSERTPKTHDR(m); 1405 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1406 1407 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1408 m_freem(m); 1409 return (ENETDOWN); 1410 } 1411 1412 rc = parse_pkt(&m); 1413 if (__predict_false(rc != 0)) { 1414 MPASS(m == NULL); /* was freed already */ 1415 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1416 return (rc); 1417 } 1418 1419 /* Select a txq. */ 1420 txq = &sc->sge.txq[pi->first_txq]; 1421 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1422 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1423 pi->rsrv_noflowq); 1424 1425 items[0] = m; 1426 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1427 if (__predict_false(rc != 0)) 1428 m_freem(m); 1429 1430 return (rc); 1431} 1432 1433static void 1434cxgbe_qflush(struct ifnet *ifp) 1435{ 1436 struct port_info *pi = ifp->if_softc; 1437 struct sge_txq *txq; 1438 int i; 1439 1440 /* queues do not exist if !PORT_INIT_DONE. */ 1441 if (pi->flags & PORT_INIT_DONE) { 1442 for_each_txq(pi, i, txq) { 1443 TXQ_LOCK(txq); 1444 txq->eq.flags &= ~EQ_ENABLED; 1445 TXQ_UNLOCK(txq); 1446 while (!mp_ring_is_idle(txq->r)) { 1447 mp_ring_check_drainage(txq->r, 0); 1448 pause("qflush", 1); 1449 } 1450 } 1451 } 1452 if_qflush(ifp); 1453} 1454 1455static int 1456cxgbe_media_change(struct ifnet *ifp) 1457{ 1458 struct port_info *pi = ifp->if_softc; 1459 1460 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1461 1462 return (EOPNOTSUPP); 1463} 1464 1465static void 1466cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1467{ 1468 struct port_info *pi = ifp->if_softc; 1469 struct ifmedia *media = NULL; 1470 struct ifmedia_entry *cur; 1471 int speed = pi->link_cfg.speed; 1472#ifdef INVARIANTS 1473 int data = (pi->port_type << 8) | pi->mod_type; 1474#endif 1475 1476 if (ifp == pi->ifp) 1477 media = &pi->media; 1478#ifdef DEV_NETMAP 1479 else if (ifp == pi->nm_ifp) 1480 media = &pi->nm_media; 1481#endif 1482 MPASS(media != NULL); 1483 1484 cur = media->ifm_cur; 1485 MPASS(cur->ifm_data == data); 1486 1487 ifmr->ifm_status = IFM_AVALID; 1488 if (!pi->link_cfg.link_ok) 1489 return; 1490 1491 ifmr->ifm_status |= IFM_ACTIVE; 1492 1493 /* active and current will differ iff current media is autoselect. */ 1494 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1495 return; 1496 1497 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1498 if (speed == SPEED_10000) 1499 ifmr->ifm_active |= IFM_10G_T; 1500 else if (speed == SPEED_1000) 1501 ifmr->ifm_active |= IFM_1000_T; 1502 else if (speed == SPEED_100) 1503 ifmr->ifm_active |= IFM_100_TX; 1504 else if (speed == SPEED_10) 1505 ifmr->ifm_active |= IFM_10_T; 1506 else 1507 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1508 speed)); 1509} 1510 1511void 1512t4_fatal_err(struct adapter *sc) 1513{ 1514 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1515 t4_intr_disable(sc); 1516 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1517 device_get_nameunit(sc->dev)); 1518} 1519 1520static int 1521map_bars_0_and_4(struct adapter *sc) 1522{ 1523 sc->regs_rid = PCIR_BAR(0); 1524 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1525 &sc->regs_rid, RF_ACTIVE); 1526 if (sc->regs_res == NULL) { 1527 device_printf(sc->dev, "cannot map registers.\n"); 1528 return (ENXIO); 1529 } 1530 sc->bt = rman_get_bustag(sc->regs_res); 1531 sc->bh = rman_get_bushandle(sc->regs_res); 1532 sc->mmio_len = rman_get_size(sc->regs_res); 1533 setbit(&sc->doorbells, DOORBELL_KDB); 1534 1535 sc->msix_rid = PCIR_BAR(4); 1536 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1537 &sc->msix_rid, RF_ACTIVE); 1538 if (sc->msix_res == NULL) { 1539 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1540 return (ENXIO); 1541 } 1542 1543 return (0); 1544} 1545 1546static int 1547map_bar_2(struct adapter *sc) 1548{ 1549 1550 /* 1551 * T4: only iWARP driver uses the userspace doorbells. There is no need 1552 * to map it if RDMA is disabled. 1553 */ 1554 if (is_t4(sc) && sc->rdmacaps == 0) 1555 return (0); 1556 1557 sc->udbs_rid = PCIR_BAR(2); 1558 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1559 &sc->udbs_rid, RF_ACTIVE); 1560 if (sc->udbs_res == NULL) { 1561 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1562 return (ENXIO); 1563 } 1564 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1565 1566 if (is_t5(sc)) { 1567 setbit(&sc->doorbells, DOORBELL_UDB); 1568#if defined(__i386__) || defined(__amd64__) 1569 if (t5_write_combine) { 1570 int rc; 1571 1572 /* 1573 * Enable write combining on BAR2. This is the 1574 * userspace doorbell BAR and is split into 128B 1575 * (UDBS_SEG_SIZE) doorbell regions, each associated 1576 * with an egress queue. The first 64B has the doorbell 1577 * and the second 64B can be used to submit a tx work 1578 * request with an implicit doorbell. 1579 */ 1580 1581 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1582 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1583 if (rc == 0) { 1584 clrbit(&sc->doorbells, DOORBELL_UDB); 1585 setbit(&sc->doorbells, DOORBELL_WCWR); 1586 setbit(&sc->doorbells, DOORBELL_UDBWC); 1587 } else { 1588 device_printf(sc->dev, 1589 "couldn't enable write combining: %d\n", 1590 rc); 1591 } 1592 1593 t4_write_reg(sc, A_SGE_STAT_CFG, 1594 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1595 } 1596#endif 1597 } 1598 1599 return (0); 1600} 1601 1602static const struct memwin t4_memwin[] = { 1603 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1604 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1605 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1606}; 1607 1608static const struct memwin t5_memwin[] = { 1609 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1610 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1611 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1612}; 1613 1614static void 1615setup_memwin(struct adapter *sc) 1616{ 1617 const struct memwin *mw; 1618 int i, n; 1619 uint32_t bar0; 1620 1621 if (is_t4(sc)) { 1622 /* 1623 * Read low 32b of bar0 indirectly via the hardware backdoor 1624 * mechanism. Works from within PCI passthrough environments 1625 * too, where rman_get_start() can return a different value. We 1626 * need to program the T4 memory window decoders with the actual 1627 * addresses that will be coming across the PCIe link. 1628 */ 1629 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1630 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1631 1632 mw = &t4_memwin[0]; 1633 n = nitems(t4_memwin); 1634 } else { 1635 /* T5 uses the relative offset inside the PCIe BAR */ 1636 bar0 = 0; 1637 1638 mw = &t5_memwin[0]; 1639 n = nitems(t5_memwin); 1640 } 1641 1642 for (i = 0; i < n; i++, mw++) { 1643 t4_write_reg(sc, 1644 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1645 (mw->base + bar0) | V_BIR(0) | 1646 V_WINDOW(ilog2(mw->aperture) - 10)); 1647 } 1648 1649 /* flush */ 1650 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1651} 1652 1653/* 1654 * Verify that the memory range specified by the addr/len pair is valid and lies 1655 * entirely within a single region (EDCx or MCx). 1656 */ 1657static int 1658validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1659{ 1660 uint32_t em, addr_len, maddr, mlen; 1661 1662 /* Memory can only be accessed in naturally aligned 4 byte units */ 1663 if (addr & 3 || len & 3 || len == 0) 1664 return (EINVAL); 1665 1666 /* Enabled memories */ 1667 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1668 if (em & F_EDRAM0_ENABLE) { 1669 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1670 maddr = G_EDRAM0_BASE(addr_len) << 20; 1671 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1672 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1673 addr + len <= maddr + mlen) 1674 return (0); 1675 } 1676 if (em & F_EDRAM1_ENABLE) { 1677 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1678 maddr = G_EDRAM1_BASE(addr_len) << 20; 1679 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1680 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1681 addr + len <= maddr + mlen) 1682 return (0); 1683 } 1684 if (em & F_EXT_MEM_ENABLE) { 1685 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1686 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1687 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1688 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1689 addr + len <= maddr + mlen) 1690 return (0); 1691 } 1692 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1693 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1694 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1695 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1696 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1697 addr + len <= maddr + mlen) 1698 return (0); 1699 } 1700 1701 return (EFAULT); 1702} 1703 1704static int 1705fwmtype_to_hwmtype(int mtype) 1706{ 1707 1708 switch (mtype) { 1709 case FW_MEMTYPE_EDC0: 1710 return (MEM_EDC0); 1711 case FW_MEMTYPE_EDC1: 1712 return (MEM_EDC1); 1713 case FW_MEMTYPE_EXTMEM: 1714 return (MEM_MC0); 1715 case FW_MEMTYPE_EXTMEM1: 1716 return (MEM_MC1); 1717 default: 1718 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1719 } 1720} 1721 1722/* 1723 * Verify that the memory range specified by the memtype/offset/len pair is 1724 * valid and lies entirely within the memtype specified. The global address of 1725 * the start of the range is returned in addr. 1726 */ 1727static int 1728validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1729 uint32_t *addr) 1730{ 1731 uint32_t em, addr_len, maddr, mlen; 1732 1733 /* Memory can only be accessed in naturally aligned 4 byte units */ 1734 if (off & 3 || len & 3 || len == 0) 1735 return (EINVAL); 1736 1737 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1738 switch (fwmtype_to_hwmtype(mtype)) { 1739 case MEM_EDC0: 1740 if (!(em & F_EDRAM0_ENABLE)) 1741 return (EINVAL); 1742 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1743 maddr = G_EDRAM0_BASE(addr_len) << 20; 1744 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1745 break; 1746 case MEM_EDC1: 1747 if (!(em & F_EDRAM1_ENABLE)) 1748 return (EINVAL); 1749 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1750 maddr = G_EDRAM1_BASE(addr_len) << 20; 1751 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1752 break; 1753 case MEM_MC: 1754 if (!(em & F_EXT_MEM_ENABLE)) 1755 return (EINVAL); 1756 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1757 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1758 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1759 break; 1760 case MEM_MC1: 1761 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1762 return (EINVAL); 1763 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1764 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1765 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1766 break; 1767 default: 1768 return (EINVAL); 1769 } 1770 1771 if (mlen > 0 && off < mlen && off + len <= mlen) { 1772 *addr = maddr + off; /* global address */ 1773 return (0); 1774 } 1775 1776 return (EFAULT); 1777} 1778 1779static void 1780memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1781{ 1782 const struct memwin *mw; 1783 1784 if (is_t4(sc)) { 1785 KASSERT(win >= 0 && win < nitems(t4_memwin), 1786 ("%s: incorrect memwin# (%d)", __func__, win)); 1787 mw = &t4_memwin[win]; 1788 } else { 1789 KASSERT(win >= 0 && win < nitems(t5_memwin), 1790 ("%s: incorrect memwin# (%d)", __func__, win)); 1791 mw = &t5_memwin[win]; 1792 } 1793 1794 if (base != NULL) 1795 *base = mw->base; 1796 if (aperture != NULL) 1797 *aperture = mw->aperture; 1798} 1799 1800/* 1801 * Positions the memory window such that it can be used to access the specified 1802 * address in the chip's address space. The return value is the offset of addr 1803 * from the start of the window. 1804 */ 1805static uint32_t 1806position_memwin(struct adapter *sc, int n, uint32_t addr) 1807{ 1808 uint32_t start, pf; 1809 uint32_t reg; 1810 1811 KASSERT(n >= 0 && n <= 3, 1812 ("%s: invalid window %d.", __func__, n)); 1813 KASSERT((addr & 3) == 0, 1814 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1815 1816 if (is_t4(sc)) { 1817 pf = 0; 1818 start = addr & ~0xf; /* start must be 16B aligned */ 1819 } else { 1820 pf = V_PFNUM(sc->pf); 1821 start = addr & ~0x7f; /* start must be 128B aligned */ 1822 } 1823 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1824 1825 t4_write_reg(sc, reg, start | pf); 1826 t4_read_reg(sc, reg); 1827 1828 return (addr - start); 1829} 1830 1831static int 1832cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1833 struct intrs_and_queues *iaq) 1834{ 1835 int rc, itype, navail, nrxq10g, nrxq1g, n; 1836 int nofldrxq10g = 0, nofldrxq1g = 0; 1837 int nnmrxq10g = 0, nnmrxq1g = 0; 1838 1839 bzero(iaq, sizeof(*iaq)); 1840 1841 iaq->ntxq10g = t4_ntxq10g; 1842 iaq->ntxq1g = t4_ntxq1g; 1843 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1844 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1845 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1846#ifdef TCP_OFFLOAD 1847 if (is_offload(sc)) { 1848 iaq->nofldtxq10g = t4_nofldtxq10g; 1849 iaq->nofldtxq1g = t4_nofldtxq1g; 1850 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1851 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1852 } 1853#endif 1854#ifdef DEV_NETMAP 1855 iaq->nnmtxq10g = t4_nnmtxq10g; 1856 iaq->nnmtxq1g = t4_nnmtxq1g; 1857 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1858 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1859#endif 1860 1861 for (itype = INTR_MSIX; itype; itype >>= 1) { 1862 1863 if ((itype & t4_intr_types) == 0) 1864 continue; /* not allowed */ 1865 1866 if (itype == INTR_MSIX) 1867 navail = pci_msix_count(sc->dev); 1868 else if (itype == INTR_MSI) 1869 navail = pci_msi_count(sc->dev); 1870 else 1871 navail = 1; 1872restart: 1873 if (navail == 0) 1874 continue; 1875 1876 iaq->intr_type = itype; 1877 iaq->intr_flags_10g = 0; 1878 iaq->intr_flags_1g = 0; 1879 1880 /* 1881 * Best option: an interrupt vector for errors, one for the 1882 * firmware event queue, and one for every rxq (NIC, TOE, and 1883 * netmap). 1884 */ 1885 iaq->nirq = T4_EXTRA_INTR; 1886 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1887 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1888 if (iaq->nirq <= navail && 1889 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1890 iaq->intr_flags_10g = INTR_ALL; 1891 iaq->intr_flags_1g = INTR_ALL; 1892 goto allocate; 1893 } 1894 1895 /* 1896 * Second best option: a vector for errors, one for the firmware 1897 * event queue, and vectors for either all the NIC rx queues or 1898 * all the TOE rx queues. The queues that don't get vectors 1899 * will forward their interrupts to those that do. 1900 * 1901 * Note: netmap rx queues cannot be created early and so they 1902 * can't be setup to receive forwarded interrupts for others. 1903 */ 1904 iaq->nirq = T4_EXTRA_INTR; 1905 if (nrxq10g >= nofldrxq10g) { 1906 iaq->intr_flags_10g = INTR_RXQ; 1907 iaq->nirq += n10g * nrxq10g; 1908#ifdef DEV_NETMAP 1909 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1910#endif 1911 } else { 1912 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1913 iaq->nirq += n10g * nofldrxq10g; 1914#ifdef DEV_NETMAP 1915 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1916#endif 1917 } 1918 if (nrxq1g >= nofldrxq1g) { 1919 iaq->intr_flags_1g = INTR_RXQ; 1920 iaq->nirq += n1g * nrxq1g; 1921#ifdef DEV_NETMAP 1922 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1923#endif 1924 } else { 1925 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1926 iaq->nirq += n1g * nofldrxq1g; 1927#ifdef DEV_NETMAP 1928 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1929#endif 1930 } 1931 if (iaq->nirq <= navail && 1932 (itype != INTR_MSI || powerof2(iaq->nirq))) 1933 goto allocate; 1934 1935 /* 1936 * Next best option: an interrupt vector for errors, one for the 1937 * firmware event queue, and at least one per port. At this 1938 * point we know we'll have to downsize nrxq and/or nofldrxq 1939 * and/or nnmrxq to fit what's available to us. 1940 */ 1941 iaq->nirq = T4_EXTRA_INTR; 1942 iaq->nirq += n10g + n1g; 1943 if (iaq->nirq <= navail) { 1944 int leftover = navail - iaq->nirq; 1945 1946 if (n10g > 0) { 1947 int target = max(nrxq10g, nofldrxq10g); 1948 1949 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1950 INTR_RXQ : INTR_OFLD_RXQ; 1951 1952 n = 1; 1953 while (n < target && leftover >= n10g) { 1954 leftover -= n10g; 1955 iaq->nirq += n10g; 1956 n++; 1957 } 1958 iaq->nrxq10g = min(n, nrxq10g); 1959#ifdef TCP_OFFLOAD 1960 iaq->nofldrxq10g = min(n, nofldrxq10g); 1961#endif 1962#ifdef DEV_NETMAP 1963 iaq->nnmrxq10g = min(n, nnmrxq10g); 1964#endif 1965 } 1966 1967 if (n1g > 0) { 1968 int target = max(nrxq1g, nofldrxq1g); 1969 1970 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1971 INTR_RXQ : INTR_OFLD_RXQ; 1972 1973 n = 1; 1974 while (n < target && leftover >= n1g) { 1975 leftover -= n1g; 1976 iaq->nirq += n1g; 1977 n++; 1978 } 1979 iaq->nrxq1g = min(n, nrxq1g); 1980#ifdef TCP_OFFLOAD 1981 iaq->nofldrxq1g = min(n, nofldrxq1g); 1982#endif 1983#ifdef DEV_NETMAP 1984 iaq->nnmrxq1g = min(n, nnmrxq1g); 1985#endif 1986 } 1987 1988 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1989 goto allocate; 1990 } 1991 1992 /* 1993 * Least desirable option: one interrupt vector for everything. 1994 */ 1995 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1996 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 1997#ifdef TCP_OFFLOAD 1998 if (is_offload(sc)) 1999 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2000#endif 2001#ifdef DEV_NETMAP 2002 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2003#endif 2004 2005allocate: 2006 navail = iaq->nirq; 2007 rc = 0; 2008 if (itype == INTR_MSIX) 2009 rc = pci_alloc_msix(sc->dev, &navail); 2010 else if (itype == INTR_MSI) 2011 rc = pci_alloc_msi(sc->dev, &navail); 2012 2013 if (rc == 0) { 2014 if (navail == iaq->nirq) 2015 return (0); 2016 2017 /* 2018 * Didn't get the number requested. Use whatever number 2019 * the kernel is willing to allocate (it's in navail). 2020 */ 2021 device_printf(sc->dev, "fewer vectors than requested, " 2022 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2023 itype, iaq->nirq, navail); 2024 pci_release_msi(sc->dev); 2025 goto restart; 2026 } 2027 2028 device_printf(sc->dev, 2029 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2030 itype, rc, iaq->nirq, navail); 2031 } 2032 2033 device_printf(sc->dev, 2034 "failed to find a usable interrupt type. " 2035 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2036 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2037 2038 return (ENXIO); 2039} 2040 2041#define FW_VERSION(chip) ( \ 2042 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2043 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2044 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2045 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2046#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2047 2048struct fw_info { 2049 uint8_t chip; 2050 char *kld_name; 2051 char *fw_mod_name; 2052 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2053} fw_info[] = { 2054 { 2055 .chip = CHELSIO_T4, 2056 .kld_name = "t4fw_cfg", 2057 .fw_mod_name = "t4fw", 2058 .fw_hdr = { 2059 .chip = FW_HDR_CHIP_T4, 2060 .fw_ver = htobe32_const(FW_VERSION(T4)), 2061 .intfver_nic = FW_INTFVER(T4, NIC), 2062 .intfver_vnic = FW_INTFVER(T4, VNIC), 2063 .intfver_ofld = FW_INTFVER(T4, OFLD), 2064 .intfver_ri = FW_INTFVER(T4, RI), 2065 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2066 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2067 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2068 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2069 }, 2070 }, { 2071 .chip = CHELSIO_T5, 2072 .kld_name = "t5fw_cfg", 2073 .fw_mod_name = "t5fw", 2074 .fw_hdr = { 2075 .chip = FW_HDR_CHIP_T5, 2076 .fw_ver = htobe32_const(FW_VERSION(T5)), 2077 .intfver_nic = FW_INTFVER(T5, NIC), 2078 .intfver_vnic = FW_INTFVER(T5, VNIC), 2079 .intfver_ofld = FW_INTFVER(T5, OFLD), 2080 .intfver_ri = FW_INTFVER(T5, RI), 2081 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2082 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2083 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2084 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2085 }, 2086 } 2087}; 2088 2089static struct fw_info * 2090find_fw_info(int chip) 2091{ 2092 int i; 2093 2094 for (i = 0; i < nitems(fw_info); i++) { 2095 if (fw_info[i].chip == chip) 2096 return (&fw_info[i]); 2097 } 2098 return (NULL); 2099} 2100 2101/* 2102 * Is the given firmware API compatible with the one the driver was compiled 2103 * with? 2104 */ 2105static int 2106fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2107{ 2108 2109 /* short circuit if it's the exact same firmware version */ 2110 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2111 return (1); 2112 2113 /* 2114 * XXX: Is this too conservative? Perhaps I should limit this to the 2115 * features that are supported in the driver. 2116 */ 2117#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2118 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2119 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2120 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2121 return (1); 2122#undef SAME_INTF 2123 2124 return (0); 2125} 2126 2127/* 2128 * The firmware in the KLD is usable, but should it be installed? This routine 2129 * explains itself in detail if it indicates the KLD firmware should be 2130 * installed. 2131 */ 2132static int 2133should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2134{ 2135 const char *reason; 2136 2137 if (!card_fw_usable) { 2138 reason = "incompatible or unusable"; 2139 goto install; 2140 } 2141 2142 if (k > c) { 2143 reason = "older than the version bundled with this driver"; 2144 goto install; 2145 } 2146 2147 if (t4_fw_install == 2 && k != c) { 2148 reason = "different than the version bundled with this driver"; 2149 goto install; 2150 } 2151 2152 return (0); 2153 2154install: 2155 if (t4_fw_install == 0) { 2156 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2157 "but the driver is prohibited from installing a different " 2158 "firmware on the card.\n", 2159 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2160 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2161 2162 return (0); 2163 } 2164 2165 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2166 "installing firmware %u.%u.%u.%u on card.\n", 2167 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2168 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2169 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2170 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2171 2172 return (1); 2173} 2174/* 2175 * Establish contact with the firmware and determine if we are the master driver 2176 * or not, and whether we are responsible for chip initialization. 2177 */ 2178static int 2179prep_firmware(struct adapter *sc) 2180{ 2181 const struct firmware *fw = NULL, *default_cfg; 2182 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2183 enum dev_state state; 2184 struct fw_info *fw_info; 2185 struct fw_hdr *card_fw; /* fw on the card */ 2186 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2187 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2188 against */ 2189 2190 /* Contact firmware. */ 2191 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2192 if (rc < 0 || state == DEV_STATE_ERR) { 2193 rc = -rc; 2194 device_printf(sc->dev, 2195 "failed to connect to the firmware: %d, %d.\n", rc, state); 2196 return (rc); 2197 } 2198 pf = rc; 2199 if (pf == sc->mbox) 2200 sc->flags |= MASTER_PF; 2201 else if (state == DEV_STATE_UNINIT) { 2202 /* 2203 * We didn't get to be the master so we definitely won't be 2204 * configuring the chip. It's a bug if someone else hasn't 2205 * configured it already. 2206 */ 2207 device_printf(sc->dev, "couldn't be master(%d), " 2208 "device not already initialized either(%d).\n", rc, state); 2209 return (EDOOFUS); 2210 } 2211 2212 /* This is the firmware whose headers the driver was compiled against */ 2213 fw_info = find_fw_info(chip_id(sc)); 2214 if (fw_info == NULL) { 2215 device_printf(sc->dev, 2216 "unable to look up firmware information for chip %d.\n", 2217 chip_id(sc)); 2218 return (EINVAL); 2219 } 2220 drv_fw = &fw_info->fw_hdr; 2221 2222 /* 2223 * The firmware KLD contains many modules. The KLD name is also the 2224 * name of the module that contains the default config file. 2225 */ 2226 default_cfg = firmware_get(fw_info->kld_name); 2227 2228 /* Read the header of the firmware on the card */ 2229 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2230 rc = -t4_read_flash(sc, FLASH_FW_START, 2231 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2232 if (rc == 0) 2233 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2234 else { 2235 device_printf(sc->dev, 2236 "Unable to read card's firmware header: %d\n", rc); 2237 card_fw_usable = 0; 2238 } 2239 2240 /* This is the firmware in the KLD */ 2241 fw = firmware_get(fw_info->fw_mod_name); 2242 if (fw != NULL) { 2243 kld_fw = (const void *)fw->data; 2244 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2245 } else { 2246 kld_fw = NULL; 2247 kld_fw_usable = 0; 2248 } 2249 2250 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2251 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2252 /* 2253 * Common case: the firmware on the card is an exact match and 2254 * the KLD is an exact match too, or the KLD is 2255 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2256 * here -- use cxgbetool loadfw if you want to reinstall the 2257 * same firmware as the one on the card. 2258 */ 2259 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2260 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2261 be32toh(card_fw->fw_ver))) { 2262 2263 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2264 if (rc != 0) { 2265 device_printf(sc->dev, 2266 "failed to install firmware: %d\n", rc); 2267 goto done; 2268 } 2269 2270 /* Installed successfully, update the cached header too. */ 2271 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2272 card_fw_usable = 1; 2273 need_fw_reset = 0; /* already reset as part of load_fw */ 2274 } 2275 2276 if (!card_fw_usable) { 2277 uint32_t d, c, k; 2278 2279 d = ntohl(drv_fw->fw_ver); 2280 c = ntohl(card_fw->fw_ver); 2281 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2282 2283 device_printf(sc->dev, "Cannot find a usable firmware: " 2284 "fw_install %d, chip state %d, " 2285 "driver compiled with %d.%d.%d.%d, " 2286 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2287 t4_fw_install, state, 2288 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2289 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2290 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2291 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2292 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2293 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2294 rc = EINVAL; 2295 goto done; 2296 } 2297 2298 /* We're using whatever's on the card and it's known to be good. */ 2299 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2300 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2301 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2302 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2303 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2304 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2305 t4_get_tp_version(sc, &sc->params.tp_vers); 2306 2307 /* Reset device */ 2308 if (need_fw_reset && 2309 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2310 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2311 if (rc != ETIMEDOUT && rc != EIO) 2312 t4_fw_bye(sc, sc->mbox); 2313 goto done; 2314 } 2315 sc->flags |= FW_OK; 2316 2317 rc = get_params__pre_init(sc); 2318 if (rc != 0) 2319 goto done; /* error message displayed already */ 2320 2321 /* Partition adapter resources as specified in the config file. */ 2322 if (state == DEV_STATE_UNINIT) { 2323 2324 KASSERT(sc->flags & MASTER_PF, 2325 ("%s: trying to change chip settings when not master.", 2326 __func__)); 2327 2328 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2329 if (rc != 0) 2330 goto done; /* error message displayed already */ 2331 2332 t4_tweak_chip_settings(sc); 2333 2334 /* get basic stuff going */ 2335 rc = -t4_fw_initialize(sc, sc->mbox); 2336 if (rc != 0) { 2337 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2338 goto done; 2339 } 2340 } else { 2341 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2342 sc->cfcsum = 0; 2343 } 2344 2345done: 2346 free(card_fw, M_CXGBE); 2347 if (fw != NULL) 2348 firmware_put(fw, FIRMWARE_UNLOAD); 2349 if (default_cfg != NULL) 2350 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2351 2352 return (rc); 2353} 2354 2355#define FW_PARAM_DEV(param) \ 2356 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2357 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2358#define FW_PARAM_PFVF(param) \ 2359 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2360 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2361 2362/* 2363 * Partition chip resources for use between various PFs, VFs, etc. 2364 */ 2365static int 2366partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2367 const char *name_prefix) 2368{ 2369 const struct firmware *cfg = NULL; 2370 int rc = 0; 2371 struct fw_caps_config_cmd caps; 2372 uint32_t mtype, moff, finicsum, cfcsum; 2373 2374 /* 2375 * Figure out what configuration file to use. Pick the default config 2376 * file for the card if the user hasn't specified one explicitly. 2377 */ 2378 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2379 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2380 /* Card specific overrides go here. */ 2381 if (pci_get_device(sc->dev) == 0x440a) 2382 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2383 if (is_fpga(sc)) 2384 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2385 } 2386 2387 /* 2388 * We need to load another module if the profile is anything except 2389 * "default" or "flash". 2390 */ 2391 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2392 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2393 char s[32]; 2394 2395 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2396 cfg = firmware_get(s); 2397 if (cfg == NULL) { 2398 if (default_cfg != NULL) { 2399 device_printf(sc->dev, 2400 "unable to load module \"%s\" for " 2401 "configuration profile \"%s\", will use " 2402 "the default config file instead.\n", 2403 s, sc->cfg_file); 2404 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2405 "%s", DEFAULT_CF); 2406 } else { 2407 device_printf(sc->dev, 2408 "unable to load module \"%s\" for " 2409 "configuration profile \"%s\", will use " 2410 "the config file on the card's flash " 2411 "instead.\n", s, sc->cfg_file); 2412 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2413 "%s", FLASH_CF); 2414 } 2415 } 2416 } 2417 2418 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2419 default_cfg == NULL) { 2420 device_printf(sc->dev, 2421 "default config file not available, will use the config " 2422 "file on the card's flash instead.\n"); 2423 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2424 } 2425 2426 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2427 u_int cflen, i, n; 2428 const uint32_t *cfdata; 2429 uint32_t param, val, addr, off, mw_base, mw_aperture; 2430 2431 KASSERT(cfg != NULL || default_cfg != NULL, 2432 ("%s: no config to upload", __func__)); 2433 2434 /* 2435 * Ask the firmware where it wants us to upload the config file. 2436 */ 2437 param = FW_PARAM_DEV(CF); 2438 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2439 if (rc != 0) { 2440 /* No support for config file? Shouldn't happen. */ 2441 device_printf(sc->dev, 2442 "failed to query config file location: %d.\n", rc); 2443 goto done; 2444 } 2445 mtype = G_FW_PARAMS_PARAM_Y(val); 2446 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2447 2448 /* 2449 * XXX: sheer laziness. We deliberately added 4 bytes of 2450 * useless stuffing/comments at the end of the config file so 2451 * it's ok to simply throw away the last remaining bytes when 2452 * the config file is not an exact multiple of 4. This also 2453 * helps with the validate_mt_off_len check. 2454 */ 2455 if (cfg != NULL) { 2456 cflen = cfg->datasize & ~3; 2457 cfdata = cfg->data; 2458 } else { 2459 cflen = default_cfg->datasize & ~3; 2460 cfdata = default_cfg->data; 2461 } 2462 2463 if (cflen > FLASH_CFG_MAX_SIZE) { 2464 device_printf(sc->dev, 2465 "config file too long (%d, max allowed is %d). " 2466 "Will try to use the config on the card, if any.\n", 2467 cflen, FLASH_CFG_MAX_SIZE); 2468 goto use_config_on_flash; 2469 } 2470 2471 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2472 if (rc != 0) { 2473 device_printf(sc->dev, 2474 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2475 "Will try to use the config on the card, if any.\n", 2476 __func__, mtype, moff, cflen, rc); 2477 goto use_config_on_flash; 2478 } 2479 2480 memwin_info(sc, 2, &mw_base, &mw_aperture); 2481 while (cflen) { 2482 off = position_memwin(sc, 2, addr); 2483 n = min(cflen, mw_aperture - off); 2484 for (i = 0; i < n; i += 4) 2485 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2486 cflen -= n; 2487 addr += n; 2488 } 2489 } else { 2490use_config_on_flash: 2491 mtype = FW_MEMTYPE_FLASH; 2492 moff = t4_flash_cfg_addr(sc); 2493 } 2494 2495 bzero(&caps, sizeof(caps)); 2496 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2497 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2498 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2499 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2500 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2501 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2502 if (rc != 0) { 2503 device_printf(sc->dev, 2504 "failed to pre-process config file: %d " 2505 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2506 goto done; 2507 } 2508 2509 finicsum = be32toh(caps.finicsum); 2510 cfcsum = be32toh(caps.cfcsum); 2511 if (finicsum != cfcsum) { 2512 device_printf(sc->dev, 2513 "WARNING: config file checksum mismatch: %08x %08x\n", 2514 finicsum, cfcsum); 2515 } 2516 sc->cfcsum = cfcsum; 2517 2518#define LIMIT_CAPS(x) do { \ 2519 caps.x &= htobe16(t4_##x##_allowed); \ 2520} while (0) 2521 2522 /* 2523 * Let the firmware know what features will (not) be used so it can tune 2524 * things accordingly. 2525 */ 2526 LIMIT_CAPS(linkcaps); 2527 LIMIT_CAPS(niccaps); 2528 LIMIT_CAPS(toecaps); 2529 LIMIT_CAPS(rdmacaps); 2530 LIMIT_CAPS(iscsicaps); 2531 LIMIT_CAPS(fcoecaps); 2532#undef LIMIT_CAPS 2533 2534 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2535 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2536 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2537 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2538 if (rc != 0) { 2539 device_printf(sc->dev, 2540 "failed to process config file: %d.\n", rc); 2541 } 2542done: 2543 if (cfg != NULL) 2544 firmware_put(cfg, FIRMWARE_UNLOAD); 2545 return (rc); 2546} 2547 2548/* 2549 * Retrieve parameters that are needed (or nice to have) very early. 2550 */ 2551static int 2552get_params__pre_init(struct adapter *sc) 2553{ 2554 int rc; 2555 uint32_t param[2], val[2]; 2556 struct fw_devlog_cmd cmd; 2557 struct devlog_params *dlog = &sc->params.devlog; 2558 2559 param[0] = FW_PARAM_DEV(PORTVEC); 2560 param[1] = FW_PARAM_DEV(CCLK); 2561 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2562 if (rc != 0) { 2563 device_printf(sc->dev, 2564 "failed to query parameters (pre_init): %d.\n", rc); 2565 return (rc); 2566 } 2567 2568 sc->params.portvec = val[0]; 2569 sc->params.nports = bitcount32(val[0]); 2570 sc->params.vpd.cclk = val[1]; 2571 2572 /* Read device log parameters. */ 2573 bzero(&cmd, sizeof(cmd)); 2574 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2575 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2576 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2577 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2578 if (rc != 0) { 2579 device_printf(sc->dev, 2580 "failed to get devlog parameters: %d.\n", rc); 2581 bzero(dlog, sizeof (*dlog)); 2582 rc = 0; /* devlog isn't critical for device operation */ 2583 } else { 2584 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2585 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2586 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2587 dlog->size = be32toh(cmd.memsize_devlog); 2588 } 2589 2590 return (rc); 2591} 2592 2593/* 2594 * Retrieve various parameters that are of interest to the driver. The device 2595 * has been initialized by the firmware at this point. 2596 */ 2597static int 2598get_params__post_init(struct adapter *sc) 2599{ 2600 int rc; 2601 uint32_t param[7], val[7]; 2602 struct fw_caps_config_cmd caps; 2603 2604 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2605 param[1] = FW_PARAM_PFVF(EQ_START); 2606 param[2] = FW_PARAM_PFVF(FILTER_START); 2607 param[3] = FW_PARAM_PFVF(FILTER_END); 2608 param[4] = FW_PARAM_PFVF(L2T_START); 2609 param[5] = FW_PARAM_PFVF(L2T_END); 2610 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2611 if (rc != 0) { 2612 device_printf(sc->dev, 2613 "failed to query parameters (post_init): %d.\n", rc); 2614 return (rc); 2615 } 2616 2617 sc->sge.iq_start = val[0]; 2618 sc->sge.eq_start = val[1]; 2619 sc->tids.ftid_base = val[2]; 2620 sc->tids.nftids = val[3] - val[2] + 1; 2621 sc->params.ftid_min = val[2]; 2622 sc->params.ftid_max = val[3]; 2623 sc->vres.l2t.start = val[4]; 2624 sc->vres.l2t.size = val[5] - val[4] + 1; 2625 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2626 ("%s: L2 table size (%u) larger than expected (%u)", 2627 __func__, sc->vres.l2t.size, L2T_SIZE)); 2628 2629 /* get capabilites */ 2630 bzero(&caps, sizeof(caps)); 2631 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2632 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2633 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2634 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2635 if (rc != 0) { 2636 device_printf(sc->dev, 2637 "failed to get card capabilities: %d.\n", rc); 2638 return (rc); 2639 } 2640 2641#define READ_CAPS(x) do { \ 2642 sc->x = htobe16(caps.x); \ 2643} while (0) 2644 READ_CAPS(linkcaps); 2645 READ_CAPS(niccaps); 2646 READ_CAPS(toecaps); 2647 READ_CAPS(rdmacaps); 2648 READ_CAPS(iscsicaps); 2649 READ_CAPS(fcoecaps); 2650 2651 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2652 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2653 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2654 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2655 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2656 if (rc != 0) { 2657 device_printf(sc->dev, 2658 "failed to query NIC parameters: %d.\n", rc); 2659 return (rc); 2660 } 2661 sc->tids.etid_base = val[0]; 2662 sc->params.etid_min = val[0]; 2663 sc->tids.netids = val[1] - val[0] + 1; 2664 sc->params.netids = sc->tids.netids; 2665 sc->params.eo_wr_cred = val[2]; 2666 sc->params.ethoffload = 1; 2667 } 2668 2669 if (sc->toecaps) { 2670 /* query offload-related parameters */ 2671 param[0] = FW_PARAM_DEV(NTID); 2672 param[1] = FW_PARAM_PFVF(SERVER_START); 2673 param[2] = FW_PARAM_PFVF(SERVER_END); 2674 param[3] = FW_PARAM_PFVF(TDDP_START); 2675 param[4] = FW_PARAM_PFVF(TDDP_END); 2676 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2677 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2678 if (rc != 0) { 2679 device_printf(sc->dev, 2680 "failed to query TOE parameters: %d.\n", rc); 2681 return (rc); 2682 } 2683 sc->tids.ntids = val[0]; 2684 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2685 sc->tids.stid_base = val[1]; 2686 sc->tids.nstids = val[2] - val[1] + 1; 2687 sc->vres.ddp.start = val[3]; 2688 sc->vres.ddp.size = val[4] - val[3] + 1; 2689 sc->params.ofldq_wr_cred = val[5]; 2690 sc->params.offload = 1; 2691 } 2692 if (sc->rdmacaps) { 2693 param[0] = FW_PARAM_PFVF(STAG_START); 2694 param[1] = FW_PARAM_PFVF(STAG_END); 2695 param[2] = FW_PARAM_PFVF(RQ_START); 2696 param[3] = FW_PARAM_PFVF(RQ_END); 2697 param[4] = FW_PARAM_PFVF(PBL_START); 2698 param[5] = FW_PARAM_PFVF(PBL_END); 2699 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2700 if (rc != 0) { 2701 device_printf(sc->dev, 2702 "failed to query RDMA parameters(1): %d.\n", rc); 2703 return (rc); 2704 } 2705 sc->vres.stag.start = val[0]; 2706 sc->vres.stag.size = val[1] - val[0] + 1; 2707 sc->vres.rq.start = val[2]; 2708 sc->vres.rq.size = val[3] - val[2] + 1; 2709 sc->vres.pbl.start = val[4]; 2710 sc->vres.pbl.size = val[5] - val[4] + 1; 2711 2712 param[0] = FW_PARAM_PFVF(SQRQ_START); 2713 param[1] = FW_PARAM_PFVF(SQRQ_END); 2714 param[2] = FW_PARAM_PFVF(CQ_START); 2715 param[3] = FW_PARAM_PFVF(CQ_END); 2716 param[4] = FW_PARAM_PFVF(OCQ_START); 2717 param[5] = FW_PARAM_PFVF(OCQ_END); 2718 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2719 if (rc != 0) { 2720 device_printf(sc->dev, 2721 "failed to query RDMA parameters(2): %d.\n", rc); 2722 return (rc); 2723 } 2724 sc->vres.qp.start = val[0]; 2725 sc->vres.qp.size = val[1] - val[0] + 1; 2726 sc->vres.cq.start = val[2]; 2727 sc->vres.cq.size = val[3] - val[2] + 1; 2728 sc->vres.ocq.start = val[4]; 2729 sc->vres.ocq.size = val[5] - val[4] + 1; 2730 } 2731 if (sc->iscsicaps) { 2732 param[0] = FW_PARAM_PFVF(ISCSI_START); 2733 param[1] = FW_PARAM_PFVF(ISCSI_END); 2734 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2735 if (rc != 0) { 2736 device_printf(sc->dev, 2737 "failed to query iSCSI parameters: %d.\n", rc); 2738 return (rc); 2739 } 2740 sc->vres.iscsi.start = val[0]; 2741 sc->vres.iscsi.size = val[1] - val[0] + 1; 2742 } 2743 2744 /* 2745 * We've got the params we wanted to query via the firmware. Now grab 2746 * some others directly from the chip. 2747 */ 2748 rc = t4_read_chip_settings(sc); 2749 2750 return (rc); 2751} 2752 2753static int 2754set_params__post_init(struct adapter *sc) 2755{ 2756 uint32_t param, val; 2757 2758 /* ask for encapsulated CPLs */ 2759 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2760 val = 1; 2761 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2762 2763 return (0); 2764} 2765 2766#undef FW_PARAM_PFVF 2767#undef FW_PARAM_DEV 2768 2769static void 2770t4_set_desc(struct adapter *sc) 2771{ 2772 char buf[128]; 2773 struct adapter_params *p = &sc->params; 2774 2775 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2776 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2777 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2778 2779 device_set_desc_copy(sc->dev, buf); 2780} 2781 2782static void 2783build_medialist(struct port_info *pi, struct ifmedia *media) 2784{ 2785 int data, m; 2786 2787 PORT_LOCK(pi); 2788 2789 ifmedia_removeall(media); 2790 2791 m = IFM_ETHER | IFM_FDX; 2792 data = (pi->port_type << 8) | pi->mod_type; 2793 2794 switch(pi->port_type) { 2795 case FW_PORT_TYPE_BT_XFI: 2796 case FW_PORT_TYPE_BT_XAUI: 2797 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2798 /* fall through */ 2799 2800 case FW_PORT_TYPE_BT_SGMII: 2801 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2802 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2803 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2804 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2805 break; 2806 2807 case FW_PORT_TYPE_CX4: 2808 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2809 ifmedia_set(media, m | IFM_10G_CX4); 2810 break; 2811 2812 case FW_PORT_TYPE_QSFP_10G: 2813 case FW_PORT_TYPE_SFP: 2814 case FW_PORT_TYPE_FIBER_XFI: 2815 case FW_PORT_TYPE_FIBER_XAUI: 2816 switch (pi->mod_type) { 2817 2818 case FW_PORT_MOD_TYPE_LR: 2819 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2820 ifmedia_set(media, m | IFM_10G_LR); 2821 break; 2822 2823 case FW_PORT_MOD_TYPE_SR: 2824 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2825 ifmedia_set(media, m | IFM_10G_SR); 2826 break; 2827 2828 case FW_PORT_MOD_TYPE_LRM: 2829 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2830 ifmedia_set(media, m | IFM_10G_LRM); 2831 break; 2832 2833 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2834 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2835 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2836 ifmedia_set(media, m | IFM_10G_TWINAX); 2837 break; 2838 2839 case FW_PORT_MOD_TYPE_NONE: 2840 m &= ~IFM_FDX; 2841 ifmedia_add(media, m | IFM_NONE, data, NULL); 2842 ifmedia_set(media, m | IFM_NONE); 2843 break; 2844 2845 case FW_PORT_MOD_TYPE_NA: 2846 case FW_PORT_MOD_TYPE_ER: 2847 default: 2848 device_printf(pi->dev, 2849 "unknown port_type (%d), mod_type (%d)\n", 2850 pi->port_type, pi->mod_type); 2851 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2852 ifmedia_set(media, m | IFM_UNKNOWN); 2853 break; 2854 } 2855 break; 2856 2857 case FW_PORT_TYPE_QSFP: 2858 switch (pi->mod_type) { 2859 2860 case FW_PORT_MOD_TYPE_LR: 2861 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2862 ifmedia_set(media, m | IFM_40G_LR4); 2863 break; 2864 2865 case FW_PORT_MOD_TYPE_SR: 2866 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2867 ifmedia_set(media, m | IFM_40G_SR4); 2868 break; 2869 2870 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2871 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2872 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2873 ifmedia_set(media, m | IFM_40G_CR4); 2874 break; 2875 2876 case FW_PORT_MOD_TYPE_NONE: 2877 m &= ~IFM_FDX; 2878 ifmedia_add(media, m | IFM_NONE, data, NULL); 2879 ifmedia_set(media, m | IFM_NONE); 2880 break; 2881 2882 default: 2883 device_printf(pi->dev, 2884 "unknown port_type (%d), mod_type (%d)\n", 2885 pi->port_type, pi->mod_type); 2886 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2887 ifmedia_set(media, m | IFM_UNKNOWN); 2888 break; 2889 } 2890 break; 2891 2892 default: 2893 device_printf(pi->dev, 2894 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2895 pi->mod_type); 2896 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2897 ifmedia_set(media, m | IFM_UNKNOWN); 2898 break; 2899 } 2900 2901 PORT_UNLOCK(pi); 2902} 2903 2904#define FW_MAC_EXACT_CHUNK 7 2905 2906/* 2907 * Program the port's XGMAC based on parameters in ifnet. The caller also 2908 * indicates which parameters should be programmed (the rest are left alone). 2909 */ 2910int 2911update_mac_settings(struct ifnet *ifp, int flags) 2912{ 2913 int rc = 0; 2914 struct port_info *pi = ifp->if_softc; 2915 struct adapter *sc = pi->adapter; 2916 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2917 uint16_t viid = 0xffff; 2918 int16_t *xact_addr_filt = NULL; 2919 2920 ASSERT_SYNCHRONIZED_OP(sc); 2921 KASSERT(flags, ("%s: not told what to update.", __func__)); 2922 2923 if (ifp == pi->ifp) { 2924 viid = pi->viid; 2925 xact_addr_filt = &pi->xact_addr_filt; 2926 } 2927#ifdef DEV_NETMAP 2928 else if (ifp == pi->nm_ifp) { 2929 viid = pi->nm_viid; 2930 xact_addr_filt = &pi->nm_xact_addr_filt; 2931 } 2932#endif 2933 if (flags & XGMAC_MTU) 2934 mtu = ifp->if_mtu; 2935 2936 if (flags & XGMAC_PROMISC) 2937 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2938 2939 if (flags & XGMAC_ALLMULTI) 2940 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2941 2942 if (flags & XGMAC_VLANEX) 2943 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2944 2945 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2946 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2947 1, vlanex, false); 2948 if (rc) { 2949 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2950 rc); 2951 return (rc); 2952 } 2953 } 2954 2955 if (flags & XGMAC_UCADDR) { 2956 uint8_t ucaddr[ETHER_ADDR_LEN]; 2957 2958 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2959 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2960 true, true); 2961 if (rc < 0) { 2962 rc = -rc; 2963 if_printf(ifp, "change_mac failed: %d\n", rc); 2964 return (rc); 2965 } else { 2966 *xact_addr_filt = rc; 2967 rc = 0; 2968 } 2969 } 2970 2971 if (flags & XGMAC_MCADDRS) { 2972 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2973 int del = 1; 2974 uint64_t hash = 0; 2975 struct ifmultiaddr *ifma; 2976 int i = 0, j; 2977 2978 if_maddr_rlock(ifp); 2979 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2980 if (ifma->ifma_addr->sa_family != AF_LINK) 2981 continue; 2982 mcaddr[i] = 2983 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2984 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 2985 i++; 2986 2987 if (i == FW_MAC_EXACT_CHUNK) { 2988 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 2989 i, mcaddr, NULL, &hash, 0); 2990 if (rc < 0) { 2991 rc = -rc; 2992 for (j = 0; j < i; j++) { 2993 if_printf(ifp, 2994 "failed to add mc address" 2995 " %02x:%02x:%02x:" 2996 "%02x:%02x:%02x rc=%d\n", 2997 mcaddr[j][0], mcaddr[j][1], 2998 mcaddr[j][2], mcaddr[j][3], 2999 mcaddr[j][4], mcaddr[j][5], 3000 rc); 3001 } 3002 goto mcfail; 3003 } 3004 del = 0; 3005 i = 0; 3006 } 3007 } 3008 if (i > 0) { 3009 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3010 mcaddr, NULL, &hash, 0); 3011 if (rc < 0) { 3012 rc = -rc; 3013 for (j = 0; j < i; j++) { 3014 if_printf(ifp, 3015 "failed to add mc address" 3016 " %02x:%02x:%02x:" 3017 "%02x:%02x:%02x rc=%d\n", 3018 mcaddr[j][0], mcaddr[j][1], 3019 mcaddr[j][2], mcaddr[j][3], 3020 mcaddr[j][4], mcaddr[j][5], 3021 rc); 3022 } 3023 goto mcfail; 3024 } 3025 } 3026 3027 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3028 if (rc != 0) 3029 if_printf(ifp, "failed to set mc address hash: %d", rc); 3030mcfail: 3031 if_maddr_runlock(ifp); 3032 } 3033 3034 return (rc); 3035} 3036 3037/* 3038 * {begin|end}_synchronized_op must be called from the same thread. 3039 */ 3040int 3041begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3042 char *wmesg) 3043{ 3044 int rc, pri; 3045 3046#ifdef WITNESS 3047 /* the caller thinks it's ok to sleep, but is it really? */ 3048 if (flags & SLEEP_OK) 3049 pause("t4slptst", 1); 3050#endif 3051 3052 if (INTR_OK) 3053 pri = PCATCH; 3054 else 3055 pri = 0; 3056 3057 ADAPTER_LOCK(sc); 3058 for (;;) { 3059 3060 if (pi && IS_DOOMED(pi)) { 3061 rc = ENXIO; 3062 goto done; 3063 } 3064 3065 if (!IS_BUSY(sc)) { 3066 rc = 0; 3067 break; 3068 } 3069 3070 if (!(flags & SLEEP_OK)) { 3071 rc = EBUSY; 3072 goto done; 3073 } 3074 3075 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3076 rc = EINTR; 3077 goto done; 3078 } 3079 } 3080 3081 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3082 SET_BUSY(sc); 3083#ifdef INVARIANTS 3084 sc->last_op = wmesg; 3085 sc->last_op_thr = curthread; 3086#endif 3087 3088done: 3089 if (!(flags & HOLD_LOCK) || rc) 3090 ADAPTER_UNLOCK(sc); 3091 3092 return (rc); 3093} 3094 3095/* 3096 * {begin|end}_synchronized_op must be called from the same thread. 3097 */ 3098void 3099end_synchronized_op(struct adapter *sc, int flags) 3100{ 3101 3102 if (flags & LOCK_HELD) 3103 ADAPTER_LOCK_ASSERT_OWNED(sc); 3104 else 3105 ADAPTER_LOCK(sc); 3106 3107 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3108 CLR_BUSY(sc); 3109 wakeup(&sc->flags); 3110 ADAPTER_UNLOCK(sc); 3111} 3112 3113static int 3114cxgbe_init_synchronized(struct port_info *pi) 3115{ 3116 struct adapter *sc = pi->adapter; 3117 struct ifnet *ifp = pi->ifp; 3118 int rc = 0, i; 3119 struct sge_txq *txq; 3120 3121 ASSERT_SYNCHRONIZED_OP(sc); 3122 3123 if (isset(&sc->open_device_map, pi->port_id)) { 3124 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3125 ("mismatch between open_device_map and if_drv_flags")); 3126 return (0); /* already running */ 3127 } 3128 3129 if (!(sc->flags & FULL_INIT_DONE) && 3130 ((rc = adapter_full_init(sc)) != 0)) 3131 return (rc); /* error message displayed already */ 3132 3133 if (!(pi->flags & PORT_INIT_DONE) && 3134 ((rc = port_full_init(pi)) != 0)) 3135 return (rc); /* error message displayed already */ 3136 3137 rc = update_mac_settings(ifp, XGMAC_ALL); 3138 if (rc) 3139 goto done; /* error message displayed already */ 3140 3141 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3142 if (rc != 0) { 3143 if_printf(ifp, "enable_vi failed: %d\n", rc); 3144 goto done; 3145 } 3146 3147 /* 3148 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3149 * if this changes. 3150 */ 3151 3152 for_each_txq(pi, i, txq) { 3153 TXQ_LOCK(txq); 3154 txq->eq.flags |= EQ_ENABLED; 3155 TXQ_UNLOCK(txq); 3156 } 3157 3158 /* 3159 * The first iq of the first port to come up is used for tracing. 3160 */ 3161 if (sc->traceq < 0) { 3162 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3163 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3164 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3165 V_QUEUENUMBER(sc->traceq)); 3166 pi->flags |= HAS_TRACEQ; 3167 } 3168 3169 /* all ok */ 3170 setbit(&sc->open_device_map, pi->port_id); 3171 PORT_LOCK(pi); 3172 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3173 PORT_UNLOCK(pi); 3174 3175 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3176done: 3177 if (rc != 0) 3178 cxgbe_uninit_synchronized(pi); 3179 3180 return (rc); 3181} 3182 3183/* 3184 * Idempotent. 3185 */ 3186static int 3187cxgbe_uninit_synchronized(struct port_info *pi) 3188{ 3189 struct adapter *sc = pi->adapter; 3190 struct ifnet *ifp = pi->ifp; 3191 int rc, i; 3192 struct sge_txq *txq; 3193 3194 ASSERT_SYNCHRONIZED_OP(sc); 3195 3196 /* 3197 * Disable the VI so that all its data in either direction is discarded 3198 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3199 * tick) intact as the TP can deliver negative advice or data that it's 3200 * holding in its RAM (for an offloaded connection) even after the VI is 3201 * disabled. 3202 */ 3203 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3204 if (rc) { 3205 if_printf(ifp, "disable_vi failed: %d\n", rc); 3206 return (rc); 3207 } 3208 3209 for_each_txq(pi, i, txq) { 3210 TXQ_LOCK(txq); 3211 txq->eq.flags &= ~EQ_ENABLED; 3212 TXQ_UNLOCK(txq); 3213 } 3214 3215 clrbit(&sc->open_device_map, pi->port_id); 3216 PORT_LOCK(pi); 3217 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3218 PORT_UNLOCK(pi); 3219 3220 pi->link_cfg.link_ok = 0; 3221 pi->link_cfg.speed = 0; 3222 pi->linkdnrc = -1; 3223 t4_os_link_changed(sc, pi->port_id, 0, -1); 3224 3225 return (0); 3226} 3227 3228/* 3229 * It is ok for this function to fail midway and return right away. t4_detach 3230 * will walk the entire sc->irq list and clean up whatever is valid. 3231 */ 3232static int 3233setup_intr_handlers(struct adapter *sc) 3234{ 3235 int rc, rid, p, q; 3236 char s[8]; 3237 struct irq *irq; 3238 struct port_info *pi; 3239 struct sge_rxq *rxq; 3240#ifdef TCP_OFFLOAD 3241 struct sge_ofld_rxq *ofld_rxq; 3242#endif 3243#ifdef DEV_NETMAP 3244 struct sge_nm_rxq *nm_rxq; 3245#endif 3246 3247 /* 3248 * Setup interrupts. 3249 */ 3250 irq = &sc->irq[0]; 3251 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3252 if (sc->intr_count == 1) 3253 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3254 3255 /* Multiple interrupts. */ 3256 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3257 ("%s: too few intr.", __func__)); 3258 3259 /* The first one is always error intr */ 3260 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3261 if (rc != 0) 3262 return (rc); 3263 irq++; 3264 rid++; 3265 3266 /* The second one is always the firmware event queue */ 3267 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3268 if (rc != 0) 3269 return (rc); 3270 irq++; 3271 rid++; 3272 3273 for_each_port(sc, p) { 3274 pi = sc->port[p]; 3275 3276 if (pi->flags & INTR_RXQ) { 3277 for_each_rxq(pi, q, rxq) { 3278 snprintf(s, sizeof(s), "%d.%d", p, q); 3279 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3280 s); 3281 if (rc != 0) 3282 return (rc); 3283 irq++; 3284 rid++; 3285 } 3286 } 3287#ifdef TCP_OFFLOAD 3288 if (pi->flags & INTR_OFLD_RXQ) { 3289 for_each_ofld_rxq(pi, q, ofld_rxq) { 3290 snprintf(s, sizeof(s), "%d,%d", p, q); 3291 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3292 ofld_rxq, s); 3293 if (rc != 0) 3294 return (rc); 3295 irq++; 3296 rid++; 3297 } 3298 } 3299#endif 3300#ifdef DEV_NETMAP 3301 if (pi->flags & INTR_NM_RXQ) { 3302 for_each_nm_rxq(pi, q, nm_rxq) { 3303 snprintf(s, sizeof(s), "%d-%d", p, q); 3304 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3305 nm_rxq, s); 3306 if (rc != 0) 3307 return (rc); 3308 irq++; 3309 rid++; 3310 } 3311 } 3312#endif 3313 } 3314 MPASS(irq == &sc->irq[sc->intr_count]); 3315 3316 return (0); 3317} 3318 3319int 3320adapter_full_init(struct adapter *sc) 3321{ 3322 int rc, i; 3323 3324 ASSERT_SYNCHRONIZED_OP(sc); 3325 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3326 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3327 ("%s: FULL_INIT_DONE already", __func__)); 3328 3329 /* 3330 * queues that belong to the adapter (not any particular port). 3331 */ 3332 rc = t4_setup_adapter_queues(sc); 3333 if (rc != 0) 3334 goto done; 3335 3336 for (i = 0; i < nitems(sc->tq); i++) { 3337 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3338 taskqueue_thread_enqueue, &sc->tq[i]); 3339 if (sc->tq[i] == NULL) { 3340 device_printf(sc->dev, 3341 "failed to allocate task queue %d\n", i); 3342 rc = ENOMEM; 3343 goto done; 3344 } 3345 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3346 device_get_nameunit(sc->dev), i); 3347 } 3348 3349 t4_intr_enable(sc); 3350 sc->flags |= FULL_INIT_DONE; 3351done: 3352 if (rc != 0) 3353 adapter_full_uninit(sc); 3354 3355 return (rc); 3356} 3357 3358int 3359adapter_full_uninit(struct adapter *sc) 3360{ 3361 int i; 3362 3363 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3364 3365 t4_teardown_adapter_queues(sc); 3366 3367 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3368 taskqueue_free(sc->tq[i]); 3369 sc->tq[i] = NULL; 3370 } 3371 3372 sc->flags &= ~FULL_INIT_DONE; 3373 3374 return (0); 3375} 3376 3377int 3378port_full_init(struct port_info *pi) 3379{ 3380 struct adapter *sc = pi->adapter; 3381 struct ifnet *ifp = pi->ifp; 3382 uint16_t *rss; 3383 struct sge_rxq *rxq; 3384 int rc, i, j; 3385 3386 ASSERT_SYNCHRONIZED_OP(sc); 3387 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3388 ("%s: PORT_INIT_DONE already", __func__)); 3389 3390 sysctl_ctx_init(&pi->ctx); 3391 pi->flags |= PORT_SYSCTL_CTX; 3392 3393 /* 3394 * Allocate tx/rx/fl queues for this port. 3395 */ 3396 rc = t4_setup_port_queues(pi); 3397 if (rc != 0) 3398 goto done; /* error message displayed already */ 3399 3400 /* 3401 * Setup RSS for this port. Save a copy of the RSS table for later use. 3402 */ 3403 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3404 for (i = 0; i < pi->rss_size;) { 3405 for_each_rxq(pi, j, rxq) { 3406 rss[i++] = rxq->iq.abs_id; 3407 if (i == pi->rss_size) 3408 break; 3409 } 3410 } 3411 3412 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3413 pi->rss_size); 3414 if (rc != 0) { 3415 if_printf(ifp, "rss_config failed: %d\n", rc); 3416 goto done; 3417 } 3418 3419 pi->rss = rss; 3420 pi->flags |= PORT_INIT_DONE; 3421done: 3422 if (rc != 0) 3423 port_full_uninit(pi); 3424 3425 return (rc); 3426} 3427 3428/* 3429 * Idempotent. 3430 */ 3431int 3432port_full_uninit(struct port_info *pi) 3433{ 3434 struct adapter *sc = pi->adapter; 3435 int i; 3436 struct sge_rxq *rxq; 3437 struct sge_txq *txq; 3438#ifdef TCP_OFFLOAD 3439 struct sge_ofld_rxq *ofld_rxq; 3440 struct sge_wrq *ofld_txq; 3441#endif 3442 3443 if (pi->flags & PORT_INIT_DONE) { 3444 3445 /* Need to quiesce queues. */ 3446 3447 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3448 3449 for_each_txq(pi, i, txq) { 3450 quiesce_txq(sc, txq); 3451 } 3452 3453#ifdef TCP_OFFLOAD 3454 for_each_ofld_txq(pi, i, ofld_txq) { 3455 quiesce_wrq(sc, ofld_txq); 3456 } 3457#endif 3458 3459 for_each_rxq(pi, i, rxq) { 3460 quiesce_iq(sc, &rxq->iq); 3461 quiesce_fl(sc, &rxq->fl); 3462 } 3463 3464#ifdef TCP_OFFLOAD 3465 for_each_ofld_rxq(pi, i, ofld_rxq) { 3466 quiesce_iq(sc, &ofld_rxq->iq); 3467 quiesce_fl(sc, &ofld_rxq->fl); 3468 } 3469#endif 3470 free(pi->rss, M_CXGBE); 3471 } 3472 3473 t4_teardown_port_queues(pi); 3474 pi->flags &= ~PORT_INIT_DONE; 3475 3476 return (0); 3477} 3478 3479static void 3480quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3481{ 3482 struct sge_eq *eq = &txq->eq; 3483 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3484 3485 (void) sc; /* unused */ 3486 3487#ifdef INVARIANTS 3488 TXQ_LOCK(txq); 3489 MPASS((eq->flags & EQ_ENABLED) == 0); 3490 TXQ_UNLOCK(txq); 3491#endif 3492 3493 /* Wait for the mp_ring to empty. */ 3494 while (!mp_ring_is_idle(txq->r)) { 3495 mp_ring_check_drainage(txq->r, 0); 3496 pause("rquiesce", 1); 3497 } 3498 3499 /* Then wait for the hardware to finish. */ 3500 while (spg->cidx != htobe16(eq->pidx)) 3501 pause("equiesce", 1); 3502 3503 /* Finally, wait for the driver to reclaim all descriptors. */ 3504 while (eq->cidx != eq->pidx) 3505 pause("dquiesce", 1); 3506} 3507 3508static void 3509quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3510{ 3511 3512 /* XXXTX */ 3513} 3514 3515static void 3516quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3517{ 3518 (void) sc; /* unused */ 3519 3520 /* Synchronize with the interrupt handler */ 3521 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3522 pause("iqfree", 1); 3523} 3524 3525static void 3526quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3527{ 3528 mtx_lock(&sc->sfl_lock); 3529 FL_LOCK(fl); 3530 fl->flags |= FL_DOOMED; 3531 FL_UNLOCK(fl); 3532 mtx_unlock(&sc->sfl_lock); 3533 3534 callout_drain(&sc->sfl_callout); 3535 KASSERT((fl->flags & FL_STARVING) == 0, 3536 ("%s: still starving", __func__)); 3537} 3538 3539static int 3540t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3541 driver_intr_t *handler, void *arg, char *name) 3542{ 3543 int rc; 3544 3545 irq->rid = rid; 3546 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3547 RF_SHAREABLE | RF_ACTIVE); 3548 if (irq->res == NULL) { 3549 device_printf(sc->dev, 3550 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3551 return (ENOMEM); 3552 } 3553 3554 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3555 NULL, handler, arg, &irq->tag); 3556 if (rc != 0) { 3557 device_printf(sc->dev, 3558 "failed to setup interrupt for rid %d, name %s: %d\n", 3559 rid, name, rc); 3560 } else if (name) 3561 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3562 3563 return (rc); 3564} 3565 3566static int 3567t4_free_irq(struct adapter *sc, struct irq *irq) 3568{ 3569 if (irq->tag) 3570 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3571 if (irq->res) 3572 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3573 3574 bzero(irq, sizeof(*irq)); 3575 3576 return (0); 3577} 3578 3579static void 3580reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3581 unsigned int end) 3582{ 3583 uint32_t *p = (uint32_t *)(buf + start); 3584 3585 for ( ; start <= end; start += sizeof(uint32_t)) 3586 *p++ = t4_read_reg(sc, start); 3587} 3588 3589static void 3590t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3591{ 3592 int i, n; 3593 const unsigned int *reg_ranges; 3594 static const unsigned int t4_reg_ranges[] = { 3595 0x1008, 0x1108, 3596 0x1180, 0x11b4, 3597 0x11fc, 0x123c, 3598 0x1300, 0x173c, 3599 0x1800, 0x18fc, 3600 0x3000, 0x30d8, 3601 0x30e0, 0x5924, 3602 0x5960, 0x59d4, 3603 0x5a00, 0x5af8, 3604 0x6000, 0x6098, 3605 0x6100, 0x6150, 3606 0x6200, 0x6208, 3607 0x6240, 0x6248, 3608 0x6280, 0x6338, 3609 0x6370, 0x638c, 3610 0x6400, 0x643c, 3611 0x6500, 0x6524, 3612 0x6a00, 0x6a38, 3613 0x6a60, 0x6a78, 3614 0x6b00, 0x6b84, 3615 0x6bf0, 0x6c84, 3616 0x6cf0, 0x6d84, 3617 0x6df0, 0x6e84, 3618 0x6ef0, 0x6f84, 3619 0x6ff0, 0x7084, 3620 0x70f0, 0x7184, 3621 0x71f0, 0x7284, 3622 0x72f0, 0x7384, 3623 0x73f0, 0x7450, 3624 0x7500, 0x7530, 3625 0x7600, 0x761c, 3626 0x7680, 0x76cc, 3627 0x7700, 0x7798, 3628 0x77c0, 0x77fc, 3629 0x7900, 0x79fc, 3630 0x7b00, 0x7c38, 3631 0x7d00, 0x7efc, 3632 0x8dc0, 0x8e1c, 3633 0x8e30, 0x8e78, 3634 0x8ea0, 0x8f6c, 3635 0x8fc0, 0x9074, 3636 0x90fc, 0x90fc, 3637 0x9400, 0x9458, 3638 0x9600, 0x96bc, 3639 0x9800, 0x9808, 3640 0x9820, 0x983c, 3641 0x9850, 0x9864, 3642 0x9c00, 0x9c6c, 3643 0x9c80, 0x9cec, 3644 0x9d00, 0x9d6c, 3645 0x9d80, 0x9dec, 3646 0x9e00, 0x9e6c, 3647 0x9e80, 0x9eec, 3648 0x9f00, 0x9f6c, 3649 0x9f80, 0x9fec, 3650 0xd004, 0xd03c, 3651 0xdfc0, 0xdfe0, 3652 0xe000, 0xea7c, 3653 0xf000, 0x11110, 3654 0x11118, 0x11190, 3655 0x19040, 0x1906c, 3656 0x19078, 0x19080, 3657 0x1908c, 0x19124, 3658 0x19150, 0x191b0, 3659 0x191d0, 0x191e8, 3660 0x19238, 0x1924c, 3661 0x193f8, 0x19474, 3662 0x19490, 0x194f8, 3663 0x19800, 0x19f30, 3664 0x1a000, 0x1a06c, 3665 0x1a0b0, 0x1a120, 3666 0x1a128, 0x1a138, 3667 0x1a190, 0x1a1c4, 3668 0x1a1fc, 0x1a1fc, 3669 0x1e040, 0x1e04c, 3670 0x1e284, 0x1e28c, 3671 0x1e2c0, 0x1e2c0, 3672 0x1e2e0, 0x1e2e0, 3673 0x1e300, 0x1e384, 3674 0x1e3c0, 0x1e3c8, 3675 0x1e440, 0x1e44c, 3676 0x1e684, 0x1e68c, 3677 0x1e6c0, 0x1e6c0, 3678 0x1e6e0, 0x1e6e0, 3679 0x1e700, 0x1e784, 3680 0x1e7c0, 0x1e7c8, 3681 0x1e840, 0x1e84c, 3682 0x1ea84, 0x1ea8c, 3683 0x1eac0, 0x1eac0, 3684 0x1eae0, 0x1eae0, 3685 0x1eb00, 0x1eb84, 3686 0x1ebc0, 0x1ebc8, 3687 0x1ec40, 0x1ec4c, 3688 0x1ee84, 0x1ee8c, 3689 0x1eec0, 0x1eec0, 3690 0x1eee0, 0x1eee0, 3691 0x1ef00, 0x1ef84, 3692 0x1efc0, 0x1efc8, 3693 0x1f040, 0x1f04c, 3694 0x1f284, 0x1f28c, 3695 0x1f2c0, 0x1f2c0, 3696 0x1f2e0, 0x1f2e0, 3697 0x1f300, 0x1f384, 3698 0x1f3c0, 0x1f3c8, 3699 0x1f440, 0x1f44c, 3700 0x1f684, 0x1f68c, 3701 0x1f6c0, 0x1f6c0, 3702 0x1f6e0, 0x1f6e0, 3703 0x1f700, 0x1f784, 3704 0x1f7c0, 0x1f7c8, 3705 0x1f840, 0x1f84c, 3706 0x1fa84, 0x1fa8c, 3707 0x1fac0, 0x1fac0, 3708 0x1fae0, 0x1fae0, 3709 0x1fb00, 0x1fb84, 3710 0x1fbc0, 0x1fbc8, 3711 0x1fc40, 0x1fc4c, 3712 0x1fe84, 0x1fe8c, 3713 0x1fec0, 0x1fec0, 3714 0x1fee0, 0x1fee0, 3715 0x1ff00, 0x1ff84, 3716 0x1ffc0, 0x1ffc8, 3717 0x20000, 0x2002c, 3718 0x20100, 0x2013c, 3719 0x20190, 0x201c8, 3720 0x20200, 0x20318, 3721 0x20400, 0x20528, 3722 0x20540, 0x20614, 3723 0x21000, 0x21040, 3724 0x2104c, 0x21060, 3725 0x210c0, 0x210ec, 3726 0x21200, 0x21268, 3727 0x21270, 0x21284, 3728 0x212fc, 0x21388, 3729 0x21400, 0x21404, 3730 0x21500, 0x21518, 3731 0x2152c, 0x2153c, 3732 0x21550, 0x21554, 3733 0x21600, 0x21600, 3734 0x21608, 0x21628, 3735 0x21630, 0x2163c, 3736 0x21700, 0x2171c, 3737 0x21780, 0x2178c, 3738 0x21800, 0x21c38, 3739 0x21c80, 0x21d7c, 3740 0x21e00, 0x21e04, 3741 0x22000, 0x2202c, 3742 0x22100, 0x2213c, 3743 0x22190, 0x221c8, 3744 0x22200, 0x22318, 3745 0x22400, 0x22528, 3746 0x22540, 0x22614, 3747 0x23000, 0x23040, 3748 0x2304c, 0x23060, 3749 0x230c0, 0x230ec, 3750 0x23200, 0x23268, 3751 0x23270, 0x23284, 3752 0x232fc, 0x23388, 3753 0x23400, 0x23404, 3754 0x23500, 0x23518, 3755 0x2352c, 0x2353c, 3756 0x23550, 0x23554, 3757 0x23600, 0x23600, 3758 0x23608, 0x23628, 3759 0x23630, 0x2363c, 3760 0x23700, 0x2371c, 3761 0x23780, 0x2378c, 3762 0x23800, 0x23c38, 3763 0x23c80, 0x23d7c, 3764 0x23e00, 0x23e04, 3765 0x24000, 0x2402c, 3766 0x24100, 0x2413c, 3767 0x24190, 0x241c8, 3768 0x24200, 0x24318, 3769 0x24400, 0x24528, 3770 0x24540, 0x24614, 3771 0x25000, 0x25040, 3772 0x2504c, 0x25060, 3773 0x250c0, 0x250ec, 3774 0x25200, 0x25268, 3775 0x25270, 0x25284, 3776 0x252fc, 0x25388, 3777 0x25400, 0x25404, 3778 0x25500, 0x25518, 3779 0x2552c, 0x2553c, 3780 0x25550, 0x25554, 3781 0x25600, 0x25600, 3782 0x25608, 0x25628, 3783 0x25630, 0x2563c, 3784 0x25700, 0x2571c, 3785 0x25780, 0x2578c, 3786 0x25800, 0x25c38, 3787 0x25c80, 0x25d7c, 3788 0x25e00, 0x25e04, 3789 0x26000, 0x2602c, 3790 0x26100, 0x2613c, 3791 0x26190, 0x261c8, 3792 0x26200, 0x26318, 3793 0x26400, 0x26528, 3794 0x26540, 0x26614, 3795 0x27000, 0x27040, 3796 0x2704c, 0x27060, 3797 0x270c0, 0x270ec, 3798 0x27200, 0x27268, 3799 0x27270, 0x27284, 3800 0x272fc, 0x27388, 3801 0x27400, 0x27404, 3802 0x27500, 0x27518, 3803 0x2752c, 0x2753c, 3804 0x27550, 0x27554, 3805 0x27600, 0x27600, 3806 0x27608, 0x27628, 3807 0x27630, 0x2763c, 3808 0x27700, 0x2771c, 3809 0x27780, 0x2778c, 3810 0x27800, 0x27c38, 3811 0x27c80, 0x27d7c, 3812 0x27e00, 0x27e04 3813 }; 3814 static const unsigned int t5_reg_ranges[] = { 3815 0x1008, 0x1148, 3816 0x1180, 0x11b4, 3817 0x11fc, 0x123c, 3818 0x1280, 0x173c, 3819 0x1800, 0x18fc, 3820 0x3000, 0x3028, 3821 0x3060, 0x30d8, 3822 0x30e0, 0x30fc, 3823 0x3140, 0x357c, 3824 0x35a8, 0x35cc, 3825 0x35ec, 0x35ec, 3826 0x3600, 0x5624, 3827 0x56cc, 0x575c, 3828 0x580c, 0x5814, 3829 0x5890, 0x58bc, 3830 0x5940, 0x59dc, 3831 0x59fc, 0x5a18, 3832 0x5a60, 0x5a9c, 3833 0x5b94, 0x5bfc, 3834 0x6000, 0x6040, 3835 0x6058, 0x614c, 3836 0x7700, 0x7798, 3837 0x77c0, 0x78fc, 3838 0x7b00, 0x7c54, 3839 0x7d00, 0x7efc, 3840 0x8dc0, 0x8de0, 3841 0x8df8, 0x8e84, 3842 0x8ea0, 0x8f84, 3843 0x8fc0, 0x90f8, 3844 0x9400, 0x9470, 3845 0x9600, 0x96f4, 3846 0x9800, 0x9808, 3847 0x9820, 0x983c, 3848 0x9850, 0x9864, 3849 0x9c00, 0x9c6c, 3850 0x9c80, 0x9cec, 3851 0x9d00, 0x9d6c, 3852 0x9d80, 0x9dec, 3853 0x9e00, 0x9e6c, 3854 0x9e80, 0x9eec, 3855 0x9f00, 0x9f6c, 3856 0x9f80, 0xa020, 3857 0xd004, 0xd03c, 3858 0xdfc0, 0xdfe0, 3859 0xe000, 0x11088, 3860 0x1109c, 0x11110, 3861 0x11118, 0x1117c, 3862 0x11190, 0x11204, 3863 0x19040, 0x1906c, 3864 0x19078, 0x19080, 3865 0x1908c, 0x19124, 3866 0x19150, 0x191b0, 3867 0x191d0, 0x191e8, 3868 0x19238, 0x19290, 3869 0x193f8, 0x19474, 3870 0x19490, 0x194cc, 3871 0x194f0, 0x194f8, 3872 0x19c00, 0x19c60, 3873 0x19c94, 0x19e10, 3874 0x19e50, 0x19f34, 3875 0x19f40, 0x19f50, 3876 0x19f90, 0x19fe4, 3877 0x1a000, 0x1a06c, 3878 0x1a0b0, 0x1a120, 3879 0x1a128, 0x1a138, 3880 0x1a190, 0x1a1c4, 3881 0x1a1fc, 0x1a1fc, 3882 0x1e008, 0x1e00c, 3883 0x1e040, 0x1e04c, 3884 0x1e284, 0x1e290, 3885 0x1e2c0, 0x1e2c0, 3886 0x1e2e0, 0x1e2e0, 3887 0x1e300, 0x1e384, 3888 0x1e3c0, 0x1e3c8, 3889 0x1e408, 0x1e40c, 3890 0x1e440, 0x1e44c, 3891 0x1e684, 0x1e690, 3892 0x1e6c0, 0x1e6c0, 3893 0x1e6e0, 0x1e6e0, 3894 0x1e700, 0x1e784, 3895 0x1e7c0, 0x1e7c8, 3896 0x1e808, 0x1e80c, 3897 0x1e840, 0x1e84c, 3898 0x1ea84, 0x1ea90, 3899 0x1eac0, 0x1eac0, 3900 0x1eae0, 0x1eae0, 3901 0x1eb00, 0x1eb84, 3902 0x1ebc0, 0x1ebc8, 3903 0x1ec08, 0x1ec0c, 3904 0x1ec40, 0x1ec4c, 3905 0x1ee84, 0x1ee90, 3906 0x1eec0, 0x1eec0, 3907 0x1eee0, 0x1eee0, 3908 0x1ef00, 0x1ef84, 3909 0x1efc0, 0x1efc8, 3910 0x1f008, 0x1f00c, 3911 0x1f040, 0x1f04c, 3912 0x1f284, 0x1f290, 3913 0x1f2c0, 0x1f2c0, 3914 0x1f2e0, 0x1f2e0, 3915 0x1f300, 0x1f384, 3916 0x1f3c0, 0x1f3c8, 3917 0x1f408, 0x1f40c, 3918 0x1f440, 0x1f44c, 3919 0x1f684, 0x1f690, 3920 0x1f6c0, 0x1f6c0, 3921 0x1f6e0, 0x1f6e0, 3922 0x1f700, 0x1f784, 3923 0x1f7c0, 0x1f7c8, 3924 0x1f808, 0x1f80c, 3925 0x1f840, 0x1f84c, 3926 0x1fa84, 0x1fa90, 3927 0x1fac0, 0x1fac0, 3928 0x1fae0, 0x1fae0, 3929 0x1fb00, 0x1fb84, 3930 0x1fbc0, 0x1fbc8, 3931 0x1fc08, 0x1fc0c, 3932 0x1fc40, 0x1fc4c, 3933 0x1fe84, 0x1fe90, 3934 0x1fec0, 0x1fec0, 3935 0x1fee0, 0x1fee0, 3936 0x1ff00, 0x1ff84, 3937 0x1ffc0, 0x1ffc8, 3938 0x30000, 0x30030, 3939 0x30100, 0x30144, 3940 0x30190, 0x301d0, 3941 0x30200, 0x30318, 3942 0x30400, 0x3052c, 3943 0x30540, 0x3061c, 3944 0x30800, 0x30834, 3945 0x308c0, 0x30908, 3946 0x30910, 0x309ac, 3947 0x30a00, 0x30a2c, 3948 0x30a44, 0x30a50, 3949 0x30a74, 0x30c24, 3950 0x30d00, 0x30d00, 3951 0x30d08, 0x30d14, 3952 0x30d1c, 0x30d20, 3953 0x30d3c, 0x30d50, 3954 0x31200, 0x3120c, 3955 0x31220, 0x31220, 3956 0x31240, 0x31240, 3957 0x31600, 0x3160c, 3958 0x31a00, 0x31a1c, 3959 0x31e00, 0x31e20, 3960 0x31e38, 0x31e3c, 3961 0x31e80, 0x31e80, 3962 0x31e88, 0x31ea8, 3963 0x31eb0, 0x31eb4, 3964 0x31ec8, 0x31ed4, 3965 0x31fb8, 0x32004, 3966 0x32200, 0x32200, 3967 0x32208, 0x32240, 3968 0x32248, 0x32280, 3969 0x32288, 0x322c0, 3970 0x322c8, 0x322fc, 3971 0x32600, 0x32630, 3972 0x32a00, 0x32abc, 3973 0x32b00, 0x32b70, 3974 0x33000, 0x33048, 3975 0x33060, 0x3309c, 3976 0x330f0, 0x33148, 3977 0x33160, 0x3319c, 3978 0x331f0, 0x332e4, 3979 0x332f8, 0x333e4, 3980 0x333f8, 0x33448, 3981 0x33460, 0x3349c, 3982 0x334f0, 0x33548, 3983 0x33560, 0x3359c, 3984 0x335f0, 0x336e4, 3985 0x336f8, 0x337e4, 3986 0x337f8, 0x337fc, 3987 0x33814, 0x33814, 3988 0x3382c, 0x3382c, 3989 0x33880, 0x3388c, 3990 0x338e8, 0x338ec, 3991 0x33900, 0x33948, 3992 0x33960, 0x3399c, 3993 0x339f0, 0x33ae4, 3994 0x33af8, 0x33b10, 3995 0x33b28, 0x33b28, 3996 0x33b3c, 0x33b50, 3997 0x33bf0, 0x33c10, 3998 0x33c28, 0x33c28, 3999 0x33c3c, 0x33c50, 4000 0x33cf0, 0x33cfc, 4001 0x34000, 0x34030, 4002 0x34100, 0x34144, 4003 0x34190, 0x341d0, 4004 0x34200, 0x34318, 4005 0x34400, 0x3452c, 4006 0x34540, 0x3461c, 4007 0x34800, 0x34834, 4008 0x348c0, 0x34908, 4009 0x34910, 0x349ac, 4010 0x34a00, 0x34a2c, 4011 0x34a44, 0x34a50, 4012 0x34a74, 0x34c24, 4013 0x34d00, 0x34d00, 4014 0x34d08, 0x34d14, 4015 0x34d1c, 0x34d20, 4016 0x34d3c, 0x34d50, 4017 0x35200, 0x3520c, 4018 0x35220, 0x35220, 4019 0x35240, 0x35240, 4020 0x35600, 0x3560c, 4021 0x35a00, 0x35a1c, 4022 0x35e00, 0x35e20, 4023 0x35e38, 0x35e3c, 4024 0x35e80, 0x35e80, 4025 0x35e88, 0x35ea8, 4026 0x35eb0, 0x35eb4, 4027 0x35ec8, 0x35ed4, 4028 0x35fb8, 0x36004, 4029 0x36200, 0x36200, 4030 0x36208, 0x36240, 4031 0x36248, 0x36280, 4032 0x36288, 0x362c0, 4033 0x362c8, 0x362fc, 4034 0x36600, 0x36630, 4035 0x36a00, 0x36abc, 4036 0x36b00, 0x36b70, 4037 0x37000, 0x37048, 4038 0x37060, 0x3709c, 4039 0x370f0, 0x37148, 4040 0x37160, 0x3719c, 4041 0x371f0, 0x372e4, 4042 0x372f8, 0x373e4, 4043 0x373f8, 0x37448, 4044 0x37460, 0x3749c, 4045 0x374f0, 0x37548, 4046 0x37560, 0x3759c, 4047 0x375f0, 0x376e4, 4048 0x376f8, 0x377e4, 4049 0x377f8, 0x377fc, 4050 0x37814, 0x37814, 4051 0x3782c, 0x3782c, 4052 0x37880, 0x3788c, 4053 0x378e8, 0x378ec, 4054 0x37900, 0x37948, 4055 0x37960, 0x3799c, 4056 0x379f0, 0x37ae4, 4057 0x37af8, 0x37b10, 4058 0x37b28, 0x37b28, 4059 0x37b3c, 0x37b50, 4060 0x37bf0, 0x37c10, 4061 0x37c28, 0x37c28, 4062 0x37c3c, 0x37c50, 4063 0x37cf0, 0x37cfc, 4064 0x38000, 0x38030, 4065 0x38100, 0x38144, 4066 0x38190, 0x381d0, 4067 0x38200, 0x38318, 4068 0x38400, 0x3852c, 4069 0x38540, 0x3861c, 4070 0x38800, 0x38834, 4071 0x388c0, 0x38908, 4072 0x38910, 0x389ac, 4073 0x38a00, 0x38a2c, 4074 0x38a44, 0x38a50, 4075 0x38a74, 0x38c24, 4076 0x38d00, 0x38d00, 4077 0x38d08, 0x38d14, 4078 0x38d1c, 0x38d20, 4079 0x38d3c, 0x38d50, 4080 0x39200, 0x3920c, 4081 0x39220, 0x39220, 4082 0x39240, 0x39240, 4083 0x39600, 0x3960c, 4084 0x39a00, 0x39a1c, 4085 0x39e00, 0x39e20, 4086 0x39e38, 0x39e3c, 4087 0x39e80, 0x39e80, 4088 0x39e88, 0x39ea8, 4089 0x39eb0, 0x39eb4, 4090 0x39ec8, 0x39ed4, 4091 0x39fb8, 0x3a004, 4092 0x3a200, 0x3a200, 4093 0x3a208, 0x3a240, 4094 0x3a248, 0x3a280, 4095 0x3a288, 0x3a2c0, 4096 0x3a2c8, 0x3a2fc, 4097 0x3a600, 0x3a630, 4098 0x3aa00, 0x3aabc, 4099 0x3ab00, 0x3ab70, 4100 0x3b000, 0x3b048, 4101 0x3b060, 0x3b09c, 4102 0x3b0f0, 0x3b148, 4103 0x3b160, 0x3b19c, 4104 0x3b1f0, 0x3b2e4, 4105 0x3b2f8, 0x3b3e4, 4106 0x3b3f8, 0x3b448, 4107 0x3b460, 0x3b49c, 4108 0x3b4f0, 0x3b548, 4109 0x3b560, 0x3b59c, 4110 0x3b5f0, 0x3b6e4, 4111 0x3b6f8, 0x3b7e4, 4112 0x3b7f8, 0x3b7fc, 4113 0x3b814, 0x3b814, 4114 0x3b82c, 0x3b82c, 4115 0x3b880, 0x3b88c, 4116 0x3b8e8, 0x3b8ec, 4117 0x3b900, 0x3b948, 4118 0x3b960, 0x3b99c, 4119 0x3b9f0, 0x3bae4, 4120 0x3baf8, 0x3bb10, 4121 0x3bb28, 0x3bb28, 4122 0x3bb3c, 0x3bb50, 4123 0x3bbf0, 0x3bc10, 4124 0x3bc28, 0x3bc28, 4125 0x3bc3c, 0x3bc50, 4126 0x3bcf0, 0x3bcfc, 4127 0x3c000, 0x3c030, 4128 0x3c100, 0x3c144, 4129 0x3c190, 0x3c1d0, 4130 0x3c200, 0x3c318, 4131 0x3c400, 0x3c52c, 4132 0x3c540, 0x3c61c, 4133 0x3c800, 0x3c834, 4134 0x3c8c0, 0x3c908, 4135 0x3c910, 0x3c9ac, 4136 0x3ca00, 0x3ca2c, 4137 0x3ca44, 0x3ca50, 4138 0x3ca74, 0x3cc24, 4139 0x3cd00, 0x3cd00, 4140 0x3cd08, 0x3cd14, 4141 0x3cd1c, 0x3cd20, 4142 0x3cd3c, 0x3cd50, 4143 0x3d200, 0x3d20c, 4144 0x3d220, 0x3d220, 4145 0x3d240, 0x3d240, 4146 0x3d600, 0x3d60c, 4147 0x3da00, 0x3da1c, 4148 0x3de00, 0x3de20, 4149 0x3de38, 0x3de3c, 4150 0x3de80, 0x3de80, 4151 0x3de88, 0x3dea8, 4152 0x3deb0, 0x3deb4, 4153 0x3dec8, 0x3ded4, 4154 0x3dfb8, 0x3e004, 4155 0x3e200, 0x3e200, 4156 0x3e208, 0x3e240, 4157 0x3e248, 0x3e280, 4158 0x3e288, 0x3e2c0, 4159 0x3e2c8, 0x3e2fc, 4160 0x3e600, 0x3e630, 4161 0x3ea00, 0x3eabc, 4162 0x3eb00, 0x3eb70, 4163 0x3f000, 0x3f048, 4164 0x3f060, 0x3f09c, 4165 0x3f0f0, 0x3f148, 4166 0x3f160, 0x3f19c, 4167 0x3f1f0, 0x3f2e4, 4168 0x3f2f8, 0x3f3e4, 4169 0x3f3f8, 0x3f448, 4170 0x3f460, 0x3f49c, 4171 0x3f4f0, 0x3f548, 4172 0x3f560, 0x3f59c, 4173 0x3f5f0, 0x3f6e4, 4174 0x3f6f8, 0x3f7e4, 4175 0x3f7f8, 0x3f7fc, 4176 0x3f814, 0x3f814, 4177 0x3f82c, 0x3f82c, 4178 0x3f880, 0x3f88c, 4179 0x3f8e8, 0x3f8ec, 4180 0x3f900, 0x3f948, 4181 0x3f960, 0x3f99c, 4182 0x3f9f0, 0x3fae4, 4183 0x3faf8, 0x3fb10, 4184 0x3fb28, 0x3fb28, 4185 0x3fb3c, 0x3fb50, 4186 0x3fbf0, 0x3fc10, 4187 0x3fc28, 0x3fc28, 4188 0x3fc3c, 0x3fc50, 4189 0x3fcf0, 0x3fcfc, 4190 0x40000, 0x4000c, 4191 0x40040, 0x40068, 4192 0x4007c, 0x40144, 4193 0x40180, 0x4018c, 4194 0x40200, 0x40298, 4195 0x402ac, 0x4033c, 4196 0x403f8, 0x403fc, 4197 0x41304, 0x413c4, 4198 0x41400, 0x4141c, 4199 0x41480, 0x414d0, 4200 0x44000, 0x44078, 4201 0x440c0, 0x44278, 4202 0x442c0, 0x44478, 4203 0x444c0, 0x44678, 4204 0x446c0, 0x44878, 4205 0x448c0, 0x449fc, 4206 0x45000, 0x45068, 4207 0x45080, 0x45084, 4208 0x450a0, 0x450b0, 4209 0x45200, 0x45268, 4210 0x45280, 0x45284, 4211 0x452a0, 0x452b0, 4212 0x460c0, 0x460e4, 4213 0x47000, 0x4708c, 4214 0x47200, 0x47250, 4215 0x47400, 0x47420, 4216 0x47600, 0x47618, 4217 0x47800, 0x47814, 4218 0x48000, 0x4800c, 4219 0x48040, 0x48068, 4220 0x4807c, 0x48144, 4221 0x48180, 0x4818c, 4222 0x48200, 0x48298, 4223 0x482ac, 0x4833c, 4224 0x483f8, 0x483fc, 4225 0x49304, 0x493c4, 4226 0x49400, 0x4941c, 4227 0x49480, 0x494d0, 4228 0x4c000, 0x4c078, 4229 0x4c0c0, 0x4c278, 4230 0x4c2c0, 0x4c478, 4231 0x4c4c0, 0x4c678, 4232 0x4c6c0, 0x4c878, 4233 0x4c8c0, 0x4c9fc, 4234 0x4d000, 0x4d068, 4235 0x4d080, 0x4d084, 4236 0x4d0a0, 0x4d0b0, 4237 0x4d200, 0x4d268, 4238 0x4d280, 0x4d284, 4239 0x4d2a0, 0x4d2b0, 4240 0x4e0c0, 0x4e0e4, 4241 0x4f000, 0x4f08c, 4242 0x4f200, 0x4f250, 4243 0x4f400, 0x4f420, 4244 0x4f600, 0x4f618, 4245 0x4f800, 0x4f814, 4246 0x50000, 0x500cc, 4247 0x50400, 0x50400, 4248 0x50800, 0x508cc, 4249 0x50c00, 0x50c00, 4250 0x51000, 0x5101c, 4251 0x51300, 0x51308, 4252 }; 4253 4254 if (is_t4(sc)) { 4255 reg_ranges = &t4_reg_ranges[0]; 4256 n = nitems(t4_reg_ranges); 4257 } else { 4258 reg_ranges = &t5_reg_ranges[0]; 4259 n = nitems(t5_reg_ranges); 4260 } 4261 4262 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4263 for (i = 0; i < n; i += 2) 4264 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4265} 4266 4267static void 4268cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4269{ 4270 struct ifnet *ifp = pi->ifp; 4271 struct sge_txq *txq; 4272 int i, drops; 4273 struct port_stats *s = &pi->stats; 4274 struct timeval tv; 4275 const struct timeval interval = {0, 250000}; /* 250ms */ 4276 4277 getmicrotime(&tv); 4278 timevalsub(&tv, &interval); 4279 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4280 return; 4281 4282 t4_get_port_stats(sc, pi->tx_chan, s); 4283 4284 ifp->if_opackets = s->tx_frames - s->tx_pause; 4285 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4286 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4287 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4288 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4289 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4290 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4291 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4292 s->rx_trunc3; 4293 for (i = 0; i < NCHAN; i++) { 4294 if (pi->rx_chan_map & (1 << i)) { 4295 uint32_t v; 4296 4297 mtx_lock(&sc->regwin_lock); 4298 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4299 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4300 mtx_unlock(&sc->regwin_lock); 4301 ifp->if_iqdrops += v; 4302 } 4303 } 4304 4305 drops = s->tx_drop; 4306 for_each_txq(pi, i, txq) 4307 drops += counter_u64_fetch(txq->r->drops); 4308 ifp->if_snd.ifq_drops = drops; 4309 4310 ifp->if_oerrors = s->tx_error_frames; 4311 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4312 s->rx_fcs_err + s->rx_len_err; 4313 4314 getmicrotime(&pi->last_refreshed); 4315} 4316 4317static void 4318cxgbe_tick(void *arg) 4319{ 4320 struct port_info *pi = arg; 4321 struct adapter *sc = pi->adapter; 4322 struct ifnet *ifp = pi->ifp; 4323 4324 PORT_LOCK(pi); 4325 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4326 PORT_UNLOCK(pi); 4327 return; /* without scheduling another callout */ 4328 } 4329 4330 cxgbe_refresh_stats(sc, pi); 4331 4332 callout_schedule(&pi->tick, hz); 4333 PORT_UNLOCK(pi); 4334} 4335 4336static void 4337cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4338{ 4339 struct ifnet *vlan; 4340 4341 if (arg != ifp || ifp->if_type != IFT_ETHER) 4342 return; 4343 4344 vlan = VLAN_DEVAT(ifp, vid); 4345 VLAN_SETCOOKIE(vlan, ifp); 4346} 4347 4348static int 4349cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4350{ 4351 4352#ifdef INVARIANTS 4353 panic("%s: opcode 0x%02x on iq %p with payload %p", 4354 __func__, rss->opcode, iq, m); 4355#else 4356 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4357 __func__, rss->opcode, iq, m); 4358 m_freem(m); 4359#endif 4360 return (EDOOFUS); 4361} 4362 4363int 4364t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4365{ 4366 uintptr_t *loc, new; 4367 4368 if (opcode >= nitems(sc->cpl_handler)) 4369 return (EINVAL); 4370 4371 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4372 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4373 atomic_store_rel_ptr(loc, new); 4374 4375 return (0); 4376} 4377 4378static int 4379an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4380{ 4381 4382#ifdef INVARIANTS 4383 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4384#else 4385 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4386 __func__, iq, ctrl); 4387#endif 4388 return (EDOOFUS); 4389} 4390 4391int 4392t4_register_an_handler(struct adapter *sc, an_handler_t h) 4393{ 4394 uintptr_t *loc, new; 4395 4396 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4397 loc = (uintptr_t *) &sc->an_handler; 4398 atomic_store_rel_ptr(loc, new); 4399 4400 return (0); 4401} 4402 4403static int 4404fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4405{ 4406 const struct cpl_fw6_msg *cpl = 4407 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4408 4409#ifdef INVARIANTS 4410 panic("%s: fw_msg type %d", __func__, cpl->type); 4411#else 4412 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4413#endif 4414 return (EDOOFUS); 4415} 4416 4417int 4418t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4419{ 4420 uintptr_t *loc, new; 4421 4422 if (type >= nitems(sc->fw_msg_handler)) 4423 return (EINVAL); 4424 4425 /* 4426 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4427 * handler dispatch table. Reject any attempt to install a handler for 4428 * this subtype. 4429 */ 4430 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4431 return (EINVAL); 4432 4433 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4434 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4435 atomic_store_rel_ptr(loc, new); 4436 4437 return (0); 4438} 4439 4440static int 4441t4_sysctls(struct adapter *sc) 4442{ 4443 struct sysctl_ctx_list *ctx; 4444 struct sysctl_oid *oid; 4445 struct sysctl_oid_list *children, *c0; 4446 static char *caps[] = { 4447 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4448 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4449 "\6HASHFILTER\7ETHOFLD", 4450 "\20\1TOE", /* caps[2] toecaps */ 4451 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4452 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4453 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4454 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4455 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4456 "\4PO_INITIAOR\5PO_TARGET" 4457 }; 4458 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4459 4460 ctx = device_get_sysctl_ctx(sc->dev); 4461 4462 /* 4463 * dev.t4nex.X. 4464 */ 4465 oid = device_get_sysctl_tree(sc->dev); 4466 c0 = children = SYSCTL_CHILDREN(oid); 4467 4468 sc->sc_do_rxcopy = 1; 4469 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4470 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4471 4472 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4473 sc->params.nports, "# of ports"); 4474 4475 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4476 NULL, chip_rev(sc), "chip hardware revision"); 4477 4478 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4479 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4480 4481 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4482 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4483 4484 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4485 sc->cfcsum, "config file checksum"); 4486 4487 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4488 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4489 sysctl_bitfield, "A", "available doorbells"); 4490 4491 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4492 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4493 sysctl_bitfield, "A", "available link capabilities"); 4494 4495 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4496 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4497 sysctl_bitfield, "A", "available NIC capabilities"); 4498 4499 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4500 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4501 sysctl_bitfield, "A", "available TCP offload capabilities"); 4502 4503 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4504 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4505 sysctl_bitfield, "A", "available RDMA capabilities"); 4506 4507 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4508 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4509 sysctl_bitfield, "A", "available iSCSI capabilities"); 4510 4511 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4512 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4513 sysctl_bitfield, "A", "available FCoE capabilities"); 4514 4515 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4516 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4517 4518 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4519 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4520 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4521 "interrupt holdoff timer values (us)"); 4522 4523 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4524 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4525 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4526 "interrupt holdoff packet counter values"); 4527 4528 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4529 NULL, sc->tids.nftids, "number of filters"); 4530 4531 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4532 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4533 "chip temperature (in Celsius)"); 4534 4535 t4_sge_sysctls(sc, ctx, children); 4536 4537 sc->lro_timeout = 100; 4538 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4539 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4540 4541#ifdef SBUF_DRAIN 4542 /* 4543 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4544 */ 4545 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4546 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4547 "logs and miscellaneous information"); 4548 children = SYSCTL_CHILDREN(oid); 4549 4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4551 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4552 sysctl_cctrl, "A", "congestion control"); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4556 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4560 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4564 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4565 4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4567 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4568 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4569 4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4571 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4572 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4573 4574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4575 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4576 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4577 4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4579 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4580 sysctl_cim_la, "A", "CIM logic analyzer"); 4581 4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4583 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4584 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4588 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4589 4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4591 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4592 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4593 4594 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4595 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4596 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4597 4598 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4599 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4600 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4601 4602 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4603 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4604 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4605 4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4607 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4608 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4609 4610 if (is_t5(sc)) { 4611 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4612 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4613 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4614 4615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4616 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4617 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4618 } 4619 4620 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4621 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4622 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4623 4624 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4625 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4626 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4627 4628 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4629 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4630 sysctl_cpl_stats, "A", "CPL statistics"); 4631 4632 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4633 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4634 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4635 4636 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4637 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4638 sysctl_devlog, "A", "firmware's device log"); 4639 4640 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4641 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4642 sysctl_fcoe_stats, "A", "FCoE statistics"); 4643 4644 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4645 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4646 sysctl_hw_sched, "A", "hardware scheduler "); 4647 4648 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4649 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4650 sysctl_l2t, "A", "hardware L2 table"); 4651 4652 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4653 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4654 sysctl_lb_stats, "A", "loopback statistics"); 4655 4656 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4657 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4658 sysctl_meminfo, "A", "memory regions"); 4659 4660 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4661 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4662 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4663 4664 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4665 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4666 sysctl_path_mtus, "A", "path MTUs"); 4667 4668 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4669 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4670 sysctl_pm_stats, "A", "PM statistics"); 4671 4672 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4673 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4674 sysctl_rdma_stats, "A", "RDMA statistics"); 4675 4676 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4677 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4678 sysctl_tcp_stats, "A", "TCP statistics"); 4679 4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4682 sysctl_tids, "A", "TID information"); 4683 4684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4685 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4686 sysctl_tp_err_stats, "A", "TP error statistics"); 4687 4688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4689 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4690 sysctl_tp_la, "A", "TP logic analyzer"); 4691 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4694 sysctl_tx_rate, "A", "Tx rate"); 4695 4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4698 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4699 4700 if (is_t5(sc)) { 4701 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4702 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4703 sysctl_wcwr_stats, "A", "write combined work requests"); 4704 } 4705#endif 4706 4707#ifdef TCP_OFFLOAD 4708 if (is_offload(sc)) { 4709 /* 4710 * dev.t4nex.X.toe. 4711 */ 4712 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4713 NULL, "TOE parameters"); 4714 children = SYSCTL_CHILDREN(oid); 4715 4716 sc->tt.sndbuf = 256 * 1024; 4717 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4718 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4719 4720 sc->tt.ddp = 0; 4721 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4722 &sc->tt.ddp, 0, "DDP allowed"); 4723 4724 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4725 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4726 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4727 4728 sc->tt.ddp_thres = 4729 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4730 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4731 &sc->tt.ddp_thres, 0, "DDP threshold"); 4732 4733 sc->tt.rx_coalesce = 1; 4734 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4735 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4736 4737 sc->tt.tx_align = 1; 4738 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4739 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4740 } 4741#endif 4742 4743 4744 return (0); 4745} 4746 4747static int 4748cxgbe_sysctls(struct port_info *pi) 4749{ 4750 struct sysctl_ctx_list *ctx; 4751 struct sysctl_oid *oid; 4752 struct sysctl_oid_list *children; 4753 struct adapter *sc = pi->adapter; 4754 4755 ctx = device_get_sysctl_ctx(pi->dev); 4756 4757 /* 4758 * dev.cxgbe.X. 4759 */ 4760 oid = device_get_sysctl_tree(pi->dev); 4761 children = SYSCTL_CHILDREN(oid); 4762 4763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4764 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4765 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4766 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4767 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4768 "PHY temperature (in Celsius)"); 4769 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4770 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4771 "PHY firmware version"); 4772 } 4773 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4774 &pi->nrxq, 0, "# of rx queues"); 4775 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4776 &pi->ntxq, 0, "# of tx queues"); 4777 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4778 &pi->first_rxq, 0, "index of first rx queue"); 4779 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4780 &pi->first_txq, 0, "index of first tx queue"); 4781 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4782 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4783 "Reserve queue 0 for non-flowid packets"); 4784 4785#ifdef TCP_OFFLOAD 4786 if (is_offload(sc)) { 4787 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4788 &pi->nofldrxq, 0, 4789 "# of rx queues for offloaded TCP connections"); 4790 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4791 &pi->nofldtxq, 0, 4792 "# of tx queues for offloaded TCP connections"); 4793 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4794 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4795 "index of first TOE rx queue"); 4796 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4797 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4798 "index of first TOE tx queue"); 4799 } 4800#endif 4801#ifdef DEV_NETMAP 4802 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4803 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4804 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4805 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4807 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4808 "index of first netmap rx queue"); 4809 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4810 CTLFLAG_RD, &pi->first_nm_txq, 0, 4811 "index of first netmap tx queue"); 4812#endif 4813 4814 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4815 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4816 "holdoff timer index"); 4817 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4818 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4819 "holdoff packet counter index"); 4820 4821 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4822 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4823 "rx queue size"); 4824 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4825 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4826 "tx queue size"); 4827 4828 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4829 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4830 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4831 4832 /* 4833 * dev.cxgbe.X.stats. 4834 */ 4835 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4836 NULL, "port statistics"); 4837 children = SYSCTL_CHILDREN(oid); 4838 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4839 &pi->tx_parse_error, 0, 4840 "# of tx packets with invalid length or # of segments"); 4841 4842#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4843 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4844 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4845 sysctl_handle_t4_reg64, "QU", desc) 4846 4847 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4848 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4849 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4850 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4851 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4853 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4854 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4855 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4856 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4857 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4858 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4859 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4860 "# of tx frames in this range", 4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4862 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4863 "# of tx frames in this range", 4864 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4865 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4866 "# of tx frames in this range", 4867 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4868 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4869 "# of tx frames in this range", 4870 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4871 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4872 "# of tx frames in this range", 4873 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4874 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4875 "# of tx frames in this range", 4876 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4877 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4878 "# of tx frames in this range", 4879 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4880 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4881 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4882 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4883 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4884 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4885 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4886 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4887 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4888 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4889 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4890 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4891 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4892 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4893 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4894 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4895 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4896 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4897 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4898 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4899 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4900 4901 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4903 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4905 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4907 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4916 "# of frames received with bad FCS", 4917 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4918 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4919 "# of frames received with length error", 4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4921 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4922 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4923 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4926 "# of rx frames in this range", 4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4928 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4929 "# of rx frames in this range", 4930 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4931 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4932 "# of rx frames in this range", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4935 "# of rx frames in this range", 4936 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4937 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4938 "# of rx frames in this range", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4941 "# of rx frames in this range", 4942 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4943 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4944 "# of rx frames in this range", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4954 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4956 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4958 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4960 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4961 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4962 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4963 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4964 4965#undef SYSCTL_ADD_T4_REG64 4966 4967#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4968 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4969 &pi->stats.name, desc) 4970 4971 /* We get these from port_stats and they may be stale by upto 1s */ 4972 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4973 "# drops due to buffer-group 0 overflows"); 4974 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4975 "# drops due to buffer-group 1 overflows"); 4976 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4977 "# drops due to buffer-group 2 overflows"); 4978 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4979 "# drops due to buffer-group 3 overflows"); 4980 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4981 "# of buffer-group 0 truncated packets"); 4982 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4983 "# of buffer-group 1 truncated packets"); 4984 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4985 "# of buffer-group 2 truncated packets"); 4986 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4987 "# of buffer-group 3 truncated packets"); 4988 4989#undef SYSCTL_ADD_T4_PORTSTAT 4990 4991 return (0); 4992} 4993 4994static int 4995sysctl_int_array(SYSCTL_HANDLER_ARGS) 4996{ 4997 int rc, *i, space = 0; 4998 struct sbuf sb; 4999 5000 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5001 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5002 if (space) 5003 sbuf_printf(&sb, " "); 5004 sbuf_printf(&sb, "%d", *i); 5005 space = 1; 5006 } 5007 sbuf_finish(&sb); 5008 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5009 sbuf_delete(&sb); 5010 return (rc); 5011} 5012 5013static int 5014sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5015{ 5016 int rc; 5017 struct sbuf *sb; 5018 5019 rc = sysctl_wire_old_buffer(req, 0); 5020 if (rc != 0) 5021 return(rc); 5022 5023 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5024 if (sb == NULL) 5025 return (ENOMEM); 5026 5027 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5028 rc = sbuf_finish(sb); 5029 sbuf_delete(sb); 5030 5031 return (rc); 5032} 5033 5034static int 5035sysctl_btphy(SYSCTL_HANDLER_ARGS) 5036{ 5037 struct port_info *pi = arg1; 5038 int op = arg2; 5039 struct adapter *sc = pi->adapter; 5040 u_int v; 5041 int rc; 5042 5043 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5044 if (rc) 5045 return (rc); 5046 /* XXX: magic numbers */ 5047 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5048 &v); 5049 end_synchronized_op(sc, 0); 5050 if (rc) 5051 return (rc); 5052 if (op == 0) 5053 v /= 256; 5054 5055 rc = sysctl_handle_int(oidp, &v, 0, req); 5056 return (rc); 5057} 5058 5059static int 5060sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5061{ 5062 struct port_info *pi = arg1; 5063 int rc, val; 5064 5065 val = pi->rsrv_noflowq; 5066 rc = sysctl_handle_int(oidp, &val, 0, req); 5067 if (rc != 0 || req->newptr == NULL) 5068 return (rc); 5069 5070 if ((val >= 1) && (pi->ntxq > 1)) 5071 pi->rsrv_noflowq = 1; 5072 else 5073 pi->rsrv_noflowq = 0; 5074 5075 return (rc); 5076} 5077 5078static int 5079sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5080{ 5081 struct port_info *pi = arg1; 5082 struct adapter *sc = pi->adapter; 5083 int idx, rc, i; 5084 struct sge_rxq *rxq; 5085#ifdef TCP_OFFLOAD 5086 struct sge_ofld_rxq *ofld_rxq; 5087#endif 5088 uint8_t v; 5089 5090 idx = pi->tmr_idx; 5091 5092 rc = sysctl_handle_int(oidp, &idx, 0, req); 5093 if (rc != 0 || req->newptr == NULL) 5094 return (rc); 5095 5096 if (idx < 0 || idx >= SGE_NTIMERS) 5097 return (EINVAL); 5098 5099 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5100 "t4tmr"); 5101 if (rc) 5102 return (rc); 5103 5104 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5105 for_each_rxq(pi, i, rxq) { 5106#ifdef atomic_store_rel_8 5107 atomic_store_rel_8(&rxq->iq.intr_params, v); 5108#else 5109 rxq->iq.intr_params = v; 5110#endif 5111 } 5112#ifdef TCP_OFFLOAD 5113 for_each_ofld_rxq(pi, i, ofld_rxq) { 5114#ifdef atomic_store_rel_8 5115 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5116#else 5117 ofld_rxq->iq.intr_params = v; 5118#endif 5119 } 5120#endif 5121 pi->tmr_idx = idx; 5122 5123 end_synchronized_op(sc, LOCK_HELD); 5124 return (0); 5125} 5126 5127static int 5128sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5129{ 5130 struct port_info *pi = arg1; 5131 struct adapter *sc = pi->adapter; 5132 int idx, rc; 5133 5134 idx = pi->pktc_idx; 5135 5136 rc = sysctl_handle_int(oidp, &idx, 0, req); 5137 if (rc != 0 || req->newptr == NULL) 5138 return (rc); 5139 5140 if (idx < -1 || idx >= SGE_NCOUNTERS) 5141 return (EINVAL); 5142 5143 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5144 "t4pktc"); 5145 if (rc) 5146 return (rc); 5147 5148 if (pi->flags & PORT_INIT_DONE) 5149 rc = EBUSY; /* cannot be changed once the queues are created */ 5150 else 5151 pi->pktc_idx = idx; 5152 5153 end_synchronized_op(sc, LOCK_HELD); 5154 return (rc); 5155} 5156 5157static int 5158sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5159{ 5160 struct port_info *pi = arg1; 5161 struct adapter *sc = pi->adapter; 5162 int qsize, rc; 5163 5164 qsize = pi->qsize_rxq; 5165 5166 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5167 if (rc != 0 || req->newptr == NULL) 5168 return (rc); 5169 5170 if (qsize < 128 || (qsize & 7)) 5171 return (EINVAL); 5172 5173 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5174 "t4rxqs"); 5175 if (rc) 5176 return (rc); 5177 5178 if (pi->flags & PORT_INIT_DONE) 5179 rc = EBUSY; /* cannot be changed once the queues are created */ 5180 else 5181 pi->qsize_rxq = qsize; 5182 5183 end_synchronized_op(sc, LOCK_HELD); 5184 return (rc); 5185} 5186 5187static int 5188sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5189{ 5190 struct port_info *pi = arg1; 5191 struct adapter *sc = pi->adapter; 5192 int qsize, rc; 5193 5194 qsize = pi->qsize_txq; 5195 5196 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5197 if (rc != 0 || req->newptr == NULL) 5198 return (rc); 5199 5200 if (qsize < 128 || qsize > 65536) 5201 return (EINVAL); 5202 5203 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5204 "t4txqs"); 5205 if (rc) 5206 return (rc); 5207 5208 if (pi->flags & PORT_INIT_DONE) 5209 rc = EBUSY; /* cannot be changed once the queues are created */ 5210 else 5211 pi->qsize_txq = qsize; 5212 5213 end_synchronized_op(sc, LOCK_HELD); 5214 return (rc); 5215} 5216 5217static int 5218sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5219{ 5220 struct port_info *pi = arg1; 5221 struct adapter *sc = pi->adapter; 5222 struct link_config *lc = &pi->link_cfg; 5223 int rc; 5224 5225 if (req->newptr == NULL) { 5226 struct sbuf *sb; 5227 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5228 5229 rc = sysctl_wire_old_buffer(req, 0); 5230 if (rc != 0) 5231 return(rc); 5232 5233 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5234 if (sb == NULL) 5235 return (ENOMEM); 5236 5237 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5238 rc = sbuf_finish(sb); 5239 sbuf_delete(sb); 5240 } else { 5241 char s[2]; 5242 int n; 5243 5244 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5245 s[1] = 0; 5246 5247 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5248 if (rc != 0) 5249 return(rc); 5250 5251 if (s[1] != 0) 5252 return (EINVAL); 5253 if (s[0] < '0' || s[0] > '9') 5254 return (EINVAL); /* not a number */ 5255 n = s[0] - '0'; 5256 if (n & ~(PAUSE_TX | PAUSE_RX)) 5257 return (EINVAL); /* some other bit is set too */ 5258 5259 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5260 if (rc) 5261 return (rc); 5262 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5263 int link_ok = lc->link_ok; 5264 5265 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5266 lc->requested_fc |= n; 5267 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5268 lc->link_ok = link_ok; /* restore */ 5269 } 5270 end_synchronized_op(sc, 0); 5271 } 5272 5273 return (rc); 5274} 5275 5276static int 5277sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5278{ 5279 struct adapter *sc = arg1; 5280 int reg = arg2; 5281 uint64_t val; 5282 5283 val = t4_read_reg64(sc, reg); 5284 5285 return (sysctl_handle_64(oidp, &val, 0, req)); 5286} 5287 5288static int 5289sysctl_temperature(SYSCTL_HANDLER_ARGS) 5290{ 5291 struct adapter *sc = arg1; 5292 int rc, t; 5293 uint32_t param, val; 5294 5295 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5296 if (rc) 5297 return (rc); 5298 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5299 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5300 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5301 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5302 end_synchronized_op(sc, 0); 5303 if (rc) 5304 return (rc); 5305 5306 /* unknown is returned as 0 but we display -1 in that case */ 5307 t = val == 0 ? -1 : val; 5308 5309 rc = sysctl_handle_int(oidp, &t, 0, req); 5310 return (rc); 5311} 5312 5313#ifdef SBUF_DRAIN 5314static int 5315sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5316{ 5317 struct adapter *sc = arg1; 5318 struct sbuf *sb; 5319 int rc, i; 5320 uint16_t incr[NMTUS][NCCTRL_WIN]; 5321 static const char *dec_fac[] = { 5322 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5323 "0.9375" 5324 }; 5325 5326 rc = sysctl_wire_old_buffer(req, 0); 5327 if (rc != 0) 5328 return (rc); 5329 5330 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5331 if (sb == NULL) 5332 return (ENOMEM); 5333 5334 t4_read_cong_tbl(sc, incr); 5335 5336 for (i = 0; i < NCCTRL_WIN; ++i) { 5337 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5338 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5339 incr[5][i], incr[6][i], incr[7][i]); 5340 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5341 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5342 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5343 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5344 } 5345 5346 rc = sbuf_finish(sb); 5347 sbuf_delete(sb); 5348 5349 return (rc); 5350} 5351 5352static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5353 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5354 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5355 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5356}; 5357 5358static int 5359sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5360{ 5361 struct adapter *sc = arg1; 5362 struct sbuf *sb; 5363 int rc, i, n, qid = arg2; 5364 uint32_t *buf, *p; 5365 char *qtype; 5366 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5367 5368 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5369 ("%s: bad qid %d\n", __func__, qid)); 5370 5371 if (qid < CIM_NUM_IBQ) { 5372 /* inbound queue */ 5373 qtype = "IBQ"; 5374 n = 4 * CIM_IBQ_SIZE; 5375 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5376 rc = t4_read_cim_ibq(sc, qid, buf, n); 5377 } else { 5378 /* outbound queue */ 5379 qtype = "OBQ"; 5380 qid -= CIM_NUM_IBQ; 5381 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5382 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5383 rc = t4_read_cim_obq(sc, qid, buf, n); 5384 } 5385 5386 if (rc < 0) { 5387 rc = -rc; 5388 goto done; 5389 } 5390 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5391 5392 rc = sysctl_wire_old_buffer(req, 0); 5393 if (rc != 0) 5394 goto done; 5395 5396 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5397 if (sb == NULL) { 5398 rc = ENOMEM; 5399 goto done; 5400 } 5401 5402 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5403 for (i = 0, p = buf; i < n; i += 16, p += 4) 5404 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5405 p[2], p[3]); 5406 5407 rc = sbuf_finish(sb); 5408 sbuf_delete(sb); 5409done: 5410 free(buf, M_CXGBE); 5411 return (rc); 5412} 5413 5414static int 5415sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5416{ 5417 struct adapter *sc = arg1; 5418 u_int cfg; 5419 struct sbuf *sb; 5420 uint32_t *buf, *p; 5421 int rc; 5422 5423 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5424 if (rc != 0) 5425 return (rc); 5426 5427 rc = sysctl_wire_old_buffer(req, 0); 5428 if (rc != 0) 5429 return (rc); 5430 5431 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5432 if (sb == NULL) 5433 return (ENOMEM); 5434 5435 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5436 M_ZERO | M_WAITOK); 5437 5438 rc = -t4_cim_read_la(sc, buf, NULL); 5439 if (rc != 0) 5440 goto done; 5441 5442 sbuf_printf(sb, "Status Data PC%s", 5443 cfg & F_UPDBGLACAPTPCONLY ? "" : 5444 " LS0Stat LS0Addr LS0Data"); 5445 5446 KASSERT((sc->params.cim_la_size & 7) == 0, 5447 ("%s: p will walk off the end of buf", __func__)); 5448 5449 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5450 if (cfg & F_UPDBGLACAPTPCONLY) { 5451 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5452 p[6], p[7]); 5453 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5454 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5455 p[4] & 0xff, p[5] >> 8); 5456 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5457 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5458 p[1] & 0xf, p[2] >> 4); 5459 } else { 5460 sbuf_printf(sb, 5461 "\n %02x %x%07x %x%07x %08x %08x " 5462 "%08x%08x%08x%08x", 5463 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5464 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5465 p[6], p[7]); 5466 } 5467 } 5468 5469 rc = sbuf_finish(sb); 5470 sbuf_delete(sb); 5471done: 5472 free(buf, M_CXGBE); 5473 return (rc); 5474} 5475 5476static int 5477sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5478{ 5479 struct adapter *sc = arg1; 5480 u_int i; 5481 struct sbuf *sb; 5482 uint32_t *buf, *p; 5483 int rc; 5484 5485 rc = sysctl_wire_old_buffer(req, 0); 5486 if (rc != 0) 5487 return (rc); 5488 5489 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5490 if (sb == NULL) 5491 return (ENOMEM); 5492 5493 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5494 M_ZERO | M_WAITOK); 5495 5496 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5497 p = buf; 5498 5499 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5500 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5501 p[1], p[0]); 5502 } 5503 5504 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5505 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5506 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5507 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5508 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5509 (p[1] >> 2) | ((p[2] & 3) << 30), 5510 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5511 p[0] & 1); 5512 } 5513 5514 rc = sbuf_finish(sb); 5515 sbuf_delete(sb); 5516 free(buf, M_CXGBE); 5517 return (rc); 5518} 5519 5520static int 5521sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5522{ 5523 struct adapter *sc = arg1; 5524 u_int i; 5525 struct sbuf *sb; 5526 uint32_t *buf, *p; 5527 int rc; 5528 5529 rc = sysctl_wire_old_buffer(req, 0); 5530 if (rc != 0) 5531 return (rc); 5532 5533 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5534 if (sb == NULL) 5535 return (ENOMEM); 5536 5537 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5538 M_ZERO | M_WAITOK); 5539 5540 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5541 p = buf; 5542 5543 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5544 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5545 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5546 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5547 p[4], p[3], p[2], p[1], p[0]); 5548 } 5549 5550 sbuf_printf(sb, "\n\nCntl ID Data"); 5551 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5552 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5553 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5554 } 5555 5556 rc = sbuf_finish(sb); 5557 sbuf_delete(sb); 5558 free(buf, M_CXGBE); 5559 return (rc); 5560} 5561 5562static int 5563sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5564{ 5565 struct adapter *sc = arg1; 5566 struct sbuf *sb; 5567 int rc, i; 5568 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5569 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5570 uint16_t thres[CIM_NUM_IBQ]; 5571 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5572 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5573 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5574 5575 if (is_t4(sc)) { 5576 cim_num_obq = CIM_NUM_OBQ; 5577 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5578 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5579 } else { 5580 cim_num_obq = CIM_NUM_OBQ_T5; 5581 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5582 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5583 } 5584 nq = CIM_NUM_IBQ + cim_num_obq; 5585 5586 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5587 if (rc == 0) 5588 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5589 if (rc != 0) 5590 return (rc); 5591 5592 t4_read_cimq_cfg(sc, base, size, thres); 5593 5594 rc = sysctl_wire_old_buffer(req, 0); 5595 if (rc != 0) 5596 return (rc); 5597 5598 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5599 if (sb == NULL) 5600 return (ENOMEM); 5601 5602 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5603 5604 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5605 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5606 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5607 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5608 G_QUEREMFLITS(p[2]) * 16); 5609 for ( ; i < nq; i++, p += 4, wr += 2) 5610 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5611 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5612 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5613 G_QUEREMFLITS(p[2]) * 16); 5614 5615 rc = sbuf_finish(sb); 5616 sbuf_delete(sb); 5617 5618 return (rc); 5619} 5620 5621static int 5622sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5623{ 5624 struct adapter *sc = arg1; 5625 struct sbuf *sb; 5626 int rc; 5627 struct tp_cpl_stats stats; 5628 5629 rc = sysctl_wire_old_buffer(req, 0); 5630 if (rc != 0) 5631 return (rc); 5632 5633 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5634 if (sb == NULL) 5635 return (ENOMEM); 5636 5637 t4_tp_get_cpl_stats(sc, &stats); 5638 5639 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5640 "channel 3\n"); 5641 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5642 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5643 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5644 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5645 5646 rc = sbuf_finish(sb); 5647 sbuf_delete(sb); 5648 5649 return (rc); 5650} 5651 5652static int 5653sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5654{ 5655 struct adapter *sc = arg1; 5656 struct sbuf *sb; 5657 int rc; 5658 struct tp_usm_stats stats; 5659 5660 rc = sysctl_wire_old_buffer(req, 0); 5661 if (rc != 0) 5662 return(rc); 5663 5664 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5665 if (sb == NULL) 5666 return (ENOMEM); 5667 5668 t4_get_usm_stats(sc, &stats); 5669 5670 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5671 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5672 sbuf_printf(sb, "Drops: %u", stats.drops); 5673 5674 rc = sbuf_finish(sb); 5675 sbuf_delete(sb); 5676 5677 return (rc); 5678} 5679 5680const char *devlog_level_strings[] = { 5681 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5682 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5683 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5684 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5685 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5686 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5687}; 5688 5689const char *devlog_facility_strings[] = { 5690 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5691 [FW_DEVLOG_FACILITY_CF] = "CF", 5692 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5693 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5694 [FW_DEVLOG_FACILITY_RES] = "RES", 5695 [FW_DEVLOG_FACILITY_HW] = "HW", 5696 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5697 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5698 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5699 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5700 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5701 [FW_DEVLOG_FACILITY_VI] = "VI", 5702 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5703 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5704 [FW_DEVLOG_FACILITY_TM] = "TM", 5705 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5706 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5707 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5708 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5709 [FW_DEVLOG_FACILITY_RI] = "RI", 5710 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5711 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5712 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5713 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5714}; 5715 5716static int 5717sysctl_devlog(SYSCTL_HANDLER_ARGS) 5718{ 5719 struct adapter *sc = arg1; 5720 struct devlog_params *dparams = &sc->params.devlog; 5721 struct fw_devlog_e *buf, *e; 5722 int i, j, rc, nentries, first = 0, m; 5723 struct sbuf *sb; 5724 uint64_t ftstamp = UINT64_MAX; 5725 5726 if (dparams->start == 0) { 5727 dparams->memtype = FW_MEMTYPE_EDC0; 5728 dparams->start = 0x84000; 5729 dparams->size = 32768; 5730 } 5731 5732 nentries = dparams->size / sizeof(struct fw_devlog_e); 5733 5734 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5735 if (buf == NULL) 5736 return (ENOMEM); 5737 5738 m = fwmtype_to_hwmtype(dparams->memtype); 5739 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5740 if (rc != 0) 5741 goto done; 5742 5743 for (i = 0; i < nentries; i++) { 5744 e = &buf[i]; 5745 5746 if (e->timestamp == 0) 5747 break; /* end */ 5748 5749 e->timestamp = be64toh(e->timestamp); 5750 e->seqno = be32toh(e->seqno); 5751 for (j = 0; j < 8; j++) 5752 e->params[j] = be32toh(e->params[j]); 5753 5754 if (e->timestamp < ftstamp) { 5755 ftstamp = e->timestamp; 5756 first = i; 5757 } 5758 } 5759 5760 if (buf[first].timestamp == 0) 5761 goto done; /* nothing in the log */ 5762 5763 rc = sysctl_wire_old_buffer(req, 0); 5764 if (rc != 0) 5765 goto done; 5766 5767 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5768 if (sb == NULL) { 5769 rc = ENOMEM; 5770 goto done; 5771 } 5772 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5773 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5774 5775 i = first; 5776 do { 5777 e = &buf[i]; 5778 if (e->timestamp == 0) 5779 break; /* end */ 5780 5781 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5782 e->seqno, e->timestamp, 5783 (e->level < nitems(devlog_level_strings) ? 5784 devlog_level_strings[e->level] : "UNKNOWN"), 5785 (e->facility < nitems(devlog_facility_strings) ? 5786 devlog_facility_strings[e->facility] : "UNKNOWN")); 5787 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5788 e->params[2], e->params[3], e->params[4], 5789 e->params[5], e->params[6], e->params[7]); 5790 5791 if (++i == nentries) 5792 i = 0; 5793 } while (i != first); 5794 5795 rc = sbuf_finish(sb); 5796 sbuf_delete(sb); 5797done: 5798 free(buf, M_CXGBE); 5799 return (rc); 5800} 5801 5802static int 5803sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5804{ 5805 struct adapter *sc = arg1; 5806 struct sbuf *sb; 5807 int rc; 5808 struct tp_fcoe_stats stats[4]; 5809 5810 rc = sysctl_wire_old_buffer(req, 0); 5811 if (rc != 0) 5812 return (rc); 5813 5814 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5815 if (sb == NULL) 5816 return (ENOMEM); 5817 5818 t4_get_fcoe_stats(sc, 0, &stats[0]); 5819 t4_get_fcoe_stats(sc, 1, &stats[1]); 5820 t4_get_fcoe_stats(sc, 2, &stats[2]); 5821 t4_get_fcoe_stats(sc, 3, &stats[3]); 5822 5823 sbuf_printf(sb, " channel 0 channel 1 " 5824 "channel 2 channel 3\n"); 5825 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5826 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5827 stats[3].octetsDDP); 5828 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5829 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5830 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5831 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5832 stats[3].framesDrop); 5833 5834 rc = sbuf_finish(sb); 5835 sbuf_delete(sb); 5836 5837 return (rc); 5838} 5839 5840static int 5841sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5842{ 5843 struct adapter *sc = arg1; 5844 struct sbuf *sb; 5845 int rc, i; 5846 unsigned int map, kbps, ipg, mode; 5847 unsigned int pace_tab[NTX_SCHED]; 5848 5849 rc = sysctl_wire_old_buffer(req, 0); 5850 if (rc != 0) 5851 return (rc); 5852 5853 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5854 if (sb == NULL) 5855 return (ENOMEM); 5856 5857 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5858 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5859 t4_read_pace_tbl(sc, pace_tab); 5860 5861 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5862 "Class IPG (0.1 ns) Flow IPG (us)"); 5863 5864 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5865 t4_get_tx_sched(sc, i, &kbps, &ipg); 5866 sbuf_printf(sb, "\n %u %-5s %u ", i, 5867 (mode & (1 << i)) ? "flow" : "class", map & 3); 5868 if (kbps) 5869 sbuf_printf(sb, "%9u ", kbps); 5870 else 5871 sbuf_printf(sb, " disabled "); 5872 5873 if (ipg) 5874 sbuf_printf(sb, "%13u ", ipg); 5875 else 5876 sbuf_printf(sb, " disabled "); 5877 5878 if (pace_tab[i]) 5879 sbuf_printf(sb, "%10u", pace_tab[i]); 5880 else 5881 sbuf_printf(sb, " disabled"); 5882 } 5883 5884 rc = sbuf_finish(sb); 5885 sbuf_delete(sb); 5886 5887 return (rc); 5888} 5889 5890static int 5891sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5892{ 5893 struct adapter *sc = arg1; 5894 struct sbuf *sb; 5895 int rc, i, j; 5896 uint64_t *p0, *p1; 5897 struct lb_port_stats s[2]; 5898 static const char *stat_name[] = { 5899 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5900 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5901 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5902 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5903 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5904 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5905 "BG2FramesTrunc:", "BG3FramesTrunc:" 5906 }; 5907 5908 rc = sysctl_wire_old_buffer(req, 0); 5909 if (rc != 0) 5910 return (rc); 5911 5912 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5913 if (sb == NULL) 5914 return (ENOMEM); 5915 5916 memset(s, 0, sizeof(s)); 5917 5918 for (i = 0; i < 4; i += 2) { 5919 t4_get_lb_stats(sc, i, &s[0]); 5920 t4_get_lb_stats(sc, i + 1, &s[1]); 5921 5922 p0 = &s[0].octets; 5923 p1 = &s[1].octets; 5924 sbuf_printf(sb, "%s Loopback %u" 5925 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5926 5927 for (j = 0; j < nitems(stat_name); j++) 5928 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5929 *p0++, *p1++); 5930 } 5931 5932 rc = sbuf_finish(sb); 5933 sbuf_delete(sb); 5934 5935 return (rc); 5936} 5937 5938static int 5939sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5940{ 5941 int rc = 0; 5942 struct port_info *pi = arg1; 5943 struct sbuf *sb; 5944 static const char *linkdnreasons[] = { 5945 "non-specific", "remote fault", "autoneg failed", "reserved3", 5946 "PHY overheated", "unknown", "rx los", "reserved7" 5947 }; 5948 5949 rc = sysctl_wire_old_buffer(req, 0); 5950 if (rc != 0) 5951 return(rc); 5952 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5953 if (sb == NULL) 5954 return (ENOMEM); 5955 5956 if (pi->linkdnrc < 0) 5957 sbuf_printf(sb, "n/a"); 5958 else if (pi->linkdnrc < nitems(linkdnreasons)) 5959 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5960 else 5961 sbuf_printf(sb, "%d", pi->linkdnrc); 5962 5963 rc = sbuf_finish(sb); 5964 sbuf_delete(sb); 5965 5966 return (rc); 5967} 5968 5969struct mem_desc { 5970 unsigned int base; 5971 unsigned int limit; 5972 unsigned int idx; 5973}; 5974 5975static int 5976mem_desc_cmp(const void *a, const void *b) 5977{ 5978 return ((const struct mem_desc *)a)->base - 5979 ((const struct mem_desc *)b)->base; 5980} 5981 5982static void 5983mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5984 unsigned int to) 5985{ 5986 unsigned int size; 5987 5988 size = to - from + 1; 5989 if (size == 0) 5990 return; 5991 5992 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5993 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5994} 5995 5996static int 5997sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5998{ 5999 struct adapter *sc = arg1; 6000 struct sbuf *sb; 6001 int rc, i, n; 6002 uint32_t lo, hi, used, alloc; 6003 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6004 static const char *region[] = { 6005 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6006 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6007 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6008 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6009 "RQUDP region:", "PBL region:", "TXPBL region:", 6010 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6011 "On-chip queues:" 6012 }; 6013 struct mem_desc avail[4]; 6014 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6015 struct mem_desc *md = mem; 6016 6017 rc = sysctl_wire_old_buffer(req, 0); 6018 if (rc != 0) 6019 return (rc); 6020 6021 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6022 if (sb == NULL) 6023 return (ENOMEM); 6024 6025 for (i = 0; i < nitems(mem); i++) { 6026 mem[i].limit = 0; 6027 mem[i].idx = i; 6028 } 6029 6030 /* Find and sort the populated memory ranges */ 6031 i = 0; 6032 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6033 if (lo & F_EDRAM0_ENABLE) { 6034 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6035 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6036 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6037 avail[i].idx = 0; 6038 i++; 6039 } 6040 if (lo & F_EDRAM1_ENABLE) { 6041 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6042 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6043 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6044 avail[i].idx = 1; 6045 i++; 6046 } 6047 if (lo & F_EXT_MEM_ENABLE) { 6048 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6049 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6050 avail[i].limit = avail[i].base + 6051 (G_EXT_MEM_SIZE(hi) << 20); 6052 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6053 i++; 6054 } 6055 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6056 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6057 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6058 avail[i].limit = avail[i].base + 6059 (G_EXT_MEM1_SIZE(hi) << 20); 6060 avail[i].idx = 4; 6061 i++; 6062 } 6063 if (!i) /* no memory available */ 6064 return 0; 6065 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6066 6067 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6068 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6069 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6070 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6071 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6072 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6073 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6074 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6075 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6076 6077 /* the next few have explicit upper bounds */ 6078 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6079 md->limit = md->base - 1 + 6080 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6081 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6082 md++; 6083 6084 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6085 md->limit = md->base - 1 + 6086 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6087 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6088 md++; 6089 6090 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6091 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6092 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6093 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6094 } else { 6095 md->base = 0; 6096 md->idx = nitems(region); /* hide it */ 6097 } 6098 md++; 6099 6100#define ulp_region(reg) \ 6101 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6102 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6103 6104 ulp_region(RX_ISCSI); 6105 ulp_region(RX_TDDP); 6106 ulp_region(TX_TPT); 6107 ulp_region(RX_STAG); 6108 ulp_region(RX_RQ); 6109 ulp_region(RX_RQUDP); 6110 ulp_region(RX_PBL); 6111 ulp_region(TX_PBL); 6112#undef ulp_region 6113 6114 md->base = 0; 6115 md->idx = nitems(region); 6116 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6117 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6118 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6119 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6120 } 6121 md++; 6122 6123 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6124 md->limit = md->base + sc->tids.ntids - 1; 6125 md++; 6126 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6127 md->limit = md->base + sc->tids.ntids - 1; 6128 md++; 6129 6130 md->base = sc->vres.ocq.start; 6131 if (sc->vres.ocq.size) 6132 md->limit = md->base + sc->vres.ocq.size - 1; 6133 else 6134 md->idx = nitems(region); /* hide it */ 6135 md++; 6136 6137 /* add any address-space holes, there can be up to 3 */ 6138 for (n = 0; n < i - 1; n++) 6139 if (avail[n].limit < avail[n + 1].base) 6140 (md++)->base = avail[n].limit; 6141 if (avail[n].limit) 6142 (md++)->base = avail[n].limit; 6143 6144 n = md - mem; 6145 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6146 6147 for (lo = 0; lo < i; lo++) 6148 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6149 avail[lo].limit - 1); 6150 6151 sbuf_printf(sb, "\n"); 6152 for (i = 0; i < n; i++) { 6153 if (mem[i].idx >= nitems(region)) 6154 continue; /* skip holes */ 6155 if (!mem[i].limit) 6156 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6157 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6158 mem[i].limit); 6159 } 6160 6161 sbuf_printf(sb, "\n"); 6162 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6163 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6164 mem_region_show(sb, "uP RAM:", lo, hi); 6165 6166 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6167 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6168 mem_region_show(sb, "uP Extmem2:", lo, hi); 6169 6170 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6171 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6172 G_PMRXMAXPAGE(lo), 6173 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6174 (lo & F_PMRXNUMCHN) ? 2 : 1); 6175 6176 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6177 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6178 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6179 G_PMTXMAXPAGE(lo), 6180 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6181 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6182 sbuf_printf(sb, "%u p-structs\n", 6183 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6184 6185 for (i = 0; i < 4; i++) { 6186 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6187 if (is_t4(sc)) { 6188 used = G_USED(lo); 6189 alloc = G_ALLOC(lo); 6190 } else { 6191 used = G_T5_USED(lo); 6192 alloc = G_T5_ALLOC(lo); 6193 } 6194 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6195 i, used, alloc); 6196 } 6197 for (i = 0; i < 4; i++) { 6198 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6199 if (is_t4(sc)) { 6200 used = G_USED(lo); 6201 alloc = G_ALLOC(lo); 6202 } else { 6203 used = G_T5_USED(lo); 6204 alloc = G_T5_ALLOC(lo); 6205 } 6206 sbuf_printf(sb, 6207 "\nLoopback %d using %u pages out of %u allocated", 6208 i, used, alloc); 6209 } 6210 6211 rc = sbuf_finish(sb); 6212 sbuf_delete(sb); 6213 6214 return (rc); 6215} 6216 6217static inline void 6218tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6219{ 6220 *mask = x | y; 6221 y = htobe64(y); 6222 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6223} 6224 6225static int 6226sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6227{ 6228 struct adapter *sc = arg1; 6229 struct sbuf *sb; 6230 int rc, i, n; 6231 6232 rc = sysctl_wire_old_buffer(req, 0); 6233 if (rc != 0) 6234 return (rc); 6235 6236 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6237 if (sb == NULL) 6238 return (ENOMEM); 6239 6240 sbuf_printf(sb, 6241 "Idx Ethernet address Mask Vld Ports PF" 6242 " VF Replication P0 P1 P2 P3 ML"); 6243 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6244 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6245 for (i = 0; i < n; i++) { 6246 uint64_t tcamx, tcamy, mask; 6247 uint32_t cls_lo, cls_hi; 6248 uint8_t addr[ETHER_ADDR_LEN]; 6249 6250 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6251 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6252 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6253 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6254 6255 if (tcamx & tcamy) 6256 continue; 6257 6258 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6259 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6260 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6261 addr[3], addr[4], addr[5], (uintmax_t)mask, 6262 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6263 G_PORTMAP(cls_hi), G_PF(cls_lo), 6264 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6265 6266 if (cls_lo & F_REPLICATE) { 6267 struct fw_ldst_cmd ldst_cmd; 6268 6269 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6270 ldst_cmd.op_to_addrspace = 6271 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6272 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6273 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6274 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6275 ldst_cmd.u.mps.fid_ctl = 6276 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6277 V_FW_LDST_CMD_CTL(i)); 6278 6279 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6280 "t4mps"); 6281 if (rc) 6282 break; 6283 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6284 sizeof(ldst_cmd), &ldst_cmd); 6285 end_synchronized_op(sc, 0); 6286 6287 if (rc != 0) { 6288 sbuf_printf(sb, 6289 " ------------ error %3u ------------", rc); 6290 rc = 0; 6291 } else { 6292 sbuf_printf(sb, " %08x %08x %08x %08x", 6293 be32toh(ldst_cmd.u.mps.rplc127_96), 6294 be32toh(ldst_cmd.u.mps.rplc95_64), 6295 be32toh(ldst_cmd.u.mps.rplc63_32), 6296 be32toh(ldst_cmd.u.mps.rplc31_0)); 6297 } 6298 } else 6299 sbuf_printf(sb, "%36s", ""); 6300 6301 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6302 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6303 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6304 } 6305 6306 if (rc) 6307 (void) sbuf_finish(sb); 6308 else 6309 rc = sbuf_finish(sb); 6310 sbuf_delete(sb); 6311 6312 return (rc); 6313} 6314 6315static int 6316sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6317{ 6318 struct adapter *sc = arg1; 6319 struct sbuf *sb; 6320 int rc; 6321 uint16_t mtus[NMTUS]; 6322 6323 rc = sysctl_wire_old_buffer(req, 0); 6324 if (rc != 0) 6325 return (rc); 6326 6327 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6328 if (sb == NULL) 6329 return (ENOMEM); 6330 6331 t4_read_mtu_tbl(sc, mtus, NULL); 6332 6333 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6334 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6335 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6336 mtus[14], mtus[15]); 6337 6338 rc = sbuf_finish(sb); 6339 sbuf_delete(sb); 6340 6341 return (rc); 6342} 6343 6344static int 6345sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6346{ 6347 struct adapter *sc = arg1; 6348 struct sbuf *sb; 6349 int rc, i; 6350 uint32_t cnt[PM_NSTATS]; 6351 uint64_t cyc[PM_NSTATS]; 6352 static const char *rx_stats[] = { 6353 "Read:", "Write bypass:", "Write mem:", "Flush:" 6354 }; 6355 static const char *tx_stats[] = { 6356 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6357 }; 6358 6359 rc = sysctl_wire_old_buffer(req, 0); 6360 if (rc != 0) 6361 return (rc); 6362 6363 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6364 if (sb == NULL) 6365 return (ENOMEM); 6366 6367 t4_pmtx_get_stats(sc, cnt, cyc); 6368 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6369 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6370 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6371 cyc[i]); 6372 6373 t4_pmrx_get_stats(sc, cnt, cyc); 6374 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6375 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6376 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6377 cyc[i]); 6378 6379 rc = sbuf_finish(sb); 6380 sbuf_delete(sb); 6381 6382 return (rc); 6383} 6384 6385static int 6386sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6387{ 6388 struct adapter *sc = arg1; 6389 struct sbuf *sb; 6390 int rc; 6391 struct tp_rdma_stats stats; 6392 6393 rc = sysctl_wire_old_buffer(req, 0); 6394 if (rc != 0) 6395 return (rc); 6396 6397 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6398 if (sb == NULL) 6399 return (ENOMEM); 6400 6401 t4_tp_get_rdma_stats(sc, &stats); 6402 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6403 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6404 6405 rc = sbuf_finish(sb); 6406 sbuf_delete(sb); 6407 6408 return (rc); 6409} 6410 6411static int 6412sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6413{ 6414 struct adapter *sc = arg1; 6415 struct sbuf *sb; 6416 int rc; 6417 struct tp_tcp_stats v4, v6; 6418 6419 rc = sysctl_wire_old_buffer(req, 0); 6420 if (rc != 0) 6421 return (rc); 6422 6423 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6424 if (sb == NULL) 6425 return (ENOMEM); 6426 6427 t4_tp_get_tcp_stats(sc, &v4, &v6); 6428 sbuf_printf(sb, 6429 " IP IPv6\n"); 6430 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6431 v4.tcpOutRsts, v6.tcpOutRsts); 6432 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6433 v4.tcpInSegs, v6.tcpInSegs); 6434 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6435 v4.tcpOutSegs, v6.tcpOutSegs); 6436 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6437 v4.tcpRetransSegs, v6.tcpRetransSegs); 6438 6439 rc = sbuf_finish(sb); 6440 sbuf_delete(sb); 6441 6442 return (rc); 6443} 6444 6445static int 6446sysctl_tids(SYSCTL_HANDLER_ARGS) 6447{ 6448 struct adapter *sc = arg1; 6449 struct sbuf *sb; 6450 int rc; 6451 struct tid_info *t = &sc->tids; 6452 6453 rc = sysctl_wire_old_buffer(req, 0); 6454 if (rc != 0) 6455 return (rc); 6456 6457 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6458 if (sb == NULL) 6459 return (ENOMEM); 6460 6461 if (t->natids) { 6462 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6463 t->atids_in_use); 6464 } 6465 6466 if (t->ntids) { 6467 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6468 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6469 6470 if (b) { 6471 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6472 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6473 t->ntids - 1); 6474 } else { 6475 sbuf_printf(sb, "TID range: %u-%u", 6476 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6477 t->ntids - 1); 6478 } 6479 } else 6480 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6481 sbuf_printf(sb, ", in use: %u\n", 6482 atomic_load_acq_int(&t->tids_in_use)); 6483 } 6484 6485 if (t->nstids) { 6486 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6487 t->stid_base + t->nstids - 1, t->stids_in_use); 6488 } 6489 6490 if (t->nftids) { 6491 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6492 t->ftid_base + t->nftids - 1); 6493 } 6494 6495 if (t->netids) { 6496 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6497 t->etid_base + t->netids - 1); 6498 } 6499 6500 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6501 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6502 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6503 6504 rc = sbuf_finish(sb); 6505 sbuf_delete(sb); 6506 6507 return (rc); 6508} 6509 6510static int 6511sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6512{ 6513 struct adapter *sc = arg1; 6514 struct sbuf *sb; 6515 int rc; 6516 struct tp_err_stats stats; 6517 6518 rc = sysctl_wire_old_buffer(req, 0); 6519 if (rc != 0) 6520 return (rc); 6521 6522 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6523 if (sb == NULL) 6524 return (ENOMEM); 6525 6526 t4_tp_get_err_stats(sc, &stats); 6527 6528 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6529 "channel 3\n"); 6530 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6531 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6532 stats.macInErrs[3]); 6533 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6534 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6535 stats.hdrInErrs[3]); 6536 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6537 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6538 stats.tcpInErrs[3]); 6539 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6540 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6541 stats.tcp6InErrs[3]); 6542 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6543 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6544 stats.tnlCongDrops[3]); 6545 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6546 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6547 stats.tnlTxDrops[3]); 6548 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6549 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6550 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6551 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6552 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6553 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6554 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6555 stats.ofldNoNeigh, stats.ofldCongDefer); 6556 6557 rc = sbuf_finish(sb); 6558 sbuf_delete(sb); 6559 6560 return (rc); 6561} 6562 6563struct field_desc { 6564 const char *name; 6565 u_int start; 6566 u_int width; 6567}; 6568 6569static void 6570field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6571{ 6572 char buf[32]; 6573 int line_size = 0; 6574 6575 while (f->name) { 6576 uint64_t mask = (1ULL << f->width) - 1; 6577 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6578 ((uintmax_t)v >> f->start) & mask); 6579 6580 if (line_size + len >= 79) { 6581 line_size = 8; 6582 sbuf_printf(sb, "\n "); 6583 } 6584 sbuf_printf(sb, "%s ", buf); 6585 line_size += len + 1; 6586 f++; 6587 } 6588 sbuf_printf(sb, "\n"); 6589} 6590 6591static struct field_desc tp_la0[] = { 6592 { "RcfOpCodeOut", 60, 4 }, 6593 { "State", 56, 4 }, 6594 { "WcfState", 52, 4 }, 6595 { "RcfOpcSrcOut", 50, 2 }, 6596 { "CRxError", 49, 1 }, 6597 { "ERxError", 48, 1 }, 6598 { "SanityFailed", 47, 1 }, 6599 { "SpuriousMsg", 46, 1 }, 6600 { "FlushInputMsg", 45, 1 }, 6601 { "FlushInputCpl", 44, 1 }, 6602 { "RssUpBit", 43, 1 }, 6603 { "RssFilterHit", 42, 1 }, 6604 { "Tid", 32, 10 }, 6605 { "InitTcb", 31, 1 }, 6606 { "LineNumber", 24, 7 }, 6607 { "Emsg", 23, 1 }, 6608 { "EdataOut", 22, 1 }, 6609 { "Cmsg", 21, 1 }, 6610 { "CdataOut", 20, 1 }, 6611 { "EreadPdu", 19, 1 }, 6612 { "CreadPdu", 18, 1 }, 6613 { "TunnelPkt", 17, 1 }, 6614 { "RcfPeerFin", 16, 1 }, 6615 { "RcfReasonOut", 12, 4 }, 6616 { "TxCchannel", 10, 2 }, 6617 { "RcfTxChannel", 8, 2 }, 6618 { "RxEchannel", 6, 2 }, 6619 { "RcfRxChannel", 5, 1 }, 6620 { "RcfDataOutSrdy", 4, 1 }, 6621 { "RxDvld", 3, 1 }, 6622 { "RxOoDvld", 2, 1 }, 6623 { "RxCongestion", 1, 1 }, 6624 { "TxCongestion", 0, 1 }, 6625 { NULL } 6626}; 6627 6628static struct field_desc tp_la1[] = { 6629 { "CplCmdIn", 56, 8 }, 6630 { "CplCmdOut", 48, 8 }, 6631 { "ESynOut", 47, 1 }, 6632 { "EAckOut", 46, 1 }, 6633 { "EFinOut", 45, 1 }, 6634 { "ERstOut", 44, 1 }, 6635 { "SynIn", 43, 1 }, 6636 { "AckIn", 42, 1 }, 6637 { "FinIn", 41, 1 }, 6638 { "RstIn", 40, 1 }, 6639 { "DataIn", 39, 1 }, 6640 { "DataInVld", 38, 1 }, 6641 { "PadIn", 37, 1 }, 6642 { "RxBufEmpty", 36, 1 }, 6643 { "RxDdp", 35, 1 }, 6644 { "RxFbCongestion", 34, 1 }, 6645 { "TxFbCongestion", 33, 1 }, 6646 { "TxPktSumSrdy", 32, 1 }, 6647 { "RcfUlpType", 28, 4 }, 6648 { "Eread", 27, 1 }, 6649 { "Ebypass", 26, 1 }, 6650 { "Esave", 25, 1 }, 6651 { "Static0", 24, 1 }, 6652 { "Cread", 23, 1 }, 6653 { "Cbypass", 22, 1 }, 6654 { "Csave", 21, 1 }, 6655 { "CPktOut", 20, 1 }, 6656 { "RxPagePoolFull", 18, 2 }, 6657 { "RxLpbkPkt", 17, 1 }, 6658 { "TxLpbkPkt", 16, 1 }, 6659 { "RxVfValid", 15, 1 }, 6660 { "SynLearned", 14, 1 }, 6661 { "SetDelEntry", 13, 1 }, 6662 { "SetInvEntry", 12, 1 }, 6663 { "CpcmdDvld", 11, 1 }, 6664 { "CpcmdSave", 10, 1 }, 6665 { "RxPstructsFull", 8, 2 }, 6666 { "EpcmdDvld", 7, 1 }, 6667 { "EpcmdFlush", 6, 1 }, 6668 { "EpcmdTrimPrefix", 5, 1 }, 6669 { "EpcmdTrimPostfix", 4, 1 }, 6670 { "ERssIp4Pkt", 3, 1 }, 6671 { "ERssIp6Pkt", 2, 1 }, 6672 { "ERssTcpUdpPkt", 1, 1 }, 6673 { "ERssFceFipPkt", 0, 1 }, 6674 { NULL } 6675}; 6676 6677static struct field_desc tp_la2[] = { 6678 { "CplCmdIn", 56, 8 }, 6679 { "MpsVfVld", 55, 1 }, 6680 { "MpsPf", 52, 3 }, 6681 { "MpsVf", 44, 8 }, 6682 { "SynIn", 43, 1 }, 6683 { "AckIn", 42, 1 }, 6684 { "FinIn", 41, 1 }, 6685 { "RstIn", 40, 1 }, 6686 { "DataIn", 39, 1 }, 6687 { "DataInVld", 38, 1 }, 6688 { "PadIn", 37, 1 }, 6689 { "RxBufEmpty", 36, 1 }, 6690 { "RxDdp", 35, 1 }, 6691 { "RxFbCongestion", 34, 1 }, 6692 { "TxFbCongestion", 33, 1 }, 6693 { "TxPktSumSrdy", 32, 1 }, 6694 { "RcfUlpType", 28, 4 }, 6695 { "Eread", 27, 1 }, 6696 { "Ebypass", 26, 1 }, 6697 { "Esave", 25, 1 }, 6698 { "Static0", 24, 1 }, 6699 { "Cread", 23, 1 }, 6700 { "Cbypass", 22, 1 }, 6701 { "Csave", 21, 1 }, 6702 { "CPktOut", 20, 1 }, 6703 { "RxPagePoolFull", 18, 2 }, 6704 { "RxLpbkPkt", 17, 1 }, 6705 { "TxLpbkPkt", 16, 1 }, 6706 { "RxVfValid", 15, 1 }, 6707 { "SynLearned", 14, 1 }, 6708 { "SetDelEntry", 13, 1 }, 6709 { "SetInvEntry", 12, 1 }, 6710 { "CpcmdDvld", 11, 1 }, 6711 { "CpcmdSave", 10, 1 }, 6712 { "RxPstructsFull", 8, 2 }, 6713 { "EpcmdDvld", 7, 1 }, 6714 { "EpcmdFlush", 6, 1 }, 6715 { "EpcmdTrimPrefix", 5, 1 }, 6716 { "EpcmdTrimPostfix", 4, 1 }, 6717 { "ERssIp4Pkt", 3, 1 }, 6718 { "ERssIp6Pkt", 2, 1 }, 6719 { "ERssTcpUdpPkt", 1, 1 }, 6720 { "ERssFceFipPkt", 0, 1 }, 6721 { NULL } 6722}; 6723 6724static void 6725tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6726{ 6727 6728 field_desc_show(sb, *p, tp_la0); 6729} 6730 6731static void 6732tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6733{ 6734 6735 if (idx) 6736 sbuf_printf(sb, "\n"); 6737 field_desc_show(sb, p[0], tp_la0); 6738 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6739 field_desc_show(sb, p[1], tp_la0); 6740} 6741 6742static void 6743tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6744{ 6745 6746 if (idx) 6747 sbuf_printf(sb, "\n"); 6748 field_desc_show(sb, p[0], tp_la0); 6749 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6750 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6751} 6752 6753static int 6754sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6755{ 6756 struct adapter *sc = arg1; 6757 struct sbuf *sb; 6758 uint64_t *buf, *p; 6759 int rc; 6760 u_int i, inc; 6761 void (*show_func)(struct sbuf *, uint64_t *, int); 6762 6763 rc = sysctl_wire_old_buffer(req, 0); 6764 if (rc != 0) 6765 return (rc); 6766 6767 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6768 if (sb == NULL) 6769 return (ENOMEM); 6770 6771 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6772 6773 t4_tp_read_la(sc, buf, NULL); 6774 p = buf; 6775 6776 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6777 case 2: 6778 inc = 2; 6779 show_func = tp_la_show2; 6780 break; 6781 case 3: 6782 inc = 2; 6783 show_func = tp_la_show3; 6784 break; 6785 default: 6786 inc = 1; 6787 show_func = tp_la_show; 6788 } 6789 6790 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6791 (*show_func)(sb, p, i); 6792 6793 rc = sbuf_finish(sb); 6794 sbuf_delete(sb); 6795 free(buf, M_CXGBE); 6796 return (rc); 6797} 6798 6799static int 6800sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6801{ 6802 struct adapter *sc = arg1; 6803 struct sbuf *sb; 6804 int rc; 6805 u64 nrate[NCHAN], orate[NCHAN]; 6806 6807 rc = sysctl_wire_old_buffer(req, 0); 6808 if (rc != 0) 6809 return (rc); 6810 6811 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6812 if (sb == NULL) 6813 return (ENOMEM); 6814 6815 t4_get_chan_txrate(sc, nrate, orate); 6816 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6817 "channel 3\n"); 6818 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6819 nrate[0], nrate[1], nrate[2], nrate[3]); 6820 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6821 orate[0], orate[1], orate[2], orate[3]); 6822 6823 rc = sbuf_finish(sb); 6824 sbuf_delete(sb); 6825 6826 return (rc); 6827} 6828 6829static int 6830sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6831{ 6832 struct adapter *sc = arg1; 6833 struct sbuf *sb; 6834 uint32_t *buf, *p; 6835 int rc, i; 6836 6837 rc = sysctl_wire_old_buffer(req, 0); 6838 if (rc != 0) 6839 return (rc); 6840 6841 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6842 if (sb == NULL) 6843 return (ENOMEM); 6844 6845 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6846 M_ZERO | M_WAITOK); 6847 6848 t4_ulprx_read_la(sc, buf); 6849 p = buf; 6850 6851 sbuf_printf(sb, " Pcmd Type Message" 6852 " Data"); 6853 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6854 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6855 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6856 } 6857 6858 rc = sbuf_finish(sb); 6859 sbuf_delete(sb); 6860 free(buf, M_CXGBE); 6861 return (rc); 6862} 6863 6864static int 6865sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6866{ 6867 struct adapter *sc = arg1; 6868 struct sbuf *sb; 6869 int rc, v; 6870 6871 rc = sysctl_wire_old_buffer(req, 0); 6872 if (rc != 0) 6873 return (rc); 6874 6875 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6876 if (sb == NULL) 6877 return (ENOMEM); 6878 6879 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6880 if (G_STATSOURCE_T5(v) == 7) { 6881 if (G_STATMODE(v) == 0) { 6882 sbuf_printf(sb, "total %d, incomplete %d", 6883 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6884 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6885 } else if (G_STATMODE(v) == 1) { 6886 sbuf_printf(sb, "total %d, data overflow %d", 6887 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6888 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6889 } 6890 } 6891 rc = sbuf_finish(sb); 6892 sbuf_delete(sb); 6893 6894 return (rc); 6895} 6896#endif 6897 6898static uint32_t 6899fconf_to_mode(uint32_t fconf) 6900{ 6901 uint32_t mode; 6902 6903 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6904 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6905 6906 if (fconf & F_FRAGMENTATION) 6907 mode |= T4_FILTER_IP_FRAGMENT; 6908 6909 if (fconf & F_MPSHITTYPE) 6910 mode |= T4_FILTER_MPS_HIT_TYPE; 6911 6912 if (fconf & F_MACMATCH) 6913 mode |= T4_FILTER_MAC_IDX; 6914 6915 if (fconf & F_ETHERTYPE) 6916 mode |= T4_FILTER_ETH_TYPE; 6917 6918 if (fconf & F_PROTOCOL) 6919 mode |= T4_FILTER_IP_PROTO; 6920 6921 if (fconf & F_TOS) 6922 mode |= T4_FILTER_IP_TOS; 6923 6924 if (fconf & F_VLAN) 6925 mode |= T4_FILTER_VLAN; 6926 6927 if (fconf & F_VNIC_ID) 6928 mode |= T4_FILTER_VNIC; 6929 6930 if (fconf & F_PORT) 6931 mode |= T4_FILTER_PORT; 6932 6933 if (fconf & F_FCOE) 6934 mode |= T4_FILTER_FCoE; 6935 6936 return (mode); 6937} 6938 6939static uint32_t 6940mode_to_fconf(uint32_t mode) 6941{ 6942 uint32_t fconf = 0; 6943 6944 if (mode & T4_FILTER_IP_FRAGMENT) 6945 fconf |= F_FRAGMENTATION; 6946 6947 if (mode & T4_FILTER_MPS_HIT_TYPE) 6948 fconf |= F_MPSHITTYPE; 6949 6950 if (mode & T4_FILTER_MAC_IDX) 6951 fconf |= F_MACMATCH; 6952 6953 if (mode & T4_FILTER_ETH_TYPE) 6954 fconf |= F_ETHERTYPE; 6955 6956 if (mode & T4_FILTER_IP_PROTO) 6957 fconf |= F_PROTOCOL; 6958 6959 if (mode & T4_FILTER_IP_TOS) 6960 fconf |= F_TOS; 6961 6962 if (mode & T4_FILTER_VLAN) 6963 fconf |= F_VLAN; 6964 6965 if (mode & T4_FILTER_VNIC) 6966 fconf |= F_VNIC_ID; 6967 6968 if (mode & T4_FILTER_PORT) 6969 fconf |= F_PORT; 6970 6971 if (mode & T4_FILTER_FCoE) 6972 fconf |= F_FCOE; 6973 6974 return (fconf); 6975} 6976 6977static uint32_t 6978fspec_to_fconf(struct t4_filter_specification *fs) 6979{ 6980 uint32_t fconf = 0; 6981 6982 if (fs->val.frag || fs->mask.frag) 6983 fconf |= F_FRAGMENTATION; 6984 6985 if (fs->val.matchtype || fs->mask.matchtype) 6986 fconf |= F_MPSHITTYPE; 6987 6988 if (fs->val.macidx || fs->mask.macidx) 6989 fconf |= F_MACMATCH; 6990 6991 if (fs->val.ethtype || fs->mask.ethtype) 6992 fconf |= F_ETHERTYPE; 6993 6994 if (fs->val.proto || fs->mask.proto) 6995 fconf |= F_PROTOCOL; 6996 6997 if (fs->val.tos || fs->mask.tos) 6998 fconf |= F_TOS; 6999 7000 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7001 fconf |= F_VLAN; 7002 7003 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7004 fconf |= F_VNIC_ID; 7005 7006 if (fs->val.iport || fs->mask.iport) 7007 fconf |= F_PORT; 7008 7009 if (fs->val.fcoe || fs->mask.fcoe) 7010 fconf |= F_FCOE; 7011 7012 return (fconf); 7013} 7014 7015static int 7016get_filter_mode(struct adapter *sc, uint32_t *mode) 7017{ 7018 int rc; 7019 uint32_t fconf; 7020 7021 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7022 "t4getfm"); 7023 if (rc) 7024 return (rc); 7025 7026 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7027 A_TP_VLAN_PRI_MAP); 7028 7029 if (sc->params.tp.vlan_pri_map != fconf) { 7030 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7031 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7032 fconf); 7033 } 7034 7035 *mode = fconf_to_mode(fconf); 7036 7037 end_synchronized_op(sc, LOCK_HELD); 7038 return (0); 7039} 7040 7041static int 7042set_filter_mode(struct adapter *sc, uint32_t mode) 7043{ 7044 uint32_t fconf; 7045 int rc; 7046 7047 fconf = mode_to_fconf(mode); 7048 7049 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7050 "t4setfm"); 7051 if (rc) 7052 return (rc); 7053 7054 if (sc->tids.ftids_in_use > 0) { 7055 rc = EBUSY; 7056 goto done; 7057 } 7058 7059#ifdef TCP_OFFLOAD 7060 if (sc->offload_map) { 7061 rc = EBUSY; 7062 goto done; 7063 } 7064#endif 7065 7066 rc = -t4_set_filter_mode(sc, fconf); 7067done: 7068 end_synchronized_op(sc, LOCK_HELD); 7069 return (rc); 7070} 7071 7072static inline uint64_t 7073get_filter_hits(struct adapter *sc, uint32_t fid) 7074{ 7075 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7076 uint64_t hits; 7077 7078 memwin_info(sc, 0, &mw_base, NULL); 7079 off = position_memwin(sc, 0, 7080 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7081 if (is_t4(sc)) { 7082 hits = t4_read_reg64(sc, mw_base + off + 16); 7083 hits = be64toh(hits); 7084 } else { 7085 hits = t4_read_reg(sc, mw_base + off + 24); 7086 hits = be32toh(hits); 7087 } 7088 7089 return (hits); 7090} 7091 7092static int 7093get_filter(struct adapter *sc, struct t4_filter *t) 7094{ 7095 int i, rc, nfilters = sc->tids.nftids; 7096 struct filter_entry *f; 7097 7098 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7099 "t4getf"); 7100 if (rc) 7101 return (rc); 7102 7103 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7104 t->idx >= nfilters) { 7105 t->idx = 0xffffffff; 7106 goto done; 7107 } 7108 7109 f = &sc->tids.ftid_tab[t->idx]; 7110 for (i = t->idx; i < nfilters; i++, f++) { 7111 if (f->valid) { 7112 t->idx = i; 7113 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7114 t->smtidx = f->smtidx; 7115 if (f->fs.hitcnts) 7116 t->hits = get_filter_hits(sc, t->idx); 7117 else 7118 t->hits = UINT64_MAX; 7119 t->fs = f->fs; 7120 7121 goto done; 7122 } 7123 } 7124 7125 t->idx = 0xffffffff; 7126done: 7127 end_synchronized_op(sc, LOCK_HELD); 7128 return (0); 7129} 7130 7131static int 7132set_filter(struct adapter *sc, struct t4_filter *t) 7133{ 7134 unsigned int nfilters, nports; 7135 struct filter_entry *f; 7136 int i, rc; 7137 7138 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7139 if (rc) 7140 return (rc); 7141 7142 nfilters = sc->tids.nftids; 7143 nports = sc->params.nports; 7144 7145 if (nfilters == 0) { 7146 rc = ENOTSUP; 7147 goto done; 7148 } 7149 7150 if (!(sc->flags & FULL_INIT_DONE)) { 7151 rc = EAGAIN; 7152 goto done; 7153 } 7154 7155 if (t->idx >= nfilters) { 7156 rc = EINVAL; 7157 goto done; 7158 } 7159 7160 /* Validate against the global filter mode */ 7161 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7162 sc->params.tp.vlan_pri_map) { 7163 rc = E2BIG; 7164 goto done; 7165 } 7166 7167 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7168 rc = EINVAL; 7169 goto done; 7170 } 7171 7172 if (t->fs.val.iport >= nports) { 7173 rc = EINVAL; 7174 goto done; 7175 } 7176 7177 /* Can't specify an iq if not steering to it */ 7178 if (!t->fs.dirsteer && t->fs.iq) { 7179 rc = EINVAL; 7180 goto done; 7181 } 7182 7183 /* IPv6 filter idx must be 4 aligned */ 7184 if (t->fs.type == 1 && 7185 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7186 rc = EINVAL; 7187 goto done; 7188 } 7189 7190 if (sc->tids.ftid_tab == NULL) { 7191 KASSERT(sc->tids.ftids_in_use == 0, 7192 ("%s: no memory allocated but filters_in_use > 0", 7193 __func__)); 7194 7195 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7196 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7197 if (sc->tids.ftid_tab == NULL) { 7198 rc = ENOMEM; 7199 goto done; 7200 } 7201 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7202 } 7203 7204 for (i = 0; i < 4; i++) { 7205 f = &sc->tids.ftid_tab[t->idx + i]; 7206 7207 if (f->pending || f->valid) { 7208 rc = EBUSY; 7209 goto done; 7210 } 7211 if (f->locked) { 7212 rc = EPERM; 7213 goto done; 7214 } 7215 7216 if (t->fs.type == 0) 7217 break; 7218 } 7219 7220 f = &sc->tids.ftid_tab[t->idx]; 7221 f->fs = t->fs; 7222 7223 rc = set_filter_wr(sc, t->idx); 7224done: 7225 end_synchronized_op(sc, 0); 7226 7227 if (rc == 0) { 7228 mtx_lock(&sc->tids.ftid_lock); 7229 for (;;) { 7230 if (f->pending == 0) { 7231 rc = f->valid ? 0 : EIO; 7232 break; 7233 } 7234 7235 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7236 PCATCH, "t4setfw", 0)) { 7237 rc = EINPROGRESS; 7238 break; 7239 } 7240 } 7241 mtx_unlock(&sc->tids.ftid_lock); 7242 } 7243 return (rc); 7244} 7245 7246static int 7247del_filter(struct adapter *sc, struct t4_filter *t) 7248{ 7249 unsigned int nfilters; 7250 struct filter_entry *f; 7251 int rc; 7252 7253 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7254 if (rc) 7255 return (rc); 7256 7257 nfilters = sc->tids.nftids; 7258 7259 if (nfilters == 0) { 7260 rc = ENOTSUP; 7261 goto done; 7262 } 7263 7264 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7265 t->idx >= nfilters) { 7266 rc = EINVAL; 7267 goto done; 7268 } 7269 7270 if (!(sc->flags & FULL_INIT_DONE)) { 7271 rc = EAGAIN; 7272 goto done; 7273 } 7274 7275 f = &sc->tids.ftid_tab[t->idx]; 7276 7277 if (f->pending) { 7278 rc = EBUSY; 7279 goto done; 7280 } 7281 if (f->locked) { 7282 rc = EPERM; 7283 goto done; 7284 } 7285 7286 if (f->valid) { 7287 t->fs = f->fs; /* extra info for the caller */ 7288 rc = del_filter_wr(sc, t->idx); 7289 } 7290 7291done: 7292 end_synchronized_op(sc, 0); 7293 7294 if (rc == 0) { 7295 mtx_lock(&sc->tids.ftid_lock); 7296 for (;;) { 7297 if (f->pending == 0) { 7298 rc = f->valid ? EIO : 0; 7299 break; 7300 } 7301 7302 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7303 PCATCH, "t4delfw", 0)) { 7304 rc = EINPROGRESS; 7305 break; 7306 } 7307 } 7308 mtx_unlock(&sc->tids.ftid_lock); 7309 } 7310 7311 return (rc); 7312} 7313 7314static void 7315clear_filter(struct filter_entry *f) 7316{ 7317 if (f->l2t) 7318 t4_l2t_release(f->l2t); 7319 7320 bzero(f, sizeof (*f)); 7321} 7322 7323static int 7324set_filter_wr(struct adapter *sc, int fidx) 7325{ 7326 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7327 struct fw_filter_wr *fwr; 7328 unsigned int ftid; 7329 struct wrq_cookie cookie; 7330 7331 ASSERT_SYNCHRONIZED_OP(sc); 7332 7333 if (f->fs.newdmac || f->fs.newvlan) { 7334 /* This filter needs an L2T entry; allocate one. */ 7335 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7336 if (f->l2t == NULL) 7337 return (EAGAIN); 7338 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7339 f->fs.dmac)) { 7340 t4_l2t_release(f->l2t); 7341 f->l2t = NULL; 7342 return (ENOMEM); 7343 } 7344 } 7345 7346 ftid = sc->tids.ftid_base + fidx; 7347 7348 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7349 if (fwr == NULL) 7350 return (ENOMEM); 7351 bzero(fwr, sizeof(*fwr)); 7352 7353 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7354 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7355 fwr->tid_to_iq = 7356 htobe32(V_FW_FILTER_WR_TID(ftid) | 7357 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7358 V_FW_FILTER_WR_NOREPLY(0) | 7359 V_FW_FILTER_WR_IQ(f->fs.iq)); 7360 fwr->del_filter_to_l2tix = 7361 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7362 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7363 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7364 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7365 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7366 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7367 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7368 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7369 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7370 f->fs.newvlan == VLAN_REWRITE) | 7371 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7372 f->fs.newvlan == VLAN_REWRITE) | 7373 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7374 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7375 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7376 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7377 fwr->ethtype = htobe16(f->fs.val.ethtype); 7378 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7379 fwr->frag_to_ovlan_vldm = 7380 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7381 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7382 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7383 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7384 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7385 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7386 fwr->smac_sel = 0; 7387 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7388 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7389 fwr->maci_to_matchtypem = 7390 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7391 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7392 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7393 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7394 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7395 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7396 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7397 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7398 fwr->ptcl = f->fs.val.proto; 7399 fwr->ptclm = f->fs.mask.proto; 7400 fwr->ttyp = f->fs.val.tos; 7401 fwr->ttypm = f->fs.mask.tos; 7402 fwr->ivlan = htobe16(f->fs.val.vlan); 7403 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7404 fwr->ovlan = htobe16(f->fs.val.vnic); 7405 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7406 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7407 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7408 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7409 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7410 fwr->lp = htobe16(f->fs.val.dport); 7411 fwr->lpm = htobe16(f->fs.mask.dport); 7412 fwr->fp = htobe16(f->fs.val.sport); 7413 fwr->fpm = htobe16(f->fs.mask.sport); 7414 if (f->fs.newsmac) 7415 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7416 7417 f->pending = 1; 7418 sc->tids.ftids_in_use++; 7419 7420 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7421 return (0); 7422} 7423 7424static int 7425del_filter_wr(struct adapter *sc, int fidx) 7426{ 7427 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7428 struct fw_filter_wr *fwr; 7429 unsigned int ftid; 7430 struct wrq_cookie cookie; 7431 7432 ftid = sc->tids.ftid_base + fidx; 7433 7434 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7435 if (fwr == NULL) 7436 return (ENOMEM); 7437 bzero(fwr, sizeof (*fwr)); 7438 7439 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7440 7441 f->pending = 1; 7442 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7443 return (0); 7444} 7445 7446int 7447t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7448{ 7449 struct adapter *sc = iq->adapter; 7450 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7451 unsigned int idx = GET_TID(rpl); 7452 unsigned int rc; 7453 struct filter_entry *f; 7454 7455 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7456 rss->opcode)); 7457 7458 if (is_ftid(sc, idx)) { 7459 7460 idx -= sc->tids.ftid_base; 7461 f = &sc->tids.ftid_tab[idx]; 7462 rc = G_COOKIE(rpl->cookie); 7463 7464 mtx_lock(&sc->tids.ftid_lock); 7465 if (rc == FW_FILTER_WR_FLT_ADDED) { 7466 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7467 __func__, idx)); 7468 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7469 f->pending = 0; /* asynchronous setup completed */ 7470 f->valid = 1; 7471 } else { 7472 if (rc != FW_FILTER_WR_FLT_DELETED) { 7473 /* Add or delete failed, display an error */ 7474 log(LOG_ERR, 7475 "filter %u setup failed with error %u\n", 7476 idx, rc); 7477 } 7478 7479 clear_filter(f); 7480 sc->tids.ftids_in_use--; 7481 } 7482 wakeup(&sc->tids.ftid_tab); 7483 mtx_unlock(&sc->tids.ftid_lock); 7484 } 7485 7486 return (0); 7487} 7488 7489static int 7490get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7491{ 7492 int rc; 7493 7494 if (cntxt->cid > M_CTXTQID) 7495 return (EINVAL); 7496 7497 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7498 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7499 return (EINVAL); 7500 7501 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7502 if (rc) 7503 return (rc); 7504 7505 if (sc->flags & FW_OK) { 7506 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7507 &cntxt->data[0]); 7508 if (rc == 0) 7509 goto done; 7510 } 7511 7512 /* 7513 * Read via firmware failed or wasn't even attempted. Read directly via 7514 * the backdoor. 7515 */ 7516 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7517done: 7518 end_synchronized_op(sc, 0); 7519 return (rc); 7520} 7521 7522static int 7523load_fw(struct adapter *sc, struct t4_data *fw) 7524{ 7525 int rc; 7526 uint8_t *fw_data; 7527 7528 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7529 if (rc) 7530 return (rc); 7531 7532 if (sc->flags & FULL_INIT_DONE) { 7533 rc = EBUSY; 7534 goto done; 7535 } 7536 7537 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7538 if (fw_data == NULL) { 7539 rc = ENOMEM; 7540 goto done; 7541 } 7542 7543 rc = copyin(fw->data, fw_data, fw->len); 7544 if (rc == 0) 7545 rc = -t4_load_fw(sc, fw_data, fw->len); 7546 7547 free(fw_data, M_CXGBE); 7548done: 7549 end_synchronized_op(sc, 0); 7550 return (rc); 7551} 7552 7553static int 7554read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7555{ 7556 uint32_t addr, off, remaining, i, n; 7557 uint32_t *buf, *b; 7558 uint32_t mw_base, mw_aperture; 7559 int rc; 7560 uint8_t *dst; 7561 7562 rc = validate_mem_range(sc, mr->addr, mr->len); 7563 if (rc != 0) 7564 return (rc); 7565 7566 memwin_info(sc, win, &mw_base, &mw_aperture); 7567 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7568 addr = mr->addr; 7569 remaining = mr->len; 7570 dst = (void *)mr->data; 7571 7572 while (remaining) { 7573 off = position_memwin(sc, win, addr); 7574 7575 /* number of bytes that we'll copy in the inner loop */ 7576 n = min(remaining, mw_aperture - off); 7577 for (i = 0; i < n; i += 4) 7578 *b++ = t4_read_reg(sc, mw_base + off + i); 7579 7580 rc = copyout(buf, dst, n); 7581 if (rc != 0) 7582 break; 7583 7584 b = buf; 7585 dst += n; 7586 remaining -= n; 7587 addr += n; 7588 } 7589 7590 free(buf, M_CXGBE); 7591 return (rc); 7592} 7593 7594static int 7595read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7596{ 7597 int rc; 7598 7599 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7600 return (EINVAL); 7601 7602 if (i2cd->len > sizeof(i2cd->data)) 7603 return (EFBIG); 7604 7605 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7606 if (rc) 7607 return (rc); 7608 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7609 i2cd->offset, i2cd->len, &i2cd->data[0]); 7610 end_synchronized_op(sc, 0); 7611 7612 return (rc); 7613} 7614 7615static int 7616in_range(int val, int lo, int hi) 7617{ 7618 7619 return (val < 0 || (val <= hi && val >= lo)); 7620} 7621 7622static int 7623set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7624{ 7625 int fw_subcmd, fw_type, rc; 7626 7627 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7628 if (rc) 7629 return (rc); 7630 7631 if (!(sc->flags & FULL_INIT_DONE)) { 7632 rc = EAGAIN; 7633 goto done; 7634 } 7635 7636 /* 7637 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7638 * sub-command and type are in common locations.) 7639 */ 7640 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7641 fw_subcmd = FW_SCHED_SC_CONFIG; 7642 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7643 fw_subcmd = FW_SCHED_SC_PARAMS; 7644 else { 7645 rc = EINVAL; 7646 goto done; 7647 } 7648 if (p->type == SCHED_CLASS_TYPE_PACKET) 7649 fw_type = FW_SCHED_TYPE_PKTSCHED; 7650 else { 7651 rc = EINVAL; 7652 goto done; 7653 } 7654 7655 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7656 /* Vet our parameters ..*/ 7657 if (p->u.config.minmax < 0) { 7658 rc = EINVAL; 7659 goto done; 7660 } 7661 7662 /* And pass the request to the firmware ...*/ 7663 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7664 goto done; 7665 } 7666 7667 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7668 int fw_level; 7669 int fw_mode; 7670 int fw_rateunit; 7671 int fw_ratemode; 7672 7673 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7674 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7675 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7676 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7677 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7678 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7679 else { 7680 rc = EINVAL; 7681 goto done; 7682 } 7683 7684 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7685 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7686 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7687 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7688 else { 7689 rc = EINVAL; 7690 goto done; 7691 } 7692 7693 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7694 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7695 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7696 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7697 else { 7698 rc = EINVAL; 7699 goto done; 7700 } 7701 7702 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7703 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7704 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7705 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7706 else { 7707 rc = EINVAL; 7708 goto done; 7709 } 7710 7711 /* Vet our parameters ... */ 7712 if (!in_range(p->u.params.channel, 0, 3) || 7713 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7714 !in_range(p->u.params.minrate, 0, 10000000) || 7715 !in_range(p->u.params.maxrate, 0, 10000000) || 7716 !in_range(p->u.params.weight, 0, 100)) { 7717 rc = ERANGE; 7718 goto done; 7719 } 7720 7721 /* 7722 * Translate any unset parameters into the firmware's 7723 * nomenclature and/or fail the call if the parameters 7724 * are required ... 7725 */ 7726 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7727 p->u.params.channel < 0 || p->u.params.cl < 0) { 7728 rc = EINVAL; 7729 goto done; 7730 } 7731 if (p->u.params.minrate < 0) 7732 p->u.params.minrate = 0; 7733 if (p->u.params.maxrate < 0) { 7734 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7735 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7736 rc = EINVAL; 7737 goto done; 7738 } else 7739 p->u.params.maxrate = 0; 7740 } 7741 if (p->u.params.weight < 0) { 7742 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7743 rc = EINVAL; 7744 goto done; 7745 } else 7746 p->u.params.weight = 0; 7747 } 7748 if (p->u.params.pktsize < 0) { 7749 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7750 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7751 rc = EINVAL; 7752 goto done; 7753 } else 7754 p->u.params.pktsize = 0; 7755 } 7756 7757 /* See what the firmware thinks of the request ... */ 7758 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7759 fw_rateunit, fw_ratemode, p->u.params.channel, 7760 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7761 p->u.params.weight, p->u.params.pktsize, 1); 7762 goto done; 7763 } 7764 7765 rc = EINVAL; 7766done: 7767 end_synchronized_op(sc, 0); 7768 return (rc); 7769} 7770 7771static int 7772set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7773{ 7774 struct port_info *pi = NULL; 7775 struct sge_txq *txq; 7776 uint32_t fw_mnem, fw_queue, fw_class; 7777 int i, rc; 7778 7779 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7780 if (rc) 7781 return (rc); 7782 7783 if (!(sc->flags & FULL_INIT_DONE)) { 7784 rc = EAGAIN; 7785 goto done; 7786 } 7787 7788 if (p->port >= sc->params.nports) { 7789 rc = EINVAL; 7790 goto done; 7791 } 7792 7793 pi = sc->port[p->port]; 7794 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7795 rc = EINVAL; 7796 goto done; 7797 } 7798 7799 /* 7800 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7801 * Scheduling Class in this case). 7802 */ 7803 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7804 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7805 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7806 7807 /* 7808 * If op.queue is non-negative, then we're only changing the scheduling 7809 * on a single specified TX queue. 7810 */ 7811 if (p->queue >= 0) { 7812 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7813 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7814 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7815 &fw_class); 7816 goto done; 7817 } 7818 7819 /* 7820 * Change the scheduling on all the TX queues for the 7821 * interface. 7822 */ 7823 for_each_txq(pi, i, txq) { 7824 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7825 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7826 &fw_class); 7827 if (rc) 7828 goto done; 7829 } 7830 7831 rc = 0; 7832done: 7833 end_synchronized_op(sc, 0); 7834 return (rc); 7835} 7836 7837int 7838t4_os_find_pci_capability(struct adapter *sc, int cap) 7839{ 7840 int i; 7841 7842 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7843} 7844 7845int 7846t4_os_pci_save_state(struct adapter *sc) 7847{ 7848 device_t dev; 7849 struct pci_devinfo *dinfo; 7850 7851 dev = sc->dev; 7852 dinfo = device_get_ivars(dev); 7853 7854 pci_cfg_save(dev, dinfo, 0); 7855 return (0); 7856} 7857 7858int 7859t4_os_pci_restore_state(struct adapter *sc) 7860{ 7861 device_t dev; 7862 struct pci_devinfo *dinfo; 7863 7864 dev = sc->dev; 7865 dinfo = device_get_ivars(dev); 7866 7867 pci_cfg_restore(dev, dinfo); 7868 return (0); 7869} 7870 7871void 7872t4_os_portmod_changed(const struct adapter *sc, int idx) 7873{ 7874 struct port_info *pi = sc->port[idx]; 7875 static const char *mod_str[] = { 7876 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7877 }; 7878 7879 build_medialist(pi, &pi->media); 7880#ifdef DEV_NETMAP 7881 build_medialist(pi, &pi->nm_media); 7882#endif 7883 7884 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7885 if_printf(pi->ifp, "transceiver unplugged.\n"); 7886 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7887 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7888 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7889 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7890 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7891 if_printf(pi->ifp, "%s transceiver inserted.\n", 7892 mod_str[pi->mod_type]); 7893 } else { 7894 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7895 pi->mod_type); 7896 } 7897} 7898 7899void 7900t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7901{ 7902 struct port_info *pi = sc->port[idx]; 7903 struct ifnet *ifp = pi->ifp; 7904 7905 if (link_stat) { 7906 pi->linkdnrc = -1; 7907 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7908 if_link_state_change(ifp, LINK_STATE_UP); 7909 } else { 7910 if (reason >= 0) 7911 pi->linkdnrc = reason; 7912 if_link_state_change(ifp, LINK_STATE_DOWN); 7913 } 7914} 7915 7916void 7917t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7918{ 7919 struct adapter *sc; 7920 7921 sx_slock(&t4_list_lock); 7922 SLIST_FOREACH(sc, &t4_list, link) { 7923 /* 7924 * func should not make any assumptions about what state sc is 7925 * in - the only guarantee is that sc->sc_lock is a valid lock. 7926 */ 7927 func(sc, arg); 7928 } 7929 sx_sunlock(&t4_list_lock); 7930} 7931 7932static int 7933t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7934{ 7935 return (0); 7936} 7937 7938static int 7939t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7940{ 7941 return (0); 7942} 7943 7944static int 7945t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7946 struct thread *td) 7947{ 7948 int rc; 7949 struct adapter *sc = dev->si_drv1; 7950 7951 rc = priv_check(td, PRIV_DRIVER); 7952 if (rc != 0) 7953 return (rc); 7954 7955 switch (cmd) { 7956 case CHELSIO_T4_GETREG: { 7957 struct t4_reg *edata = (struct t4_reg *)data; 7958 7959 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7960 return (EFAULT); 7961 7962 if (edata->size == 4) 7963 edata->val = t4_read_reg(sc, edata->addr); 7964 else if (edata->size == 8) 7965 edata->val = t4_read_reg64(sc, edata->addr); 7966 else 7967 return (EINVAL); 7968 7969 break; 7970 } 7971 case CHELSIO_T4_SETREG: { 7972 struct t4_reg *edata = (struct t4_reg *)data; 7973 7974 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7975 return (EFAULT); 7976 7977 if (edata->size == 4) { 7978 if (edata->val & 0xffffffff00000000) 7979 return (EINVAL); 7980 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7981 } else if (edata->size == 8) 7982 t4_write_reg64(sc, edata->addr, edata->val); 7983 else 7984 return (EINVAL); 7985 break; 7986 } 7987 case CHELSIO_T4_REGDUMP: { 7988 struct t4_regdump *regs = (struct t4_regdump *)data; 7989 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7990 uint8_t *buf; 7991 7992 if (regs->len < reglen) { 7993 regs->len = reglen; /* hint to the caller */ 7994 return (ENOBUFS); 7995 } 7996 7997 regs->len = reglen; 7998 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 7999 t4_get_regs(sc, regs, buf); 8000 rc = copyout(buf, regs->data, reglen); 8001 free(buf, M_CXGBE); 8002 break; 8003 } 8004 case CHELSIO_T4_GET_FILTER_MODE: 8005 rc = get_filter_mode(sc, (uint32_t *)data); 8006 break; 8007 case CHELSIO_T4_SET_FILTER_MODE: 8008 rc = set_filter_mode(sc, *(uint32_t *)data); 8009 break; 8010 case CHELSIO_T4_GET_FILTER: 8011 rc = get_filter(sc, (struct t4_filter *)data); 8012 break; 8013 case CHELSIO_T4_SET_FILTER: 8014 rc = set_filter(sc, (struct t4_filter *)data); 8015 break; 8016 case CHELSIO_T4_DEL_FILTER: 8017 rc = del_filter(sc, (struct t4_filter *)data); 8018 break; 8019 case CHELSIO_T4_GET_SGE_CONTEXT: 8020 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8021 break; 8022 case CHELSIO_T4_LOAD_FW: 8023 rc = load_fw(sc, (struct t4_data *)data); 8024 break; 8025 case CHELSIO_T4_GET_MEM: 8026 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8027 break; 8028 case CHELSIO_T4_GET_I2C: 8029 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8030 break; 8031 case CHELSIO_T4_CLEAR_STATS: { 8032 int i; 8033 u_int port_id = *(uint32_t *)data; 8034 struct port_info *pi; 8035 8036 if (port_id >= sc->params.nports) 8037 return (EINVAL); 8038 pi = sc->port[port_id]; 8039 8040 /* MAC stats */ 8041 t4_clr_port_stats(sc, pi->tx_chan); 8042 pi->tx_parse_error = 0; 8043 8044 if (pi->flags & PORT_INIT_DONE) { 8045 struct sge_rxq *rxq; 8046 struct sge_txq *txq; 8047 struct sge_wrq *wrq; 8048 8049 for_each_rxq(pi, i, rxq) { 8050#if defined(INET) || defined(INET6) 8051 rxq->lro.lro_queued = 0; 8052 rxq->lro.lro_flushed = 0; 8053#endif 8054 rxq->rxcsum = 0; 8055 rxq->vlan_extraction = 0; 8056 } 8057 8058 for_each_txq(pi, i, txq) { 8059 txq->txcsum = 0; 8060 txq->tso_wrs = 0; 8061 txq->vlan_insertion = 0; 8062 txq->imm_wrs = 0; 8063 txq->sgl_wrs = 0; 8064 txq->txpkt_wrs = 0; 8065 txq->txpkts0_wrs = 0; 8066 txq->txpkts1_wrs = 0; 8067 txq->txpkts0_pkts = 0; 8068 txq->txpkts1_pkts = 0; 8069 mp_ring_reset_stats(txq->r); 8070 } 8071 8072#ifdef TCP_OFFLOAD 8073 /* nothing to clear for each ofld_rxq */ 8074 8075 for_each_ofld_txq(pi, i, wrq) { 8076 wrq->tx_wrs_direct = 0; 8077 wrq->tx_wrs_copied = 0; 8078 } 8079#endif 8080 wrq = &sc->sge.ctrlq[pi->port_id]; 8081 wrq->tx_wrs_direct = 0; 8082 wrq->tx_wrs_copied = 0; 8083 } 8084 break; 8085 } 8086 case CHELSIO_T4_SCHED_CLASS: 8087 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8088 break; 8089 case CHELSIO_T4_SCHED_QUEUE: 8090 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8091 break; 8092 case CHELSIO_T4_GET_TRACER: 8093 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8094 break; 8095 case CHELSIO_T4_SET_TRACER: 8096 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8097 break; 8098 default: 8099 rc = EINVAL; 8100 } 8101 8102 return (rc); 8103} 8104 8105#ifdef TCP_OFFLOAD 8106void 8107t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8108 const unsigned int *pgsz_order) 8109{ 8110 struct port_info *pi = ifp->if_softc; 8111 struct adapter *sc = pi->adapter; 8112 8113 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8114 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8115 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8116 V_HPZ3(pgsz_order[3])); 8117} 8118 8119static int 8120toe_capability(struct port_info *pi, int enable) 8121{ 8122 int rc; 8123 struct adapter *sc = pi->adapter; 8124 8125 ASSERT_SYNCHRONIZED_OP(sc); 8126 8127 if (!is_offload(sc)) 8128 return (ENODEV); 8129 8130 if (enable) { 8131 /* 8132 * We need the port's queues around so that we're able to send 8133 * and receive CPLs to/from the TOE even if the ifnet for this 8134 * port has never been UP'd administratively. 8135 */ 8136 if (!(pi->flags & PORT_INIT_DONE)) { 8137 rc = cxgbe_init_synchronized(pi); 8138 if (rc) 8139 return (rc); 8140 } 8141 8142 if (isset(&sc->offload_map, pi->port_id)) 8143 return (0); 8144 8145 if (!(sc->flags & TOM_INIT_DONE)) { 8146 rc = t4_activate_uld(sc, ULD_TOM); 8147 if (rc == EAGAIN) { 8148 log(LOG_WARNING, 8149 "You must kldload t4_tom.ko before trying " 8150 "to enable TOE on a cxgbe interface.\n"); 8151 } 8152 if (rc != 0) 8153 return (rc); 8154 KASSERT(sc->tom_softc != NULL, 8155 ("%s: TOM activated but softc NULL", __func__)); 8156 KASSERT(sc->flags & TOM_INIT_DONE, 8157 ("%s: TOM activated but flag not set", __func__)); 8158 } 8159 8160 setbit(&sc->offload_map, pi->port_id); 8161 } else { 8162 if (!isset(&sc->offload_map, pi->port_id)) 8163 return (0); 8164 8165 KASSERT(sc->flags & TOM_INIT_DONE, 8166 ("%s: TOM never initialized?", __func__)); 8167 clrbit(&sc->offload_map, pi->port_id); 8168 } 8169 8170 return (0); 8171} 8172 8173/* 8174 * Add an upper layer driver to the global list. 8175 */ 8176int 8177t4_register_uld(struct uld_info *ui) 8178{ 8179 int rc = 0; 8180 struct uld_info *u; 8181 8182 sx_xlock(&t4_uld_list_lock); 8183 SLIST_FOREACH(u, &t4_uld_list, link) { 8184 if (u->uld_id == ui->uld_id) { 8185 rc = EEXIST; 8186 goto done; 8187 } 8188 } 8189 8190 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8191 ui->refcount = 0; 8192done: 8193 sx_xunlock(&t4_uld_list_lock); 8194 return (rc); 8195} 8196 8197int 8198t4_unregister_uld(struct uld_info *ui) 8199{ 8200 int rc = EINVAL; 8201 struct uld_info *u; 8202 8203 sx_xlock(&t4_uld_list_lock); 8204 8205 SLIST_FOREACH(u, &t4_uld_list, link) { 8206 if (u == ui) { 8207 if (ui->refcount > 0) { 8208 rc = EBUSY; 8209 goto done; 8210 } 8211 8212 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8213 rc = 0; 8214 goto done; 8215 } 8216 } 8217done: 8218 sx_xunlock(&t4_uld_list_lock); 8219 return (rc); 8220} 8221 8222int 8223t4_activate_uld(struct adapter *sc, int id) 8224{ 8225 int rc = EAGAIN; 8226 struct uld_info *ui; 8227 8228 ASSERT_SYNCHRONIZED_OP(sc); 8229 8230 sx_slock(&t4_uld_list_lock); 8231 8232 SLIST_FOREACH(ui, &t4_uld_list, link) { 8233 if (ui->uld_id == id) { 8234 if (!(sc->flags & FULL_INIT_DONE)) { 8235 rc = adapter_full_init(sc); 8236 if (rc != 0) 8237 goto done; 8238 } 8239 8240 rc = ui->activate(sc); 8241 if (rc == 0) 8242 ui->refcount++; 8243 goto done; 8244 } 8245 } 8246done: 8247 sx_sunlock(&t4_uld_list_lock); 8248 8249 return (rc); 8250} 8251 8252int 8253t4_deactivate_uld(struct adapter *sc, int id) 8254{ 8255 int rc = EINVAL; 8256 struct uld_info *ui; 8257 8258 ASSERT_SYNCHRONIZED_OP(sc); 8259 8260 sx_slock(&t4_uld_list_lock); 8261 8262 SLIST_FOREACH(ui, &t4_uld_list, link) { 8263 if (ui->uld_id == id) { 8264 rc = ui->deactivate(sc); 8265 if (rc == 0) 8266 ui->refcount--; 8267 goto done; 8268 } 8269 } 8270done: 8271 sx_sunlock(&t4_uld_list_lock); 8272 8273 return (rc); 8274} 8275#endif 8276 8277/* 8278 * Come up with reasonable defaults for some of the tunables, provided they're 8279 * not set by the user (in which case we'll use the values as is). 8280 */ 8281static void 8282tweak_tunables(void) 8283{ 8284 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8285 8286 if (t4_ntxq10g < 1) 8287 t4_ntxq10g = min(nc, NTXQ_10G); 8288 8289 if (t4_ntxq1g < 1) 8290 t4_ntxq1g = min(nc, NTXQ_1G); 8291 8292 if (t4_nrxq10g < 1) 8293 t4_nrxq10g = min(nc, NRXQ_10G); 8294 8295 if (t4_nrxq1g < 1) 8296 t4_nrxq1g = min(nc, NRXQ_1G); 8297 8298#ifdef TCP_OFFLOAD 8299 if (t4_nofldtxq10g < 1) 8300 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8301 8302 if (t4_nofldtxq1g < 1) 8303 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8304 8305 if (t4_nofldrxq10g < 1) 8306 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8307 8308 if (t4_nofldrxq1g < 1) 8309 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8310 8311 if (t4_toecaps_allowed == -1) 8312 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8313#else 8314 if (t4_toecaps_allowed == -1) 8315 t4_toecaps_allowed = 0; 8316#endif 8317 8318#ifdef DEV_NETMAP 8319 if (t4_nnmtxq10g < 1) 8320 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8321 8322 if (t4_nnmtxq1g < 1) 8323 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8324 8325 if (t4_nnmrxq10g < 1) 8326 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8327 8328 if (t4_nnmrxq1g < 1) 8329 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8330#endif 8331 8332 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8333 t4_tmr_idx_10g = TMR_IDX_10G; 8334 8335 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8336 t4_pktc_idx_10g = PKTC_IDX_10G; 8337 8338 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8339 t4_tmr_idx_1g = TMR_IDX_1G; 8340 8341 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8342 t4_pktc_idx_1g = PKTC_IDX_1G; 8343 8344 if (t4_qsize_txq < 128) 8345 t4_qsize_txq = 128; 8346 8347 if (t4_qsize_rxq < 128) 8348 t4_qsize_rxq = 128; 8349 while (t4_qsize_rxq & 7) 8350 t4_qsize_rxq++; 8351 8352 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8353} 8354 8355static struct sx mlu; /* mod load unload */ 8356SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8357 8358static int 8359mod_event(module_t mod, int cmd, void *arg) 8360{ 8361 int rc = 0; 8362 static int loaded = 0; 8363 8364 switch (cmd) { 8365 case MOD_LOAD: 8366 sx_xlock(&mlu); 8367 if (loaded++ == 0) { 8368 t4_sge_modload(); 8369 sx_init(&t4_list_lock, "T4/T5 adapters"); 8370 SLIST_INIT(&t4_list); 8371#ifdef TCP_OFFLOAD 8372 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8373 SLIST_INIT(&t4_uld_list); 8374#endif 8375 t4_tracer_modload(); 8376 tweak_tunables(); 8377 } 8378 sx_xunlock(&mlu); 8379 break; 8380 8381 case MOD_UNLOAD: 8382 sx_xlock(&mlu); 8383 if (--loaded == 0) { 8384 int tries; 8385 8386 sx_slock(&t4_list_lock); 8387 if (!SLIST_EMPTY(&t4_list)) { 8388 rc = EBUSY; 8389 sx_sunlock(&t4_list_lock); 8390 goto done_unload; 8391 } 8392#ifdef TCP_OFFLOAD 8393 sx_slock(&t4_uld_list_lock); 8394 if (!SLIST_EMPTY(&t4_uld_list)) { 8395 rc = EBUSY; 8396 sx_sunlock(&t4_uld_list_lock); 8397 sx_sunlock(&t4_list_lock); 8398 goto done_unload; 8399 } 8400#endif 8401 tries = 0; 8402 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8403 uprintf("%ju clusters with custom free routine " 8404 "still is use.\n", t4_sge_extfree_refs()); 8405 pause("t4unload", 2 * hz); 8406 } 8407#ifdef TCP_OFFLOAD 8408 sx_sunlock(&t4_uld_list_lock); 8409#endif 8410 sx_sunlock(&t4_list_lock); 8411 8412 if (t4_sge_extfree_refs() == 0) { 8413 t4_tracer_modunload(); 8414#ifdef TCP_OFFLOAD 8415 sx_destroy(&t4_uld_list_lock); 8416#endif 8417 sx_destroy(&t4_list_lock); 8418 t4_sge_modunload(); 8419 loaded = 0; 8420 } else { 8421 rc = EBUSY; 8422 loaded++; /* undo earlier decrement */ 8423 } 8424 } 8425done_unload: 8426 sx_xunlock(&mlu); 8427 break; 8428 } 8429 8430 return (rc); 8431} 8432 8433static devclass_t t4_devclass, t5_devclass; 8434static devclass_t cxgbe_devclass, cxl_devclass; 8435 8436DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8437MODULE_VERSION(t4nex, 1); 8438MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8439 8440DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8441MODULE_VERSION(t5nex, 1); 8442MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8443 8444DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8445MODULE_VERSION(cxgbe, 1); 8446 8447DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8448MODULE_VERSION(cxl, 1); 8449