t4_vf.c revision 306664
1/*- 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: John Baldwin <jhb@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/t4_vf.c 306664 2016-10-03 23:49:05Z jhb $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/bus.h> 36#include <sys/conf.h> 37#include <sys/kernel.h> 38#include <sys/module.h> 39#include <sys/priv.h> 40#include <dev/pci/pcivar.h> 41#if defined(__i386__) || defined(__amd64__) 42#include <vm/vm.h> 43#include <vm/pmap.h> 44#endif 45 46#include "common/common.h" 47#include "common/t4_regs.h" 48#include "t4_ioctl.h" 49#include "t4_mp_ring.h" 50 51/* 52 * Some notes: 53 * 54 * The Virtual Interfaces are connected to an internal switch on the chip 55 * which allows VIs attached to the same port to talk to each other even when 56 * the port link is down. As a result, we might want to always report a 57 * VF's link as being "up". 58 * 59 * XXX: Add a TUNABLE and possible per-device sysctl for this? 60 */ 61 62struct intrs_and_queues { 63 uint16_t intr_type; /* MSI, or MSI-X */ 64 uint16_t nirq; /* Total # of vectors */ 65 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 66 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 67 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 68 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 69 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 70 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 71}; 72 73struct { 74 uint16_t device; 75 char *desc; 76} t4vf_pciids[] = { 77 {0x4800, "Chelsio T440-dbg VF"}, 78 {0x4801, "Chelsio T420-CR VF"}, 79 {0x4802, "Chelsio T422-CR VF"}, 80 {0x4803, "Chelsio T440-CR VF"}, 81 {0x4804, "Chelsio T420-BCH VF"}, 82 {0x4805, "Chelsio T440-BCH VF"}, 83 {0x4806, "Chelsio T440-CH VF"}, 84 {0x4807, "Chelsio T420-SO VF"}, 85 {0x4808, "Chelsio T420-CX VF"}, 86 {0x4809, "Chelsio T420-BT VF"}, 87 {0x480a, "Chelsio T404-BT VF"}, 88 {0x480e, "Chelsio T440-LP-CR VF"}, 89}, t5vf_pciids[] = { 90 {0x5800, "Chelsio T580-dbg VF"}, 91 {0x5801, "Chelsio T520-CR VF"}, /* 2 x 10G */ 92 {0x5802, "Chelsio T522-CR VF"}, /* 2 x 10G, 2 X 1G */ 93 {0x5803, "Chelsio T540-CR VF"}, /* 4 x 10G */ 94 {0x5807, "Chelsio T520-SO VF"}, /* 2 x 10G, nomem */ 95 {0x5809, "Chelsio T520-BT VF"}, /* 2 x 10GBaseT */ 96 {0x580a, "Chelsio T504-BT VF"}, /* 4 x 1G */ 97 {0x580d, "Chelsio T580-CR VF"}, /* 2 x 40G */ 98 {0x580e, "Chelsio T540-LP-CR VF"}, /* 4 x 10G */ 99 {0x5810, "Chelsio T580-LP-CR VF"}, /* 2 x 40G */ 100 {0x5811, "Chelsio T520-LL-CR VF"}, /* 2 x 10G */ 101 {0x5812, "Chelsio T560-CR VF"}, /* 1 x 40G, 2 x 10G */ 102 {0x5814, "Chelsio T580-LP-SO-CR VF"}, /* 2 x 40G, nomem */ 103 {0x5815, "Chelsio T502-BT VF"}, /* 2 x 1G */ 104#ifdef notyet 105 {0x5804, "Chelsio T520-BCH VF"}, 106 {0x5805, "Chelsio T540-BCH VF"}, 107 {0x5806, "Chelsio T540-CH VF"}, 108 {0x5808, "Chelsio T520-CX VF"}, 109 {0x580b, "Chelsio B520-SR VF"}, 110 {0x580c, "Chelsio B504-BT VF"}, 111 {0x580f, "Chelsio Amsterdam VF"}, 112 {0x5813, "Chelsio T580-CHR VF"}, 113#endif 114}; 115 116static d_ioctl_t t4vf_ioctl; 117 118static struct cdevsw t4vf_cdevsw = { 119 .d_version = D_VERSION, 120 .d_ioctl = t4vf_ioctl, 121 .d_name = "t4vf", 122}; 123 124static int 125t4vf_probe(device_t dev) 126{ 127 uint16_t d; 128 size_t i; 129 130 d = pci_get_device(dev); 131 for (i = 0; i < nitems(t4vf_pciids); i++) { 132 if (d == t4vf_pciids[i].device) { 133 device_set_desc(dev, t4vf_pciids[i].desc); 134 return (BUS_PROBE_DEFAULT); 135 } 136 } 137 return (ENXIO); 138} 139 140static int 141t5vf_probe(device_t dev) 142{ 143 uint16_t d; 144 size_t i; 145 146 d = pci_get_device(dev); 147 for (i = 0; i < nitems(t5vf_pciids); i++) { 148 if (d == t5vf_pciids[i].device) { 149 device_set_desc(dev, t5vf_pciids[i].desc); 150 return (BUS_PROBE_DEFAULT); 151 } 152 } 153 return (ENXIO); 154} 155 156#define FW_PARAM_DEV(param) \ 157 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 158 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 159#define FW_PARAM_PFVF(param) \ 160 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 161 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 162 163static int 164get_params__pre_init(struct adapter *sc) 165{ 166 int rc; 167 uint32_t param[3], val[3]; 168 169 param[0] = FW_PARAM_DEV(FWREV); 170 param[1] = FW_PARAM_DEV(TPREV); 171 param[2] = FW_PARAM_DEV(CCLK); 172 rc = -t4vf_query_params(sc, nitems(param), param, val); 173 if (rc != 0) { 174 device_printf(sc->dev, 175 "failed to query parameters (pre_init): %d.\n", rc); 176 return (rc); 177 } 178 179 sc->params.fw_vers = val[0]; 180 sc->params.tp_vers = val[1]; 181 sc->params.vpd.cclk = val[2]; 182 183 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 184 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 185 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 186 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 187 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 188 189 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 190 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 191 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 192 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 193 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 194 195 return (0); 196} 197 198static int 199get_params__post_init(struct adapter *sc) 200{ 201 int rc; 202 203 rc = -t4vf_get_sge_params(sc); 204 if (rc != 0) { 205 device_printf(sc->dev, 206 "unable to retrieve adapter SGE parameters: %d\n", rc); 207 return (rc); 208 } 209 210 rc = -t4vf_get_rss_glb_config(sc); 211 if (rc != 0) { 212 device_printf(sc->dev, 213 "unable to retrieve adapter RSS parameters: %d\n", rc); 214 return (rc); 215 } 216 if (sc->params.rss.mode != FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { 217 device_printf(sc->dev, 218 "unable to operate with global RSS mode %d\n", 219 sc->params.rss.mode); 220 return (EINVAL); 221 } 222 223 rc = t4_read_chip_settings(sc); 224 if (rc != 0) 225 return (rc); 226 227 /* 228 * Grab our Virtual Interface resource allocation, extract the 229 * features that we're interested in and do a bit of sanity testing on 230 * what we discover. 231 */ 232 rc = -t4vf_get_vfres(sc); 233 if (rc != 0) { 234 device_printf(sc->dev, 235 "unable to get virtual interface resources: %d\n", rc); 236 return (rc); 237 } 238 239 /* 240 * Check for various parameter sanity issues. 241 */ 242 if (sc->params.vfres.pmask == 0) { 243 device_printf(sc->dev, "no port access configured/usable!\n"); 244 return (EINVAL); 245 } 246 if (sc->params.vfres.nvi == 0) { 247 device_printf(sc->dev, 248 "no virtual interfaces configured/usable!\n"); 249 return (EINVAL); 250 } 251 sc->params.portvec = sc->params.vfres.pmask; 252 253 return (0); 254} 255 256static int 257set_params__post_init(struct adapter *sc) 258{ 259 uint32_t param, val; 260 261 /* ask for encapsulated CPLs */ 262 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 263 val = 1; 264 (void)t4vf_set_params(sc, 1, ¶m, &val); 265 266 return (0); 267} 268 269#undef FW_PARAM_PFVF 270#undef FW_PARAM_DEV 271 272static int 273cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 274 struct intrs_and_queues *iaq) 275{ 276 struct vf_resources *vfres; 277 int nrxq10g, nrxq1g, nrxq; 278 int ntxq10g, ntxq1g, ntxq; 279 int itype, iq_avail, navail, rc; 280 281 /* 282 * Figure out the layout of queues across our VIs and ensure 283 * we can allocate enough interrupts for our layout. 284 */ 285 vfres = &sc->params.vfres; 286 bzero(iaq, sizeof(*iaq)); 287 288 for (itype = INTR_MSIX; itype != 0; itype >>= 1) { 289 if (itype == INTR_INTX) 290 continue; 291 292 if (itype == INTR_MSIX) 293 navail = pci_msix_count(sc->dev); 294 else 295 navail = pci_msi_count(sc->dev); 296 297 if (navail == 0) 298 continue; 299 300 iaq->intr_type = itype; 301 iaq->intr_flags_10g = 0; 302 iaq->intr_flags_1g = 0; 303 304 /* 305 * XXX: The Linux driver reserves an Ingress Queue for 306 * forwarded interrupts when using MSI (but not MSI-X). 307 * It seems it just always asks for 2 interrupts and 308 * forwards all rxqs to the forwarded interrupt. 309 * 310 * We must reserve one IRQ for the for the firmware 311 * event queue. 312 * 313 * Every rxq requires an ingress queue with a free 314 * list and interrupts and an egress queue. Every txq 315 * requires an ETH egress queue. 316 */ 317 iaq->nirq = T4VF_EXTRA_INTR; 318 319 /* 320 * First, determine how many queues we can allocate. 321 * Start by finding the upper bound on rxqs from the 322 * limit on ingress queues. 323 */ 324 iq_avail = vfres->niqflint - iaq->nirq; 325 if (iq_avail < n10g + n1g) { 326 device_printf(sc->dev, 327 "Not enough ingress queues (%d) for %d ports\n", 328 vfres->niqflint, n10g + n1g); 329 return (ENXIO); 330 } 331 332 /* 333 * Try to honor the cap on interrupts. If there aren't 334 * enough interrupts for at least one interrupt per 335 * port, then don't bother, we will just forward all 336 * interrupts to one interrupt in that case. 337 */ 338 if (iaq->nirq + n10g + n1g <= navail) { 339 if (iq_avail > navail - iaq->nirq) 340 iq_avail = navail - iaq->nirq; 341 } 342 343 nrxq10g = t4_nrxq10g; 344 nrxq1g = t4_nrxq1g; 345 nrxq = n10g * nrxq10g + n1g * nrxq1g; 346 if (nrxq > iq_avail && nrxq1g > 1) { 347 /* Too many ingress queues. Try just 1 for 1G. */ 348 nrxq1g = 1; 349 nrxq = n10g * nrxq10g + n1g * nrxq1g; 350 } 351 if (nrxq > iq_avail) { 352 /* 353 * Still too many ingress queues. Use what we 354 * can for each 10G port. 355 */ 356 nrxq10g = (iq_avail - n1g) / n10g; 357 nrxq = n10g * nrxq10g + n1g * nrxq1g; 358 } 359 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 360 361 /* 362 * Next, determine the upper bound on txqs from the limit 363 * on ETH queues. 364 */ 365 if (vfres->nethctrl < n10g + n1g) { 366 device_printf(sc->dev, 367 "Not enough ETH queues (%d) for %d ports\n", 368 vfres->nethctrl, n10g + n1g); 369 return (ENXIO); 370 } 371 372 ntxq10g = t4_ntxq10g; 373 ntxq1g = t4_ntxq1g; 374 ntxq = n10g * ntxq10g + n1g * ntxq1g; 375 if (ntxq > vfres->nethctrl) { 376 /* Too many ETH queues. Try just 1 for 1G. */ 377 ntxq1g = 1; 378 ntxq = n10g * ntxq10g + n1g * ntxq1g; 379 } 380 if (ntxq > vfres->nethctrl) { 381 /* 382 * Still too many ETH queues. Use what we 383 * can for each 10G port. 384 */ 385 ntxq10g = (vfres->nethctrl - n1g) / n10g; 386 ntxq = n10g * ntxq10g + n1g * ntxq1g; 387 } 388 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 389 390 /* 391 * Finally, ensure we have enough egress queues. 392 */ 393 if (vfres->neq < (n10g + n1g) * 2) { 394 device_printf(sc->dev, 395 "Not enough egress queues (%d) for %d ports\n", 396 vfres->neq, n10g + n1g); 397 return (ENXIO); 398 } 399 if (nrxq + ntxq > vfres->neq) { 400 /* Just punt and use 1 for everything. */ 401 nrxq1g = ntxq1g = nrxq10g = ntxq10g = 1; 402 nrxq = n10g * nrxq10g + n1g * nrxq1g; 403 ntxq = n10g * ntxq10g + n1g * ntxq1g; 404 } 405 KASSERT(nrxq <= iq_avail, ("too many ingress queues")); 406 KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); 407 KASSERT(nrxq + ntxq <= vfres->neq, ("too many egress queues")); 408 409 /* 410 * Do we have enough interrupts? For MSI the interrupts 411 * have to be a power of 2 as well. 412 */ 413 iaq->nirq += nrxq; 414 iaq->ntxq10g = ntxq10g; 415 iaq->ntxq1g = ntxq1g; 416 iaq->nrxq10g = nrxq10g; 417 iaq->nrxq1g = nrxq1g; 418 if (iaq->nirq <= navail && 419 (itype != INTR_MSI || powerof2(iaq->nirq))) { 420 navail = iaq->nirq; 421 if (itype == INTR_MSIX) 422 rc = pci_alloc_msix(sc->dev, &navail); 423 else 424 rc = pci_alloc_msi(sc->dev, &navail); 425 if (rc != 0) { 426 device_printf(sc->dev, 427 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 428 itype, rc, iaq->nirq, navail); 429 return (rc); 430 } 431 if (navail == iaq->nirq) { 432 iaq->intr_flags_10g = INTR_RXQ; 433 iaq->intr_flags_1g = INTR_RXQ; 434 return (0); 435 } 436 pci_release_msi(sc->dev); 437 } 438 439 /* Fall back to a single interrupt. */ 440 iaq->nirq = 1; 441 navail = iaq->nirq; 442 if (itype == INTR_MSIX) 443 rc = pci_alloc_msix(sc->dev, &navail); 444 else 445 rc = pci_alloc_msi(sc->dev, &navail); 446 if (rc != 0) 447 device_printf(sc->dev, 448 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 449 itype, rc, iaq->nirq, navail); 450 iaq->intr_flags_10g = 0; 451 iaq->intr_flags_1g = 0; 452 return (rc); 453 } 454 455 device_printf(sc->dev, 456 "failed to find a usable interrupt type. " 457 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 458 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 459 460 return (ENXIO); 461} 462 463static int 464t4vf_attach(device_t dev) 465{ 466 struct adapter *sc; 467 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 468 struct make_dev_args mda; 469 struct intrs_and_queues iaq; 470 struct sge *s; 471 472 sc = device_get_softc(dev); 473 sc->dev = dev; 474 pci_enable_busmaster(dev); 475 pci_set_max_read_req(dev, 4096); 476 sc->params.pci.mps = pci_get_max_payload(dev); 477 478 sc->flags |= IS_VF; 479 480 sc->sge_gts_reg = VF_SGE_REG(A_SGE_VF_GTS); 481 sc->sge_kdoorbell_reg = VF_SGE_REG(A_SGE_VF_KDOORBELL); 482 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 483 device_get_nameunit(dev)); 484 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 485 t4_add_adapter(sc); 486 487 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 488 TAILQ_INIT(&sc->sfl); 489 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 490 491 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 492 493 rc = t4_map_bars_0_and_4(sc); 494 if (rc != 0) 495 goto done; /* error message displayed already */ 496 497 rc = -t4vf_prep_adapter(sc); 498 if (rc != 0) 499 goto done; 500 501 /* 502 * Leave the 'pf' and 'mbox' values as zero. This ensures 503 * that various firmware messages do not set the fields which 504 * is the correct thing to do for a VF. 505 */ 506 507 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 508 509 make_dev_args_init(&mda); 510 mda.mda_devsw = &t4vf_cdevsw; 511 mda.mda_uid = UID_ROOT; 512 mda.mda_gid = GID_WHEEL; 513 mda.mda_mode = 0600; 514 mda.mda_si_drv1 = sc; 515 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); 516 if (rc != 0) 517 device_printf(dev, "failed to create nexus char device: %d.\n", 518 rc); 519 520#if defined(__i386__) 521 if ((cpu_feature & CPUID_CX8) == 0) { 522 device_printf(dev, "64 bit atomics not available.\n"); 523 rc = ENOTSUP; 524 goto done; 525 } 526#endif 527 528 /* 529 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux 530 * 2.6.31 and later we can't call pci_reset_function() in order to 531 * issue an FLR because of a self- deadlock on the device semaphore. 532 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the 533 * cases where they're needed -- for instance, some versions of KVM 534 * fail to reset "Assigned Devices" when the VM reboots. Therefore we 535 * use the firmware based reset in order to reset any per function 536 * state. 537 */ 538 rc = -t4vf_fw_reset(sc); 539 if (rc != 0) { 540 device_printf(dev, "FW reset failed: %d\n", rc); 541 goto done; 542 } 543 sc->flags |= FW_OK; 544 545 /* 546 * Grab basic operational parameters. These will predominantly have 547 * been set up by the Physical Function Driver or will be hard coded 548 * into the adapter. We just have to live with them ... Note that 549 * we _must_ get our VPD parameters before our SGE parameters because 550 * we need to know the adapter's core clock from the VPD in order to 551 * properly decode the SGE Timer Values. 552 */ 553 rc = get_params__pre_init(sc); 554 if (rc != 0) 555 goto done; /* error message displayed already */ 556 rc = get_params__post_init(sc); 557 if (rc != 0) 558 goto done; /* error message displayed already */ 559 560 rc = set_params__post_init(sc); 561 if (rc != 0) 562 goto done; /* error message displayed already */ 563 564 rc = t4_map_bar_2(sc); 565 if (rc != 0) 566 goto done; /* error message displayed already */ 567 568 rc = t4_create_dma_tag(sc); 569 if (rc != 0) 570 goto done; /* error message displayed already */ 571 572 /* 573 * The number of "ports" which we support is equal to the number of 574 * Virtual Interfaces with which we've been provisioned. 575 */ 576 sc->params.nports = imin(sc->params.vfres.nvi, MAX_NPORTS); 577 578 /* 579 * We may have been provisioned with more VIs than the number of 580 * ports we're allowed to access (our Port Access Rights Mask). 581 * Just use a single VI for each port. 582 */ 583 sc->params.nports = imin(sc->params.nports, 584 bitcount32(sc->params.vfres.pmask)); 585 586#ifdef notyet 587 /* 588 * XXX: The Linux VF driver will lower nports if it thinks there 589 * are too few resources in vfres (niqflint, nethctrl, neq). 590 */ 591#endif 592 593 /* 594 * First pass over all the ports - allocate VIs and initialize some 595 * basic parameters like mac address, port type, etc. We also figure 596 * out whether a port is 10G or 1G and use that information when 597 * calculating how many interrupts to attempt to allocate. 598 */ 599 n10g = n1g = 0; 600 for_each_port(sc, i) { 601 struct port_info *pi; 602 603 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 604 sc->port[i] = pi; 605 606 /* These must be set before t4_port_init */ 607 pi->adapter = sc; 608 pi->port_id = i; 609 pi->nvi = 1; 610 pi->vi = malloc(sizeof(struct vi_info) * pi->nvi, M_CXGBE, 611 M_ZERO | M_WAITOK); 612 613 /* 614 * Allocate the "main" VI and initialize parameters 615 * like mac addr. 616 */ 617 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 618 if (rc != 0) { 619 device_printf(dev, "unable to initialize port %d: %d\n", 620 i, rc); 621 free(pi->vi, M_CXGBE); 622 free(pi, M_CXGBE); 623 sc->port[i] = NULL; 624 goto done; 625 } 626 627 /* No t4_link_start. */ 628 629 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 630 device_get_nameunit(dev), i); 631 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 632 sc->chan_map[pi->tx_chan] = i; 633 634 pi->tc = malloc(sizeof(struct tx_sched_class) * 635 sc->chip_params->nsched_cls, M_CXGBE, M_ZERO | M_WAITOK); 636 637 if (is_10G_port(pi) || is_40G_port(pi)) { 638 n10g++; 639 } else { 640 n1g++; 641 } 642 643 pi->linkdnrc = -1; 644 645 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbev" : "cxlv", 646 -1); 647 if (pi->dev == NULL) { 648 device_printf(dev, 649 "failed to add device for port %d.\n", i); 650 rc = ENXIO; 651 goto done; 652 } 653 pi->vi[0].dev = pi->dev; 654 device_set_softc(pi->dev, pi); 655 } 656 657 /* 658 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 659 */ 660 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 661 if (rc != 0) 662 goto done; /* error message displayed already */ 663 664 sc->intr_type = iaq.intr_type; 665 sc->intr_count = iaq.nirq; 666 667 s = &sc->sge; 668 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 669 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 670 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 671 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 672 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 673 674 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 675 M_ZERO | M_WAITOK); 676 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 677 M_ZERO | M_WAITOK); 678 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 679 M_ZERO | M_WAITOK); 680 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 681 M_ZERO | M_WAITOK); 682 683 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 684 M_ZERO | M_WAITOK); 685 686 /* 687 * Second pass over the ports. This time we know the number of rx and 688 * tx queues that each port should get. 689 */ 690 rqidx = tqidx = 0; 691 for_each_port(sc, i) { 692 struct port_info *pi = sc->port[i]; 693 struct vi_info *vi; 694 695 if (pi == NULL) 696 continue; 697 698 for_each_vi(pi, j, vi) { 699 vi->pi = pi; 700 vi->qsize_rxq = t4_qsize_rxq; 701 vi->qsize_txq = t4_qsize_txq; 702 703 vi->first_rxq = rqidx; 704 vi->first_txq = tqidx; 705 if (is_10G_port(pi) || is_40G_port(pi)) { 706 vi->tmr_idx = t4_tmr_idx_10g; 707 vi->pktc_idx = t4_pktc_idx_10g; 708 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 709 vi->nrxq = j == 0 ? iaq.nrxq10g : 1; 710 vi->ntxq = j == 0 ? iaq.ntxq10g : 1; 711 } else { 712 vi->tmr_idx = t4_tmr_idx_1g; 713 vi->pktc_idx = t4_pktc_idx_1g; 714 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 715 vi->nrxq = j == 0 ? iaq.nrxq1g : 1; 716 vi->ntxq = j == 0 ? iaq.ntxq1g : 1; 717 } 718 rqidx += vi->nrxq; 719 tqidx += vi->ntxq; 720 721 vi->rsrv_noflowq = 0; 722 } 723 } 724 725 rc = t4_setup_intr_handlers(sc); 726 if (rc != 0) { 727 device_printf(dev, 728 "failed to setup interrupt handlers: %d\n", rc); 729 goto done; 730 } 731 732 rc = bus_generic_attach(dev); 733 if (rc != 0) { 734 device_printf(dev, 735 "failed to attach all child ports: %d\n", rc); 736 goto done; 737 } 738 739 device_printf(dev, 740 "%d ports, %d %s interrupt%s, %d eq, %d iq\n", 741 sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? 742 "MSI-X" : "MSI", sc->intr_count > 1 ? "s" : "", sc->sge.neq, 743 sc->sge.niq); 744 745done: 746 if (rc != 0) 747 t4_detach_common(dev); 748 else 749 t4_sysctls(sc); 750 751 return (rc); 752} 753 754static void 755get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 756{ 757 758 /* 0x3f is used as the revision for VFs. */ 759 regs->version = chip_id(sc) | (0x3f << 10); 760 t4_get_regs(sc, buf, regs->len); 761} 762 763static void 764t4_clr_vi_stats(struct adapter *sc) 765{ 766 int reg; 767 768 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 769 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 770 t4_write_reg(sc, VF_MPS_REG(reg), 0); 771} 772 773static int 774t4vf_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 775 struct thread *td) 776{ 777 int rc; 778 struct adapter *sc = dev->si_drv1; 779 780 rc = priv_check(td, PRIV_DRIVER); 781 if (rc != 0) 782 return (rc); 783 784 switch (cmd) { 785 case CHELSIO_T4_GETREG: { 786 struct t4_reg *edata = (struct t4_reg *)data; 787 788 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 789 return (EFAULT); 790 791 if (edata->size == 4) 792 edata->val = t4_read_reg(sc, edata->addr); 793 else if (edata->size == 8) 794 edata->val = t4_read_reg64(sc, edata->addr); 795 else 796 return (EINVAL); 797 798 break; 799 } 800 case CHELSIO_T4_SETREG: { 801 struct t4_reg *edata = (struct t4_reg *)data; 802 803 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 804 return (EFAULT); 805 806 if (edata->size == 4) { 807 if (edata->val & 0xffffffff00000000) 808 return (EINVAL); 809 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 810 } else if (edata->size == 8) 811 t4_write_reg64(sc, edata->addr, edata->val); 812 else 813 return (EINVAL); 814 break; 815 } 816 case CHELSIO_T4_REGDUMP: { 817 struct t4_regdump *regs = (struct t4_regdump *)data; 818 int reglen = t4_get_regs_len(sc); 819 uint8_t *buf; 820 821 if (regs->len < reglen) { 822 regs->len = reglen; /* hint to the caller */ 823 return (ENOBUFS); 824 } 825 826 regs->len = reglen; 827 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 828 get_regs(sc, regs, buf); 829 rc = copyout(buf, regs->data, reglen); 830 free(buf, M_CXGBE); 831 break; 832 } 833 case CHELSIO_T4_CLEAR_STATS: { 834 int i, v; 835 u_int port_id = *(uint32_t *)data; 836 struct port_info *pi; 837 struct vi_info *vi; 838 839 if (port_id >= sc->params.nports) 840 return (EINVAL); 841 pi = sc->port[port_id]; 842 843 /* MAC stats */ 844 pi->tx_parse_error = 0; 845 t4_clr_vi_stats(sc); 846 847 /* 848 * Since this command accepts a port, clear stats for 849 * all VIs on this port. 850 */ 851 for_each_vi(pi, v, vi) { 852 if (vi->flags & VI_INIT_DONE) { 853 struct sge_rxq *rxq; 854 struct sge_txq *txq; 855 856 for_each_rxq(vi, i, rxq) { 857#if defined(INET) || defined(INET6) 858 rxq->lro.lro_queued = 0; 859 rxq->lro.lro_flushed = 0; 860#endif 861 rxq->rxcsum = 0; 862 rxq->vlan_extraction = 0; 863 } 864 865 for_each_txq(vi, i, txq) { 866 txq->txcsum = 0; 867 txq->tso_wrs = 0; 868 txq->vlan_insertion = 0; 869 txq->imm_wrs = 0; 870 txq->sgl_wrs = 0; 871 txq->txpkt_wrs = 0; 872 txq->txpkts0_wrs = 0; 873 txq->txpkts1_wrs = 0; 874 txq->txpkts0_pkts = 0; 875 txq->txpkts1_pkts = 0; 876 mp_ring_reset_stats(txq->r); 877 } 878 } 879 } 880 break; 881 } 882 case CHELSIO_T4_SCHED_CLASS: 883 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); 884 break; 885 case CHELSIO_T4_SCHED_QUEUE: 886 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); 887 break; 888 default: 889 rc = ENOTTY; 890 } 891 892 return (rc); 893} 894 895static device_method_t t4vf_methods[] = { 896 DEVMETHOD(device_probe, t4vf_probe), 897 DEVMETHOD(device_attach, t4vf_attach), 898 DEVMETHOD(device_detach, t4_detach_common), 899 900 DEVMETHOD_END 901}; 902 903static driver_t t4vf_driver = { 904 "t4vf", 905 t4vf_methods, 906 sizeof(struct adapter) 907}; 908 909static device_method_t t5vf_methods[] = { 910 DEVMETHOD(device_probe, t5vf_probe), 911 DEVMETHOD(device_attach, t4vf_attach), 912 DEVMETHOD(device_detach, t4_detach_common), 913 914 DEVMETHOD_END 915}; 916 917static driver_t t5vf_driver = { 918 "t5vf", 919 t5vf_methods, 920 sizeof(struct adapter) 921}; 922 923static driver_t cxgbev_driver = { 924 "cxgbev", 925 cxgbe_methods, 926 sizeof(struct port_info) 927}; 928 929static driver_t cxlv_driver = { 930 "cxlv", 931 cxgbe_methods, 932 sizeof(struct port_info) 933}; 934 935static devclass_t t4vf_devclass, t5vf_devclass; 936static devclass_t cxgbev_devclass, cxlv_devclass; 937 938DRIVER_MODULE(t4vf, pci, t4vf_driver, t4vf_devclass, 0, 0); 939MODULE_VERSION(t4vf, 1); 940MODULE_DEPEND(t4vf, t4nex, 1, 1, 1); 941 942DRIVER_MODULE(t5vf, pci, t5vf_driver, t5vf_devclass, 0, 0); 943MODULE_VERSION(t5vf, 1); 944MODULE_DEPEND(t5vf, t5nex, 1, 1, 1); 945 946DRIVER_MODULE(cxgbev, t4vf, cxgbev_driver, cxgbev_devclass, 0, 0); 947MODULE_VERSION(cxgbev, 1); 948 949DRIVER_MODULE(cxlv, t5vf, cxlv_driver, cxlv_devclass, 0, 0); 950MODULE_VERSION(cxlv, 1); 951