efx_tx.c revision 284555
1115396Skan/*- 2115396Skan * Copyright (c) 2007-2015 Solarflare Communications Inc. 3115396Skan * All rights reserved. 4115396Skan * 5115396Skan * Redistribution and use in source and binary forms, with or without 6115396Skan * modification, are permitted provided that the following conditions are met: 7115396Skan * 8115396Skan * 1. Redistributions of source code must retain the above copyright notice, 9115396Skan * this list of conditions and the following disclaimer. 10115396Skan * 2. Redistributions in binary form must reproduce the above copyright notice, 11115396Skan * this list of conditions and the following disclaimer in the documentation 12115396Skan * and/or other materials provided with the distribution. 13115396Skan * 14115396Skan * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15115396Skan * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16115396Skan * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17115396Skan * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18115396Skan * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19115396Skan * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20115396Skan * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21115396Skan * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22115396Skan * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23115396Skan * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24115396Skan * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25115396Skan * 26115396Skan * The views and conclusions contained in the software and documentation are 27115396Skan * those of the authors and should not be interpreted as representing official 28115396Skan * policies, either expressed or implied, of the FreeBSD Project. 29115396Skan */ 30115396Skan 31115396Skan#include <sys/cdefs.h> 32185558Skib__FBSDID("$FreeBSD: stable/10/sys/dev/sfxge/common/efx_tx.c 284555 2015-06-18 15:46:39Z arybchik $"); 33115396Skan 34115396Skan#include "efsys.h" 35115396Skan#include "efx.h" 36115396Skan#include "efx_types.h" 37116557Smdodd#include "efx_regs.h" 38115396Skan#include "efx_impl.h" 39115396Skan 40115396Skan#if EFSYS_OPT_QSTATS 41115396Skan#define EFX_TX_QSTAT_INCR(_etp, _stat) \ 42115396Skan do { \ 43115396Skan (_etp)->et_stat[_stat]++; \ 44116557Smdodd _NOTE(CONSTANTCONDITION) \ 45115396Skan } while (B_FALSE) 46115396Skan#else 47281453Skib#define EFX_TX_QSTAT_INCR(_etp, _stat) 48281453Skib#endif 49281453Skib 50115396Skan#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 51115396Skan 52115396Skanstatic __checkReturn int 53115396Skanfalconsiena_tx_init( 54115396Skan __in efx_nic_t *enp); 55115396Skan 56115396Skanstatic void 57115396Skanfalconsiena_tx_fini( 58178807Skib __in efx_nic_t *enp); 59115396Skan 60216695Skibstatic __checkReturn int 61216695Skibfalconsiena_tx_qcreate( 62216695Skib __in efx_nic_t *enp, 63115396Skan __in unsigned int index, 64216695Skib __in unsigned int label, 65216695Skib __in efsys_mem_t *esmp, 66216695Skib __in size_t n, 67216695Skib __in uint32_t id, 68216695Skib __in uint16_t flags, 69216695Skib __in efx_evq_t *eep, 70216695Skib __in efx_txq_t *etp, 71216695Skib __out unsigned int *addedp); 72216695Skib 73115396Skanstatic void 74115396Skanfalconsiena_tx_qdestroy( 75115396Skan __in efx_txq_t *etp); 76 77static __checkReturn int 78falconsiena_tx_qpost( 79 __in efx_txq_t *etp, 80 __in_ecount(n) efx_buffer_t *eb, 81 __in unsigned int n, 82 __in unsigned int completed, 83 __inout unsigned int *addedp); 84 85static void 86falconsiena_tx_qpush( 87 __in efx_txq_t *etp, 88 __in unsigned int added, 89 __in unsigned int pushed); 90 91static __checkReturn int 92falconsiena_tx_qpace( 93 __in efx_txq_t *etp, 94 __in unsigned int ns); 95 96static __checkReturn int 97falconsiena_tx_qflush( 98 __in efx_txq_t *etp); 99 100static void 101falconsiena_tx_qenable( 102 __in efx_txq_t *etp); 103 104 __checkReturn int 105falconsiena_tx_qdesc_post( 106 __in efx_txq_t *etp, 107 __in_ecount(n) efx_desc_t *ed, 108 __in unsigned int n, 109 __in unsigned int completed, 110 __inout unsigned int *addedp); 111 112 void 113falconsiena_tx_qdesc_dma_create( 114 __in efx_txq_t *etp, 115 __in efsys_dma_addr_t addr, 116 __in size_t size, 117 __in boolean_t eop, 118 __out efx_desc_t *edp); 119 120#if EFSYS_OPT_QSTATS 121static void 122falconsiena_tx_qstats_update( 123 __in efx_txq_t *etp, 124 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 125#endif 126 127#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 128 129 130#if EFSYS_OPT_FALCON 131static efx_tx_ops_t __efx_tx_falcon_ops = { 132 falconsiena_tx_init, /* etxo_init */ 133 falconsiena_tx_fini, /* etxo_fini */ 134 falconsiena_tx_qcreate, /* etxo_qcreate */ 135 falconsiena_tx_qdestroy, /* etxo_qdestroy */ 136 falconsiena_tx_qpost, /* etxo_qpost */ 137 falconsiena_tx_qpush, /* etxo_qpush */ 138 falconsiena_tx_qpace, /* etxo_qpace */ 139 falconsiena_tx_qflush, /* etxo_qflush */ 140 falconsiena_tx_qenable, /* etxo_qenable */ 141 NULL, /* etxo_qpio_enable */ 142 NULL, /* etxo_qpio_disable */ 143 NULL, /* etxo_qpio_write */ 144 NULL, /* etxo_qpio_post */ 145 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 146 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 147 NULL, /* etxo_qdesc_tso_create */ 148 NULL, /* etxo_qdesc_vlantci_create */ 149#if EFSYS_OPT_QSTATS 150 falconsiena_tx_qstats_update, /* etxo_qstats_update */ 151#endif 152}; 153#endif /* EFSYS_OPT_FALCON */ 154 155#if EFSYS_OPT_SIENA 156static efx_tx_ops_t __efx_tx_siena_ops = { 157 falconsiena_tx_init, /* etxo_init */ 158 falconsiena_tx_fini, /* etxo_fini */ 159 falconsiena_tx_qcreate, /* etxo_qcreate */ 160 falconsiena_tx_qdestroy, /* etxo_qdestroy */ 161 falconsiena_tx_qpost, /* etxo_qpost */ 162 falconsiena_tx_qpush, /* etxo_qpush */ 163 falconsiena_tx_qpace, /* etxo_qpace */ 164 falconsiena_tx_qflush, /* etxo_qflush */ 165 falconsiena_tx_qenable, /* etxo_qenable */ 166 NULL, /* etxo_qpio_enable */ 167 NULL, /* etxo_qpio_disable */ 168 NULL, /* etxo_qpio_write */ 169 NULL, /* etxo_qpio_post */ 170 falconsiena_tx_qdesc_post, /* etxo_qdesc_post */ 171 falconsiena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 172 NULL, /* etxo_qdesc_tso_create */ 173 NULL, /* etxo_qdesc_vlantci_create */ 174#if EFSYS_OPT_QSTATS 175 falconsiena_tx_qstats_update, /* etxo_qstats_update */ 176#endif 177}; 178#endif /* EFSYS_OPT_SIENA */ 179 180#if EFSYS_OPT_HUNTINGTON 181static efx_tx_ops_t __efx_tx_hunt_ops = { 182 hunt_tx_init, /* etxo_init */ 183 hunt_tx_fini, /* etxo_fini */ 184 hunt_tx_qcreate, /* etxo_qcreate */ 185 hunt_tx_qdestroy, /* etxo_qdestroy */ 186 hunt_tx_qpost, /* etxo_qpost */ 187 hunt_tx_qpush, /* etxo_qpush */ 188 hunt_tx_qpace, /* etxo_qpace */ 189 hunt_tx_qflush, /* etxo_qflush */ 190 hunt_tx_qenable, /* etxo_qenable */ 191 hunt_tx_qpio_enable, /* etxo_qpio_enable */ 192 hunt_tx_qpio_disable, /* etxo_qpio_disable */ 193 hunt_tx_qpio_write, /* etxo_qpio_write */ 194 hunt_tx_qpio_post, /* etxo_qpio_post */ 195 hunt_tx_qdesc_post, /* etxo_qdesc_post */ 196 hunt_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 197 hunt_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 198 hunt_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 199#if EFSYS_OPT_QSTATS 200 hunt_tx_qstats_update, /* etxo_qstats_update */ 201#endif 202}; 203#endif /* EFSYS_OPT_HUNTINGTON */ 204 205 __checkReturn int 206efx_tx_init( 207 __in efx_nic_t *enp) 208{ 209 efx_tx_ops_t *etxop; 210 int rc; 211 212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 214 215 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 216 rc = EINVAL; 217 goto fail1; 218 } 219 220 if (enp->en_mod_flags & EFX_MOD_TX) { 221 rc = EINVAL; 222 goto fail2; 223 } 224 225 switch (enp->en_family) { 226#if EFSYS_OPT_FALCON 227 case EFX_FAMILY_FALCON: 228 etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops; 229 break; 230#endif /* EFSYS_OPT_FALCON */ 231 232#if EFSYS_OPT_SIENA 233 case EFX_FAMILY_SIENA: 234 etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops; 235 break; 236#endif /* EFSYS_OPT_SIENA */ 237 238#if EFSYS_OPT_HUNTINGTON 239 case EFX_FAMILY_HUNTINGTON: 240 etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops; 241 break; 242#endif /* EFSYS_OPT_HUNTINGTON */ 243 244 default: 245 EFSYS_ASSERT(0); 246 rc = ENOTSUP; 247 goto fail3; 248 } 249 250 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 251 252 if ((rc = etxop->etxo_init(enp)) != 0) 253 goto fail4; 254 255 enp->en_etxop = etxop; 256 enp->en_mod_flags |= EFX_MOD_TX; 257 return (0); 258 259fail4: 260 EFSYS_PROBE(fail4); 261fail3: 262 EFSYS_PROBE(fail3); 263fail2: 264 EFSYS_PROBE(fail2); 265fail1: 266 EFSYS_PROBE1(fail1, int, rc); 267 268 enp->en_etxop = NULL; 269 enp->en_mod_flags &= ~EFX_MOD_TX; 270 return (rc); 271} 272 273 void 274efx_tx_fini( 275 __in efx_nic_t *enp) 276{ 277 efx_tx_ops_t *etxop = enp->en_etxop; 278 279 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 280 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 283 284 etxop->etxo_fini(enp); 285 286 enp->en_etxop = NULL; 287 enp->en_mod_flags &= ~EFX_MOD_TX; 288} 289 290 __checkReturn int 291efx_tx_qcreate( 292 __in efx_nic_t *enp, 293 __in unsigned int index, 294 __in unsigned int label, 295 __in efsys_mem_t *esmp, 296 __in size_t n, 297 __in uint32_t id, 298 __in uint16_t flags, 299 __in efx_evq_t *eep, 300 __deref_out efx_txq_t **etpp, 301 __out unsigned int *addedp) 302{ 303 efx_tx_ops_t *etxop = enp->en_etxop; 304 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 305 efx_txq_t *etp; 306 int rc; 307 308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 310 311 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 312 313 /* Allocate an TXQ object */ 314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 315 316 if (etp == NULL) { 317 rc = ENOMEM; 318 goto fail1; 319 } 320 321 etp->et_magic = EFX_TXQ_MAGIC; 322 etp->et_enp = enp; 323 etp->et_index = index; 324 etp->et_mask = n - 1; 325 etp->et_esmp = esmp; 326 327 /* Initial descriptor index may be modified by etxo_qcreate */ 328 *addedp = 0; 329 330 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 331 n, id, flags, eep, etp, addedp)) != 0) 332 goto fail2; 333 334 enp->en_tx_qcount++; 335 *etpp = etp; 336 337 return (0); 338 339fail2: 340 EFSYS_PROBE(fail2); 341 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 342fail1: 343 EFSYS_PROBE1(fail1, int, rc); 344 return (rc); 345} 346 347 void 348efx_tx_qdestroy( 349 __in efx_txq_t *etp) 350{ 351 efx_nic_t *enp = etp->et_enp; 352 efx_tx_ops_t *etxop = enp->en_etxop; 353 354 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 355 356 EFSYS_ASSERT(enp->en_tx_qcount != 0); 357 --enp->en_tx_qcount; 358 359 etxop->etxo_qdestroy(etp); 360 361 /* Free the TXQ object */ 362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 363} 364 365 __checkReturn int 366efx_tx_qpost( 367 __in efx_txq_t *etp, 368 __in_ecount(n) efx_buffer_t *eb, 369 __in unsigned int n, 370 __in unsigned int completed, 371 __inout unsigned int *addedp) 372{ 373 efx_nic_t *enp = etp->et_enp; 374 efx_tx_ops_t *etxop = enp->en_etxop; 375 int rc; 376 377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 378 379 if ((rc = etxop->etxo_qpost(etp, eb, 380 n, completed, addedp)) != 0) 381 goto fail1; 382 383 return (0); 384 385fail1: 386 EFSYS_PROBE1(fail1, int, rc); 387 return (rc); 388} 389 390 void 391efx_tx_qpush( 392 __in efx_txq_t *etp, 393 __in unsigned int added, 394 __in unsigned int pushed) 395{ 396 efx_nic_t *enp = etp->et_enp; 397 efx_tx_ops_t *etxop = enp->en_etxop; 398 399 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 400 401 etxop->etxo_qpush(etp, added, pushed); 402} 403 404 __checkReturn int 405efx_tx_qpace( 406 __in efx_txq_t *etp, 407 __in unsigned int ns) 408{ 409 efx_nic_t *enp = etp->et_enp; 410 efx_tx_ops_t *etxop = enp->en_etxop; 411 int rc; 412 413 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 414 415 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 416 goto fail1; 417 418 return (0); 419 420fail1: 421 EFSYS_PROBE1(fail1, int, rc); 422 return (rc); 423} 424 425 __checkReturn int 426efx_tx_qflush( 427 __in efx_txq_t *etp) 428{ 429 efx_nic_t *enp = etp->et_enp; 430 efx_tx_ops_t *etxop = enp->en_etxop; 431 int rc; 432 433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 434 435 if ((rc = etxop->etxo_qflush(etp)) != 0) 436 goto fail1; 437 438 return (0); 439 440fail1: 441 EFSYS_PROBE1(fail1, int, rc); 442 return (rc); 443} 444 445 void 446efx_tx_qenable( 447 __in efx_txq_t *etp) 448{ 449 efx_nic_t *enp = etp->et_enp; 450 efx_tx_ops_t *etxop = enp->en_etxop; 451 452 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 453 454 etxop->etxo_qenable(etp); 455} 456 457 __checkReturn int 458efx_tx_qpio_enable( 459 __in efx_txq_t *etp) 460{ 461 efx_nic_t *enp = etp->et_enp; 462 efx_tx_ops_t *etxop = enp->en_etxop; 463 int rc; 464 465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 466 467 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 468 rc = ENOTSUP; 469 goto fail1; 470 } 471 if (etxop->etxo_qpio_enable == NULL) { 472 rc = ENOTSUP; 473 goto fail2; 474 } 475 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 476 goto fail3; 477 478 return (0); 479 480fail3: 481 EFSYS_PROBE(fail3); 482fail2: 483 EFSYS_PROBE(fail2); 484fail1: 485 EFSYS_PROBE1(fail1, int, rc); 486 return (rc); 487} 488 489 void 490efx_tx_qpio_disable( 491 __in efx_txq_t *etp) 492{ 493 efx_nic_t *enp = etp->et_enp; 494 efx_tx_ops_t *etxop = enp->en_etxop; 495 496 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 497 498 if (etxop->etxo_qpio_disable != NULL) 499 etxop->etxo_qpio_disable(etp); 500} 501 502 __checkReturn int 503efx_tx_qpio_write( 504 __in efx_txq_t *etp, 505 __in_ecount(buf_length) uint8_t *buffer, 506 __in size_t buf_length, 507 __in size_t pio_buf_offset) 508{ 509 efx_nic_t *enp = etp->et_enp; 510 efx_tx_ops_t *etxop = enp->en_etxop; 511 int rc; 512 513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 514 515 if (etxop->etxo_qpio_write != NULL) { 516 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 517 pio_buf_offset)) != 0) 518 goto fail1; 519 return (0); 520 } 521 522 return (ENOTSUP); 523 524fail1: 525 EFSYS_PROBE1(fail1, int, rc); 526 return (rc); 527} 528 529 __checkReturn int 530efx_tx_qpio_post( 531 __in efx_txq_t *etp, 532 __in size_t pkt_length, 533 __in unsigned int completed, 534 __inout unsigned int *addedp) 535{ 536 efx_nic_t *enp = etp->et_enp; 537 efx_tx_ops_t *etxop = enp->en_etxop; 538 int rc; 539 540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 541 542 if (etxop->etxo_qpio_post != NULL) { 543 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 544 addedp)) != 0) 545 goto fail1; 546 return (0); 547 } 548 549 return (ENOTSUP); 550 551fail1: 552 EFSYS_PROBE1(fail1, int, rc); 553 return (rc); 554} 555 556 __checkReturn int 557efx_tx_qdesc_post( 558 __in efx_txq_t *etp, 559 __in_ecount(n) efx_desc_t *ed, 560 __in unsigned int n, 561 __in unsigned int completed, 562 __inout unsigned int *addedp) 563{ 564 efx_nic_t *enp = etp->et_enp; 565 efx_tx_ops_t *etxop = enp->en_etxop; 566 int rc; 567 568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 569 570 if ((rc = etxop->etxo_qdesc_post(etp, ed, 571 n, completed, addedp)) != 0) 572 goto fail1; 573 574 return (0); 575 576fail1: 577 EFSYS_PROBE1(fail1, int, rc); 578 return (rc); 579} 580 581 void 582efx_tx_qdesc_dma_create( 583 __in efx_txq_t *etp, 584 __in efsys_dma_addr_t addr, 585 __in size_t size, 586 __in boolean_t eop, 587 __out efx_desc_t *edp) 588{ 589 efx_nic_t *enp = etp->et_enp; 590 efx_tx_ops_t *etxop = enp->en_etxop; 591 592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 594 595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 596} 597 598 void 599efx_tx_qdesc_tso_create( 600 __in efx_txq_t *etp, 601 __in uint16_t ipv4_id, 602 __in uint32_t tcp_seq, 603 __in uint8_t tcp_flags, 604 __out efx_desc_t *edp) 605{ 606 efx_nic_t *enp = etp->et_enp; 607 efx_tx_ops_t *etxop = enp->en_etxop; 608 609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 611 612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 613} 614 615 void 616efx_tx_qdesc_vlantci_create( 617 __in efx_txq_t *etp, 618 __in uint16_t tci, 619 __out efx_desc_t *edp) 620{ 621 efx_nic_t *enp = etp->et_enp; 622 efx_tx_ops_t *etxop = enp->en_etxop; 623 624 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 625 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 626 627 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 628} 629 630 631#if EFSYS_OPT_QSTATS 632 void 633efx_tx_qstats_update( 634 __in efx_txq_t *etp, 635 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 636{ 637 efx_nic_t *enp = etp->et_enp; 638 efx_tx_ops_t *etxop = enp->en_etxop; 639 640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 641 642 etxop->etxo_qstats_update(etp, stat); 643} 644#endif 645 646 647#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 648 649static __checkReturn int 650falconsiena_tx_init( 651 __in efx_nic_t *enp) 652{ 653 efx_oword_t oword; 654 655 /* 656 * Disable the timer-based TX DMA backoff and allow TX DMA to be 657 * controlled by the RX FIFO fill level (although always allow a 658 * minimal trickle). 659 */ 660 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 661 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 662 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 663 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 664 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 665 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 666 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 667 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 668 669 /* 670 * Filter all packets less than 14 bytes to avoid parsing 671 * errors. 672 */ 673 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 674 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 675 676 /* 677 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 678 * descriptors (which is bad). 679 */ 680 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 682 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 683 684 return (0); 685} 686 687#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 688 do { \ 689 unsigned int id; \ 690 size_t offset; \ 691 efx_qword_t qword; \ 692 \ 693 id = (_added)++ & (_etp)->et_mask; \ 694 offset = id * sizeof (efx_qword_t); \ 695 \ 696 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 697 unsigned int, id, efsys_dma_addr_t, (_addr), \ 698 size_t, (_size), boolean_t, (_eop)); \ 699 \ 700 EFX_POPULATE_QWORD_4(qword, \ 701 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 702 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 703 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 704 (uint32_t)((_addr) & 0xffffffff), \ 705 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 706 (uint32_t)((_addr) >> 32)); \ 707 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 708 \ 709 _NOTE(CONSTANTCONDITION) \ 710 } while (B_FALSE) 711 712static __checkReturn int 713falconsiena_tx_qpost( 714 __in efx_txq_t *etp, 715 __in_ecount(n) efx_buffer_t *eb, 716 __in unsigned int n, 717 __in unsigned int completed, 718 __inout unsigned int *addedp) 719{ 720 unsigned int added = *addedp; 721 unsigned int i; 722 int rc = ENOSPC; 723 724 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 725 goto fail1; 726 727 for (i = 0; i < n; i++) { 728 efx_buffer_t *ebp = &eb[i]; 729 efsys_dma_addr_t start = ebp->eb_addr; 730 size_t size = ebp->eb_size; 731 efsys_dma_addr_t end = start + size; 732 733 /* Fragments must not span 4k boundaries. */ 734 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 735 736 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 737 } 738 739 EFX_TX_QSTAT_INCR(etp, TX_POST); 740 741 *addedp = added; 742 return (0); 743 744fail1: 745 EFSYS_PROBE1(fail1, int, rc); 746 747 return (rc); 748} 749 750static void 751falconsiena_tx_qpush( 752 __in efx_txq_t *etp, 753 __in unsigned int added, 754 __in unsigned int pushed) 755{ 756 efx_nic_t *enp = etp->et_enp; 757 uint32_t wptr; 758 efx_dword_t dword; 759 efx_oword_t oword; 760 761 /* Push the populated descriptors out */ 762 wptr = added & etp->et_mask; 763 764 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 765 766 /* Only write the third DWORD */ 767 EFX_POPULATE_DWORD_1(dword, 768 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 769 770 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 771 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 772 wptr, pushed & etp->et_mask); 773 EFSYS_PIO_WRITE_BARRIER(); 774 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 775 etp->et_index, &dword, B_FALSE); 776} 777 778#define EFX_MAX_PACE_VALUE 20 779#define EFX_TX_PACE_CLOCK_BASE 104 780 781static __checkReturn int 782falconsiena_tx_qpace( 783 __in efx_txq_t *etp, 784 __in unsigned int ns) 785{ 786 efx_nic_t *enp = etp->et_enp; 787 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 788 efx_oword_t oword; 789 unsigned int pace_val; 790 unsigned int timer_period; 791 int rc; 792 793 if (ns == 0) { 794 pace_val = 0; 795 } else { 796 /* 797 * The pace_val to write into the table is s.t 798 * ns <= timer_period * (2 ^ pace_val) 799 */ 800 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 801 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 802 if ((timer_period << pace_val) >= ns) 803 break; 804 } 805 } 806 if (pace_val > EFX_MAX_PACE_VALUE) { 807 rc = EINVAL; 808 goto fail1; 809 } 810 811 /* Update the pacing table */ 812 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 813 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 814 &oword, B_TRUE); 815 816 return (0); 817 818fail1: 819 EFSYS_PROBE1(fail1, int, rc); 820 821 return (rc); 822} 823 824static __checkReturn int 825falconsiena_tx_qflush( 826 __in efx_txq_t *etp) 827{ 828 efx_nic_t *enp = etp->et_enp; 829 efx_oword_t oword; 830 uint32_t label; 831 832 efx_tx_qpace(etp, 0); 833 834 label = etp->et_index; 835 836 /* Flush the queue */ 837 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 838 FRF_AZ_TX_FLUSH_DESCQ, label); 839 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 840 841 return (0); 842} 843 844static void 845falconsiena_tx_qenable( 846 __in efx_txq_t *etp) 847{ 848 efx_nic_t *enp = etp->et_enp; 849 efx_oword_t oword; 850 851 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 852 etp->et_index, &oword, B_TRUE); 853 854 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 855 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 856 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 857 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 858 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 859 860 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 861 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 862 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 863 864 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 865 etp->et_index, &oword, B_TRUE); 866} 867 868static __checkReturn int 869falconsiena_tx_qcreate( 870 __in efx_nic_t *enp, 871 __in unsigned int index, 872 __in unsigned int label, 873 __in efsys_mem_t *esmp, 874 __in size_t n, 875 __in uint32_t id, 876 __in uint16_t flags, 877 __in efx_evq_t *eep, 878 __in efx_txq_t *etp, 879 __out unsigned int *addedp) 880{ 881 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 882 efx_oword_t oword; 883 uint32_t size; 884 int rc; 885 886 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 887 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 888 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 889 890 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp))); 891 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 892 893 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) { 894 rc = EINVAL; 895 goto fail1; 896 } 897 if (index >= encp->enc_txq_limit) { 898 rc = EINVAL; 899 goto fail2; 900 } 901 for (size = 0; 902 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS); 903 size++) 904 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 905 break; 906 if (id + (1 << size) >= encp->enc_buftbl_limit) { 907 rc = EINVAL; 908 goto fail3; 909 } 910 911 /* Set up the new descriptor queue */ 912 EFX_POPULATE_OWORD_6(oword, 913 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 914 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 915 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 916 FRF_AZ_TX_DESCQ_LABEL, label, 917 FRF_AZ_TX_DESCQ_SIZE, size, 918 FRF_AZ_TX_DESCQ_TYPE, 0); 919 920 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 921 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 922 (flags & EFX_CKSUM_IPV4) ? 0 : 1); 923 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 924 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 925 926 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 927 etp->et_index, &oword, B_TRUE); 928 929 return (0); 930 931fail3: 932 EFSYS_PROBE(fail3); 933fail2: 934 EFSYS_PROBE(fail2); 935fail1: 936 EFSYS_PROBE1(fail1, int, rc); 937 938 return (rc); 939} 940 941 __checkReturn int 942falconsiena_tx_qdesc_post( 943 __in efx_txq_t *etp, 944 __in_ecount(n) efx_desc_t *ed, 945 __in unsigned int n, 946 __in unsigned int completed, 947 __inout unsigned int *addedp) 948{ 949 unsigned int added = *addedp; 950 unsigned int i; 951 int rc; 952 953 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 954 rc = ENOSPC; 955 goto fail1; 956 } 957 958 for (i = 0; i < n; i++) { 959 efx_desc_t *edp = &ed[i]; 960 unsigned int id; 961 size_t offset; 962 963 id = added++ & etp->et_mask; 964 offset = id * sizeof (efx_desc_t); 965 966 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 967 } 968 969 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 970 unsigned int, added, unsigned int, n); 971 972 EFX_TX_QSTAT_INCR(etp, TX_POST); 973 974 *addedp = added; 975 return (0); 976 977fail1: 978 EFSYS_PROBE1(fail1, int, rc); 979 return (rc); 980} 981 982 void 983falconsiena_tx_qdesc_dma_create( 984 __in efx_txq_t *etp, 985 __in efsys_dma_addr_t addr, 986 __in size_t size, 987 __in boolean_t eop, 988 __out efx_desc_t *edp) 989{ 990 /* Fragments must not span 4k boundaries. */ 991 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size); 992 993 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 994 efsys_dma_addr_t, addr, 995 size_t, size, boolean_t, eop); 996 997 EFX_POPULATE_QWORD_4(edp->ed_eq, 998 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 999 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1000 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1001 (uint32_t)(addr & 0xffffffff), 1002 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1003 (uint32_t)(addr >> 32)); 1004} 1005 1006#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1007 1008#if EFSYS_OPT_QSTATS 1009#if EFSYS_OPT_NAMES 1010/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */ 1011static const char *__efx_tx_qstat_name[] = { 1012 "post", 1013 "post_pio", 1014}; 1015/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1016 1017 const char * 1018efx_tx_qstat_name( 1019 __in efx_nic_t *enp, 1020 __in unsigned int id) 1021{ 1022 _NOTE(ARGUNUSED(enp)) 1023 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1024 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1025 1026 return (__efx_tx_qstat_name[id]); 1027} 1028#endif /* EFSYS_OPT_NAMES */ 1029#endif /* EFSYS_OPT_QSTATS */ 1030 1031#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA 1032 1033#if EFSYS_OPT_QSTATS 1034static void 1035falconsiena_tx_qstats_update( 1036 __in efx_txq_t *etp, 1037 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1038{ 1039 unsigned int id; 1040 1041 for (id = 0; id < TX_NQSTATS; id++) { 1042 efsys_stat_t *essp = &stat[id]; 1043 1044 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1045 etp->et_stat[id] = 0; 1046 } 1047} 1048#endif /* EFSYS_OPT_QSTATS */ 1049 1050static void 1051falconsiena_tx_qdestroy( 1052 __in efx_txq_t *etp) 1053{ 1054 efx_nic_t *enp = etp->et_enp; 1055 efx_oword_t oword; 1056 1057 /* Purge descriptor queue */ 1058 EFX_ZERO_OWORD(oword); 1059 1060 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1061 etp->et_index, &oword, B_TRUE); 1062} 1063 1064static void 1065falconsiena_tx_fini( 1066 __in efx_nic_t *enp) 1067{ 1068 _NOTE(ARGUNUSED(enp)) 1069} 1070 1071#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */ 1072