tcp_tlro.c revision 291184
1202181Sthompsa/*- 2202181Sthompsa * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 3202181Sthompsa * 4202181Sthompsa * Redistribution and use in source and binary forms, with or without 5202181Sthompsa * modification, are permitted provided that the following conditions 6202181Sthompsa * are met: 7202181Sthompsa * 1. Redistributions of source code must retain the above copyright 8202181Sthompsa * notice, this list of conditions and the following disclaimer. 9202181Sthompsa * 2. Redistributions in binary form must reproduce the above copyright 10202181Sthompsa * notice, this list of conditions and the following disclaimer in the 11202181Sthompsa * documentation and/or other materials provided with the distribution. 12202181Sthompsa * 13202181Sthompsa * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14202181Sthompsa * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15202181Sthompsa * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16202181Sthompsa * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17202181Sthompsa * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18202181Sthompsa * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19202181Sthompsa * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20202181Sthompsa * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21202181Sthompsa * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22202181Sthompsa * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23202181Sthompsa * SUCH DAMAGE. 24202181Sthompsa */ 25202181Sthompsa 26210283Strasz#include <sys/cdefs.h> 27206622Suqs__FBSDID("$FreeBSD: stable/10/sys/dev/mlx5/mlx5_en/tcp_tlro.c 291184 2015-11-23 09:32:32Z hselasky $"); 28202181Sthompsa 29202181Sthompsa#include "opt_inet.h" 30202243Sthompsa#include "opt_inet6.h" 31202181Sthompsa 32202181Sthompsa#include <sys/param.h> 33202181Sthompsa#include <sys/libkern.h> 34202181Sthompsa#include <sys/mbuf.h> 35202181Sthompsa#include <sys/lock.h> 36202181Sthompsa#include <sys/mutex.h> 37202181Sthompsa#include <sys/sysctl.h> 38202181Sthompsa#include <sys/malloc.h> 39202181Sthompsa#include <sys/kernel.h> 40202181Sthompsa#include <sys/endian.h> 41202181Sthompsa#include <sys/socket.h> 42202181Sthompsa#include <sys/sockopt.h> 43202181Sthompsa#include <sys/smp.h> 44202181Sthompsa 45202181Sthompsa#include <net/if.h> 46210275Sthompsa#include <net/if_var.h> 47210275Sthompsa#include <net/ethernet.h> 48210275Sthompsa 49202181Sthompsa#if defined(INET) || defined(INET6) 50202181Sthompsa#include <netinet/in.h> 51202270Sthompsa#endif 52202270Sthompsa 53202181Sthompsa#ifdef INET 54202181Sthompsa#include <netinet/ip.h> 55202181Sthompsa#endif 56202243Sthompsa 57202181Sthompsa#ifdef INET6 58202181Sthompsa#include <netinet/ip6.h> 59202181Sthompsa#endif 60202181Sthompsa 61202181Sthompsa#include <netinet/tcp_var.h> 62202181Sthompsa 63202181Sthompsa#include "tcp_tlro.h" 64210275Sthompsa 65210275Sthompsa#ifndef M_HASHTYPE_LRO_TCP 66210275Sthompsa#ifndef KLD_MODULE 67210275Sthompsa#warning "M_HASHTYPE_LRO_TCP is not defined" 68210275Sthompsa#endif 69210275Sthompsa#define M_HASHTYPE_LRO_TCP 254 70210275Sthompsa#endif 71210275Sthompsa 72202181Sthompsastatic SYSCTL_NODE(_net_inet_tcp, OID_AUTO, tlro, 73202181Sthompsa CTLFLAG_RW, 0, "TCP turbo LRO parameters"); 74202181Sthompsa 75210275Sthompsastatic MALLOC_DEFINE(M_TLRO, "TLRO", "Turbo LRO"); 76210275Sthompsa 77202181Sthompsastatic int tlro_min_rate = 20; /* Hz */ 78202181Sthompsa 79202181SthompsaSYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, min_rate, CTLFLAG_RWTUN, 80202181Sthompsa &tlro_min_rate, 0, "Minimum serving rate in Hz"); 81202181Sthompsa 82225037Shselaskystatic int tlro_max_packet = IP_MAXPACKET; 83225037Shselasky 84202181SthompsaSYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, max_packet, CTLFLAG_RWTUN, 85202243Sthompsa &tlro_max_packet, 0, "Maximum packet size in bytes"); 86210275Sthompsa 87210275Sthompsatypedef struct { 88202243Sthompsa uint32_t value; 89202181Sthompsa} __packed uint32_p_t; 90202181Sthompsa 91202181Sthompsastatic uint16_t 92202181Sthompsatcp_tlro_csum(const uint32_p_t *p, size_t l) 93202270Sthompsa{ 94202243Sthompsa const uint32_p_t *pend = p + (l / 4); 95202243Sthompsa uint64_t cs; 96202243Sthompsa 97202243Sthompsa for (cs = 0; p != pend; p++) 98202243Sthompsa cs += le32toh(p->value); 99202243Sthompsa while (cs > 0xffff) 100202243Sthompsa cs = (cs >> 16) + (cs & 0xffff); 101202243Sthompsa return (cs); 102202243Sthompsa} 103235317Sgjb 104208028Suqsstatic void * 105208028Suqstcp_tlro_get_header(const struct mbuf *m, const u_int off, 106208028Suqs const u_int len) 107208028Suqs{ 108202181Sthompsa if (m->m_len < (off + len)) 109210275Sthompsa return (NULL); 110210275Sthompsa return (mtod(m, char *) + off); 111202181Sthompsa} 112202181Sthompsa 113202181Sthompsastatic uint8_t 114202181Sthompsatcp_tlro_info_save_timestamp(struct tlro_mbuf_data *pinfo) 115202181Sthompsa{ 116202181Sthompsa struct tcphdr *tcp = pinfo->tcp; 117202181Sthompsa uint32_t *ts_ptr; 118202181Sthompsa 119202181Sthompsa if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2)) 120202181Sthompsa return (0); 121202181Sthompsa 122202181Sthompsa ts_ptr = (uint32_t *)(tcp + 1); 123202181Sthompsa if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 124202181Sthompsa (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) 125202181Sthompsa return (0); 126202181Sthompsa 127202181Sthompsa /* Save timestamps */ 128202181Sthompsa pinfo->tcp_ts = ts_ptr[1]; 129202181Sthompsa pinfo->tcp_ts_reply = ts_ptr[2]; 130202181Sthompsa return (1); 131202181Sthompsa} 132202181Sthompsa 133202181Sthompsastatic void 134202181Sthompsatcp_tlro_info_restore_timestamp(struct tlro_mbuf_data *pinfoa, 135210283Strasz struct tlro_mbuf_data *pinfob) 136202181Sthompsa{ 137202181Sthompsa struct tcphdr *tcp = pinfoa->tcp; 138202181Sthompsa uint32_t *ts_ptr; 139202181Sthompsa 140202181Sthompsa if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2)) 141202181Sthompsa return; 142202181Sthompsa 143 ts_ptr = (uint32_t *)(tcp + 1); 144 if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 145 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) 146 return; 147 148 /* Restore timestamps */ 149 ts_ptr[1] = pinfob->tcp_ts; 150 ts_ptr[2] = pinfob->tcp_ts_reply; 151} 152 153static void 154tcp_tlro_extract_header(struct tlro_mbuf_data *pinfo, struct mbuf *m, int seq) 155{ 156 uint8_t *phdr = (uint8_t *)pinfo->buf; 157 struct ether_header *eh; 158 struct ether_vlan_header *vlan; 159#ifdef INET 160 struct ip *ip; 161#endif 162#ifdef INET6 163 struct ip6_hdr *ip6; 164#endif 165 struct tcphdr *tcp; 166 uint16_t etype; 167 int diff; 168 int off; 169 170 /* Fill in information */ 171 pinfo->head = m; 172 pinfo->last_tick = ticks; 173 pinfo->sequence = seq; 174 pinfo->pprev = &m_last(m)->m_next; 175 176 off = sizeof(*eh); 177 if (m->m_len < off) 178 goto error; 179 eh = tcp_tlro_get_header(m, 0, sizeof(*eh)); 180 if (eh == NULL) 181 goto error; 182 memcpy(phdr, &eh->ether_dhost, ETHER_ADDR_LEN); 183 phdr += ETHER_ADDR_LEN; 184 memcpy(phdr, &eh->ether_type, sizeof(eh->ether_type)); 185 phdr += sizeof(eh->ether_type); 186 etype = ntohs(eh->ether_type); 187 188 if (etype == ETHERTYPE_VLAN) { 189 vlan = tcp_tlro_get_header(m, off, sizeof(*vlan)); 190 if (vlan == NULL) 191 goto error; 192 memcpy(phdr, &vlan->evl_tag, sizeof(vlan->evl_tag) + 193 sizeof(vlan->evl_proto)); 194 phdr += sizeof(vlan->evl_tag) + sizeof(vlan->evl_proto); 195 etype = ntohs(vlan->evl_proto); 196 off += sizeof(*vlan) - sizeof(*eh); 197 } 198 switch (etype) { 199#ifdef INET 200 case ETHERTYPE_IP: 201 /* 202 * Cannot LRO: 203 * - Non-IP packets 204 * - Fragmented packets 205 * - Packets with IPv4 options 206 * - Non-TCP packets 207 */ 208 ip = tcp_tlro_get_header(m, off, sizeof(*ip)); 209 if (ip == NULL || 210 (ip->ip_off & htons(IP_MF | IP_OFFMASK)) != 0 || 211 (ip->ip_p != IPPROTO_TCP) || 212 (ip->ip_hl << 2) != sizeof(*ip)) 213 goto error; 214 215 /* Legacy IP has a header checksum that needs to be correct */ 216 if (!(m->m_pkthdr.csum_flags & CSUM_IP_CHECKED)) { 217 /* Verify IP header */ 218 if (tcp_tlro_csum((uint32_p_t *)ip, sizeof(*ip)) != 0xFFFF) 219 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; 220 else 221 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | 222 CSUM_IP_VALID; 223 } 224 /* Only accept valid checksums */ 225 if (!(m->m_pkthdr.csum_flags & CSUM_IP_VALID) || 226 !(m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) 227 goto error; 228 memcpy(phdr, &ip->ip_src, sizeof(ip->ip_src) + 229 sizeof(ip->ip_dst)); 230 phdr += sizeof(ip->ip_src) + sizeof(ip->ip_dst); 231 if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP) 232 pinfo->ip_len = m->m_pkthdr.len - off; 233 else 234 pinfo->ip_len = ntohs(ip->ip_len); 235 pinfo->ip_hdrlen = sizeof(*ip); 236 pinfo->ip.v4 = ip; 237 pinfo->ip_version = 4; 238 off += sizeof(*ip); 239 break; 240#endif 241#ifdef INET6 242 case ETHERTYPE_IPV6: 243 /* 244 * Cannot LRO: 245 * - Non-IP packets 246 * - Packets with IPv6 options 247 * - Non-TCP packets 248 */ 249 ip6 = tcp_tlro_get_header(m, off, sizeof(*ip6)); 250 if (ip6 == NULL || ip6->ip6_nxt != IPPROTO_TCP) 251 goto error; 252 if (!(m->m_pkthdr.csum_flags & CSUM_DATA_VALID)) 253 goto error; 254 memcpy(phdr, &ip6->ip6_src, sizeof(struct in6_addr) + 255 sizeof(struct in6_addr)); 256 phdr += sizeof(struct in6_addr) + sizeof(struct in6_addr); 257 if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP) 258 pinfo->ip_len = m->m_pkthdr.len - off; 259 else 260 pinfo->ip_len = ntohs(ip6->ip6_plen) + sizeof(*ip6); 261 pinfo->ip_hdrlen = sizeof(*ip6); 262 pinfo->ip.v6 = ip6; 263 pinfo->ip_version = 6; 264 off += sizeof(*ip6); 265 break; 266#endif 267 default: 268 goto error; 269 } 270 tcp = tcp_tlro_get_header(m, off, sizeof(*tcp)); 271 if (tcp == NULL) 272 goto error; 273 memcpy(phdr, &tcp->th_sport, sizeof(tcp->th_sport) + 274 sizeof(tcp->th_dport)); 275 phdr += sizeof(tcp->th_sport) + 276 sizeof(tcp->th_dport); 277 /* Store TCP header length */ 278 *phdr++ = tcp->th_off; 279 if (tcp->th_off < (sizeof(*tcp) >> 2)) 280 goto error; 281 282 /* Compute offset to data payload */ 283 pinfo->tcp_len = (tcp->th_off << 2); 284 off += pinfo->tcp_len; 285 286 /* Store more info */ 287 pinfo->data_off = off; 288 pinfo->tcp = tcp; 289 290 /* Try to save timestamp, if any */ 291 *phdr++ = tcp_tlro_info_save_timestamp(pinfo); 292 293 /* Verify offset and IP/TCP length */ 294 if (off > m->m_pkthdr.len || 295 pinfo->ip_len < pinfo->tcp_len) 296 goto error; 297 298 /* Compute data payload length */ 299 pinfo->data_len = (pinfo->ip_len - pinfo->tcp_len - pinfo->ip_hdrlen); 300 301 /* Trim any padded data */ 302 diff = (m->m_pkthdr.len - off) - pinfo->data_len; 303 if (diff != 0) { 304 if (diff < 0) 305 goto error; 306 else 307 m_adj(m, -diff); 308 } 309 /* Compute header length */ 310 pinfo->buf_length = phdr - (uint8_t *)pinfo->buf; 311 /* Zero-pad rest of buffer */ 312 memset(phdr, 0, TLRO_MAX_HEADER - pinfo->buf_length); 313 return; 314error: 315 pinfo->buf_length = 0; 316} 317 318static int 319tcp_tlro_cmp64(const uint64_t *pa, const uint64_t *pb) 320{ 321 int64_t diff = 0; 322 unsigned x; 323 324 for (x = 0; x != TLRO_MAX_HEADER / 8; x++) { 325 /* 326 * NOTE: Endianness does not matter in this 327 * comparisation: 328 */ 329 diff = pa[x] - pb[x]; 330 if (diff != 0) 331 goto done; 332 } 333done: 334 if (diff < 0) 335 return (-1); 336 else if (diff > 0) 337 return (1); 338 return (0); 339} 340 341static int 342tcp_tlro_compare_header(const void *_ppa, const void *_ppb) 343{ 344 const struct tlro_mbuf_ptr *ppa = _ppa; 345 const struct tlro_mbuf_ptr *ppb = _ppb; 346 struct tlro_mbuf_data *pinfoa = ppa->data; 347 struct tlro_mbuf_data *pinfob = ppb->data; 348 int ret; 349 350 ret = (pinfoa->head == NULL) - (pinfob->head == NULL); 351 if (ret != 0) 352 goto done; 353 354 ret = pinfoa->buf_length - pinfob->buf_length; 355 if (ret != 0) 356 goto done; 357 if (pinfoa->buf_length != 0) { 358 ret = tcp_tlro_cmp64(pinfoa->buf, pinfob->buf); 359 if (ret != 0) 360 goto done; 361 ret = ntohl(pinfoa->tcp->th_seq) - ntohl(pinfob->tcp->th_seq); 362 if (ret != 0) 363 goto done; 364 ret = ntohl(pinfoa->tcp->th_ack) - ntohl(pinfob->tcp->th_ack); 365 if (ret != 0) 366 goto done; 367 ret = pinfoa->sequence - pinfob->sequence; 368 if (ret != 0) 369 goto done; 370 } 371done: 372 return (ret); 373} 374 375static void 376tcp_tlro_sort(struct tlro_ctrl *tlro) 377{ 378 if (tlro->curr == 0) 379 return; 380 381 qsort(tlro->mbuf, tlro->curr, sizeof(struct tlro_mbuf_ptr), 382 &tcp_tlro_compare_header); 383} 384 385static int 386tcp_tlro_get_ticks(void) 387{ 388 int to = tlro_min_rate; 389 390 if (to < 1) 391 to = 1; 392 to = hz / to; 393 if (to < 1) 394 to = 1; 395 return (to); 396} 397 398static void 399tcp_tlro_combine(struct tlro_ctrl *tlro, int force) 400{ 401 struct tlro_mbuf_data *pinfoa; 402 struct tlro_mbuf_data *pinfob; 403 uint32_t cs; 404 int curr_ticks = ticks; 405 int ticks_limit = tcp_tlro_get_ticks(); 406 unsigned x; 407 unsigned y; 408 unsigned z; 409 int temp; 410 411 if (tlro->curr == 0) 412 return; 413 414 for (y = 0; y != tlro->curr;) { 415 struct mbuf *m; 416 417 pinfoa = tlro->mbuf[y].data; 418 for (x = y + 1; x != tlro->curr; x++) { 419 pinfob = tlro->mbuf[x].data; 420 if (pinfoa->buf_length != pinfob->buf_length || 421 tcp_tlro_cmp64(pinfoa->buf, pinfob->buf) != 0) 422 break; 423 } 424 if (pinfoa->buf_length == 0) { 425 /* Forward traffic which cannot be combined */ 426 for (z = y; z != x; z++) { 427 /* Just forward packets */ 428 pinfob = tlro->mbuf[z].data; 429 430 m = pinfob->head; 431 432 /* Reset info structure */ 433 pinfob->head = NULL; 434 pinfob->buf_length = 0; 435 436 /* Do stats */ 437 tlro->lro_flushed++; 438 439 /* Input packet to network layer */ 440 (*tlro->ifp->if_input) (tlro->ifp, m); 441 } 442 y = z; 443 continue; 444 } 445 446 /* Compute current checksum subtracted some header parts */ 447 temp = (pinfoa->ip_len - pinfoa->ip_hdrlen); 448 cs = ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) + 449 tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len); 450 451 /* Append all fragments into one block */ 452 for (z = y + 1; z != x; z++) { 453 454 pinfob = tlro->mbuf[z].data; 455 456 /* Check for command packets */ 457 if ((pinfoa->tcp->th_flags & ~(TH_ACK | TH_PUSH)) || 458 (pinfob->tcp->th_flags & ~(TH_ACK | TH_PUSH))) 459 break; 460 461 /* Check if there is enough space */ 462 if ((pinfoa->ip_len + pinfob->data_len) > tlro_max_packet) 463 break; 464 465 /* Try to append the new segment */ 466 temp = ntohl(pinfoa->tcp->th_seq) + pinfoa->data_len; 467 if (temp != (int)ntohl(pinfob->tcp->th_seq)) 468 break; 469 470 temp = pinfob->ip_len - pinfob->ip_hdrlen; 471 cs += ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) + 472 tcp_tlro_csum((uint32_p_t *)pinfob->tcp, pinfob->tcp_len); 473 /* Remove fields which appear twice */ 474 cs += (IPPROTO_TCP << 8); 475 if (pinfob->ip_version == 4) { 476 cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_src, 4); 477 cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_dst, 4); 478 } else { 479 cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_src, 16); 480 cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_dst, 16); 481 } 482 /* Remainder computation */ 483 while (cs > 0xffff) 484 cs = (cs >> 16) + (cs & 0xffff); 485 486 /* Update window and ack sequence number */ 487 pinfoa->tcp->th_ack = pinfob->tcp->th_ack; 488 pinfoa->tcp->th_win = pinfob->tcp->th_win; 489 490 /* Check if we should restore the timestamp */ 491 tcp_tlro_info_restore_timestamp(pinfoa, pinfob); 492 493 /* Accumulate TCP flags */ 494 pinfoa->tcp->th_flags |= pinfob->tcp->th_flags; 495 496 /* update lengths */ 497 pinfoa->ip_len += pinfob->data_len; 498 pinfoa->data_len += pinfob->data_len; 499 500 /* Clear mbuf pointer - packet is accumulated */ 501 m = pinfob->head; 502 503 /* Reset info structure */ 504 pinfob->head = NULL; 505 pinfob->buf_length = 0; 506 507 /* Append data to mbuf [y] */ 508 m_adj(m, pinfob->data_off); 509 /* Delete mbuf tags, if any */ 510 m_tag_delete_chain(m, NULL); 511 /* Clear packet header flag */ 512 m->m_flags &= ~M_PKTHDR; 513 514 /* Concat mbuf(s) to end of list */ 515 pinfoa->pprev[0] = m; 516 m = m_last(m); 517 pinfoa->pprev = &m->m_next; 518 pinfoa->head->m_pkthdr.len += pinfob->data_len; 519 } 520 /* Compute new TCP header checksum */ 521 pinfoa->tcp->th_sum = 0; 522 523 temp = pinfoa->ip_len - pinfoa->ip_hdrlen; 524 cs = (cs ^ 0xFFFF) + 525 tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len) + 526 ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8); 527 528 /* Remainder computation */ 529 while (cs > 0xffff) 530 cs = (cs >> 16) + (cs & 0xffff); 531 532 /* Update new checksum */ 533 pinfoa->tcp->th_sum = ~htole16(cs); 534 535 /* Update IP length, if any */ 536 if (pinfoa->ip_version == 4) { 537 if (pinfoa->ip_len > IP_MAXPACKET) { 538 M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP); 539 pinfoa->ip.v4->ip_len = htons(IP_MAXPACKET); 540 } else { 541 pinfoa->ip.v4->ip_len = htons(pinfoa->ip_len); 542 } 543 } else { 544 if (pinfoa->ip_len > (IP_MAXPACKET + sizeof(*pinfoa->ip.v6))) { 545 M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP); 546 pinfoa->ip.v6->ip6_plen = htons(IP_MAXPACKET); 547 } else { 548 temp = pinfoa->ip_len - sizeof(*pinfoa->ip.v6); 549 pinfoa->ip.v6->ip6_plen = htons(temp); 550 } 551 } 552 553 temp = curr_ticks - pinfoa->last_tick; 554 /* Check if packet should be forwarded */ 555 if (force != 0 || z != x || temp >= ticks_limit || 556 pinfoa->data_len == 0) { 557 558 /* Compute new IPv4 header checksum */ 559 if (pinfoa->ip_version == 4) { 560 pinfoa->ip.v4->ip_sum = 0; 561 cs = tcp_tlro_csum((uint32_p_t *)pinfoa->ip.v4, 562 sizeof(*pinfoa->ip.v4)); 563 pinfoa->ip.v4->ip_sum = ~htole16(cs); 564 } 565 /* Forward packet */ 566 m = pinfoa->head; 567 568 /* Reset info structure */ 569 pinfoa->head = NULL; 570 pinfoa->buf_length = 0; 571 572 /* Do stats */ 573 tlro->lro_flushed++; 574 575 /* Input packet to network layer */ 576 (*tlro->ifp->if_input) (tlro->ifp, m); 577 } 578 y = z; 579 } 580 581 /* Cleanup all NULL heads */ 582 for (y = 0; y != tlro->curr; y++) { 583 if (tlro->mbuf[y].data->head == NULL) { 584 for (z = y + 1; z != tlro->curr; z++) { 585 struct tlro_mbuf_ptr ptemp; 586 if (tlro->mbuf[z].data->head == NULL) 587 continue; 588 ptemp = tlro->mbuf[y]; 589 tlro->mbuf[y] = tlro->mbuf[z]; 590 tlro->mbuf[z] = ptemp; 591 y++; 592 } 593 break; 594 } 595 } 596 tlro->curr = y; 597} 598 599static void 600tcp_tlro_cleanup(struct tlro_ctrl *tlro) 601{ 602 while (tlro->curr != 0 && 603 tlro->mbuf[tlro->curr - 1].data->head == NULL) 604 tlro->curr--; 605} 606 607void 608tcp_tlro_flush(struct tlro_ctrl *tlro, int force) 609{ 610 if (tlro->curr == 0) 611 return; 612 613 tcp_tlro_sort(tlro); 614 tcp_tlro_cleanup(tlro); 615 tcp_tlro_combine(tlro, force); 616} 617 618int 619tcp_tlro_init(struct tlro_ctrl *tlro, struct ifnet *ifp, 620 int max_mbufs) 621{ 622 ssize_t size; 623 uint32_t x; 624 625 /* Set zero defaults */ 626 memset(tlro, 0, sizeof(*tlro)); 627 628 /* Compute size needed for data */ 629 size = (sizeof(struct tlro_mbuf_ptr) * max_mbufs) + 630 (sizeof(struct tlro_mbuf_data) * max_mbufs); 631 632 /* Range check */ 633 if (max_mbufs <= 0 || size <= 0 || ifp == NULL) 634 return (EINVAL); 635 636 /* Setup tlro control structure */ 637 tlro->mbuf = malloc(size, M_TLRO, M_WAITOK | M_ZERO); 638 tlro->max = max_mbufs; 639 tlro->ifp = ifp; 640 641 /* Setup pointer array */ 642 for (x = 0; x != tlro->max; x++) { 643 tlro->mbuf[x].data = ((struct tlro_mbuf_data *) 644 &tlro->mbuf[max_mbufs]) + x; 645 } 646 return (0); 647} 648 649void 650tcp_tlro_free(struct tlro_ctrl *tlro) 651{ 652 struct tlro_mbuf_data *pinfo; 653 struct mbuf *m; 654 uint32_t y; 655 656 /* Check if not setup */ 657 if (tlro->mbuf == NULL) 658 return; 659 /* Free MBUF array and any leftover MBUFs */ 660 for (y = 0; y != tlro->max; y++) { 661 662 pinfo = tlro->mbuf[y].data; 663 664 m = pinfo->head; 665 666 /* Reset info structure */ 667 pinfo->head = NULL; 668 pinfo->buf_length = 0; 669 670 m_freem(m); 671 } 672 free(tlro->mbuf, M_TLRO); 673 /* Reset buffer */ 674 memset(tlro, 0, sizeof(*tlro)); 675} 676 677void 678tcp_tlro_rx(struct tlro_ctrl *tlro, struct mbuf *m) 679{ 680 if (m->m_len > 0 && tlro->curr < tlro->max) { 681 /* do stats */ 682 tlro->lro_queued++; 683 684 /* extract header */ 685 tcp_tlro_extract_header(tlro->mbuf[tlro->curr++].data, 686 m, tlro->sequence++); 687 } else if (tlro->ifp != NULL) { 688 /* do stats */ 689 tlro->lro_flushed++; 690 691 /* input packet to network layer */ 692 (*tlro->ifp->if_input) (tlro->ifp, m); 693 } else { 694 /* packet drop */ 695 m_freem(m); 696 } 697} 698