1/* 2 * Copyright (c) 1999-2013 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */ 29/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ 30/* 31 * Mach Operating System 32 * Copyright (c) 1987 Carnegie-Mellon University 33 * All rights reserved. The CMU software License Agreement specifies 34 * the terms and conditions for use and redistribution. 35 */ 36/* 37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved. 38 * 39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California. 40 * All rights reserved. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Berkeley and its contributors. 54 * 4. Neither the name of the University nor the names of its contributors 55 * may be used to endorse or promote products derived from this software 56 * without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 * 70 * @(#)mbuf.h 8.3 (Berkeley) 1/21/94 71 */ 72/* 73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce 74 * support for mandatory and extensible security protections. This notice 75 * is included in support of clause 2.2 (b) of the Apple Public License, 76 * Version 2.0. 77 */ 78 79#ifndef _SYS_MBUF_H_ 80#define _SYS_MBUF_H_ 81 82#include <sys/cdefs.h> 83#include <sys/appleapiopts.h> 84 85#ifdef XNU_KERNEL_PRIVATE 86 87#include <sys/lock.h> 88#include <sys/queue.h> 89#include <machine/endian.h> 90/* 91 * Mbufs are of a single size, MSIZE (machine/param.h), which 92 * includes overhead. An mbuf may add a single "mbuf cluster" of size 93 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has 94 * no additional overhead and is used instead of the internal data area; 95 * this is done when at least MINCLSIZE of data must be stored. 96 */ 97 98/* 99 * The following _MLEN and _MHLEN macros are private to xnu. Private code 100 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since 101 * the sizes of the structures are dependent upon specific xnu configs. 102 */ 103#define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */ 104#define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ 105 106#define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT) 107#define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */ 108 109#define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT) 110#define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */ 111 112#define NMBPCLSHIFT (NMBPBGSHIFT - NCLPBGSHIFT) 113#define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */ 114 115#define NCLPJCLSHIFT ((M16KCLSHIFT - MBIGCLSHIFT) + NCLPBGSHIFT) 116#define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */ 117 118/* 119 * Macros for type conversion 120 * mtod(m,t) - convert mbuf pointer to data pointer of correct type 121 * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX) 122 */ 123#define mtod(m, t) ((t)m_mtod(m)) 124#define dtom(x) m_dtom(x) 125 126/* header at beginning of each mbuf: */ 127struct m_hdr { 128 struct mbuf *mh_next; /* next buffer in chain */ 129 struct mbuf *mh_nextpkt; /* next chain in queue/record */ 130 caddr_t mh_data; /* location of data */ 131 int32_t mh_len; /* amount of data in this mbuf */ 132 u_int16_t mh_type; /* type of data in this mbuf */ 133 u_int16_t mh_flags; /* flags; see below */ 134}; 135 136/* 137 * Packet tag structure (see below for details). 138 */ 139struct m_tag { 140 u_int64_t m_tag_cookie; /* Error checking */ 141#ifndef __LP64__ 142 u_int32_t pad; /* For structure alignment */ 143#endif /* !__LP64__ */ 144 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ 145 u_int16_t m_tag_type; /* Module specific type */ 146 u_int16_t m_tag_len; /* Length of data */ 147 u_int32_t m_tag_id; /* Module ID */ 148}; 149 150#define M_TAG_ALIGN(len) \ 151 (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag)) 152 153#define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL 154#define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL 155 156/* 157 * Packet tag header structure (at the top of mbuf). Pointers are 158 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded. 159 */ 160struct m_taghdr { 161#ifndef __LP64__ 162 u_int32_t pad; /* For structure alignment */ 163#endif /* !__LP64__ */ 164 u_int64_t refcnt; /* Number of tags in this mbuf */ 165}; 166 167/* 168 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX). 169 */ 170struct m_drvaux_tag { 171 u_int32_t da_family; /* IFNET_FAMILY values */ 172 u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */ 173 u_int32_t da_reserved; /* for future */ 174 u_int32_t da_length; /* length of following data */ 175}; 176 177/* Values for pftag_flags (16-bit wide) */ 178#define PF_TAG_GENERATED 0x1 /* pkt generated by PF */ 179#define PF_TAG_FRAGCACHE 0x2 180#define PF_TAG_TRANSLATE_LOCALHOST 0x4 181#if PF_ECN 182#define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */ 183#define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */ 184#endif /* PF_ECN */ 185/* 186 * PF mbuf tag 187 */ 188struct pf_mtag { 189 u_int16_t pftag_flags; /* PF_TAG flags */ 190 u_int16_t pftag_rtableid; /* alternate routing table id */ 191 u_int16_t pftag_tag; 192 u_int16_t pftag_routed; 193#if PF_ALTQ 194 u_int32_t pftag_qid; 195#endif /* PF_ALTQ */ 196#if PF_ECN 197 void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ 198#endif /* PF_ECN */ 199}; 200 201/* 202 * TCP mbuf tag 203 */ 204struct tcp_pktinfo { 205 union { 206 struct { 207 u_int32_t segsz; /* segment size (actual MSS) */ 208 } __tx; 209 struct { 210 u_int16_t lro_pktlen; /* max seg size encountered */ 211 u_int8_t lro_npkts; /* # of coalesced TCP pkts */ 212 u_int8_t lro_timediff; /* time spent in LRO */ 213 } __rx; 214 } __offload; 215 union { 216 u_int32_t pri; /* send msg priority */ 217 u_int32_t seq; /* recv msg sequence # */ 218 } __msgattr; 219#define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz 220#define lro_pktlen proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_pktlen 221#define lro_npkts proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_npkts 222#define lro_elapsed proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.lro_timediff 223#define msg_pri proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.pri 224#define msg_seq proto_mtag.__pr_u.tcp.tm_tcp.__msgattr.seq 225}; 226 227/* 228 * MPTCP mbuf tag 229 */ 230struct mptcp_pktinfo { 231 u_int64_t mtpi_dsn; /* MPTCP Data Sequence Number */ 232 union { 233 u_int64_t mtpi_dan; /* MPTCP Data Ack Number */ 234 struct { 235 u_int32_t mtpi_rel_seq; /* Relative Seq Number */ 236 u_int32_t mtpi_length; /* Length of mapping */ 237 } mtpi_subf; 238 }; 239#define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn 240#define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_rel_seq 241#define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_length 242#define mp_dack proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_subf.mtpi_dan 243}; 244 245/* 246 * TCP specific mbuf tag. Note that the current implementation uses 247 * MPTCP metadata strictly between MPTCP and the TCP subflow layers, 248 * hence tm_tcp and tm_mptcp are mutually exclusive. This also means 249 * that TCP messages functionality is currently incompatible with MPTCP. 250 */ 251struct tcp_mtag { 252 union { 253 struct tcp_pktinfo tm_tcp; /* TCP and below */ 254 struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */ 255 }; 256}; 257 258/* 259 * IPSec mbuf tag 260 */ 261struct ipsec_mtag { 262 uint32_t policy_id; 263#define ipsec_policy proto_mtag.__pr_u.ipsec.policy_id 264}; 265 266/* 267 * Protocol specific mbuf tag (at most one protocol metadata per mbuf). 268 * 269 * Care must be taken to ensure that they are mutually exclusive, e.g. 270 * IPSec policy ID implies no TCP segment offload (which is fine given 271 * that the former is used on the virtual ipsec interface that does 272 * not advertise the TSO capability.) 273 */ 274struct proto_mtag { 275 union { 276 struct tcp_mtag tcp; /* TCP specific */ 277 struct ipsec_mtag ipsec; /* IPSec specific */ 278 } __pr_u; 279}; 280 281/* 282 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set. 283 */ 284struct pkthdr { 285 struct ifnet *rcvif; /* rcv interface */ 286 /* variables for ip and tcp reassembly */ 287 void *pkt_hdr; /* pointer to packet header */ 288 int32_t len; /* total packet length */ 289 /* variables for hardware checksum */ 290 /* Note: csum_flags is used for hardware checksum and VLAN */ 291 u_int32_t csum_flags; /* flags regarding checksum */ 292 union { 293 struct { 294 u_int16_t val; /* checksum value */ 295 u_int16_t start; /* checksum start offset */ 296 } _csum_rx; 297#define csum_rx_val _csum_rx.val 298#define csum_rx_start _csum_rx.start 299 struct { 300 u_int16_t start; /* checksum start offset */ 301 u_int16_t stuff; /* checksum stuff offset */ 302 } _csum_tx; 303#define csum_tx_start _csum_tx.start 304#define csum_tx_stuff _csum_tx.stuff 305 u_int32_t csum_data; /* data field used by csum routines */ 306 }; 307 u_int16_t vlan_tag; /* VLAN tag, host byte order */ 308 /* 309 * Packet classifier info 310 * 311 * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value 312 * means the packet has been classified by one of the flow sources. 313 * It is also a prerequisite for flow control advisory, which is 314 * enabled by additionally setting PKTF_FLOW_ADV. 315 * 316 * The protocol value is a best-effort representation of the payload. 317 * It is opportunistically updated and used only for optimization. 318 * It is not a substitute for parsing the protocol header(s); use it 319 * only as a hint. 320 * 321 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the 322 * indices of interfaces which own the source and/or destination 323 * addresses of the packet. For the local/loopback case (PKTF_LOOP), 324 * both should be valid, and thus allows for the receiving end to 325 * quickly determine the actual interfaces used by the the addresses; 326 * they may not necessarily be the same or refer to the loopback 327 * interface. Otherwise, in the non-local/loopback case, the indices 328 * are opportunistically set, and because of that only one may be set 329 * (0 means the index has not been determined.) In addition, the 330 * interface address flags are also recorded. This allows us to avoid 331 * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally 332 * this would be a superset of {ia,ia6}_flags, but the namespaces are 333 * overlapping at present, so we'll need a new set of values in future 334 * to achieve this. For now, we will just rely on the address family 335 * related code paths examining this mbuf to interpret the flags. 336 */ 337 u_int8_t pkt_proto; /* IPPROTO value */ 338 u_int8_t pkt_flowsrc; /* FLOWSRC values */ 339 u_int32_t pkt_flowid; /* flow ID */ 340 u_int32_t pkt_flags; /* PKTF flags (see below) */ 341 u_int32_t pkt_svc; /* MBUF_SVC value */ 342 union { 343 struct { 344 u_int16_t src; /* ifindex of src addr i/f */ 345 u_int16_t src_flags; /* src PKT_IFAIFF flags */ 346 u_int16_t dst; /* ifindex of dst addr i/f */ 347 u_int16_t dst_flags; /* dst PKT_IFAIFF flags */ 348 } _pkt_iaif; 349#define src_ifindex _pkt_iaif.src 350#define src_iff _pkt_iaif.src_flags 351#define dst_ifindex _pkt_iaif.dst 352#define dst_iff _pkt_iaif.dst_flags 353 u_int64_t pkt_ifainfo; /* data field used by ifainfo */ 354 }; 355#if MEASURE_BW 356 u_int64_t pkt_bwseq; /* sequence # */ 357#endif /* MEASURE_BW */ 358 /* 359 * Tags (external and built-in) 360 */ 361 SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */ 362 struct proto_mtag proto_mtag; /* built-in protocol-specific tag */ 363 struct pf_mtag pf_mtag; /* built-in PF tag */ 364 /* 365 * Module private scratch space (32-bit aligned), currently 16-bytes 366 * large. Anything stored here is not guaranteed to survive across 367 * modules. This should be the penultimate structure right before 368 * the red zone. Add new fields above this. 369 */ 370 struct { 371 union { 372 u_int8_t __mpriv8[16]; 373 u_int16_t __mpriv16[8]; 374 struct { 375 union { 376 u_int8_t __val8[4]; 377 u_int16_t __val16[2]; 378 u_int32_t __val32; 379 } __mpriv32_u; 380 } __mpriv32[4]; 381 u_int64_t __mpriv64[2]; 382 } __mpriv_u; 383 } pkt_mpriv __attribute__((aligned(4))); 384 u_int32_t redzone; /* red zone */ 385}; 386 387/* 388 * Flow data source type. A data source module is responsible for generating 389 * a unique flow ID and associating it to each data flow as pkt_flowid. 390 * This is required for flow control/advisory, as it allows the output queue 391 * to identify the data source object and inform that it can resume its 392 * transmission (in the event it was flow controlled.) 393 */ 394#define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */ 395#define FLOWSRC_IFNET 2 /* flow ID generated by interface */ 396#define FLOWSRC_PF 3 /* flow ID generated by PF */ 397 398/* 399 * Packet flags. Unlike m_flags, all packet flags are copied along when 400 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags 401 * (and other classifier info) will be cleared during DLIL input. 402 * 403 * Some notes about M_LOOP and PKTF_LOOP: 404 * 405 * - M_LOOP flag is overloaded, and its use is discouraged. Historically, 406 * that flag was used by the KAME implementation for allowing certain 407 * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this 408 * was originally meant to be set as the packet is looped back to the 409 * system, and in some circumstances temporarily set in ip6_output(). 410 * Over time, this flag was used by the pre-output routines to indicate 411 * to the DLIL frameout and output routines, that the packet may be 412 * looped back to the system under the right conditions. In addition, 413 * this is an mbuf flag rather than an mbuf packet header flag. 414 * 415 * - PKTF_LOOP is an mbuf packet header flag, which is set if and only 416 * if the packet was looped back to the system. This flag should be 417 * used instead for newer code. 418 */ 419#define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */ 420#define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */ 421#define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */ 422#define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */ 423#define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */ 424#define PKTF_PROXY_DST 0x20 /* processed but not locally destined */ 425#define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */ 426#define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */ 427#define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */ 428#define PKTF_SW_LRO_PKT 0x200 /* pkt is a large coalesced pkt */ 429#define PKTF_SW_LRO_DID_CSUM 0x400 /* IP and TCP checksums done by LRO */ 430#define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */ 431#define PKTF_MPSO 0x1000 /* MPTCP socket meta data */ 432#define PKTF_LOOP 0x2000 /* loopbacked packet */ 433#define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */ 434#define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */ 435#define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */ 436/* flags related to flow control/advisory and identification */ 437#define PKTF_FLOW_MASK \ 438 (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK) 439 440/* 441 * Description of external storage mapped into mbuf, valid only if M_EXT set. 442 */ 443struct m_ext { 444 caddr_t ext_buf; /* start of buffer */ 445 void (*ext_free) /* free routine if not the usual */ 446 (caddr_t, u_int, caddr_t); 447 u_int ext_size; /* size of buffer, for ext_free */ 448 caddr_t ext_arg; /* additional ext_free argument */ 449 struct ext_refsq { /* references held */ 450 struct ext_refsq *forward, *backward; 451 } ext_refs; 452 struct ext_ref { 453 u_int32_t refcnt; 454 u_int32_t flags; 455 } *ext_refflags; 456}; 457 458/* define m_ext to a type since it gets redefined below */ 459typedef struct m_ext _m_ext_t; 460 461/* 462 * The mbuf object 463 */ 464struct mbuf { 465 struct m_hdr m_hdr; 466 union { 467 struct { 468 struct pkthdr MH_pkthdr; /* M_PKTHDR set */ 469 union { 470 struct m_ext MH_ext; /* M_EXT set */ 471 char MH_databuf[_MHLEN]; 472 } MH_dat; 473 } MH; 474 char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ 475 } M_dat; 476}; 477 478#define m_next m_hdr.mh_next 479#define m_len m_hdr.mh_len 480#define m_data m_hdr.mh_data 481#define m_type m_hdr.mh_type 482#define m_flags m_hdr.mh_flags 483#define m_nextpkt m_hdr.mh_nextpkt 484#define m_act m_nextpkt 485#define m_pkthdr M_dat.MH.MH_pkthdr 486#define m_ext M_dat.MH.MH_dat.MH_ext 487#define m_pktdat M_dat.MH.MH_dat.MH_databuf 488#define m_dat M_dat.M_databuf 489#define m_pktlen(_m) ((_m)->m_pkthdr.len) 490#define m_pftag(_m) (&(_m)->m_pkthdr.pf_mtag) 491 492/* mbuf flags (private) */ 493#define M_EXT 0x0001 /* has associated external storage */ 494#define M_PKTHDR 0x0002 /* start of record */ 495#define M_EOR 0x0004 /* end of record */ 496#define M_PROTO1 0x0008 /* protocol-specific */ 497#define M_PROTO2 0x0010 /* protocol-specific */ 498#define M_PROTO3 0x0020 /* protocol-specific */ 499#define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */ 500#define M_PROTO5 0x0080 /* protocol-specific */ 501 502/* mbuf pkthdr flags, also in m_flags (private) */ 503#define M_BCAST 0x0100 /* send/received as link-level broadcast */ 504#define M_MCAST 0x0200 /* send/received as link-level multicast */ 505#define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ 506#define M_FIRSTFRAG 0x0800 /* packet is first fragment */ 507#define M_LASTFRAG 0x1000 /* packet is last fragment */ 508#define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */ 509#define M_HASFCS 0x4000 /* packet has FCS */ 510#define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */ 511 512/* 513 * Flags to purge when crossing layers. 514 */ 515#define M_PROTOFLAGS \ 516 (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5) 517 518/* flags copied when copying m_pkthdr */ 519#define M_COPYFLAGS \ 520 (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \ 521 M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \ 522 M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS) 523 524/* flags indicating hw checksum support and sw checksum requirements */ 525#define CSUM_IP 0x0001 /* will csum IP */ 526#define CSUM_TCP 0x0002 /* will csum TCP */ 527#define CSUM_UDP 0x0004 /* will csum UDP */ 528#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ 529#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ 530#define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */ 531#define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */ 532#define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */ 533 534#define CSUM_IP_CHECKED 0x0100 /* did csum IP */ 535#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ 536#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ 537#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ 538#define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ 539 540#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) 541#define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ 542#define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6) 543#define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */ 544 545#define CSUM_TX_FLAGS \ 546 (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ 547 CSUM_DATA_VALID | CSUM_PARTIAL) 548 549#define CSUM_RX_FLAGS \ 550 (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ 551 CSUM_DATA_VALID | CSUM_PARTIAL) 552 553/* 554 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h> 555 */ 556 557/* VLAN tag present */ 558#define CSUM_VLAN_TAG_VALID 0x10000 /* vlan_tag field is valid */ 559 560/* TCP Segment Offloading requested on this mbuf */ 561#define CSUM_TSO_IPV4 0x100000 /* This mbuf needs to be segmented by the NIC */ 562#define CSUM_TSO_IPV6 0x200000 /* This mbuf needs to be segmented by the NIC */ 563 564#define TSO_IPV4_OK(_ifp, _m) \ 565 (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ 566 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ 567 568#define TSO_IPV4_NOTOK(_ifp, _m) \ 569 (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ 570 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ 571 572#define TSO_IPV6_OK(_ifp, _m) \ 573 (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ 574 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ 575 576#define TSO_IPV6_NOTOK(_ifp, _m) \ 577 (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ 578 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ 579 580#endif /* XNU_KERNEL_PRIVATE */ 581 582/* mbuf types */ 583#define MT_FREE 0 /* should be on free list */ 584#define MT_DATA 1 /* dynamic (data) allocation */ 585#define MT_HEADER 2 /* packet header */ 586#define MT_SOCKET 3 /* socket structure */ 587#define MT_PCB 4 /* protocol control block */ 588#define MT_RTABLE 5 /* routing tables */ 589#define MT_HTABLE 6 /* IMP host tables */ 590#define MT_ATABLE 7 /* address resolution tables */ 591#define MT_SONAME 8 /* socket name */ 592#define MT_SOOPTS 10 /* socket options */ 593#define MT_FTABLE 11 /* fragment reassembly header */ 594#define MT_RIGHTS 12 /* access rights */ 595#define MT_IFADDR 13 /* interface address */ 596#define MT_CONTROL 14 /* extra-data protocol message */ 597#define MT_OOBDATA 15 /* expedited data */ 598#define MT_TAG 16 /* volatile metadata associated to pkts */ 599#define MT_MAX 32 /* enough? */ 600 601#ifdef XNU_KERNEL_PRIVATE 602/* 603 * mbuf allocation/deallocation macros: 604 * 605 * MGET(struct mbuf *m, int how, int type) 606 * allocates an mbuf and initializes it to contain internal data. 607 * 608 * MGETHDR(struct mbuf *m, int how, int type) 609 * allocates an mbuf and initializes it to contain a packet header 610 * and internal data. 611 */ 612 613#if 1 614#define MCHECK(m) m_mcheck(m) 615#else 616#define MCHECK(m) 617#endif 618 619#define MGET(m, how, type) ((m) = m_get((how), (type))) 620 621#define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) 622 623/* 624 * Mbuf cluster macros. 625 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster. 626 * MCLGET adds such clusters to a normal mbuf; 627 * the flag M_EXT is set upon success. 628 * MCLFREE releases a reference to a cluster allocated by MCLALLOC, 629 * freeing the cluster if the reference count has reached 0. 630 * 631 * Normal mbuf clusters are normally treated as character arrays 632 * after allocation, but use the first word of the buffer as a free list 633 * pointer while on the free list. 634 */ 635union mcluster { 636 union mcluster *mcl_next; 637 char mcl_buf[MCLBYTES]; 638}; 639 640#define MCLALLOC(p, how) ((p) = m_mclalloc(how)) 641 642#define MCLFREE(p) m_mclfree(p) 643 644#define MCLGET(m, how) ((m) = m_mclget(m, how)) 645 646/* 647 * Mbuf big cluster 648 */ 649union mbigcluster { 650 union mbigcluster *mbc_next; 651 char mbc_buf[MBIGCLBYTES]; 652}; 653 654/* 655 * Mbuf jumbo cluster 656 */ 657union m16kcluster { 658 union m16kcluster *m16kcl_next; 659 char m16kcl_buf[M16KCLBYTES]; 660}; 661 662#define MCLHASREFERENCE(m) m_mclhasreference(m) 663 664/* 665 * MFREE(struct mbuf *m, struct mbuf *n) 666 * Free a single mbuf and associated external storage. 667 * Place the successor, if any, in n. 668 */ 669 670#define MFREE(m, n) ((n) = m_free(m)) 671 672/* 673 * Copy mbuf pkthdr from from to to. 674 * from must have M_PKTHDR set, and to must be empty. 675 * aux pointer will be moved to `to'. 676 */ 677#define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from) 678 679#define M_COPY_PFTAG(to, from) m_copy_pftag(to, from) 680 681#define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from) 682 683/* 684 * Set the m_data pointer of a newly-allocated mbuf (m_get/MGET) to place 685 * an object of the specified size at the end of the mbuf, longword aligned. 686 */ 687#define M_ALIGN(m, len) \ 688do { \ 689 (m)->m_data += (MLEN - (len)) &~ (sizeof (long) - 1); \ 690} while (0) 691 692/* 693 * As above, for mbufs allocated with m_gethdr/MGETHDR 694 * or initialized by M_COPY_PKTHDR. 695 */ 696#define MH_ALIGN(m, len) \ 697do { \ 698 (m)->m_data += (MHLEN - (len)) &~ (sizeof (long) - 1); \ 699} while (0) 700 701/* 702 * Compute the amount of space available 703 * before the current start of data in an mbuf. 704 * Subroutine - data not available if certain references. 705 */ 706#define M_LEADINGSPACE(m) m_leadingspace(m) 707 708/* 709 * Compute the amount of space available 710 * after the end of data in an mbuf. 711 * Subroutine - data not available if certain references. 712 */ 713#define M_TRAILINGSPACE(m) m_trailingspace(m) 714 715/* 716 * Arrange to prepend space of size plen to mbuf m. 717 * If a new mbuf must be allocated, how specifies whether to wait. 718 * If how is M_DONTWAIT and allocation fails, the original mbuf chain 719 * is freed and m is set to NULL. 720 */ 721#define M_PREPEND(m, plen, how) ((m) = m_prepend_2((m), (plen), (how))) 722 723/* change mbuf to new type */ 724#define MCHTYPE(m, t) m_mchtype(m, t) 725 726/* compatiblity with 4.3 */ 727#define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) 728 729#define MBSHIFT 20 /* 1MB */ 730#define MBSIZE (1 << MBSHIFT) 731#define GBSHIFT 30 /* 1GB */ 732#define GBSIZE (1 << GBSHIFT) 733 734/* 735 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to 736 * "off+len") is located in single mbuf, on contiguous memory region. 737 * The pointer to the region will be returned to pointer variable "val", 738 * with type "typ". 739 * 740 * M_STRUCT_GET0 does the same, except that it aligns the structure at 741 * very top of mbuf. GET0 is likely to make memory copy than GET. 742 */ 743#define M_STRUCT_GET(val, typ, m, off, len) \ 744do { \ 745 struct mbuf *t; \ 746 int tmp; \ 747 \ 748 if ((m)->m_len >= (off) + (len)) { \ 749 (val) = (typ)(mtod((m), caddr_t) + (off)); \ 750 } else { \ 751 t = m_pulldown((m), (off), (len), &tmp); \ 752 if (t != NULL) { \ 753 if (t->m_len < tmp + (len)) \ 754 panic("m_pulldown malfunction"); \ 755 (val) = (typ)(mtod(t, caddr_t) + tmp); \ 756 } else { \ 757 (val) = (typ)NULL; \ 758 (m) = NULL; \ 759 } \ 760 } \ 761} while (0) 762 763#define M_STRUCT_GET0(val, typ, m, off, len) \ 764do { \ 765 struct mbuf *t; \ 766 \ 767 if ((off) == 0 && ((m)->m_len >= (len))) { \ 768 (val) = (typ)(void *)mtod(m, caddr_t); \ 769 } else { \ 770 t = m_pulldown((m), (off), (len), NULL); \ 771 if (t != NULL) { \ 772 if (t->m_len < (len)) \ 773 panic("m_pulldown malfunction"); \ 774 (val) = (typ)(void *)mtod(t, caddr_t); \ 775 } else { \ 776 (val) = (typ)NULL; \ 777 (m) = NULL; \ 778 } \ 779 } \ 780} while (0) 781 782#define MBUF_INPUT_CHECK(m, rcvif) \ 783do { \ 784 if (!(m->m_flags & MBUF_PKTHDR) || \ 785 m->m_len < 0 || \ 786 m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \ 787 m->m_type == MT_FREE || \ 788 ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \ 789 panic_plain("Failed mbuf validity check: mbuf %p len %d " \ 790 "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \ 791 m, m->m_len, m->m_type, m->m_flags, \ 792 ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \ 793 if_name(rcvif), \ 794 (rcvif->if_flags & 0xffff)); \ 795 } \ 796} while (0) 797 798/* 799 * Simple mbuf queueing system 800 * 801 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using 802 * m_nextpkt instead of field.sqe_next). 803 * 804 * m_next is ignored, so queueing chains of mbufs is possible 805 */ 806#define MBUFQ_HEAD(name) \ 807struct name { \ 808 struct mbuf *mq_first; /* first packet */ \ 809 struct mbuf **mq_last; /* addr of last next packet */ \ 810} 811 812#define MBUFQ_INIT(q) do { \ 813 MBUFQ_FIRST(q) = NULL; \ 814 (q)->mq_last = &MBUFQ_FIRST(q); \ 815} while (0) 816 817#define MBUFQ_PREPEND(q, m) do { \ 818 if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \ 819 (q)->mq_last = &MBUFQ_NEXT(m); \ 820 MBUFQ_FIRST(q) = (m); \ 821} while (0) 822 823#define MBUFQ_ENQUEUE(q, m) do { \ 824 MBUFQ_NEXT(m) = NULL; \ 825 *(q)->mq_last = (m); \ 826 (q)->mq_last = &MBUFQ_NEXT(m); \ 827} while (0) 828 829#define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \ 830 MBUFQ_NEXT(n) = NULL; \ 831 *(q)->mq_last = (m); \ 832 (q)->mq_last = &MBUFQ_NEXT(n); \ 833} while (0) 834 835#define MBUFQ_DEQUEUE(q, m) do { \ 836 if (((m) = MBUFQ_FIRST(q)) != NULL) { \ 837 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \ 838 (q)->mq_last = &MBUFQ_FIRST(q); \ 839 else \ 840 MBUFQ_NEXT(m) = NULL; \ 841 } \ 842} while (0) 843 844#define MBUFQ_REMOVE(q, m) do { \ 845 if (MBUFQ_FIRST(q) == (m)) { \ 846 MBUFQ_DEQUEUE(q, m); \ 847 } else { \ 848 struct mbuf *_m = MBUFQ_FIRST(q); \ 849 while (MBUFQ_NEXT(_m) != (m)) \ 850 _m = MBUFQ_NEXT(_m); \ 851 if ((MBUFQ_NEXT(_m) = \ 852 MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \ 853 (q)->mq_last = &MBUFQ_NEXT(_m); \ 854 } \ 855} while (0) 856 857#define MBUFQ_DRAIN(q) do { \ 858 struct mbuf *__m0; \ 859 while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ 860 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ 861 MBUFQ_NEXT(__m0) = NULL; \ 862 m_freem(__m0); \ 863 } \ 864 (q)->mq_last = &MBUFQ_FIRST(q); \ 865} while (0) 866 867#define MBUFQ_FOREACH(m, q) \ 868 for ((m) = MBUFQ_FIRST(q); \ 869 (m); \ 870 (m) = MBUFQ_NEXT(m)) 871 872#define MBUFQ_FOREACH_SAFE(m, q, tvar) \ 873 for ((m) = MBUFQ_FIRST(q); \ 874 (m) && ((tvar) = MBUFQ_NEXT(m), 1); \ 875 (m) = (tvar)) 876 877#define MBUFQ_EMPTY(q) ((q)->mq_first == NULL) 878#define MBUFQ_FIRST(q) ((q)->mq_first) 879#define MBUFQ_NEXT(m) ((m)->m_nextpkt) 880#define MBUFQ_LAST(q) (*(q)->mq_last) 881 882#define max_linkhdr P2ROUNDUP(_max_linkhdr, sizeof (u_int32_t)) 883#define max_protohdr P2ROUNDUP(_max_protohdr, sizeof (u_int32_t)) 884#endif /* XNU_KERNEL_PRIVATE */ 885 886/* 887 * Mbuf statistics (legacy). 888 */ 889struct mbstat { 890 u_int32_t m_mbufs; /* mbufs obtained from page pool */ 891 u_int32_t m_clusters; /* clusters obtained from page pool */ 892 u_int32_t m_spare; /* spare field */ 893 u_int32_t m_clfree; /* free clusters */ 894 u_int32_t m_drops; /* times failed to find space */ 895 u_int32_t m_wait; /* times waited for space */ 896 u_int32_t m_drain; /* times drained protocols for space */ 897 u_short m_mtypes[256]; /* type specific mbuf allocations */ 898 u_int32_t m_mcfail; /* times m_copym failed */ 899 u_int32_t m_mpfail; /* times m_pullup failed */ 900 u_int32_t m_msize; /* length of an mbuf */ 901 u_int32_t m_mclbytes; /* length of an mbuf cluster */ 902 u_int32_t m_minclsize; /* min length of data to allocate a cluster */ 903 u_int32_t m_mlen; /* length of data in an mbuf */ 904 u_int32_t m_mhlen; /* length of data in a header mbuf */ 905 u_int32_t m_bigclusters; /* clusters obtained from page pool */ 906 u_int32_t m_bigclfree; /* free clusters */ 907 u_int32_t m_bigmclbytes; /* length of an mbuf cluster */ 908}; 909 910/* Compatibillity with 10.3 */ 911struct ombstat { 912 u_int32_t m_mbufs; /* mbufs obtained from page pool */ 913 u_int32_t m_clusters; /* clusters obtained from page pool */ 914 u_int32_t m_spare; /* spare field */ 915 u_int32_t m_clfree; /* free clusters */ 916 u_int32_t m_drops; /* times failed to find space */ 917 u_int32_t m_wait; /* times waited for space */ 918 u_int32_t m_drain; /* times drained protocols for space */ 919 u_short m_mtypes[256]; /* type specific mbuf allocations */ 920 u_int32_t m_mcfail; /* times m_copym failed */ 921 u_int32_t m_mpfail; /* times m_pullup failed */ 922 u_int32_t m_msize; /* length of an mbuf */ 923 u_int32_t m_mclbytes; /* length of an mbuf cluster */ 924 u_int32_t m_minclsize; /* min length of data to allocate a cluster */ 925 u_int32_t m_mlen; /* length of data in an mbuf */ 926 u_int32_t m_mhlen; /* length of data in a header mbuf */ 927}; 928 929/* 930 * mbuf class statistics. 931 */ 932#define MAX_MBUF_CNAME 15 933 934#if defined(XNU_KERNEL_PRIVATE) 935/* For backwards compatibility with 32-bit userland process */ 936struct omb_class_stat { 937 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ 938 u_int32_t mbcl_size; /* buffer size */ 939 u_int32_t mbcl_total; /* # of buffers created */ 940 u_int32_t mbcl_active; /* # of active buffers */ 941 u_int32_t mbcl_infree; /* # of available buffers */ 942 u_int32_t mbcl_slab_cnt; /* # of available slabs */ 943 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ 944 u_int64_t mbcl_free_cnt; /* # of times free is called */ 945 u_int64_t mbcl_notified; /* # of notified wakeups */ 946 u_int64_t mbcl_purge_cnt; /* # of purges so far */ 947 u_int64_t mbcl_fail_cnt; /* # of allocation failures */ 948 u_int32_t mbcl_ctotal; /* total only for this class */ 949 /* 950 * Cache layer statistics 951 */ 952 u_int32_t mbcl_mc_state; /* cache state (see below) */ 953 u_int32_t mbcl_mc_cached; /* # of cached buffers */ 954 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ 955 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ 956 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ 957 u_int64_t mbcl_reserved[4]; /* for future use */ 958} __attribute__((__packed__)); 959#endif /* XNU_KERNEL_PRIVATE */ 960 961typedef struct mb_class_stat { 962 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ 963 u_int32_t mbcl_size; /* buffer size */ 964 u_int32_t mbcl_total; /* # of buffers created */ 965 u_int32_t mbcl_active; /* # of active buffers */ 966 u_int32_t mbcl_infree; /* # of available buffers */ 967 u_int32_t mbcl_slab_cnt; /* # of available slabs */ 968#if defined(KERNEL) || defined(__LP64__) 969 u_int32_t mbcl_pad; /* padding */ 970#endif /* KERNEL || __LP64__ */ 971 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ 972 u_int64_t mbcl_free_cnt; /* # of times free is called */ 973 u_int64_t mbcl_notified; /* # of notified wakeups */ 974 u_int64_t mbcl_purge_cnt; /* # of purges so far */ 975 u_int64_t mbcl_fail_cnt; /* # of allocation failures */ 976 u_int32_t mbcl_ctotal; /* total only for this class */ 977 /* 978 * Cache layer statistics 979 */ 980 u_int32_t mbcl_mc_state; /* cache state (see below) */ 981 u_int32_t mbcl_mc_cached; /* # of cached buffers */ 982 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ 983 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ 984 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ 985 u_int64_t mbcl_reserved[4]; /* for future use */ 986} mb_class_stat_t; 987 988#define MCS_DISABLED 0 /* cache is permanently disabled */ 989#define MCS_ONLINE 1 /* cache is online */ 990#define MCS_PURGING 2 /* cache is being purged */ 991#define MCS_OFFLINE 3 /* cache is offline (resizing) */ 992 993#if defined(XNU_KERNEL_PRIVATE) 994/* For backwards compatibility with 32-bit userland process */ 995struct omb_stat { 996 u_int32_t mbs_cnt; /* number of classes */ 997 struct omb_class_stat mbs_class[1]; /* class array */ 998} __attribute__((__packed__)); 999#endif /* XNU_KERNEL_PRIVATE */ 1000 1001typedef struct mb_stat { 1002 u_int32_t mbs_cnt; /* number of classes */ 1003#if defined(KERNEL) || defined(__LP64__) 1004 u_int32_t mbs_pad; /* padding */ 1005#endif /* KERNEL || __LP64__ */ 1006 mb_class_stat_t mbs_class[1]; /* class array */ 1007} mb_stat_t; 1008 1009#ifdef PRIVATE 1010#define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */ 1011 1012typedef struct mleak_trace_stat { 1013 u_int64_t mltr_collisions; 1014 u_int64_t mltr_hitcount; 1015 u_int64_t mltr_allocs; 1016 u_int64_t mltr_depth; 1017 u_int64_t mltr_addr[MLEAK_STACK_DEPTH]; 1018} mleak_trace_stat_t; 1019 1020typedef struct mleak_stat { 1021 u_int32_t ml_isaddr64; /* 64-bit KVA? */ 1022 u_int32_t ml_cnt; /* number of traces */ 1023 mleak_trace_stat_t ml_trace[1]; /* trace array */ 1024} mleak_stat_t; 1025 1026struct mleak_table { 1027 u_int32_t mleak_capture; /* sampling capture counter */ 1028 u_int32_t mleak_sample_factor; /* sample factor */ 1029 1030 /* Times two active records want to occupy the same spot */ 1031 u_int64_t alloc_collisions; 1032 u_int64_t trace_collisions; 1033 1034 /* Times new record lands on spot previously occupied by freed alloc */ 1035 u_int64_t alloc_overwrites; 1036 u_int64_t trace_overwrites; 1037 1038 /* Times a new alloc or trace is put into the hash table */ 1039 u_int64_t alloc_recorded; 1040 u_int64_t trace_recorded; 1041 1042 /* Total number of outstanding allocs */ 1043 u_int64_t outstanding_allocs; 1044 1045 /* Times mleak_log returned false because couldn't acquire the lock */ 1046 u_int64_t total_conflicts; 1047}; 1048#endif /* PRIVATE */ 1049 1050#ifdef KERNEL_PRIVATE 1051__BEGIN_DECLS 1052 1053/* 1054 * Exported (private) 1055 */ 1056 1057extern struct mbstat mbstat; /* statistics */ 1058 1059__END_DECLS 1060#endif /* KERNEL_PRIVATE */ 1061 1062#ifdef XNU_KERNEL_PRIVATE 1063__BEGIN_DECLS 1064 1065/* 1066 * Not exported (xnu private) 1067 */ 1068 1069/* flags to m_get/MGET */ 1070/* Need to include malloc.h to get right options for malloc */ 1071#include <sys/malloc.h> 1072 1073struct mbuf; 1074 1075/* length to m_copy to copy all */ 1076#define M_COPYALL 1000000000 1077 1078#define M_DONTWAIT M_NOWAIT 1079#define M_WAIT M_WAITOK 1080 1081/* modes for m_copym and variants */ 1082#define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */ 1083#define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */ 1084#define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */ 1085 1086/* 1087 * These macros are mapped to the appropriate KPIs, so that private code 1088 * can be simply recompiled in order to be forward-compatible with future 1089 * changes toward the struture sizes. 1090 */ 1091#define MLEN mbuf_get_mlen() /* normal data len */ 1092#define MHLEN mbuf_get_mhlen() /* data len w/pkthdr */ 1093 1094#define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */ 1095 1096extern void m_freem(struct mbuf *); 1097extern u_int64_t mcl_to_paddr(char *); 1098extern void m_adj(struct mbuf *, int); 1099extern void m_cat(struct mbuf *, struct mbuf *); 1100extern void m_copydata(struct mbuf *, int, int, void *); 1101extern struct mbuf *m_copym(struct mbuf *, int, int, int); 1102extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, uint32_t); 1103extern struct mbuf *m_get(int, int); 1104extern struct mbuf *m_gethdr(int, int); 1105extern struct mbuf *m_getpacket(void); 1106extern struct mbuf *m_getpackets(int, int, int); 1107extern struct mbuf *m_mclget(struct mbuf *, int); 1108extern void *m_mtod(struct mbuf *); 1109extern struct mbuf *m_prepend_2(struct mbuf *, int, int); 1110extern struct mbuf *m_pullup(struct mbuf *, int); 1111extern struct mbuf *m_split(struct mbuf *, int, int); 1112extern void m_mclfree(caddr_t p); 1113 1114/* 1115 * On platforms which require strict alignment (currently for anything but 1116 * i386 or x86_64), this macro checks whether the data pointer of an mbuf 1117 * is 32-bit aligned (this is the expected minimum alignment for protocol 1118 * headers), and assert otherwise. 1119 */ 1120#if defined(__i386__) || defined(__x86_64__) 1121#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) 1122#else /* !__i386__ && !__x86_64__ */ 1123#define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \ 1124 if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \ 1125 if (((_m)->m_flags & M_PKTHDR) && \ 1126 (_m)->m_pkthdr.rcvif != NULL) { \ 1127 panic_plain("\n%s: mbuf %p data ptr %p is not " \ 1128 "32-bit aligned [%s: alignerrs=%lld]\n", \ 1129 __func__, (_m), (_m)->m_data, \ 1130 if_name((_m)->m_pkthdr.rcvif), \ 1131 (_m)->m_pkthdr.rcvif->if_alignerrs); \ 1132 } else { \ 1133 panic_plain("\n%s: mbuf %p data ptr %p is not " \ 1134 "32-bit aligned\n", \ 1135 __func__, (_m), (_m)->m_data); \ 1136 } \ 1137 } \ 1138} while (0) 1139#endif /* !__i386__ && !__x86_64__ */ 1140 1141/* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */ 1142#define MBUF_SC_MAX_CLASSES 10 1143 1144/* 1145 * These conversion macros rely on the corresponding MBUF_SC and 1146 * MBUF_TC values in order to establish the following mapping: 1147 * 1148 * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK 1149 * MBUF_SC_BK ] 1150 * 1151 * MBUF_SC_BE ] ==> MBUF_TC_BE 1152 * MBUF_SC_RD ] 1153 * MBUF_SC_OAM ] 1154 * 1155 * MBUF_SC_AV ] ==> MBUF_TC_VI 1156 * MBUF_SC_RV ] 1157 * MBUF_SC_VI ] 1158 * 1159 * MBUF_SC_VO ] ==> MBUF_TC_VO 1160 * MBUF_SC_CTL ] 1161 * 1162 * The values assigned to each service class allows for a fast mapping to 1163 * the corresponding MBUF_TC traffic class values, as well as to retrieve the 1164 * assigned index; therefore care must be taken when comparing against these 1165 * values. Use the corresponding class and index macros to retrieve the 1166 * corresponding portion, and never assume that a higher class corresponds 1167 * to a higher index. 1168 */ 1169#define MBUF_SCVAL(x) ((x) & 0xffff) 1170#define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3) 1171#define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7) 1172#define MBUF_TC2SCVAL(_tc) ((_tc) << 7) 1173#define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \ 1174 ((_sc) == MBUF_SC_BK)) 1175 1176#define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS) 1177#define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK) 1178#define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE) 1179#define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD) 1180#define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM) 1181#define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV) 1182#define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV) 1183#define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI) 1184#define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO) 1185#define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL) 1186 1187#define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS) 1188#define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK) 1189#define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE) 1190#define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD) 1191#define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM) 1192#define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV) 1193#define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV) 1194#define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI) 1195#define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO) 1196#define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL) 1197 1198#define MBUF_VALID_SC(c) \ 1199 (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \ 1200 c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \ 1201 c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_VO || \ 1202 c == MBUF_SC_CTL) 1203 1204#define MBUF_VALID_SCIDX(c) \ 1205 (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \ 1206 c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \ 1207 c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_VO || \ 1208 c == SCIDX_CTL) 1209 1210#define MBUF_VALID_SCVAL(c) \ 1211 (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \ 1212 c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \ 1213 c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_VO || \ 1214 c == SCVAL_CTL) 1215 1216extern union mbigcluster *mbutl; /* start VA of mbuf pool */ 1217extern union mbigcluster *embutl; /* end VA of mbuf pool */ 1218extern unsigned int nmbclusters; /* number of mapped clusters */ 1219extern int njcl; /* # of jumbo clusters */ 1220extern int njclbytes; /* size of a jumbo cluster */ 1221extern int max_hdr; /* largest link+protocol header */ 1222extern int max_datalen; /* MHLEN - max_hdr */ 1223 1224/* Use max_linkhdr instead of _max_linkhdr */ 1225extern int _max_linkhdr; /* largest link-level header */ 1226 1227/* Use max_protohdr instead of _max_protohdr */ 1228extern int _max_protohdr; /* largest protocol header */ 1229 1230__private_extern__ unsigned int mbuf_default_ncl(int, u_int64_t); 1231__private_extern__ void mbinit(void); 1232__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t, 1233 void (*)(caddr_t, u_int, caddr_t), u_int, caddr_t, int); 1234__private_extern__ caddr_t m_bigalloc(int); 1235__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t); 1236__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int); 1237__private_extern__ caddr_t m_16kalloc(int); 1238__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t); 1239__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int); 1240__private_extern__ int m_reinit(struct mbuf *, int); 1241__private_extern__ struct mbuf *m_free(struct mbuf *); 1242__private_extern__ struct mbuf *m_getclr(int, int); 1243__private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *); 1244__private_extern__ unsigned int m_length(struct mbuf *); 1245__private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **); 1246__private_extern__ unsigned int m_fixhdr(struct mbuf *); 1247__private_extern__ struct mbuf *m_defrag(struct mbuf *, int); 1248__private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int); 1249__private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int); 1250__private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int); 1251__private_extern__ struct mbuf *m_retry(int, int); 1252__private_extern__ struct mbuf *m_retryhdr(int, int); 1253__private_extern__ int m_freem_list(struct mbuf *); 1254__private_extern__ int m_append(struct mbuf *, int, caddr_t); 1255__private_extern__ struct mbuf *m_last(struct mbuf *); 1256__private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *, 1257 void (*)(const void *, void *, size_t)); 1258__private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *); 1259 1260__private_extern__ struct mbuf *m_getcl(int, int, int); 1261__private_extern__ caddr_t m_mclalloc(int); 1262__private_extern__ int m_mclhasreference(struct mbuf *); 1263__private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *); 1264__private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *); 1265__private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *); 1266 1267__private_extern__ struct mbuf *m_dtom(void *); 1268__private_extern__ int m_mtocl(void *); 1269__private_extern__ union mcluster *m_cltom(int); 1270 1271__private_extern__ int m_trailingspace(struct mbuf *); 1272__private_extern__ int m_leadingspace(struct mbuf *); 1273 1274__private_extern__ struct mbuf *m_normalize(struct mbuf *m); 1275__private_extern__ void m_mchtype(struct mbuf *m, int t); 1276__private_extern__ void m_mcheck(struct mbuf *); 1277 1278__private_extern__ void m_copyback(struct mbuf *, int, int, const void *); 1279__private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int, 1280 const void *, int); 1281__private_extern__ int m_makewritable(struct mbuf **, int, int, int); 1282__private_extern__ struct mbuf *m_dup(struct mbuf *m, int how); 1283__private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int, 1284 struct mbuf **, int *, uint32_t); 1285__private_extern__ struct mbuf *m_getpackethdrs(int, int); 1286__private_extern__ struct mbuf *m_getpacket_how(int); 1287__private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int, 1288 int, int, size_t); 1289__private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t, 1290 unsigned int *, int, int, size_t); 1291 1292/* 1293 * Packets may have annotations attached by affixing a list of "packet 1294 * tags" to the pkthdr structure. Packet tags are dynamically allocated 1295 * semi-opaque data structures that have a fixed header (struct m_tag) 1296 * that specifies the size of the memory block and an <id,type> pair that 1297 * identifies it. The id identifies the module and the type identifies the 1298 * type of data for that module. The id of zero is reserved for the kernel. 1299 * 1300 * Note that the packet tag returned by m_tag_allocate has the default 1301 * memory alignment implemented by malloc. To reference private data one 1302 * can use a construct like: 1303 * 1304 * struct m_tag *mtag = m_tag_allocate(...); 1305 * struct foo *p = (struct foo *)(mtag+1); 1306 * 1307 * if the alignment of struct m_tag is sufficient for referencing members 1308 * of struct foo. Otherwise it is necessary to embed struct m_tag within 1309 * the private data structure to insure proper alignment; e.g. 1310 * 1311 * struct foo { 1312 * struct m_tag tag; 1313 * ... 1314 * }; 1315 * struct foo *p = (struct foo *) m_tag_allocate(...); 1316 * struct m_tag *mtag = &p->tag; 1317 */ 1318 1319#define KERNEL_MODULE_TAG_ID 0 1320 1321enum { 1322 KERNEL_TAG_TYPE_NONE = 0, 1323 KERNEL_TAG_TYPE_DUMMYNET = 1, 1324 KERNEL_TAG_TYPE_DIVERT = 2, 1325 KERNEL_TAG_TYPE_IPFORWARD = 3, 1326 KERNEL_TAG_TYPE_IPFILT = 4, 1327 KERNEL_TAG_TYPE_MACLABEL = 5, 1328 KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6, 1329 KERNEL_TAG_TYPE_ENCAP = 8, 1330 KERNEL_TAG_TYPE_INET6 = 9, 1331 KERNEL_TAG_TYPE_IPSEC = 10, 1332 KERNEL_TAG_TYPE_DRVAUX = 11 1333}; 1334 1335/* Packet tag routines */ 1336__private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int); 1337__private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int, 1338 struct mbuf *); 1339__private_extern__ void m_tag_free(struct m_tag *); 1340__private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *); 1341__private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *); 1342__private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *); 1343__private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *); 1344__private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t, 1345 u_int16_t, struct m_tag *); 1346__private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int); 1347__private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int); 1348__private_extern__ void m_tag_init(struct mbuf *, int); 1349__private_extern__ struct m_tag *m_tag_first(struct mbuf *); 1350__private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); 1351 1352__END_DECLS 1353#endif /* XNU_KERNEL_PRIVATE */ 1354#ifdef KERNEL 1355#include <sys/kpi_mbuf.h> 1356#ifdef XNU_KERNEL_PRIVATE 1357__BEGIN_DECLS 1358 1359__private_extern__ void m_scratch_init(struct mbuf *); 1360__private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **); 1361 1362__private_extern__ void m_classifier_init(struct mbuf *, uint32_t); 1363 1364__private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t); 1365__private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *); 1366__private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t); 1367__private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t); 1368__private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t); 1369__private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); 1370 1371#define ADDCARRY(_x) do { \ 1372 while (((_x) >> 16) != 0) \ 1373 (_x) = ((_x) >> 16) + ((_x) & 0xffff); \ 1374} while (0) 1375 1376__private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, 1377 u_int32_t, u_int32_t); 1378__private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); 1379 1380__END_DECLS 1381#endif /* XNU_KERNEL_PRIVATE */ 1382#endif /* KERNEL */ 1383#endif /* !_SYS_MBUF_H_ */ 1384