uipc_sockbuf.c revision 274043
175584Sru/*- 275584Sru * Copyright (c) 1982, 1986, 1988, 1990, 1993 375584Sru * The Regents of the University of California. All rights reserved. 4104862Sru * 575584Sru * Redistribution and use in source and binary forms, with or without 675584Sru * modification, are permitted provided that the following conditions 775584Sru * are met: 875584Sru * 1. Redistributions of source code must retain the above copyright 975584Sru * notice, this list of conditions and the following disclaimer. 1075584Sru * 2. Redistributions in binary form must reproduce the above copyright 1175584Sru * notice, this list of conditions and the following disclaimer in the 1275584Sru * documentation and/or other materials provided with the distribution. 1375584Sru * 4. Neither the name of the University nor the names of its contributors 1475584Sru * may be used to endorse or promote products derived from this software 15104862Sru * without specific prior written permission. 1675584Sru * 1775584Sru * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 1875584Sru * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1975584Sru * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2075584Sru * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2175584Sru * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2275584Sru * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2375584Sru * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2475584Sru * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2575584Sru * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2675584Sru * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2775584Sru * SUCH DAMAGE. 2875584Sru * 2975584Sru * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93 3075584Sru */ 3175584Sru 3275584Sru#include <sys/cdefs.h> 3375584Sru__FBSDID("$FreeBSD: stable/10/sys/kern/uipc_sockbuf.c 274043 2014-11-03 12:38:29Z hselasky $"); 3475584Sru 3575584Sru#include "opt_param.h" 3675584Sru 3775584Sru#include <sys/param.h> 3875584Sru#include <sys/aio.h> /* for aio_swake proto */ 3975584Sru#include <sys/kernel.h> 4075584Sru#include <sys/lock.h> 4175584Sru#include <sys/mbuf.h> 4275584Sru#include <sys/mutex.h> 4375584Sru#include <sys/proc.h> 4475584Sru#include <sys/protosw.h> 4575584Sru#include <sys/resourcevar.h> 4675584Sru#include <sys/signalvar.h> 4775584Sru#include <sys/socket.h> 4875584Sru#include <sys/socketvar.h> 4975584Sru#include <sys/sx.h> 5075584Sru#include <sys/sysctl.h> 5175584Sru 5275584Sru/* 5375584Sru * Function pointer set by the AIO routines so that the socket buffer code 5475584Sru * can call back into the AIO module if it is loaded. 5575584Sru */ 5675584Sruvoid (*aio_swake)(struct socket *, struct sockbuf *); 5775584Sru 5875584Sru/* 5975584Sru * Primitive routines for operating on socket buffers 6075584Sru */ 6175584Sru 6275584Sruu_long sb_max = SB_MAX; 6375584Sruu_long sb_max_adj = 6475584Sru (quad_t)SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */ 6575584Sru 6675584Srustatic u_long sb_efficiency = 8; /* parameter for sbreserve() */ 6775584Sru 6875584Srustatic struct mbuf *sbcut_internal(struct sockbuf *sb, int len); 6975584Srustatic void sbflush_internal(struct sockbuf *sb); 7075584Sru 7175584Sru/* 7275584Sru * Socantsendmore indicates that no more data will be sent on the socket; it 7375584Sru * would normally be applied to a socket when the user informs the system 7475584Sru * that no more data is to be sent, by the protocol code (in case 7575584Sru * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be 7675584Sru * received, and will normally be applied to the socket by a protocol when it 7775584Sru * detects that the peer will send no more data. Data queued for reading in 7875584Sru * the socket may yet be read. 7975584Sru */ 8075584Sruvoid 8175584Srusocantsendmore_locked(struct socket *so) 8275584Sru{ 8375584Sru 8475584Sru SOCKBUF_LOCK_ASSERT(&so->so_snd); 8575584Sru 8675584Sru so->so_snd.sb_state |= SBS_CANTSENDMORE; 8775584Sru sowwakeup_locked(so); 8875584Sru mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 8975584Sru} 9075584Sru 9175584Sruvoid 9275584Srusocantsendmore(struct socket *so) 9375584Sru{ 9475584Sru 9575584Sru SOCKBUF_LOCK(&so->so_snd); 9675584Sru socantsendmore_locked(so); 9775584Sru mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 9875584Sru} 9975584Sru 10075584Sruvoid 10175584Srusocantrcvmore_locked(struct socket *so) 10275584Sru{ 10375584Sru 10475584Sru SOCKBUF_LOCK_ASSERT(&so->so_rcv); 10575584Sru 10675584Sru so->so_rcv.sb_state |= SBS_CANTRCVMORE; 10775584Sru sorwakeup_locked(so); 10875584Sru mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 10975584Sru} 11075584Sru 11175584Sruvoid 11275584Srusocantrcvmore(struct socket *so) 11375584Sru{ 11475584Sru 11575584Sru SOCKBUF_LOCK(&so->so_rcv); 11675584Sru socantrcvmore_locked(so); 117104862Sru mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 11875584Sru} 11975584Sru 12075584Sru/* 12175584Sru * Wait for data to arrive at/drain from a socket buffer. 12275584Sru */ 12375584Sruint 12475584Srusbwait(struct sockbuf *sb) 12575584Sru{ 12675584Sru 12775584Sru SOCKBUF_LOCK_ASSERT(sb); 12875584Sru 129114402Sru sb->sb_flags |= SB_WAIT; 13075584Sru return (msleep_sbt(&sb->sb_cc, &sb->sb_mtx, 13175584Sru (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", 13275584Sru sb->sb_timeo, 0, 0)); 13375584Sru} 13475584Sru 135104862Sruint 13675584Srusblock(struct sockbuf *sb, int flags) 13775584Sru{ 13875584Sru 139104862Sru KASSERT((flags & SBL_VALID) == flags, 14075584Sru ("sblock: flags invalid (0x%x)", flags)); 14175584Sru 14275584Sru if (flags & SBL_WAIT) { 14375584Sru if ((sb->sb_flags & SB_NOINTR) || 14475584Sru (flags & SBL_NOINTR)) { 14575584Sru sx_xlock(&sb->sb_sx); 14675584Sru return (0); 14775584Sru } 14875584Sru return (sx_xlock_sig(&sb->sb_sx)); 14975584Sru } else { 15075584Sru if (sx_try_xlock(&sb->sb_sx) == 0) 15175584Sru return (EWOULDBLOCK); 15275584Sru return (0); 15375584Sru } 15475584Sru} 15575584Sru 15675584Sruvoid 15775584Srusbunlock(struct sockbuf *sb) 15875584Sru{ 15975584Sru 16075584Sru sx_xunlock(&sb->sb_sx); 16175584Sru} 16275584Sru 16375584Sru/* 16475584Sru * Wakeup processes waiting on a socket buffer. Do asynchronous notification 16575584Sru * via SIGIO if the socket has the SS_ASYNC flag set. 16675584Sru * 16775584Sru * Called with the socket buffer lock held; will release the lock by the end 16875584Sru * of the function. This allows the caller to acquire the socket buffer lock 16975584Sru * while testing for the need for various sorts of wakeup and hold it through 17075584Sru * to the point where it's no longer required. We currently hold the lock 17175584Sru * through calls out to other subsystems (with the exception of kqueue), and 17275584Sru * then release it to avoid lock order issues. It's not clear that's 173104862Sru * correct. 17475584Sru */ 17575584Sruvoid 17675584Srusowakeup(struct socket *so, struct sockbuf *sb) 17775584Sru{ 17875584Sru int ret; 17975584Sru 18075584Sru SOCKBUF_LOCK_ASSERT(sb); 18175584Sru 18275584Sru selwakeuppri(&sb->sb_sel, PSOCK); 18375584Sru if (!SEL_WAITING(&sb->sb_sel)) 18475584Sru sb->sb_flags &= ~SB_SEL; 18575584Sru if (sb->sb_flags & SB_WAIT) { 18675584Sru sb->sb_flags &= ~SB_WAIT; 18775584Sru wakeup(&sb->sb_cc); 18875584Sru } 18975584Sru KNOTE_LOCKED(&sb->sb_sel.si_note, 0); 19075584Sru if (sb->sb_upcall != NULL) { 19175584Sru ret = sb->sb_upcall(so, sb->sb_upcallarg, M_NOWAIT); 19275584Sru if (ret == SU_ISCONNECTED) { 19375584Sru KASSERT(sb == &so->so_rcv, 19475584Sru ("SO_SND upcall returned SU_ISCONNECTED")); 19575584Sru soupcall_clear(so, SO_RCV); 19675584Sru } 19775584Sru } else 19875584Sru ret = SU_OK; 19975584Sru if (sb->sb_flags & SB_AIO) 20075584Sru aio_swake(so, sb); 20175584Sru SOCKBUF_UNLOCK(sb); 20275584Sru if (ret == SU_ISCONNECTED) 20375584Sru soisconnected(so); 20475584Sru if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL) 205104862Sru pgsigio(&so->so_sigio, SIGIO, 0); 20675584Sru mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 20775584Sru} 20875584Sru 20975584Sru/* 21075584Sru * Socket buffer (struct sockbuf) utility routines. 21175584Sru * 21275584Sru * Each socket contains two socket buffers: one for sending data and one for 21375584Sru * receiving data. Each buffer contains a queue of mbufs, information about 214 * the number of mbufs and amount of data in the queue, and other fields 215 * allowing select() statements and notification on data availability to be 216 * implemented. 217 * 218 * Data stored in a socket buffer is maintained as a list of records. Each 219 * record is a list of mbufs chained together with the m_next field. Records 220 * are chained together with the m_nextpkt field. The upper level routine 221 * soreceive() expects the following conventions to be observed when placing 222 * information in the receive buffer: 223 * 224 * 1. If the protocol requires each message be preceded by the sender's name, 225 * then a record containing that name must be present before any 226 * associated data (mbuf's must be of type MT_SONAME). 227 * 2. If the protocol supports the exchange of ``access rights'' (really just 228 * additional data associated with the message), and there are ``rights'' 229 * to be received, then a record containing this data should be present 230 * (mbuf's must be of type MT_RIGHTS). 231 * 3. If a name or rights record exists, then it must be followed by a data 232 * record, perhaps of zero length. 233 * 234 * Before using a new socket structure it is first necessary to reserve 235 * buffer space to the socket, by calling sbreserve(). This should commit 236 * some of the available buffer space in the system buffer pool for the 237 * socket (currently, it does nothing but enforce limits). The space should 238 * be released by calling sbrelease() when the socket is destroyed. 239 */ 240int 241soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 242{ 243 struct thread *td = curthread; 244 245 SOCKBUF_LOCK(&so->so_snd); 246 SOCKBUF_LOCK(&so->so_rcv); 247 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 248 goto bad; 249 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 250 goto bad2; 251 if (so->so_rcv.sb_lowat == 0) 252 so->so_rcv.sb_lowat = 1; 253 if (so->so_snd.sb_lowat == 0) 254 so->so_snd.sb_lowat = MCLBYTES; 255 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 256 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 257 SOCKBUF_UNLOCK(&so->so_rcv); 258 SOCKBUF_UNLOCK(&so->so_snd); 259 return (0); 260bad2: 261 sbrelease_locked(&so->so_snd, so); 262bad: 263 SOCKBUF_UNLOCK(&so->so_rcv); 264 SOCKBUF_UNLOCK(&so->so_snd); 265 return (ENOBUFS); 266} 267 268static int 269sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 270{ 271 int error = 0; 272 u_long tmp_sb_max = sb_max; 273 274 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 275 if (error || !req->newptr) 276 return (error); 277 if (tmp_sb_max < MSIZE + MCLBYTES) 278 return (EINVAL); 279 sb_max = tmp_sb_max; 280 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 281 return (0); 282} 283 284/* 285 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 286 * become limiting if buffering efficiency is near the normal case. 287 */ 288int 289sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 290 struct thread *td) 291{ 292 rlim_t sbsize_limit; 293 294 SOCKBUF_LOCK_ASSERT(sb); 295 296 /* 297 * When a thread is passed, we take into account the thread's socket 298 * buffer size limit. The caller will generally pass curthread, but 299 * in the TCP input path, NULL will be passed to indicate that no 300 * appropriate thread resource limits are available. In that case, 301 * we don't apply a process limit. 302 */ 303 if (cc > sb_max_adj) 304 return (0); 305 if (td != NULL) { 306 PROC_LOCK(td->td_proc); 307 sbsize_limit = lim_cur(td->td_proc, RLIMIT_SBSIZE); 308 PROC_UNLOCK(td->td_proc); 309 } else 310 sbsize_limit = RLIM_INFINITY; 311 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 312 sbsize_limit)) 313 return (0); 314 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 315 if (sb->sb_lowat > sb->sb_hiwat) 316 sb->sb_lowat = sb->sb_hiwat; 317 return (1); 318} 319 320int 321sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 322 struct thread *td) 323{ 324 int error; 325 326 SOCKBUF_LOCK(sb); 327 error = sbreserve_locked(sb, cc, so, td); 328 SOCKBUF_UNLOCK(sb); 329 return (error); 330} 331 332/* 333 * Free mbufs held by a socket, and reserved mbuf space. 334 */ 335void 336sbrelease_internal(struct sockbuf *sb, struct socket *so) 337{ 338 339 sbflush_internal(sb); 340 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 341 RLIM_INFINITY); 342 sb->sb_mbmax = 0; 343} 344 345void 346sbrelease_locked(struct sockbuf *sb, struct socket *so) 347{ 348 349 SOCKBUF_LOCK_ASSERT(sb); 350 351 sbrelease_internal(sb, so); 352} 353 354void 355sbrelease(struct sockbuf *sb, struct socket *so) 356{ 357 358 SOCKBUF_LOCK(sb); 359 sbrelease_locked(sb, so); 360 SOCKBUF_UNLOCK(sb); 361} 362 363void 364sbdestroy(struct sockbuf *sb, struct socket *so) 365{ 366 367 sbrelease_internal(sb, so); 368} 369 370/* 371 * Routines to add and remove data from an mbuf queue. 372 * 373 * The routines sbappend() or sbappendrecord() are normally called to append 374 * new mbufs to a socket buffer, after checking that adequate space is 375 * available, comparing the function sbspace() with the amount of data to be 376 * added. sbappendrecord() differs from sbappend() in that data supplied is 377 * treated as the beginning of a new record. To place a sender's address, 378 * optional access rights, and data in a socket receive buffer, 379 * sbappendaddr() should be used. To place access rights and data in a 380 * socket receive buffer, sbappendrights() should be used. In either case, 381 * the new data begins a new record. Note that unlike sbappend() and 382 * sbappendrecord(), these routines check for the caller that there will be 383 * enough space to store the data. Each fails if there is not enough space, 384 * or if it cannot find mbufs to store additional information in. 385 * 386 * Reliable protocols may use the socket send buffer to hold data awaiting 387 * acknowledgement. Data is normally copied from a socket send buffer in a 388 * protocol with m_copy for output to a peer, and then removing the data from 389 * the socket buffer with sbdrop() or sbdroprecord() when the data is 390 * acknowledged by the peer. 391 */ 392#ifdef SOCKBUF_DEBUG 393void 394sblastrecordchk(struct sockbuf *sb, const char *file, int line) 395{ 396 struct mbuf *m = sb->sb_mb; 397 398 SOCKBUF_LOCK_ASSERT(sb); 399 400 while (m && m->m_nextpkt) 401 m = m->m_nextpkt; 402 403 if (m != sb->sb_lastrecord) { 404 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 405 __func__, sb->sb_mb, sb->sb_lastrecord, m); 406 printf("packet chain:\n"); 407 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 408 printf("\t%p\n", m); 409 panic("%s from %s:%u", __func__, file, line); 410 } 411} 412 413void 414sblastmbufchk(struct sockbuf *sb, const char *file, int line) 415{ 416 struct mbuf *m = sb->sb_mb; 417 struct mbuf *n; 418 419 SOCKBUF_LOCK_ASSERT(sb); 420 421 while (m && m->m_nextpkt) 422 m = m->m_nextpkt; 423 424 while (m && m->m_next) 425 m = m->m_next; 426 427 if (m != sb->sb_mbtail) { 428 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 429 __func__, sb->sb_mb, sb->sb_mbtail, m); 430 printf("packet tree:\n"); 431 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 432 printf("\t"); 433 for (n = m; n != NULL; n = n->m_next) 434 printf("%p ", n); 435 printf("\n"); 436 } 437 panic("%s from %s:%u", __func__, file, line); 438 } 439} 440#endif /* SOCKBUF_DEBUG */ 441 442#define SBLINKRECORD(sb, m0) do { \ 443 SOCKBUF_LOCK_ASSERT(sb); \ 444 if ((sb)->sb_lastrecord != NULL) \ 445 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 446 else \ 447 (sb)->sb_mb = (m0); \ 448 (sb)->sb_lastrecord = (m0); \ 449} while (/*CONSTCOND*/0) 450 451/* 452 * Append mbuf chain m to the last record in the socket buffer sb. The 453 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 454 * are discarded and mbufs are compacted where possible. 455 */ 456void 457sbappend_locked(struct sockbuf *sb, struct mbuf *m) 458{ 459 struct mbuf *n; 460 461 SOCKBUF_LOCK_ASSERT(sb); 462 463 if (m == 0) 464 return; 465 466 SBLASTRECORDCHK(sb); 467 n = sb->sb_mb; 468 if (n) { 469 while (n->m_nextpkt) 470 n = n->m_nextpkt; 471 do { 472 if (n->m_flags & M_EOR) { 473 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 474 return; 475 } 476 } while (n->m_next && (n = n->m_next)); 477 } else { 478 /* 479 * XXX Would like to simply use sb_mbtail here, but 480 * XXX I need to verify that I won't miss an EOR that 481 * XXX way. 482 */ 483 if ((n = sb->sb_lastrecord) != NULL) { 484 do { 485 if (n->m_flags & M_EOR) { 486 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 487 return; 488 } 489 } while (n->m_next && (n = n->m_next)); 490 } else { 491 /* 492 * If this is the first record in the socket buffer, 493 * it's also the last record. 494 */ 495 sb->sb_lastrecord = m; 496 } 497 } 498 sbcompress(sb, m, n); 499 SBLASTRECORDCHK(sb); 500} 501 502/* 503 * Append mbuf chain m to the last record in the socket buffer sb. The 504 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 505 * are discarded and mbufs are compacted where possible. 506 */ 507void 508sbappend(struct sockbuf *sb, struct mbuf *m) 509{ 510 511 SOCKBUF_LOCK(sb); 512 sbappend_locked(sb, m); 513 SOCKBUF_UNLOCK(sb); 514} 515 516/* 517 * This version of sbappend() should only be used when the caller absolutely 518 * knows that there will never be more than one record in the socket buffer, 519 * that is, a stream protocol (such as TCP). 520 */ 521void 522sbappendstream_locked(struct sockbuf *sb, struct mbuf *m) 523{ 524 SOCKBUF_LOCK_ASSERT(sb); 525 526 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 527 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 528 529 SBLASTMBUFCHK(sb); 530 531 /* Remove all packet headers and mbuf tags to get a pure data chain. */ 532 m_demote(m, 1); 533 534 sbcompress(sb, m, sb->sb_mbtail); 535 536 sb->sb_lastrecord = sb->sb_mb; 537 SBLASTRECORDCHK(sb); 538} 539 540/* 541 * This version of sbappend() should only be used when the caller absolutely 542 * knows that there will never be more than one record in the socket buffer, 543 * that is, a stream protocol (such as TCP). 544 */ 545void 546sbappendstream(struct sockbuf *sb, struct mbuf *m) 547{ 548 549 SOCKBUF_LOCK(sb); 550 sbappendstream_locked(sb, m); 551 SOCKBUF_UNLOCK(sb); 552} 553 554#ifdef SOCKBUF_DEBUG 555void 556sbcheck(struct sockbuf *sb) 557{ 558 struct mbuf *m; 559 struct mbuf *n = 0; 560 u_long len = 0, mbcnt = 0; 561 562 SOCKBUF_LOCK_ASSERT(sb); 563 564 for (m = sb->sb_mb; m; m = n) { 565 n = m->m_nextpkt; 566 for (; m; m = m->m_next) { 567 len += m->m_len; 568 mbcnt += MSIZE; 569 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 570 mbcnt += m->m_ext.ext_size; 571 } 572 } 573 if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) { 574 printf("cc %ld != %u || mbcnt %ld != %u\n", len, sb->sb_cc, 575 mbcnt, sb->sb_mbcnt); 576 panic("sbcheck"); 577 } 578} 579#endif 580 581/* 582 * As above, except the mbuf chain begins a new record. 583 */ 584void 585sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 586{ 587 struct mbuf *m; 588 589 SOCKBUF_LOCK_ASSERT(sb); 590 591 if (m0 == 0) 592 return; 593 /* 594 * Put the first mbuf on the queue. Note this permits zero length 595 * records. 596 */ 597 sballoc(sb, m0); 598 SBLASTRECORDCHK(sb); 599 SBLINKRECORD(sb, m0); 600 sb->sb_mbtail = m0; 601 m = m0->m_next; 602 m0->m_next = 0; 603 if (m && (m0->m_flags & M_EOR)) { 604 m0->m_flags &= ~M_EOR; 605 m->m_flags |= M_EOR; 606 } 607 /* always call sbcompress() so it can do SBLASTMBUFCHK() */ 608 sbcompress(sb, m, m0); 609} 610 611/* 612 * As above, except the mbuf chain begins a new record. 613 */ 614void 615sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 616{ 617 618 SOCKBUF_LOCK(sb); 619 sbappendrecord_locked(sb, m0); 620 SOCKBUF_UNLOCK(sb); 621} 622 623/* Helper routine that appends data, control, and address to a sockbuf. */ 624static int 625sbappendaddr_locked_internal(struct sockbuf *sb, const struct sockaddr *asa, 626 struct mbuf *m0, struct mbuf *control, struct mbuf *ctrl_last) 627{ 628 struct mbuf *m, *n, *nlast; 629#if MSIZE <= 256 630 if (asa->sa_len > MLEN) 631 return (0); 632#endif 633 m = m_get(M_NOWAIT, MT_SONAME); 634 if (m == NULL) 635 return (0); 636 m->m_len = asa->sa_len; 637 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 638 if (ctrl_last) 639 ctrl_last->m_next = m0; /* concatenate data to control */ 640 else 641 control = m0; 642 m->m_next = control; 643 for (n = m; n->m_next != NULL; n = n->m_next) 644 sballoc(sb, n); 645 sballoc(sb, n); 646 nlast = n; 647 SBLINKRECORD(sb, m); 648 649 sb->sb_mbtail = nlast; 650 SBLASTMBUFCHK(sb); 651 652 SBLASTRECORDCHK(sb); 653 return (1); 654} 655 656/* 657 * Append address and data, and optionally, control (ancillary) data to the 658 * receive queue of a socket. If present, m0 must include a packet header 659 * with total length. Returns 0 if no space in sockbuf or insufficient 660 * mbufs. 661 */ 662int 663sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 664 struct mbuf *m0, struct mbuf *control) 665{ 666 struct mbuf *ctrl_last; 667 int space = asa->sa_len; 668 669 SOCKBUF_LOCK_ASSERT(sb); 670 671 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 672 panic("sbappendaddr_locked"); 673 if (m0) 674 space += m0->m_pkthdr.len; 675 space += m_length(control, &ctrl_last); 676 677 if (space > sbspace(sb)) 678 return (0); 679 return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last)); 680} 681 682/* 683 * Append address and data, and optionally, control (ancillary) data to the 684 * receive queue of a socket. If present, m0 must include a packet header 685 * with total length. Returns 0 if insufficient mbufs. Does not validate space 686 * on the receiving sockbuf. 687 */ 688int 689sbappendaddr_nospacecheck_locked(struct sockbuf *sb, const struct sockaddr *asa, 690 struct mbuf *m0, struct mbuf *control) 691{ 692 struct mbuf *ctrl_last; 693 694 SOCKBUF_LOCK_ASSERT(sb); 695 696 ctrl_last = (control == NULL) ? NULL : m_last(control); 697 return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last)); 698} 699 700/* 701 * Append address and data, and optionally, control (ancillary) data to the 702 * receive queue of a socket. If present, m0 must include a packet header 703 * with total length. Returns 0 if no space in sockbuf or insufficient 704 * mbufs. 705 */ 706int 707sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 708 struct mbuf *m0, struct mbuf *control) 709{ 710 int retval; 711 712 SOCKBUF_LOCK(sb); 713 retval = sbappendaddr_locked(sb, asa, m0, control); 714 SOCKBUF_UNLOCK(sb); 715 return (retval); 716} 717 718int 719sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 720 struct mbuf *control) 721{ 722 struct mbuf *m, *n, *mlast; 723 int space; 724 725 SOCKBUF_LOCK_ASSERT(sb); 726 727 if (control == 0) 728 panic("sbappendcontrol_locked"); 729 space = m_length(control, &n) + m_length(m0, NULL); 730 731 if (space > sbspace(sb)) 732 return (0); 733 n->m_next = m0; /* concatenate data to control */ 734 735 SBLASTRECORDCHK(sb); 736 737 for (m = control; m->m_next; m = m->m_next) 738 sballoc(sb, m); 739 sballoc(sb, m); 740 mlast = m; 741 SBLINKRECORD(sb, control); 742 743 sb->sb_mbtail = mlast; 744 SBLASTMBUFCHK(sb); 745 746 SBLASTRECORDCHK(sb); 747 return (1); 748} 749 750int 751sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 752{ 753 int retval; 754 755 SOCKBUF_LOCK(sb); 756 retval = sbappendcontrol_locked(sb, m0, control); 757 SOCKBUF_UNLOCK(sb); 758 return (retval); 759} 760 761/* 762 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 763 * (n). If (n) is NULL, the buffer is presumed empty. 764 * 765 * When the data is compressed, mbufs in the chain may be handled in one of 766 * three ways: 767 * 768 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 769 * record boundary, and no change in data type). 770 * 771 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 772 * an mbuf already in the socket buffer. This can occur if an 773 * appropriate mbuf exists, there is room, and no merging of data types 774 * will occur. 775 * 776 * (3) The mbuf may be appended to the end of the existing mbuf chain. 777 * 778 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 779 * end-of-record. 780 */ 781void 782sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 783{ 784 int eor = 0; 785 struct mbuf *o; 786 787 SOCKBUF_LOCK_ASSERT(sb); 788 789 while (m) { 790 eor |= m->m_flags & M_EOR; 791 if (m->m_len == 0 && 792 (eor == 0 || 793 (((o = m->m_next) || (o = n)) && 794 o->m_type == m->m_type))) { 795 if (sb->sb_lastrecord == m) 796 sb->sb_lastrecord = m->m_next; 797 m = m_free(m); 798 continue; 799 } 800 if (n && (n->m_flags & M_EOR) == 0 && 801 M_WRITABLE(n) && 802 ((sb->sb_flags & SB_NOCOALESCE) == 0) && 803 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 804 m->m_len <= M_TRAILINGSPACE(n) && 805 n->m_type == m->m_type) { 806 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, 807 (unsigned)m->m_len); 808 n->m_len += m->m_len; 809 sb->sb_cc += m->m_len; 810 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 811 /* XXX: Probably don't need.*/ 812 sb->sb_ctl += m->m_len; 813 m = m_free(m); 814 continue; 815 } 816 if (n) 817 n->m_next = m; 818 else 819 sb->sb_mb = m; 820 sb->sb_mbtail = m; 821 sballoc(sb, m); 822 n = m; 823 m->m_flags &= ~M_EOR; 824 m = m->m_next; 825 n->m_next = 0; 826 } 827 if (eor) { 828 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 829 n->m_flags |= eor; 830 } 831 SBLASTMBUFCHK(sb); 832} 833 834/* 835 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 836 */ 837static void 838sbflush_internal(struct sockbuf *sb) 839{ 840 841 while (sb->sb_mbcnt) { 842 /* 843 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty: 844 * we would loop forever. Panic instead. 845 */ 846 if (!sb->sb_cc && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 847 break; 848 m_freem(sbcut_internal(sb, (int)sb->sb_cc)); 849 } 850 if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt) 851 panic("sbflush_internal: cc %u || mb %p || mbcnt %u", 852 sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt); 853} 854 855void 856sbflush_locked(struct sockbuf *sb) 857{ 858 859 SOCKBUF_LOCK_ASSERT(sb); 860 sbflush_internal(sb); 861} 862 863void 864sbflush(struct sockbuf *sb) 865{ 866 867 SOCKBUF_LOCK(sb); 868 sbflush_locked(sb); 869 SOCKBUF_UNLOCK(sb); 870} 871 872/* 873 * Cut data from (the front of) a sockbuf. 874 */ 875static struct mbuf * 876sbcut_internal(struct sockbuf *sb, int len) 877{ 878 struct mbuf *m, *n, *next, *mfree; 879 880 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 881 mfree = NULL; 882 883 while (len > 0) { 884 if (m == 0) { 885 if (next == 0) 886 panic("sbdrop"); 887 m = next; 888 next = m->m_nextpkt; 889 continue; 890 } 891 if (m->m_len > len) { 892 m->m_len -= len; 893 m->m_data += len; 894 sb->sb_cc -= len; 895 if (sb->sb_sndptroff != 0) 896 sb->sb_sndptroff -= len; 897 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 898 sb->sb_ctl -= len; 899 break; 900 } 901 len -= m->m_len; 902 sbfree(sb, m); 903 n = m->m_next; 904 m->m_next = mfree; 905 mfree = m; 906 m = n; 907 } 908 while (m && m->m_len == 0) { 909 sbfree(sb, m); 910 n = m->m_next; 911 m->m_next = mfree; 912 mfree = m; 913 m = n; 914 } 915 if (m) { 916 sb->sb_mb = m; 917 m->m_nextpkt = next; 918 } else 919 sb->sb_mb = next; 920 /* 921 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 922 * sb_lastrecord is up-to-date if we dropped part of the last record. 923 */ 924 m = sb->sb_mb; 925 if (m == NULL) { 926 sb->sb_mbtail = NULL; 927 sb->sb_lastrecord = NULL; 928 } else if (m->m_nextpkt == NULL) { 929 sb->sb_lastrecord = m; 930 } 931 932 return (mfree); 933} 934 935/* 936 * Drop data from (the front of) a sockbuf. 937 */ 938void 939sbdrop_locked(struct sockbuf *sb, int len) 940{ 941 942 SOCKBUF_LOCK_ASSERT(sb); 943 m_freem(sbcut_internal(sb, len)); 944} 945 946/* 947 * Drop data from (the front of) a sockbuf, 948 * and return it to caller. 949 */ 950struct mbuf * 951sbcut_locked(struct sockbuf *sb, int len) 952{ 953 954 SOCKBUF_LOCK_ASSERT(sb); 955 return (sbcut_internal(sb, len)); 956} 957 958void 959sbdrop(struct sockbuf *sb, int len) 960{ 961 struct mbuf *mfree; 962 963 SOCKBUF_LOCK(sb); 964 mfree = sbcut_internal(sb, len); 965 SOCKBUF_UNLOCK(sb); 966 967 m_freem(mfree); 968} 969 970/* 971 * Maintain a pointer and offset pair into the socket buffer mbuf chain to 972 * avoid traversal of the entire socket buffer for larger offsets. 973 */ 974struct mbuf * 975sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) 976{ 977 struct mbuf *m, *ret; 978 979 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 980 KASSERT(off + len <= sb->sb_cc, ("%s: beyond sb", __func__)); 981 KASSERT(sb->sb_sndptroff <= sb->sb_cc, ("%s: sndptroff broken", __func__)); 982 983 /* 984 * Is off below stored offset? Happens on retransmits. 985 * Just return, we can't help here. 986 */ 987 if (sb->sb_sndptroff > off) { 988 *moff = off; 989 return (sb->sb_mb); 990 } 991 992 /* Return closest mbuf in chain for current offset. */ 993 *moff = off - sb->sb_sndptroff; 994 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb; 995 if (*moff == m->m_len) { 996 *moff = 0; 997 sb->sb_sndptroff += m->m_len; 998 m = ret = m->m_next; 999 KASSERT(ret->m_len > 0, 1000 ("mbuf %p in sockbuf %p chain has no valid data", ret, sb)); 1001 } 1002 1003 /* Advance by len to be as close as possible for the next transmit. */ 1004 for (off = off - sb->sb_sndptroff + len - 1; 1005 off > 0 && m != NULL && off >= m->m_len; 1006 m = m->m_next) { 1007 sb->sb_sndptroff += m->m_len; 1008 off -= m->m_len; 1009 } 1010 if (off > 0 && m == NULL) 1011 panic("%s: sockbuf %p and mbuf %p clashing", __func__, sb, ret); 1012 sb->sb_sndptr = m; 1013 1014 return (ret); 1015} 1016 1017/* 1018 * Return the first mbuf and the mbuf data offset for the provided 1019 * send offset without changing the "sb_sndptroff" field. 1020 */ 1021struct mbuf * 1022sbsndmbuf(struct sockbuf *sb, u_int off, u_int *moff) 1023{ 1024 struct mbuf *m; 1025 1026 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 1027 1028 /* 1029 * If the "off" is below the stored offset, which happens on 1030 * retransmits, just use "sb_mb": 1031 */ 1032 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { 1033 m = sb->sb_mb; 1034 } else { 1035 m = sb->sb_sndptr; 1036 off -= sb->sb_sndptroff; 1037 } 1038 while (off > 0 && m != NULL) { 1039 if (off < m->m_len) 1040 break; 1041 off -= m->m_len; 1042 m = m->m_next; 1043 } 1044 *moff = off; 1045 return (m); 1046} 1047 1048/* 1049 * Drop a record off the front of a sockbuf and move the next record to the 1050 * front. 1051 */ 1052void 1053sbdroprecord_locked(struct sockbuf *sb) 1054{ 1055 struct mbuf *m; 1056 1057 SOCKBUF_LOCK_ASSERT(sb); 1058 1059 m = sb->sb_mb; 1060 if (m) { 1061 sb->sb_mb = m->m_nextpkt; 1062 do { 1063 sbfree(sb, m); 1064 m = m_free(m); 1065 } while (m); 1066 } 1067 SB_EMPTY_FIXUP(sb); 1068} 1069 1070/* 1071 * Drop a record off the front of a sockbuf and move the next record to the 1072 * front. 1073 */ 1074void 1075sbdroprecord(struct sockbuf *sb) 1076{ 1077 1078 SOCKBUF_LOCK(sb); 1079 sbdroprecord_locked(sb); 1080 SOCKBUF_UNLOCK(sb); 1081} 1082 1083/* 1084 * Create a "control" mbuf containing the specified data with the specified 1085 * type for presentation on a socket buffer. 1086 */ 1087struct mbuf * 1088sbcreatecontrol(caddr_t p, int size, int type, int level) 1089{ 1090 struct cmsghdr *cp; 1091 struct mbuf *m; 1092 1093 if (CMSG_SPACE((u_int)size) > MCLBYTES) 1094 return ((struct mbuf *) NULL); 1095 if (CMSG_SPACE((u_int)size) > MLEN) 1096 m = m_getcl(M_NOWAIT, MT_CONTROL, 0); 1097 else 1098 m = m_get(M_NOWAIT, MT_CONTROL); 1099 if (m == NULL) 1100 return ((struct mbuf *) NULL); 1101 cp = mtod(m, struct cmsghdr *); 1102 m->m_len = 0; 1103 KASSERT(CMSG_SPACE((u_int)size) <= M_TRAILINGSPACE(m), 1104 ("sbcreatecontrol: short mbuf")); 1105 /* 1106 * Don't leave the padding between the msg header and the 1107 * cmsg data and the padding after the cmsg data un-initialized. 1108 */ 1109 bzero(cp, CMSG_SPACE((u_int)size)); 1110 if (p != NULL) 1111 (void)memcpy(CMSG_DATA(cp), p, size); 1112 m->m_len = CMSG_SPACE(size); 1113 cp->cmsg_len = CMSG_LEN(size); 1114 cp->cmsg_level = level; 1115 cp->cmsg_type = type; 1116 return (m); 1117} 1118 1119/* 1120 * This does the same for socket buffers that sotoxsocket does for sockets: 1121 * generate an user-format data structure describing the socket buffer. Note 1122 * that the xsockbuf structure, since it is always embedded in a socket, does 1123 * not include a self pointer nor a length. We make this entry point public 1124 * in case some other mechanism needs it. 1125 */ 1126void 1127sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) 1128{ 1129 1130 xsb->sb_cc = sb->sb_cc; 1131 xsb->sb_hiwat = sb->sb_hiwat; 1132 xsb->sb_mbcnt = sb->sb_mbcnt; 1133 xsb->sb_mcnt = sb->sb_mcnt; 1134 xsb->sb_ccnt = sb->sb_ccnt; 1135 xsb->sb_mbmax = sb->sb_mbmax; 1136 xsb->sb_lowat = sb->sb_lowat; 1137 xsb->sb_flags = sb->sb_flags; 1138 xsb->sb_timeo = sb->sb_timeo; 1139} 1140 1141/* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */ 1142static int dummy; 1143SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW, &dummy, 0, ""); 1144SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_ULONG|CTLFLAG_RW, 1145 &sb_max, 0, sysctl_handle_sb_max, "LU", "Maximum socket buffer size"); 1146SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, 1147 &sb_efficiency, 0, "Socket buffer size waste factor"); 1148