uipc_syscalls.c revision 339068
1/*- 2 * Copyright (c) 1982, 1986, 1989, 1990, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * sendfile(2) and related extensions: 6 * Copyright (c) 1998, David Greenman. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_syscalls.c 8.4 (Berkeley) 2/21/94 33 */ 34 35#include <sys/cdefs.h> 36__FBSDID("$FreeBSD: stable/10/sys/kern/uipc_syscalls.c 339068 2018-10-01 18:00:52Z asomers $"); 37 38#include "opt_capsicum.h" 39#include "opt_inet.h" 40#include "opt_inet6.h" 41#include "opt_compat.h" 42#include "opt_ktrace.h" 43 44#include <sys/param.h> 45#include <sys/systm.h> 46#include <sys/capsicum.h> 47#include <sys/condvar.h> 48#include <sys/kernel.h> 49#include <sys/lock.h> 50#include <sys/mutex.h> 51#include <sys/sysproto.h> 52#include <sys/malloc.h> 53#include <sys/filedesc.h> 54#include <sys/event.h> 55#include <sys/proc.h> 56#include <sys/fcntl.h> 57#include <sys/file.h> 58#include <sys/filio.h> 59#include <sys/jail.h> 60#include <sys/mman.h> 61#include <sys/mount.h> 62#include <sys/mbuf.h> 63#include <sys/protosw.h> 64#include <sys/rwlock.h> 65#include <sys/sf_buf.h> 66#include <sys/sysent.h> 67#include <sys/socket.h> 68#include <sys/socketvar.h> 69#include <sys/signalvar.h> 70#include <sys/syscallsubr.h> 71#include <sys/sysctl.h> 72#include <sys/uio.h> 73#include <sys/un.h> 74#include <sys/unpcb.h> 75#include <sys/vnode.h> 76#ifdef KTRACE 77#include <sys/ktrace.h> 78#endif 79#ifdef COMPAT_FREEBSD32 80#include <compat/freebsd32/freebsd32_util.h> 81#endif 82 83#include <net/vnet.h> 84 85#include <security/audit/audit.h> 86#include <security/mac/mac_framework.h> 87 88#include <vm/vm.h> 89#include <vm/vm_param.h> 90#include <vm/vm_object.h> 91#include <vm/vm_page.h> 92#include <vm/vm_pager.h> 93#include <vm/vm_kern.h> 94#include <vm/vm_extern.h> 95 96/* 97 * Flags for accept1() and kern_accept4(), in addition to SOCK_CLOEXEC 98 * and SOCK_NONBLOCK. 99 */ 100#define ACCEPT4_INHERIT 0x1 101#define ACCEPT4_COMPAT 0x2 102 103static int sendit(struct thread *td, int s, struct msghdr *mp, int flags); 104static int recvit(struct thread *td, int s, struct msghdr *mp, void *namelenp); 105 106static int accept1(struct thread *td, int s, struct sockaddr *uname, 107 socklen_t *anamelen, int flags); 108static int do_sendfile(struct thread *td, struct sendfile_args *uap, 109 int compat); 110static int getsockname1(struct thread *td, struct getsockname_args *uap, 111 int compat); 112static int getpeername1(struct thread *td, struct getpeername_args *uap, 113 int compat); 114 115counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)]; 116 117/* 118 * sendfile(2)-related variables and associated sysctls 119 */ 120static SYSCTL_NODE(_kern_ipc, OID_AUTO, sendfile, CTLFLAG_RW, 0, 121 "sendfile(2) tunables"); 122static int sfreadahead = 1; 123SYSCTL_INT(_kern_ipc_sendfile, OID_AUTO, readahead, CTLFLAG_RW, 124 &sfreadahead, 0, "Number of sendfile(2) read-ahead MAXBSIZE blocks"); 125 126 127static void 128sfstat_init(const void *unused) 129{ 130 131 COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t), 132 M_WAITOK); 133} 134SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL); 135 136static int 137sfstat_sysctl(SYSCTL_HANDLER_ARGS) 138{ 139 struct sfstat s; 140 141 COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t)); 142 if (req->newptr) 143 COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t)); 144 return (SYSCTL_OUT(req, &s, sizeof(s))); 145} 146SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW, 147 NULL, 0, sfstat_sysctl, "I", "sendfile statistics"); 148 149/* 150 * Convert a user file descriptor to a kernel file entry and check if required 151 * capability rights are present. 152 * A reference on the file entry is held upon returning. 153 */ 154int 155getsock_cap(struct thread *td, int fd, cap_rights_t *rightsp, 156 struct file **fpp, u_int *fflagp) 157{ 158 struct file *fp; 159 int error; 160 161 error = fget_unlocked(td->td_proc->p_fd, fd, rightsp, 0, &fp, NULL); 162 if (error != 0) 163 return (error); 164 if (fp->f_type != DTYPE_SOCKET) { 165 fdrop(fp, td); 166 return (ENOTSOCK); 167 } 168 if (fflagp != NULL) 169 *fflagp = fp->f_flag; 170 *fpp = fp; 171 return (0); 172} 173 174/* 175 * System call interface to the socket abstraction. 176 */ 177#if defined(COMPAT_43) 178#define COMPAT_OLDSOCK 179#endif 180 181int 182sys_socket(td, uap) 183 struct thread *td; 184 struct socket_args /* { 185 int domain; 186 int type; 187 int protocol; 188 } */ *uap; 189{ 190 struct socket *so; 191 struct file *fp; 192 int fd, error, type, oflag, fflag; 193 194 AUDIT_ARG_SOCKET(uap->domain, uap->type, uap->protocol); 195 196 type = uap->type; 197 oflag = 0; 198 fflag = 0; 199 if ((type & SOCK_CLOEXEC) != 0) { 200 type &= ~SOCK_CLOEXEC; 201 oflag |= O_CLOEXEC; 202 } 203 if ((type & SOCK_NONBLOCK) != 0) { 204 type &= ~SOCK_NONBLOCK; 205 fflag |= FNONBLOCK; 206 } 207 208#ifdef MAC 209 error = mac_socket_check_create(td->td_ucred, uap->domain, type, 210 uap->protocol); 211 if (error != 0) 212 return (error); 213#endif 214 error = falloc(td, &fp, &fd, oflag); 215 if (error != 0) 216 return (error); 217 /* An extra reference on `fp' has been held for us by falloc(). */ 218 error = socreate(uap->domain, &so, type, uap->protocol, 219 td->td_ucred, td); 220 if (error != 0) { 221 fdclose(td, fp, fd); 222 } else { 223 finit(fp, FREAD | FWRITE | fflag, DTYPE_SOCKET, so, &socketops); 224 if ((fflag & FNONBLOCK) != 0) 225 (void) fo_ioctl(fp, FIONBIO, &fflag, td->td_ucred, td); 226 td->td_retval[0] = fd; 227 } 228 fdrop(fp, td); 229 return (error); 230} 231 232/* ARGSUSED */ 233int 234sys_bind(td, uap) 235 struct thread *td; 236 struct bind_args /* { 237 int s; 238 caddr_t name; 239 int namelen; 240 } */ *uap; 241{ 242 struct sockaddr *sa; 243 int error; 244 245 error = getsockaddr(&sa, uap->name, uap->namelen); 246 if (error == 0) { 247 error = kern_bind(td, uap->s, sa); 248 free(sa, M_SONAME); 249 } 250 return (error); 251} 252 253static int 254kern_bindat(struct thread *td, int dirfd, int fd, struct sockaddr *sa) 255{ 256 struct socket *so; 257 struct file *fp; 258 cap_rights_t rights; 259 int error; 260 261 AUDIT_ARG_FD(fd); 262 AUDIT_ARG_SOCKADDR(td, dirfd, sa); 263 error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_BIND), 264 &fp, NULL); 265 if (error != 0) 266 return (error); 267 so = fp->f_data; 268#ifdef KTRACE 269 if (KTRPOINT(td, KTR_STRUCT)) 270 ktrsockaddr(sa); 271#endif 272#ifdef MAC 273 error = mac_socket_check_bind(td->td_ucred, so, sa); 274 if (error == 0) { 275#endif 276 if (dirfd == AT_FDCWD) 277 error = sobind(so, sa, td); 278 else 279 error = sobindat(dirfd, so, sa, td); 280#ifdef MAC 281 } 282#endif 283 fdrop(fp, td); 284 return (error); 285} 286 287int 288kern_bind(struct thread *td, int fd, struct sockaddr *sa) 289{ 290 291 return (kern_bindat(td, AT_FDCWD, fd, sa)); 292} 293 294/* ARGSUSED */ 295int 296sys_bindat(td, uap) 297 struct thread *td; 298 struct bindat_args /* { 299 int fd; 300 int s; 301 caddr_t name; 302 int namelen; 303 } */ *uap; 304{ 305 struct sockaddr *sa; 306 int error; 307 308 error = getsockaddr(&sa, uap->name, uap->namelen); 309 if (error == 0) { 310 error = kern_bindat(td, uap->fd, uap->s, sa); 311 free(sa, M_SONAME); 312 } 313 return (error); 314} 315 316/* ARGSUSED */ 317int 318sys_listen(td, uap) 319 struct thread *td; 320 struct listen_args /* { 321 int s; 322 int backlog; 323 } */ *uap; 324{ 325 struct socket *so; 326 struct file *fp; 327 cap_rights_t rights; 328 int error; 329 330 AUDIT_ARG_FD(uap->s); 331 error = getsock_cap(td, uap->s, cap_rights_init(&rights, CAP_LISTEN), 332 &fp, NULL); 333 if (error == 0) { 334 so = fp->f_data; 335#ifdef MAC 336 error = mac_socket_check_listen(td->td_ucred, so); 337 if (error == 0) 338#endif 339 error = solisten(so, uap->backlog, td); 340 fdrop(fp, td); 341 } 342 return(error); 343} 344 345/* 346 * accept1() 347 */ 348static int 349accept1(td, s, uname, anamelen, flags) 350 struct thread *td; 351 int s; 352 struct sockaddr *uname; 353 socklen_t *anamelen; 354 int flags; 355{ 356 struct sockaddr *name; 357 socklen_t namelen; 358 struct file *fp; 359 int error; 360 361 if (uname == NULL) 362 return (kern_accept4(td, s, NULL, NULL, flags, NULL)); 363 364 error = copyin(anamelen, &namelen, sizeof (namelen)); 365 if (error != 0) 366 return (error); 367 368 error = kern_accept4(td, s, &name, &namelen, flags, &fp); 369 370 /* 371 * return a namelen of zero for older code which might 372 * ignore the return value from accept. 373 */ 374 if (error != 0) { 375 (void) copyout(&namelen, anamelen, sizeof(*anamelen)); 376 return (error); 377 } 378 379 if (error == 0 && uname != NULL) { 380#ifdef COMPAT_OLDSOCK 381 if (flags & ACCEPT4_COMPAT) 382 ((struct osockaddr *)name)->sa_family = 383 name->sa_family; 384#endif 385 error = copyout(name, uname, namelen); 386 } 387 if (error == 0) 388 error = copyout(&namelen, anamelen, 389 sizeof(namelen)); 390 if (error != 0) 391 fdclose(td, fp, td->td_retval[0]); 392 fdrop(fp, td); 393 free(name, M_SONAME); 394 return (error); 395} 396 397int 398kern_accept(struct thread *td, int s, struct sockaddr **name, 399 socklen_t *namelen, struct file **fp) 400{ 401 return (kern_accept4(td, s, name, namelen, ACCEPT4_INHERIT, fp)); 402} 403 404int 405kern_accept4(struct thread *td, int s, struct sockaddr **name, 406 socklen_t *namelen, int flags, struct file **fp) 407{ 408 struct file *headfp, *nfp = NULL; 409 struct sockaddr *sa = NULL; 410 struct socket *head, *so; 411 cap_rights_t rights; 412 u_int fflag; 413 pid_t pgid; 414 int error, fd, tmp; 415 416 if (name != NULL) 417 *name = NULL; 418 419 AUDIT_ARG_FD(s); 420 error = getsock_cap(td, s, cap_rights_init(&rights, CAP_ACCEPT), 421 &headfp, &fflag); 422 if (error != 0) 423 return (error); 424 head = headfp->f_data; 425 if ((head->so_options & SO_ACCEPTCONN) == 0) { 426 error = EINVAL; 427 goto done; 428 } 429#ifdef MAC 430 error = mac_socket_check_accept(td->td_ucred, head); 431 if (error != 0) 432 goto done; 433#endif 434 error = falloc(td, &nfp, &fd, (flags & SOCK_CLOEXEC) ? O_CLOEXEC : 0); 435 if (error != 0) 436 goto done; 437 ACCEPT_LOCK(); 438 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->so_comp)) { 439 ACCEPT_UNLOCK(); 440 error = EWOULDBLOCK; 441 goto noconnection; 442 } 443 while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) { 444 if (head->so_rcv.sb_state & SBS_CANTRCVMORE) { 445 head->so_error = ECONNABORTED; 446 break; 447 } 448 error = msleep(&head->so_timeo, &accept_mtx, PSOCK | PCATCH, 449 "accept", 0); 450 if (error != 0) { 451 ACCEPT_UNLOCK(); 452 goto noconnection; 453 } 454 } 455 if (head->so_error) { 456 error = head->so_error; 457 head->so_error = 0; 458 ACCEPT_UNLOCK(); 459 goto noconnection; 460 } 461 so = TAILQ_FIRST(&head->so_comp); 462 KASSERT(!(so->so_qstate & SQ_INCOMP), ("accept1: so SQ_INCOMP")); 463 KASSERT(so->so_qstate & SQ_COMP, ("accept1: so not SQ_COMP")); 464 465 /* 466 * Before changing the flags on the socket, we have to bump the 467 * reference count. Otherwise, if the protocol calls sofree(), 468 * the socket will be released due to a zero refcount. 469 */ 470 SOCK_LOCK(so); /* soref() and so_state update */ 471 soref(so); /* file descriptor reference */ 472 473 TAILQ_REMOVE(&head->so_comp, so, so_list); 474 head->so_qlen--; 475 if (flags & ACCEPT4_INHERIT) 476 so->so_state |= (head->so_state & SS_NBIO); 477 else 478 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 479 so->so_qstate &= ~SQ_COMP; 480 so->so_head = NULL; 481 482 SOCK_UNLOCK(so); 483 ACCEPT_UNLOCK(); 484 485 /* An extra reference on `nfp' has been held for us by falloc(). */ 486 td->td_retval[0] = fd; 487 488 /* connection has been removed from the listen queue */ 489 KNOTE_UNLOCKED(&head->so_rcv.sb_sel.si_note, 0); 490 491 if (flags & ACCEPT4_INHERIT) { 492 pgid = fgetown(&head->so_sigio); 493 if (pgid != 0) 494 fsetown(pgid, &so->so_sigio); 495 } else { 496 fflag &= ~(FNONBLOCK | FASYNC); 497 if (flags & SOCK_NONBLOCK) 498 fflag |= FNONBLOCK; 499 } 500 501 finit(nfp, fflag, DTYPE_SOCKET, so, &socketops); 502 /* Sync socket nonblocking/async state with file flags */ 503 tmp = fflag & FNONBLOCK; 504 (void) fo_ioctl(nfp, FIONBIO, &tmp, td->td_ucred, td); 505 tmp = fflag & FASYNC; 506 (void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td); 507 sa = 0; 508 error = soaccept(so, &sa); 509 if (error != 0) { 510 /* 511 * return a namelen of zero for older code which might 512 * ignore the return value from accept. 513 */ 514 if (name) 515 *namelen = 0; 516 goto noconnection; 517 } 518 if (sa == NULL) { 519 if (name) 520 *namelen = 0; 521 goto done; 522 } 523 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, sa); 524 if (name) { 525 /* check sa_len before it is destroyed */ 526 if (*namelen > sa->sa_len) 527 *namelen = sa->sa_len; 528#ifdef KTRACE 529 if (KTRPOINT(td, KTR_STRUCT)) 530 ktrsockaddr(sa); 531#endif 532 *name = sa; 533 sa = NULL; 534 } 535noconnection: 536 free(sa, M_SONAME); 537 538 /* 539 * close the new descriptor, assuming someone hasn't ripped it 540 * out from under us. 541 */ 542 if (error != 0) 543 fdclose(td, nfp, fd); 544 545 /* 546 * Release explicitly held references before returning. We return 547 * a reference on nfp to the caller on success if they request it. 548 */ 549done: 550 if (fp != NULL) { 551 if (error == 0) { 552 *fp = nfp; 553 nfp = NULL; 554 } else 555 *fp = NULL; 556 } 557 if (nfp != NULL) 558 fdrop(nfp, td); 559 fdrop(headfp, td); 560 return (error); 561} 562 563int 564sys_accept(td, uap) 565 struct thread *td; 566 struct accept_args *uap; 567{ 568 569 return (accept1(td, uap->s, uap->name, uap->anamelen, ACCEPT4_INHERIT)); 570} 571 572int 573sys_accept4(td, uap) 574 struct thread *td; 575 struct accept4_args *uap; 576{ 577 578 if (uap->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 579 return (EINVAL); 580 581 return (accept1(td, uap->s, uap->name, uap->anamelen, uap->flags)); 582} 583 584#ifdef COMPAT_OLDSOCK 585int 586oaccept(td, uap) 587 struct thread *td; 588 struct accept_args *uap; 589{ 590 591 return (accept1(td, uap->s, uap->name, uap->anamelen, 592 ACCEPT4_INHERIT | ACCEPT4_COMPAT)); 593} 594#endif /* COMPAT_OLDSOCK */ 595 596/* ARGSUSED */ 597int 598sys_connect(td, uap) 599 struct thread *td; 600 struct connect_args /* { 601 int s; 602 caddr_t name; 603 int namelen; 604 } */ *uap; 605{ 606 struct sockaddr *sa; 607 int error; 608 609 error = getsockaddr(&sa, uap->name, uap->namelen); 610 if (error == 0) { 611 error = kern_connect(td, uap->s, sa); 612 free(sa, M_SONAME); 613 } 614 return (error); 615} 616 617static int 618kern_connectat(struct thread *td, int dirfd, int fd, struct sockaddr *sa) 619{ 620 struct socket *so; 621 struct file *fp; 622 cap_rights_t rights; 623 int error, interrupted = 0; 624 625 AUDIT_ARG_FD(fd); 626 AUDIT_ARG_SOCKADDR(td, dirfd, sa); 627 error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_CONNECT), 628 &fp, NULL); 629 if (error != 0) 630 return (error); 631 so = fp->f_data; 632 if (so->so_state & SS_ISCONNECTING) { 633 error = EALREADY; 634 goto done1; 635 } 636#ifdef KTRACE 637 if (KTRPOINT(td, KTR_STRUCT)) 638 ktrsockaddr(sa); 639#endif 640#ifdef MAC 641 error = mac_socket_check_connect(td->td_ucred, so, sa); 642 if (error != 0) 643 goto bad; 644#endif 645 if (dirfd == AT_FDCWD) 646 error = soconnect(so, sa, td); 647 else 648 error = soconnectat(dirfd, so, sa, td); 649 if (error != 0) 650 goto bad; 651 if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) { 652 error = EINPROGRESS; 653 goto done1; 654 } 655 SOCK_LOCK(so); 656 while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) { 657 error = msleep(&so->so_timeo, SOCK_MTX(so), PSOCK | PCATCH, 658 "connec", 0); 659 if (error != 0) { 660 if (error == EINTR || error == ERESTART) 661 interrupted = 1; 662 break; 663 } 664 } 665 if (error == 0) { 666 error = so->so_error; 667 so->so_error = 0; 668 } 669 SOCK_UNLOCK(so); 670bad: 671 if (!interrupted) 672 so->so_state &= ~SS_ISCONNECTING; 673 if (error == ERESTART) 674 error = EINTR; 675done1: 676 fdrop(fp, td); 677 return (error); 678} 679 680int 681kern_connect(struct thread *td, int fd, struct sockaddr *sa) 682{ 683 684 return (kern_connectat(td, AT_FDCWD, fd, sa)); 685} 686 687/* ARGSUSED */ 688int 689sys_connectat(td, uap) 690 struct thread *td; 691 struct connectat_args /* { 692 int fd; 693 int s; 694 caddr_t name; 695 int namelen; 696 } */ *uap; 697{ 698 struct sockaddr *sa; 699 int error; 700 701 error = getsockaddr(&sa, uap->name, uap->namelen); 702 if (error == 0) { 703 error = kern_connectat(td, uap->fd, uap->s, sa); 704 free(sa, M_SONAME); 705 } 706 return (error); 707} 708 709int 710kern_socketpair(struct thread *td, int domain, int type, int protocol, 711 int *rsv) 712{ 713 struct file *fp1, *fp2; 714 struct socket *so1, *so2; 715 int fd, error, oflag, fflag; 716 717 AUDIT_ARG_SOCKET(domain, type, protocol); 718 719 oflag = 0; 720 fflag = 0; 721 if ((type & SOCK_CLOEXEC) != 0) { 722 type &= ~SOCK_CLOEXEC; 723 oflag |= O_CLOEXEC; 724 } 725 if ((type & SOCK_NONBLOCK) != 0) { 726 type &= ~SOCK_NONBLOCK; 727 fflag |= FNONBLOCK; 728 } 729#ifdef MAC 730 /* We might want to have a separate check for socket pairs. */ 731 error = mac_socket_check_create(td->td_ucred, domain, type, 732 protocol); 733 if (error != 0) 734 return (error); 735#endif 736 error = socreate(domain, &so1, type, protocol, td->td_ucred, td); 737 if (error != 0) 738 return (error); 739 error = socreate(domain, &so2, type, protocol, td->td_ucred, td); 740 if (error != 0) 741 goto free1; 742 /* On success extra reference to `fp1' and 'fp2' is set by falloc. */ 743 error = falloc(td, &fp1, &fd, oflag); 744 if (error != 0) 745 goto free2; 746 rsv[0] = fd; 747 fp1->f_data = so1; /* so1 already has ref count */ 748 error = falloc(td, &fp2, &fd, oflag); 749 if (error != 0) 750 goto free3; 751 fp2->f_data = so2; /* so2 already has ref count */ 752 rsv[1] = fd; 753 error = soconnect2(so1, so2); 754 if (error != 0) 755 goto free4; 756 if (type == SOCK_DGRAM) { 757 /* 758 * Datagram socket connection is asymmetric. 759 */ 760 error = soconnect2(so2, so1); 761 if (error != 0) 762 goto free4; 763 } else if (so1->so_proto->pr_flags & PR_CONNREQUIRED) { 764 struct unpcb *unp, *unp2; 765 unp = sotounpcb(so1); 766 unp2 = sotounpcb(so2); 767 /* 768 * No need to lock the unps, because the sockets are brand-new. 769 * No other threads can be using them yet 770 */ 771 unp_copy_peercred(td, unp, unp2, unp); 772 } 773 finit(fp1, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp1->f_data, 774 &socketops); 775 finit(fp2, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp2->f_data, 776 &socketops); 777 if ((fflag & FNONBLOCK) != 0) { 778 (void) fo_ioctl(fp1, FIONBIO, &fflag, td->td_ucred, td); 779 (void) fo_ioctl(fp2, FIONBIO, &fflag, td->td_ucred, td); 780 } 781 fdrop(fp1, td); 782 fdrop(fp2, td); 783 return (0); 784free4: 785 fdclose(td, fp2, rsv[1]); 786 fdrop(fp2, td); 787free3: 788 fdclose(td, fp1, rsv[0]); 789 fdrop(fp1, td); 790free2: 791 if (so2 != NULL) 792 (void)soclose(so2); 793free1: 794 if (so1 != NULL) 795 (void)soclose(so1); 796 return (error); 797} 798 799int 800sys_socketpair(struct thread *td, struct socketpair_args *uap) 801{ 802 int error, sv[2]; 803 804 error = kern_socketpair(td, uap->domain, uap->type, 805 uap->protocol, sv); 806 if (error != 0) 807 return (error); 808 error = copyout(sv, uap->rsv, 2 * sizeof(int)); 809 if (error != 0) { 810 (void)kern_close(td, sv[0]); 811 (void)kern_close(td, sv[1]); 812 } 813 return (error); 814} 815 816static int 817sendit(td, s, mp, flags) 818 struct thread *td; 819 int s; 820 struct msghdr *mp; 821 int flags; 822{ 823 struct mbuf *control; 824 struct sockaddr *to; 825 int error; 826 827#ifdef CAPABILITY_MODE 828 if (IN_CAPABILITY_MODE(td) && (mp->msg_name != NULL)) 829 return (ECAPMODE); 830#endif 831 832 if (mp->msg_name != NULL) { 833 error = getsockaddr(&to, mp->msg_name, mp->msg_namelen); 834 if (error != 0) { 835 to = NULL; 836 goto bad; 837 } 838 mp->msg_name = to; 839 } else { 840 to = NULL; 841 } 842 843 if (mp->msg_control) { 844 if (mp->msg_controllen < sizeof(struct cmsghdr) 845#ifdef COMPAT_OLDSOCK 846 && mp->msg_flags != MSG_COMPAT 847#endif 848 ) { 849 error = EINVAL; 850 goto bad; 851 } 852 error = sockargs(&control, mp->msg_control, 853 mp->msg_controllen, MT_CONTROL); 854 if (error != 0) 855 goto bad; 856#ifdef COMPAT_OLDSOCK 857 if (mp->msg_flags == MSG_COMPAT) { 858 struct cmsghdr *cm; 859 860 M_PREPEND(control, sizeof(*cm), M_WAITOK); 861 cm = mtod(control, struct cmsghdr *); 862 cm->cmsg_len = control->m_len; 863 cm->cmsg_level = SOL_SOCKET; 864 cm->cmsg_type = SCM_RIGHTS; 865 } 866#endif 867 } else { 868 control = NULL; 869 } 870 871 error = kern_sendit(td, s, mp, flags, control, UIO_USERSPACE); 872 873bad: 874 free(to, M_SONAME); 875 return (error); 876} 877 878int 879kern_sendit(td, s, mp, flags, control, segflg) 880 struct thread *td; 881 int s; 882 struct msghdr *mp; 883 int flags; 884 struct mbuf *control; 885 enum uio_seg segflg; 886{ 887 struct file *fp; 888 struct uio auio; 889 struct iovec *iov; 890 struct socket *so; 891 cap_rights_t rights; 892#ifdef KTRACE 893 struct uio *ktruio = NULL; 894#endif 895 ssize_t len; 896 int i, error; 897 898 AUDIT_ARG_FD(s); 899 cap_rights_init(&rights, CAP_SEND); 900 if (mp->msg_name != NULL) { 901 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, mp->msg_name); 902 cap_rights_set(&rights, CAP_CONNECT); 903 } 904 error = getsock_cap(td, s, &rights, &fp, NULL); 905 if (error != 0) 906 return (error); 907 so = (struct socket *)fp->f_data; 908 909#ifdef KTRACE 910 if (mp->msg_name != NULL && KTRPOINT(td, KTR_STRUCT)) 911 ktrsockaddr(mp->msg_name); 912#endif 913#ifdef MAC 914 if (mp->msg_name != NULL) { 915 error = mac_socket_check_connect(td->td_ucred, so, 916 mp->msg_name); 917 if (error != 0) 918 goto bad; 919 } 920 error = mac_socket_check_send(td->td_ucred, so); 921 if (error != 0) 922 goto bad; 923#endif 924 925 auio.uio_iov = mp->msg_iov; 926 auio.uio_iovcnt = mp->msg_iovlen; 927 auio.uio_segflg = segflg; 928 auio.uio_rw = UIO_WRITE; 929 auio.uio_td = td; 930 auio.uio_offset = 0; /* XXX */ 931 auio.uio_resid = 0; 932 iov = mp->msg_iov; 933 for (i = 0; i < mp->msg_iovlen; i++, iov++) { 934 if ((auio.uio_resid += iov->iov_len) < 0) { 935 error = EINVAL; 936 goto bad; 937 } 938 } 939#ifdef KTRACE 940 if (KTRPOINT(td, KTR_GENIO)) 941 ktruio = cloneuio(&auio); 942#endif 943 len = auio.uio_resid; 944 error = sosend(so, mp->msg_name, &auio, 0, control, flags, td); 945 if (error != 0) { 946 if (auio.uio_resid != len && (error == ERESTART || 947 error == EINTR || error == EWOULDBLOCK)) 948 error = 0; 949 /* Generation of SIGPIPE can be controlled per socket */ 950 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) && 951 !(flags & MSG_NOSIGNAL)) { 952 PROC_LOCK(td->td_proc); 953 tdsignal(td, SIGPIPE); 954 PROC_UNLOCK(td->td_proc); 955 } 956 } 957 if (error == 0) 958 td->td_retval[0] = len - auio.uio_resid; 959#ifdef KTRACE 960 if (ktruio != NULL) { 961 ktruio->uio_resid = td->td_retval[0]; 962 ktrgenio(s, UIO_WRITE, ktruio, error); 963 } 964#endif 965bad: 966 fdrop(fp, td); 967 return (error); 968} 969 970int 971sys_sendto(td, uap) 972 struct thread *td; 973 struct sendto_args /* { 974 int s; 975 caddr_t buf; 976 size_t len; 977 int flags; 978 caddr_t to; 979 int tolen; 980 } */ *uap; 981{ 982 struct msghdr msg; 983 struct iovec aiov; 984 985 msg.msg_name = uap->to; 986 msg.msg_namelen = uap->tolen; 987 msg.msg_iov = &aiov; 988 msg.msg_iovlen = 1; 989 msg.msg_control = 0; 990#ifdef COMPAT_OLDSOCK 991 msg.msg_flags = 0; 992#endif 993 aiov.iov_base = uap->buf; 994 aiov.iov_len = uap->len; 995 return (sendit(td, uap->s, &msg, uap->flags)); 996} 997 998#ifdef COMPAT_OLDSOCK 999int 1000osend(td, uap) 1001 struct thread *td; 1002 struct osend_args /* { 1003 int s; 1004 caddr_t buf; 1005 int len; 1006 int flags; 1007 } */ *uap; 1008{ 1009 struct msghdr msg; 1010 struct iovec aiov; 1011 1012 msg.msg_name = 0; 1013 msg.msg_namelen = 0; 1014 msg.msg_iov = &aiov; 1015 msg.msg_iovlen = 1; 1016 aiov.iov_base = uap->buf; 1017 aiov.iov_len = uap->len; 1018 msg.msg_control = 0; 1019 msg.msg_flags = 0; 1020 return (sendit(td, uap->s, &msg, uap->flags)); 1021} 1022 1023int 1024osendmsg(td, uap) 1025 struct thread *td; 1026 struct osendmsg_args /* { 1027 int s; 1028 caddr_t msg; 1029 int flags; 1030 } */ *uap; 1031{ 1032 struct msghdr msg; 1033 struct iovec *iov; 1034 int error; 1035 1036 error = copyin(uap->msg, &msg, sizeof (struct omsghdr)); 1037 if (error != 0) 1038 return (error); 1039 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1040 if (error != 0) 1041 return (error); 1042 msg.msg_iov = iov; 1043 msg.msg_flags = MSG_COMPAT; 1044 error = sendit(td, uap->s, &msg, uap->flags); 1045 free(iov, M_IOV); 1046 return (error); 1047} 1048#endif 1049 1050int 1051sys_sendmsg(td, uap) 1052 struct thread *td; 1053 struct sendmsg_args /* { 1054 int s; 1055 caddr_t msg; 1056 int flags; 1057 } */ *uap; 1058{ 1059 struct msghdr msg; 1060 struct iovec *iov; 1061 int error; 1062 1063 error = copyin(uap->msg, &msg, sizeof (msg)); 1064 if (error != 0) 1065 return (error); 1066 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1067 if (error != 0) 1068 return (error); 1069 msg.msg_iov = iov; 1070#ifdef COMPAT_OLDSOCK 1071 msg.msg_flags = 0; 1072#endif 1073 error = sendit(td, uap->s, &msg, uap->flags); 1074 free(iov, M_IOV); 1075 return (error); 1076} 1077 1078int 1079kern_recvit(td, s, mp, fromseg, controlp) 1080 struct thread *td; 1081 int s; 1082 struct msghdr *mp; 1083 enum uio_seg fromseg; 1084 struct mbuf **controlp; 1085{ 1086 struct uio auio; 1087 struct iovec *iov; 1088 struct mbuf *m, *control = NULL; 1089 caddr_t ctlbuf; 1090 struct file *fp; 1091 struct socket *so; 1092 struct sockaddr *fromsa = NULL; 1093 cap_rights_t rights; 1094#ifdef KTRACE 1095 struct uio *ktruio = NULL; 1096#endif 1097 ssize_t len; 1098 int error, i; 1099 1100 if (controlp != NULL) 1101 *controlp = NULL; 1102 1103 AUDIT_ARG_FD(s); 1104 error = getsock_cap(td, s, cap_rights_init(&rights, CAP_RECV), 1105 &fp, NULL); 1106 if (error != 0) 1107 return (error); 1108 so = fp->f_data; 1109 1110#ifdef MAC 1111 error = mac_socket_check_receive(td->td_ucred, so); 1112 if (error != 0) { 1113 fdrop(fp, td); 1114 return (error); 1115 } 1116#endif 1117 1118 auio.uio_iov = mp->msg_iov; 1119 auio.uio_iovcnt = mp->msg_iovlen; 1120 auio.uio_segflg = UIO_USERSPACE; 1121 auio.uio_rw = UIO_READ; 1122 auio.uio_td = td; 1123 auio.uio_offset = 0; /* XXX */ 1124 auio.uio_resid = 0; 1125 iov = mp->msg_iov; 1126 for (i = 0; i < mp->msg_iovlen; i++, iov++) { 1127 if ((auio.uio_resid += iov->iov_len) < 0) { 1128 fdrop(fp, td); 1129 return (EINVAL); 1130 } 1131 } 1132#ifdef KTRACE 1133 if (KTRPOINT(td, KTR_GENIO)) 1134 ktruio = cloneuio(&auio); 1135#endif 1136 len = auio.uio_resid; 1137 error = soreceive(so, &fromsa, &auio, NULL, 1138 (mp->msg_control || controlp) ? &control : NULL, 1139 &mp->msg_flags); 1140 if (error != 0) { 1141 if (auio.uio_resid != len && (error == ERESTART || 1142 error == EINTR || error == EWOULDBLOCK)) 1143 error = 0; 1144 } 1145 if (fromsa != NULL) 1146 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, fromsa); 1147#ifdef KTRACE 1148 if (ktruio != NULL) { 1149 ktruio->uio_resid = len - auio.uio_resid; 1150 ktrgenio(s, UIO_READ, ktruio, error); 1151 } 1152#endif 1153 if (error != 0) 1154 goto out; 1155 td->td_retval[0] = len - auio.uio_resid; 1156 if (mp->msg_name) { 1157 len = mp->msg_namelen; 1158 if (len <= 0 || fromsa == NULL) 1159 len = 0; 1160 else { 1161 /* save sa_len before it is destroyed by MSG_COMPAT */ 1162 len = MIN(len, fromsa->sa_len); 1163#ifdef COMPAT_OLDSOCK 1164 if (mp->msg_flags & MSG_COMPAT) 1165 ((struct osockaddr *)fromsa)->sa_family = 1166 fromsa->sa_family; 1167#endif 1168 if (fromseg == UIO_USERSPACE) { 1169 error = copyout(fromsa, mp->msg_name, 1170 (unsigned)len); 1171 if (error != 0) 1172 goto out; 1173 } else 1174 bcopy(fromsa, mp->msg_name, len); 1175 } 1176 mp->msg_namelen = len; 1177 } 1178 if (mp->msg_control && controlp == NULL) { 1179#ifdef COMPAT_OLDSOCK 1180 /* 1181 * We assume that old recvmsg calls won't receive access 1182 * rights and other control info, esp. as control info 1183 * is always optional and those options didn't exist in 4.3. 1184 * If we receive rights, trim the cmsghdr; anything else 1185 * is tossed. 1186 */ 1187 if (control && mp->msg_flags & MSG_COMPAT) { 1188 if (mtod(control, struct cmsghdr *)->cmsg_level != 1189 SOL_SOCKET || 1190 mtod(control, struct cmsghdr *)->cmsg_type != 1191 SCM_RIGHTS) { 1192 mp->msg_controllen = 0; 1193 goto out; 1194 } 1195 control->m_len -= sizeof (struct cmsghdr); 1196 control->m_data += sizeof (struct cmsghdr); 1197 } 1198#endif 1199 len = mp->msg_controllen; 1200 m = control; 1201 mp->msg_controllen = 0; 1202 ctlbuf = mp->msg_control; 1203 1204 while (m && len > 0) { 1205 unsigned int tocopy; 1206 1207 if (len >= m->m_len) 1208 tocopy = m->m_len; 1209 else { 1210 mp->msg_flags |= MSG_CTRUNC; 1211 tocopy = len; 1212 } 1213 1214 if ((error = copyout(mtod(m, caddr_t), 1215 ctlbuf, tocopy)) != 0) 1216 goto out; 1217 1218 ctlbuf += tocopy; 1219 len -= tocopy; 1220 m = m->m_next; 1221 } 1222 mp->msg_controllen = ctlbuf - (caddr_t)mp->msg_control; 1223 } 1224out: 1225 fdrop(fp, td); 1226#ifdef KTRACE 1227 if (fromsa && KTRPOINT(td, KTR_STRUCT)) 1228 ktrsockaddr(fromsa); 1229#endif 1230 free(fromsa, M_SONAME); 1231 1232 if (error == 0 && controlp != NULL) 1233 *controlp = control; 1234 else if (control) 1235 m_freem(control); 1236 1237 return (error); 1238} 1239 1240static int 1241recvit(td, s, mp, namelenp) 1242 struct thread *td; 1243 int s; 1244 struct msghdr *mp; 1245 void *namelenp; 1246{ 1247 int error; 1248 1249 error = kern_recvit(td, s, mp, UIO_USERSPACE, NULL); 1250 if (error != 0) 1251 return (error); 1252 if (namelenp != NULL) { 1253 error = copyout(&mp->msg_namelen, namelenp, sizeof (socklen_t)); 1254#ifdef COMPAT_OLDSOCK 1255 if (mp->msg_flags & MSG_COMPAT) 1256 error = 0; /* old recvfrom didn't check */ 1257#endif 1258 } 1259 return (error); 1260} 1261 1262int 1263sys_recvfrom(td, uap) 1264 struct thread *td; 1265 struct recvfrom_args /* { 1266 int s; 1267 caddr_t buf; 1268 size_t len; 1269 int flags; 1270 struct sockaddr * __restrict from; 1271 socklen_t * __restrict fromlenaddr; 1272 } */ *uap; 1273{ 1274 struct msghdr msg; 1275 struct iovec aiov; 1276 int error; 1277 1278 if (uap->fromlenaddr) { 1279 error = copyin(uap->fromlenaddr, 1280 &msg.msg_namelen, sizeof (msg.msg_namelen)); 1281 if (error != 0) 1282 goto done2; 1283 } else { 1284 msg.msg_namelen = 0; 1285 } 1286 msg.msg_name = uap->from; 1287 msg.msg_iov = &aiov; 1288 msg.msg_iovlen = 1; 1289 aiov.iov_base = uap->buf; 1290 aiov.iov_len = uap->len; 1291 msg.msg_control = 0; 1292 msg.msg_flags = uap->flags; 1293 error = recvit(td, uap->s, &msg, uap->fromlenaddr); 1294done2: 1295 return (error); 1296} 1297 1298#ifdef COMPAT_OLDSOCK 1299int 1300orecvfrom(td, uap) 1301 struct thread *td; 1302 struct recvfrom_args *uap; 1303{ 1304 1305 uap->flags |= MSG_COMPAT; 1306 return (sys_recvfrom(td, uap)); 1307} 1308#endif 1309 1310#ifdef COMPAT_OLDSOCK 1311int 1312orecv(td, uap) 1313 struct thread *td; 1314 struct orecv_args /* { 1315 int s; 1316 caddr_t buf; 1317 int len; 1318 int flags; 1319 } */ *uap; 1320{ 1321 struct msghdr msg; 1322 struct iovec aiov; 1323 1324 msg.msg_name = 0; 1325 msg.msg_namelen = 0; 1326 msg.msg_iov = &aiov; 1327 msg.msg_iovlen = 1; 1328 aiov.iov_base = uap->buf; 1329 aiov.iov_len = uap->len; 1330 msg.msg_control = 0; 1331 msg.msg_flags = uap->flags; 1332 return (recvit(td, uap->s, &msg, NULL)); 1333} 1334 1335/* 1336 * Old recvmsg. This code takes advantage of the fact that the old msghdr 1337 * overlays the new one, missing only the flags, and with the (old) access 1338 * rights where the control fields are now. 1339 */ 1340int 1341orecvmsg(td, uap) 1342 struct thread *td; 1343 struct orecvmsg_args /* { 1344 int s; 1345 struct omsghdr *msg; 1346 int flags; 1347 } */ *uap; 1348{ 1349 struct msghdr msg; 1350 struct iovec *iov; 1351 int error; 1352 1353 error = copyin(uap->msg, &msg, sizeof (struct omsghdr)); 1354 if (error != 0) 1355 return (error); 1356 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1357 if (error != 0) 1358 return (error); 1359 msg.msg_flags = uap->flags | MSG_COMPAT; 1360 msg.msg_iov = iov; 1361 error = recvit(td, uap->s, &msg, &uap->msg->msg_namelen); 1362 if (msg.msg_controllen && error == 0) 1363 error = copyout(&msg.msg_controllen, 1364 &uap->msg->msg_accrightslen, sizeof (int)); 1365 free(iov, M_IOV); 1366 return (error); 1367} 1368#endif 1369 1370int 1371sys_recvmsg(td, uap) 1372 struct thread *td; 1373 struct recvmsg_args /* { 1374 int s; 1375 struct msghdr *msg; 1376 int flags; 1377 } */ *uap; 1378{ 1379 struct msghdr msg; 1380 struct iovec *uiov, *iov; 1381 int error; 1382 1383 error = copyin(uap->msg, &msg, sizeof (msg)); 1384 if (error != 0) 1385 return (error); 1386 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1387 if (error != 0) 1388 return (error); 1389 msg.msg_flags = uap->flags; 1390#ifdef COMPAT_OLDSOCK 1391 msg.msg_flags &= ~MSG_COMPAT; 1392#endif 1393 uiov = msg.msg_iov; 1394 msg.msg_iov = iov; 1395 error = recvit(td, uap->s, &msg, NULL); 1396 if (error == 0) { 1397 msg.msg_iov = uiov; 1398 error = copyout(&msg, uap->msg, sizeof(msg)); 1399 } 1400 free(iov, M_IOV); 1401 return (error); 1402} 1403 1404/* ARGSUSED */ 1405int 1406sys_shutdown(td, uap) 1407 struct thread *td; 1408 struct shutdown_args /* { 1409 int s; 1410 int how; 1411 } */ *uap; 1412{ 1413 struct socket *so; 1414 struct file *fp; 1415 cap_rights_t rights; 1416 int error; 1417 1418 AUDIT_ARG_FD(uap->s); 1419 error = getsock_cap(td, uap->s, cap_rights_init(&rights, CAP_SHUTDOWN), 1420 &fp, NULL); 1421 if (error == 0) { 1422 so = fp->f_data; 1423 error = soshutdown(so, uap->how); 1424 fdrop(fp, td); 1425 } 1426 return (error); 1427} 1428 1429/* ARGSUSED */ 1430int 1431sys_setsockopt(td, uap) 1432 struct thread *td; 1433 struct setsockopt_args /* { 1434 int s; 1435 int level; 1436 int name; 1437 caddr_t val; 1438 int valsize; 1439 } */ *uap; 1440{ 1441 1442 return (kern_setsockopt(td, uap->s, uap->level, uap->name, 1443 uap->val, UIO_USERSPACE, uap->valsize)); 1444} 1445 1446int 1447kern_setsockopt(td, s, level, name, val, valseg, valsize) 1448 struct thread *td; 1449 int s; 1450 int level; 1451 int name; 1452 void *val; 1453 enum uio_seg valseg; 1454 socklen_t valsize; 1455{ 1456 struct socket *so; 1457 struct file *fp; 1458 struct sockopt sopt; 1459 cap_rights_t rights; 1460 int error; 1461 1462 if (val == NULL && valsize != 0) 1463 return (EFAULT); 1464 if ((int)valsize < 0) 1465 return (EINVAL); 1466 1467 sopt.sopt_dir = SOPT_SET; 1468 sopt.sopt_level = level; 1469 sopt.sopt_name = name; 1470 sopt.sopt_val = val; 1471 sopt.sopt_valsize = valsize; 1472 switch (valseg) { 1473 case UIO_USERSPACE: 1474 sopt.sopt_td = td; 1475 break; 1476 case UIO_SYSSPACE: 1477 sopt.sopt_td = NULL; 1478 break; 1479 default: 1480 panic("kern_setsockopt called with bad valseg"); 1481 } 1482 1483 AUDIT_ARG_FD(s); 1484 error = getsock_cap(td, s, cap_rights_init(&rights, CAP_SETSOCKOPT), 1485 &fp, NULL); 1486 if (error == 0) { 1487 so = fp->f_data; 1488 error = sosetopt(so, &sopt); 1489 fdrop(fp, td); 1490 } 1491 return(error); 1492} 1493 1494/* ARGSUSED */ 1495int 1496sys_getsockopt(td, uap) 1497 struct thread *td; 1498 struct getsockopt_args /* { 1499 int s; 1500 int level; 1501 int name; 1502 void * __restrict val; 1503 socklen_t * __restrict avalsize; 1504 } */ *uap; 1505{ 1506 socklen_t valsize; 1507 int error; 1508 1509 if (uap->val) { 1510 error = copyin(uap->avalsize, &valsize, sizeof (valsize)); 1511 if (error != 0) 1512 return (error); 1513 } 1514 1515 error = kern_getsockopt(td, uap->s, uap->level, uap->name, 1516 uap->val, UIO_USERSPACE, &valsize); 1517 1518 if (error == 0) 1519 error = copyout(&valsize, uap->avalsize, sizeof (valsize)); 1520 return (error); 1521} 1522 1523/* 1524 * Kernel version of getsockopt. 1525 * optval can be a userland or userspace. optlen is always a kernel pointer. 1526 */ 1527int 1528kern_getsockopt(td, s, level, name, val, valseg, valsize) 1529 struct thread *td; 1530 int s; 1531 int level; 1532 int name; 1533 void *val; 1534 enum uio_seg valseg; 1535 socklen_t *valsize; 1536{ 1537 struct socket *so; 1538 struct file *fp; 1539 struct sockopt sopt; 1540 cap_rights_t rights; 1541 int error; 1542 1543 if (val == NULL) 1544 *valsize = 0; 1545 if ((int)*valsize < 0) 1546 return (EINVAL); 1547 1548 sopt.sopt_dir = SOPT_GET; 1549 sopt.sopt_level = level; 1550 sopt.sopt_name = name; 1551 sopt.sopt_val = val; 1552 sopt.sopt_valsize = (size_t)*valsize; /* checked non-negative above */ 1553 switch (valseg) { 1554 case UIO_USERSPACE: 1555 sopt.sopt_td = td; 1556 break; 1557 case UIO_SYSSPACE: 1558 sopt.sopt_td = NULL; 1559 break; 1560 default: 1561 panic("kern_getsockopt called with bad valseg"); 1562 } 1563 1564 AUDIT_ARG_FD(s); 1565 error = getsock_cap(td, s, cap_rights_init(&rights, CAP_GETSOCKOPT), 1566 &fp, NULL); 1567 if (error == 0) { 1568 so = fp->f_data; 1569 error = sogetopt(so, &sopt); 1570 *valsize = sopt.sopt_valsize; 1571 fdrop(fp, td); 1572 } 1573 return (error); 1574} 1575 1576/* 1577 * getsockname1() - Get socket name. 1578 */ 1579/* ARGSUSED */ 1580static int 1581getsockname1(td, uap, compat) 1582 struct thread *td; 1583 struct getsockname_args /* { 1584 int fdes; 1585 struct sockaddr * __restrict asa; 1586 socklen_t * __restrict alen; 1587 } */ *uap; 1588 int compat; 1589{ 1590 struct sockaddr *sa; 1591 socklen_t len; 1592 int error; 1593 1594 error = copyin(uap->alen, &len, sizeof(len)); 1595 if (error != 0) 1596 return (error); 1597 1598 error = kern_getsockname(td, uap->fdes, &sa, &len); 1599 if (error != 0) 1600 return (error); 1601 1602 if (len != 0) { 1603#ifdef COMPAT_OLDSOCK 1604 if (compat) 1605 ((struct osockaddr *)sa)->sa_family = sa->sa_family; 1606#endif 1607 error = copyout(sa, uap->asa, (u_int)len); 1608 } 1609 free(sa, M_SONAME); 1610 if (error == 0) 1611 error = copyout(&len, uap->alen, sizeof(len)); 1612 return (error); 1613} 1614 1615int 1616kern_getsockname(struct thread *td, int fd, struct sockaddr **sa, 1617 socklen_t *alen) 1618{ 1619 struct socket *so; 1620 struct file *fp; 1621 cap_rights_t rights; 1622 socklen_t len; 1623 int error; 1624 1625 AUDIT_ARG_FD(fd); 1626 error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_GETSOCKNAME), 1627 &fp, NULL); 1628 if (error != 0) 1629 return (error); 1630 so = fp->f_data; 1631 *sa = NULL; 1632 CURVNET_SET(so->so_vnet); 1633 error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, sa); 1634 CURVNET_RESTORE(); 1635 if (error != 0) 1636 goto bad; 1637 if (*sa == NULL) 1638 len = 0; 1639 else 1640 len = MIN(*alen, (*sa)->sa_len); 1641 *alen = len; 1642#ifdef KTRACE 1643 if (KTRPOINT(td, KTR_STRUCT)) 1644 ktrsockaddr(*sa); 1645#endif 1646bad: 1647 fdrop(fp, td); 1648 if (error != 0 && *sa != NULL) { 1649 free(*sa, M_SONAME); 1650 *sa = NULL; 1651 } 1652 return (error); 1653} 1654 1655int 1656sys_getsockname(td, uap) 1657 struct thread *td; 1658 struct getsockname_args *uap; 1659{ 1660 1661 return (getsockname1(td, uap, 0)); 1662} 1663 1664#ifdef COMPAT_OLDSOCK 1665int 1666ogetsockname(td, uap) 1667 struct thread *td; 1668 struct getsockname_args *uap; 1669{ 1670 1671 return (getsockname1(td, uap, 1)); 1672} 1673#endif /* COMPAT_OLDSOCK */ 1674 1675/* 1676 * getpeername1() - Get name of peer for connected socket. 1677 */ 1678/* ARGSUSED */ 1679static int 1680getpeername1(td, uap, compat) 1681 struct thread *td; 1682 struct getpeername_args /* { 1683 int fdes; 1684 struct sockaddr * __restrict asa; 1685 socklen_t * __restrict alen; 1686 } */ *uap; 1687 int compat; 1688{ 1689 struct sockaddr *sa; 1690 socklen_t len; 1691 int error; 1692 1693 error = copyin(uap->alen, &len, sizeof (len)); 1694 if (error != 0) 1695 return (error); 1696 1697 error = kern_getpeername(td, uap->fdes, &sa, &len); 1698 if (error != 0) 1699 return (error); 1700 1701 if (len != 0) { 1702#ifdef COMPAT_OLDSOCK 1703 if (compat) 1704 ((struct osockaddr *)sa)->sa_family = sa->sa_family; 1705#endif 1706 error = copyout(sa, uap->asa, (u_int)len); 1707 } 1708 free(sa, M_SONAME); 1709 if (error == 0) 1710 error = copyout(&len, uap->alen, sizeof(len)); 1711 return (error); 1712} 1713 1714int 1715kern_getpeername(struct thread *td, int fd, struct sockaddr **sa, 1716 socklen_t *alen) 1717{ 1718 struct socket *so; 1719 struct file *fp; 1720 cap_rights_t rights; 1721 socklen_t len; 1722 int error; 1723 1724 AUDIT_ARG_FD(fd); 1725 error = getsock_cap(td, fd, cap_rights_init(&rights, CAP_GETPEERNAME), 1726 &fp, NULL); 1727 if (error != 0) 1728 return (error); 1729 so = fp->f_data; 1730 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) { 1731 error = ENOTCONN; 1732 goto done; 1733 } 1734 *sa = NULL; 1735 CURVNET_SET(so->so_vnet); 1736 error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, sa); 1737 CURVNET_RESTORE(); 1738 if (error != 0) 1739 goto bad; 1740 if (*sa == NULL) 1741 len = 0; 1742 else 1743 len = MIN(*alen, (*sa)->sa_len); 1744 *alen = len; 1745#ifdef KTRACE 1746 if (KTRPOINT(td, KTR_STRUCT)) 1747 ktrsockaddr(*sa); 1748#endif 1749bad: 1750 if (error != 0 && *sa != NULL) { 1751 free(*sa, M_SONAME); 1752 *sa = NULL; 1753 } 1754done: 1755 fdrop(fp, td); 1756 return (error); 1757} 1758 1759int 1760sys_getpeername(td, uap) 1761 struct thread *td; 1762 struct getpeername_args *uap; 1763{ 1764 1765 return (getpeername1(td, uap, 0)); 1766} 1767 1768#ifdef COMPAT_OLDSOCK 1769int 1770ogetpeername(td, uap) 1771 struct thread *td; 1772 struct ogetpeername_args *uap; 1773{ 1774 1775 /* XXX uap should have type `getpeername_args *' to begin with. */ 1776 return (getpeername1(td, (struct getpeername_args *)uap, 1)); 1777} 1778#endif /* COMPAT_OLDSOCK */ 1779 1780int 1781sockargs(mp, buf, buflen, type) 1782 struct mbuf **mp; 1783 caddr_t buf; 1784 int buflen, type; 1785{ 1786 struct sockaddr *sa; 1787 struct mbuf *m; 1788 int error; 1789 1790 if (buflen < 0) 1791 return (EINVAL); 1792 1793 if (buflen > MLEN) { 1794#ifdef COMPAT_OLDSOCK 1795 if (type == MT_SONAME && buflen <= 112) 1796 buflen = MLEN; /* unix domain compat. hack */ 1797 else 1798#endif 1799 if (buflen > MCLBYTES) 1800 return (EINVAL); 1801 } 1802 m = m_get2(buflen, M_WAITOK, type, 0); 1803 m->m_len = buflen; 1804 error = copyin(buf, mtod(m, caddr_t), (u_int)buflen); 1805 if (error != 0) 1806 (void) m_free(m); 1807 else { 1808 *mp = m; 1809 if (type == MT_SONAME) { 1810 sa = mtod(m, struct sockaddr *); 1811 1812#if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN 1813 if (sa->sa_family == 0 && sa->sa_len < AF_MAX) 1814 sa->sa_family = sa->sa_len; 1815#endif 1816 sa->sa_len = buflen; 1817 } 1818 } 1819 return (error); 1820} 1821 1822int 1823getsockaddr(namp, uaddr, len) 1824 struct sockaddr **namp; 1825 caddr_t uaddr; 1826 size_t len; 1827{ 1828 struct sockaddr *sa; 1829 int error; 1830 1831 if (len > SOCK_MAXADDRLEN) 1832 return (ENAMETOOLONG); 1833 if (len < offsetof(struct sockaddr, sa_data[0])) 1834 return (EINVAL); 1835 sa = malloc(len, M_SONAME, M_WAITOK); 1836 error = copyin(uaddr, sa, len); 1837 if (error != 0) { 1838 free(sa, M_SONAME); 1839 } else { 1840#if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN 1841 if (sa->sa_family == 0 && sa->sa_len < AF_MAX) 1842 sa->sa_family = sa->sa_len; 1843#endif 1844 sa->sa_len = len; 1845 *namp = sa; 1846 } 1847 return (error); 1848} 1849 1850struct sendfile_sync { 1851 struct mtx mtx; 1852 struct cv cv; 1853 unsigned count; 1854}; 1855 1856/* 1857 * Detach mapped page and release resources back to the system. 1858 */ 1859int 1860sf_buf_mext(struct mbuf *mb, void *addr, void *args) 1861{ 1862 vm_page_t m; 1863 struct sendfile_sync *sfs; 1864 1865 m = sf_buf_page(args); 1866 sf_buf_free(args); 1867 vm_page_lock(m); 1868 vm_page_unwire(m, 0); 1869 /* 1870 * Check for the object going away on us. This can 1871 * happen since we don't hold a reference to it. 1872 * If so, we're responsible for freeing the page. 1873 */ 1874 if (m->wire_count == 0 && m->object == NULL) 1875 vm_page_free(m); 1876 vm_page_unlock(m); 1877 if (addr == NULL) 1878 return (EXT_FREE_OK); 1879 sfs = addr; 1880 mtx_lock(&sfs->mtx); 1881 KASSERT(sfs->count> 0, ("Sendfile sync botchup count == 0")); 1882 if (--sfs->count == 0) 1883 cv_signal(&sfs->cv); 1884 mtx_unlock(&sfs->mtx); 1885 return (EXT_FREE_OK); 1886} 1887 1888/* 1889 * sendfile(2) 1890 * 1891 * int sendfile(int fd, int s, off_t offset, size_t nbytes, 1892 * struct sf_hdtr *hdtr, off_t *sbytes, int flags) 1893 * 1894 * Send a file specified by 'fd' and starting at 'offset' to a socket 1895 * specified by 's'. Send only 'nbytes' of the file or until EOF if nbytes == 1896 * 0. Optionally add a header and/or trailer to the socket output. If 1897 * specified, write the total number of bytes sent into *sbytes. 1898 */ 1899int 1900sys_sendfile(struct thread *td, struct sendfile_args *uap) 1901{ 1902 1903 return (do_sendfile(td, uap, 0)); 1904} 1905 1906static int 1907do_sendfile(struct thread *td, struct sendfile_args *uap, int compat) 1908{ 1909 struct sf_hdtr hdtr; 1910 struct uio *hdr_uio, *trl_uio; 1911 struct file *fp; 1912 cap_rights_t rights; 1913 int error; 1914 1915 /* 1916 * File offset must be positive. If it goes beyond EOF 1917 * we send only the header/trailer and no payload data. 1918 */ 1919 if (uap->offset < 0) 1920 return (EINVAL); 1921 1922 hdr_uio = trl_uio = NULL; 1923 1924 if (uap->hdtr != NULL) { 1925 error = copyin(uap->hdtr, &hdtr, sizeof(hdtr)); 1926 if (error != 0) 1927 goto out; 1928 if (hdtr.headers != NULL) { 1929 error = copyinuio(hdtr.headers, hdtr.hdr_cnt, &hdr_uio); 1930 if (error != 0) 1931 goto out; 1932 } 1933 if (hdtr.trailers != NULL) { 1934 error = copyinuio(hdtr.trailers, hdtr.trl_cnt, &trl_uio); 1935 if (error != 0) 1936 goto out; 1937 1938 } 1939 } 1940 1941 AUDIT_ARG_FD(uap->fd); 1942 1943 /* 1944 * sendfile(2) can start at any offset within a file so we require 1945 * CAP_READ+CAP_SEEK = CAP_PREAD. 1946 */ 1947 if ((error = fget_read(td, uap->fd, 1948 cap_rights_init(&rights, CAP_PREAD), &fp)) != 0) { 1949 goto out; 1950 } 1951 1952 error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, uap->offset, 1953 uap->nbytes, uap->sbytes, uap->flags, compat ? SFK_COMPAT : 0, td); 1954 fdrop(fp, td); 1955 1956out: 1957 free(hdr_uio, M_IOV); 1958 free(trl_uio, M_IOV); 1959 return (error); 1960} 1961 1962#ifdef COMPAT_FREEBSD4 1963int 1964freebsd4_sendfile(struct thread *td, struct freebsd4_sendfile_args *uap) 1965{ 1966 struct sendfile_args args; 1967 1968 args.fd = uap->fd; 1969 args.s = uap->s; 1970 args.offset = uap->offset; 1971 args.nbytes = uap->nbytes; 1972 args.hdtr = uap->hdtr; 1973 args.sbytes = uap->sbytes; 1974 args.flags = uap->flags; 1975 1976 return (do_sendfile(td, &args, 1)); 1977} 1978#endif /* COMPAT_FREEBSD4 */ 1979 1980static int 1981sendfile_readpage(vm_object_t obj, struct vnode *vp, int nd, 1982 off_t off, int xfsize, int bsize, struct thread *td, vm_page_t *res) 1983{ 1984 vm_page_t m; 1985 vm_pindex_t pindex; 1986 ssize_t resid; 1987 int error, readahead, rv; 1988 1989 pindex = OFF_TO_IDX(off); 1990 VM_OBJECT_WLOCK(obj); 1991 m = vm_page_grab(obj, pindex, (vp != NULL ? VM_ALLOC_NOBUSY | 1992 VM_ALLOC_IGN_SBUSY : 0) | VM_ALLOC_WIRED | VM_ALLOC_NORMAL); 1993 1994 /* 1995 * Check if page is valid for what we need, otherwise initiate I/O. 1996 * 1997 * The non-zero nd argument prevents disk I/O, instead we 1998 * return the caller what he specified in nd. In particular, 1999 * if we already turned some pages into mbufs, nd == EAGAIN 2000 * and the main function send them the pages before we come 2001 * here again and block. 2002 */ 2003 if (m->valid != 0 && vm_page_is_valid(m, off & PAGE_MASK, xfsize)) { 2004 if (vp == NULL) 2005 vm_page_xunbusy(m); 2006 VM_OBJECT_WUNLOCK(obj); 2007 *res = m; 2008 return (0); 2009 } else if (nd != 0) { 2010 if (vp == NULL) 2011 vm_page_xunbusy(m); 2012 error = nd; 2013 goto free_page; 2014 } 2015 2016 /* 2017 * Get the page from backing store. 2018 */ 2019 error = 0; 2020 if (vp != NULL) { 2021 VM_OBJECT_WUNLOCK(obj); 2022 readahead = sfreadahead * MAXBSIZE; 2023 2024 /* 2025 * Use vn_rdwr() instead of the pager interface for 2026 * the vnode, to allow the read-ahead. 2027 * 2028 * XXXMAC: Because we don't have fp->f_cred here, we 2029 * pass in NOCRED. This is probably wrong, but is 2030 * consistent with our original implementation. 2031 */ 2032 error = vn_rdwr(UIO_READ, vp, NULL, readahead, trunc_page(off), 2033 UIO_NOCOPY, IO_NODELOCKED | IO_VMIO | ((readahead / 2034 bsize) << IO_SEQSHIFT), td->td_ucred, NOCRED, &resid, td); 2035 SFSTAT_INC(sf_iocnt); 2036 VM_OBJECT_WLOCK(obj); 2037 } else { 2038 if (vm_pager_has_page(obj, pindex, NULL, NULL)) { 2039 rv = vm_pager_get_pages(obj, &m, 1, 0); 2040 SFSTAT_INC(sf_iocnt); 2041 m = vm_page_lookup(obj, pindex); 2042 if (m == NULL) 2043 error = EIO; 2044 else if (rv != VM_PAGER_OK) { 2045 vm_page_lock(m); 2046 vm_page_free(m); 2047 vm_page_unlock(m); 2048 m = NULL; 2049 error = EIO; 2050 } 2051 } else { 2052 pmap_zero_page(m); 2053 m->valid = VM_PAGE_BITS_ALL; 2054 m->dirty = 0; 2055 } 2056 if (m != NULL) 2057 vm_page_xunbusy(m); 2058 } 2059 if (error == 0) { 2060 *res = m; 2061 } else if (m != NULL) { 2062free_page: 2063 vm_page_lock(m); 2064 vm_page_unwire(m, 0); 2065 2066 /* 2067 * See if anyone else might know about this page. If 2068 * not and it is not valid, then free it. 2069 */ 2070 if (m->wire_count == 0 && m->valid == 0 && !vm_page_busied(m)) 2071 vm_page_free(m); 2072 vm_page_unlock(m); 2073 } 2074 KASSERT(error != 0 || (m->wire_count > 0 && 2075 vm_page_is_valid(m, off & PAGE_MASK, xfsize)), 2076 ("wrong page state m %p off %#jx xfsize %d", m, (uintmax_t)off, 2077 xfsize)); 2078 VM_OBJECT_WUNLOCK(obj); 2079 return (error); 2080} 2081 2082static int 2083sendfile_getobj(struct thread *td, struct file *fp, vm_object_t *obj_res, 2084 struct vnode **vp_res, struct shmfd **shmfd_res, off_t *obj_size, 2085 int *bsize) 2086{ 2087 struct vattr va; 2088 vm_object_t obj; 2089 struct vnode *vp; 2090 struct shmfd *shmfd; 2091 int error; 2092 2093 vp = *vp_res = NULL; 2094 obj = NULL; 2095 shmfd = *shmfd_res = NULL; 2096 *bsize = 0; 2097 2098 /* 2099 * The file descriptor must be a regular file and have a 2100 * backing VM object. 2101 */ 2102 if (fp->f_type == DTYPE_VNODE) { 2103 vp = fp->f_vnode; 2104 vn_lock(vp, LK_SHARED | LK_RETRY); 2105 if (vp->v_type != VREG) { 2106 error = EINVAL; 2107 goto out; 2108 } 2109 *bsize = vp->v_mount->mnt_stat.f_iosize; 2110 error = VOP_GETATTR(vp, &va, td->td_ucred); 2111 if (error != 0) 2112 goto out; 2113 *obj_size = va.va_size; 2114 obj = vp->v_object; 2115 if (obj == NULL) { 2116 error = EINVAL; 2117 goto out; 2118 } 2119 } else if (fp->f_type == DTYPE_SHM) { 2120 error = 0; 2121 shmfd = fp->f_data; 2122 obj = shmfd->shm_object; 2123 *obj_size = shmfd->shm_size; 2124 } else { 2125 error = EINVAL; 2126 goto out; 2127 } 2128 2129 VM_OBJECT_WLOCK(obj); 2130 if ((obj->flags & OBJ_DEAD) != 0) { 2131 VM_OBJECT_WUNLOCK(obj); 2132 error = EBADF; 2133 goto out; 2134 } 2135 2136 /* 2137 * Temporarily increase the backing VM object's reference 2138 * count so that a forced reclamation of its vnode does not 2139 * immediately destroy it. 2140 */ 2141 vm_object_reference_locked(obj); 2142 VM_OBJECT_WUNLOCK(obj); 2143 *obj_res = obj; 2144 *vp_res = vp; 2145 *shmfd_res = shmfd; 2146 2147out: 2148 if (vp != NULL) 2149 VOP_UNLOCK(vp, 0); 2150 return (error); 2151} 2152 2153static int 2154kern_sendfile_getsock(struct thread *td, int s, struct file **sock_fp, 2155 struct socket **so) 2156{ 2157 cap_rights_t rights; 2158 int error; 2159 2160 *sock_fp = NULL; 2161 *so = NULL; 2162 2163 /* 2164 * The socket must be a stream socket and connected. 2165 */ 2166 error = getsock_cap(td, s, cap_rights_init(&rights, CAP_SEND), 2167 sock_fp, NULL); 2168 if (error != 0) 2169 return (error); 2170 *so = (*sock_fp)->f_data; 2171 if ((*so)->so_type != SOCK_STREAM) 2172 return (EINVAL); 2173 if (((*so)->so_state & SS_ISCONNECTED) == 0) 2174 return (ENOTCONN); 2175 return (0); 2176} 2177 2178int 2179vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio, 2180 struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags, 2181 int kflags, struct thread *td) 2182{ 2183 struct file *sock_fp; 2184 struct vnode *vp; 2185 struct vm_object *obj; 2186 struct socket *so; 2187 struct mbuf *m; 2188 struct sf_buf *sf; 2189 struct vm_page *pg; 2190 struct shmfd *shmfd; 2191 struct sendfile_sync *sfs; 2192 struct vattr va; 2193 off_t off, xfsize, fsbytes, sbytes, rem, obj_size; 2194 int error, bsize, nd, hdrlen, mnw; 2195 bool inflight_called; 2196 2197 pg = NULL; 2198 obj = NULL; 2199 so = NULL; 2200 m = NULL; 2201 sfs = NULL; 2202 fsbytes = sbytes = 0; 2203 hdrlen = mnw = 0; 2204 rem = nbytes; 2205 obj_size = 0; 2206 inflight_called = false; 2207 2208 error = sendfile_getobj(td, fp, &obj, &vp, &shmfd, &obj_size, &bsize); 2209 if (error != 0) 2210 return (error); 2211 if (rem == 0) 2212 rem = obj_size; 2213 2214 error = kern_sendfile_getsock(td, sockfd, &sock_fp, &so); 2215 if (error != 0) 2216 goto out; 2217 2218 /* 2219 * Do not wait on memory allocations but return ENOMEM for 2220 * caller to retry later. 2221 * XXX: Experimental. 2222 */ 2223 if (flags & SF_MNOWAIT) 2224 mnw = 1; 2225 2226 if (flags & SF_SYNC) { 2227 sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO); 2228 mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF); 2229 cv_init(&sfs->cv, "sendfile"); 2230 } 2231 2232#ifdef MAC 2233 error = mac_socket_check_send(td->td_ucred, so); 2234 if (error != 0) 2235 goto out; 2236#endif 2237 2238 /* If headers are specified copy them into mbufs. */ 2239 if (hdr_uio != NULL) { 2240 hdr_uio->uio_td = td; 2241 hdr_uio->uio_rw = UIO_WRITE; 2242 if (hdr_uio->uio_resid > 0) { 2243 /* 2244 * In FBSD < 5.0 the nbytes to send also included 2245 * the header. If compat is specified subtract the 2246 * header size from nbytes. 2247 */ 2248 if (kflags & SFK_COMPAT) { 2249 if (nbytes > hdr_uio->uio_resid) 2250 nbytes -= hdr_uio->uio_resid; 2251 else 2252 nbytes = 0; 2253 } 2254 m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK), 2255 0, 0, 0); 2256 if (m == NULL) { 2257 error = mnw ? EAGAIN : ENOBUFS; 2258 goto out; 2259 } 2260 hdrlen = m_length(m, NULL); 2261 } 2262 } 2263 2264 /* 2265 * Protect against multiple writers to the socket. 2266 * 2267 * XXXRW: Historically this has assumed non-interruptibility, so now 2268 * we implement that, but possibly shouldn't. 2269 */ 2270 (void)sblock(&so->so_snd, SBL_WAIT | SBL_NOINTR); 2271 2272 /* 2273 * Loop through the pages of the file, starting with the requested 2274 * offset. Get a file page (do I/O if necessary), map the file page 2275 * into an sf_buf, attach an mbuf header to the sf_buf, and queue 2276 * it on the socket. 2277 * This is done in two loops. The inner loop turns as many pages 2278 * as it can, up to available socket buffer space, without blocking 2279 * into mbufs to have it bulk delivered into the socket send buffer. 2280 * The outer loop checks the state and available space of the socket 2281 * and takes care of the overall progress. 2282 */ 2283 for (off = offset; ; ) { 2284 struct mbuf *mtail; 2285 int loopbytes; 2286 int space; 2287 int done; 2288 2289 if ((nbytes != 0 && nbytes == fsbytes) || 2290 (nbytes == 0 && obj_size == fsbytes)) 2291 break; 2292 2293 mtail = NULL; 2294 loopbytes = 0; 2295 space = 0; 2296 done = 0; 2297 2298 /* 2299 * Check the socket state for ongoing connection, 2300 * no errors and space in socket buffer. 2301 * If space is low allow for the remainder of the 2302 * file to be processed if it fits the socket buffer. 2303 * Otherwise block in waiting for sufficient space 2304 * to proceed, or if the socket is nonblocking, return 2305 * to userland with EAGAIN while reporting how far 2306 * we've come. 2307 * We wait until the socket buffer has significant free 2308 * space to do bulk sends. This makes good use of file 2309 * system read ahead and allows packet segmentation 2310 * offloading hardware to take over lots of work. If 2311 * we were not careful here we would send off only one 2312 * sfbuf at a time. 2313 */ 2314 SOCKBUF_LOCK(&so->so_snd); 2315 if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2) 2316 so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2; 2317retry_space: 2318 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2319 error = EPIPE; 2320 SOCKBUF_UNLOCK(&so->so_snd); 2321 goto done; 2322 } else if (so->so_error) { 2323 error = so->so_error; 2324 so->so_error = 0; 2325 SOCKBUF_UNLOCK(&so->so_snd); 2326 goto done; 2327 } 2328 space = sbspace(&so->so_snd); 2329 if (space < rem && 2330 (space <= 0 || 2331 space < so->so_snd.sb_lowat)) { 2332 if (so->so_state & SS_NBIO) { 2333 SOCKBUF_UNLOCK(&so->so_snd); 2334 error = EAGAIN; 2335 goto done; 2336 } 2337 /* 2338 * sbwait drops the lock while sleeping. 2339 * When we loop back to retry_space the 2340 * state may have changed and we retest 2341 * for it. 2342 */ 2343 error = sbwait(&so->so_snd); 2344 /* 2345 * An error from sbwait usually indicates that we've 2346 * been interrupted by a signal. If we've sent anything 2347 * then return bytes sent, otherwise return the error. 2348 */ 2349 if (error != 0) { 2350 SOCKBUF_UNLOCK(&so->so_snd); 2351 goto done; 2352 } 2353 goto retry_space; 2354 } 2355 SOCKBUF_UNLOCK(&so->so_snd); 2356 2357 /* 2358 * Reduce space in the socket buffer by the size of 2359 * the header mbuf chain. 2360 * hdrlen is set to 0 after the first loop. 2361 */ 2362 space -= hdrlen; 2363 2364 if (vp != NULL) { 2365 error = vn_lock(vp, LK_SHARED); 2366 if (error != 0) 2367 goto done; 2368 error = VOP_GETATTR(vp, &va, td->td_ucred); 2369 if (error != 0 || off >= va.va_size) { 2370 VOP_UNLOCK(vp, 0); 2371 goto done; 2372 } 2373 obj_size = va.va_size; 2374 } 2375 2376 /* 2377 * Loop and construct maximum sized mbuf chain to be bulk 2378 * dumped into socket buffer. 2379 */ 2380 while (space > loopbytes) { 2381 vm_offset_t pgoff; 2382 struct mbuf *m0; 2383 2384 /* 2385 * Calculate the amount to transfer. 2386 * Not to exceed a page, the EOF, 2387 * or the passed in nbytes. 2388 */ 2389 pgoff = (vm_offset_t)(off & PAGE_MASK); 2390 rem = obj_size - offset; 2391 if (nbytes != 0) 2392 rem = omin(rem, nbytes); 2393 rem -= fsbytes + loopbytes; 2394 xfsize = omin(PAGE_SIZE - pgoff, rem); 2395 xfsize = omin(space - loopbytes, xfsize); 2396 if (xfsize <= 0) { 2397 done = 1; /* all data sent */ 2398 break; 2399 } 2400 2401 /* 2402 * Attempt to look up the page. Allocate 2403 * if not found or wait and loop if busy. 2404 */ 2405 if (m != NULL) 2406 nd = EAGAIN; /* send what we already got */ 2407 else if ((flags & SF_NODISKIO) != 0) 2408 nd = EBUSY; 2409 else 2410 nd = 0; 2411 error = sendfile_readpage(obj, vp, nd, off, 2412 xfsize, bsize, td, &pg); 2413 if (error != 0) { 2414 if (error == EAGAIN) 2415 error = 0; /* not a real error */ 2416 break; 2417 } 2418 2419 /* 2420 * Get a sendfile buf. When allocating the 2421 * first buffer for mbuf chain, we usually 2422 * wait as long as necessary, but this wait 2423 * can be interrupted. For consequent 2424 * buffers, do not sleep, since several 2425 * threads might exhaust the buffers and then 2426 * deadlock. 2427 */ 2428 sf = sf_buf_alloc(pg, (mnw || m != NULL) ? SFB_NOWAIT : 2429 SFB_CATCH); 2430 if (sf == NULL) { 2431 SFSTAT_INC(sf_allocfail); 2432 vm_page_lock(pg); 2433 vm_page_unwire(pg, 0); 2434 KASSERT(pg->object != NULL, 2435 ("%s: object disappeared", __func__)); 2436 vm_page_unlock(pg); 2437 if (m == NULL) 2438 error = (mnw ? EAGAIN : EINTR); 2439 break; 2440 } 2441 2442 /* 2443 * Get an mbuf and set it up as having 2444 * external storage. 2445 */ 2446 m0 = m_get((mnw ? M_NOWAIT : M_WAITOK), MT_DATA); 2447 if (m0 == NULL) { 2448 error = (mnw ? EAGAIN : ENOBUFS); 2449 (void)sf_buf_mext(NULL, NULL, sf); 2450 break; 2451 } 2452 if (m_extadd(m0, (caddr_t )sf_buf_kva(sf), PAGE_SIZE, 2453 sf_buf_mext, sfs, sf, M_RDONLY, EXT_SFBUF, 2454 (mnw ? M_NOWAIT : M_WAITOK)) != 0) { 2455 error = (mnw ? EAGAIN : ENOBUFS); 2456 (void)sf_buf_mext(NULL, NULL, sf); 2457 m_freem(m0); 2458 break; 2459 } 2460 m0->m_data = (char *)sf_buf_kva(sf) + pgoff; 2461 m0->m_len = xfsize; 2462 2463 /* Append to mbuf chain. */ 2464 if (mtail != NULL) 2465 mtail->m_next = m0; 2466 else if (m != NULL) 2467 m_last(m)->m_next = m0; 2468 else 2469 m = m0; 2470 mtail = m0; 2471 2472 /* Keep track of bits processed. */ 2473 loopbytes += xfsize; 2474 off += xfsize; 2475 2476 if (sfs != NULL) { 2477 mtx_lock(&sfs->mtx); 2478 sfs->count++; 2479 mtx_unlock(&sfs->mtx); 2480 } 2481 } 2482 2483 if (vp != NULL) 2484 VOP_UNLOCK(vp, 0); 2485 2486 /* Add the buffer chain to the socket buffer. */ 2487 if (m != NULL) { 2488 int mlen, err; 2489 2490 mlen = m_length(m, NULL); 2491 SOCKBUF_LOCK(&so->so_snd); 2492 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2493 error = EPIPE; 2494 SOCKBUF_UNLOCK(&so->so_snd); 2495 goto done; 2496 } 2497 SOCKBUF_UNLOCK(&so->so_snd); 2498 CURVNET_SET(so->so_vnet); 2499 /* Avoid error aliasing. */ 2500 err = (*so->so_proto->pr_usrreqs->pru_send) 2501 (so, 0, m, NULL, NULL, td); 2502 CURVNET_RESTORE(); 2503 if (err == 0) { 2504 /* 2505 * We need two counters to get the 2506 * file offset and nbytes to send 2507 * right: 2508 * - sbytes contains the total amount 2509 * of bytes sent, including headers. 2510 * - fsbytes contains the total amount 2511 * of bytes sent from the file. 2512 */ 2513 sbytes += mlen; 2514 fsbytes += mlen; 2515 if (hdrlen) { 2516 fsbytes -= hdrlen; 2517 hdrlen = 0; 2518 } 2519 } else if (error == 0) 2520 error = err; 2521 m = NULL; /* pru_send always consumes */ 2522 } 2523 2524 /* Quit outer loop on error or when we're done. */ 2525 if (done) 2526 break; 2527 if (error != 0) 2528 goto done; 2529 } 2530 2531 /* 2532 * Send trailers. Wimp out and use writev(2). 2533 */ 2534 if (trl_uio != NULL) { 2535 sbunlock(&so->so_snd); 2536 error = kern_writev(td, sockfd, trl_uio); 2537 if (error == 0) 2538 sbytes += td->td_retval[0]; 2539 goto out; 2540 } 2541 2542done: 2543 sbunlock(&so->so_snd); 2544out: 2545 /* 2546 * If there was no error we have to clear td->td_retval[0] 2547 * because it may have been set by writev. 2548 */ 2549 if (error == 0) { 2550 td->td_retval[0] = 0; 2551 } 2552 if (sent != NULL) { 2553 copyout(&sbytes, sent, sizeof(off_t)); 2554 } 2555 if (obj != NULL) 2556 vm_object_deallocate(obj); 2557 if (so) 2558 fdrop(sock_fp, td); 2559 if (m) 2560 m_freem(m); 2561 2562 if (sfs != NULL) { 2563 mtx_lock(&sfs->mtx); 2564 if (sfs->count != 0) 2565 cv_wait(&sfs->cv, &sfs->mtx); 2566 KASSERT(sfs->count == 0, ("sendfile sync still busy")); 2567 cv_destroy(&sfs->cv); 2568 mtx_destroy(&sfs->mtx); 2569 free(sfs, M_TEMP); 2570 } 2571 2572 if (error == ERESTART) 2573 error = EINTR; 2574 2575 return (error); 2576} 2577