1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28#include "opt_inet.h" 29#include "opt_inet6.h" 30 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/buf_ring.h> 34#include <sys/epoch.h> 35#include <sys/file.h> 36#include <sys/filedesc.h> 37#include <sys/kernel.h> 38#include <sys/malloc.h> 39#include <sys/mbuf.h> 40#include <sys/module.h> 41#include <sys/nv.h> 42#include <sys/priv.h> 43#include <sys/protosw.h> 44#include <sys/rmlock.h> 45#include <sys/sdt.h> 46#include <sys/smp.h> 47#include <sys/socket.h> 48#include <sys/socketvar.h> 49#include <sys/sockio.h> 50#include <sys/sysctl.h> 51#include <sys/time.h> 52 53#include <machine/atomic.h> 54 55#include <net/bpf.h> 56#include <net/if.h> 57#include <net/if_clone.h> 58#include <net/if_types.h> 59#include <net/if_var.h> 60#include <net/if_private.h> 61#include <net/netisr.h> 62#include <net/route/nhop.h> 63 64#include <netinet/in.h> 65#include <netinet/in_fib.h> 66#include <netinet/ip.h> 67#include <netinet/ip6.h> 68#include <netinet/ip_var.h> 69#include <netinet/udp.h> 70#include <netinet/udp_var.h> 71 72#include <netinet6/ip6_var.h> 73#include <netinet6/in6_fib.h> 74 75#include <machine/in_cksum.h> 76 77#include <opencrypto/cryptodev.h> 78 79#include "if_ovpn.h" 80 81struct ovpn_kkey_dir { 82 int refcount; 83 uint8_t key[32]; 84 uint8_t keylen; 85 uint8_t nonce[8]; 86 uint8_t noncelen; 87 enum ovpn_key_cipher cipher; 88 crypto_session_t cryptoid; 89 90 struct mtx replay_mtx; 91 /* 92 * Last seen gapless sequence number. New rx seq numbers must be 93 * strictly higher than this. 94 */ 95 uint32_t rx_seq; 96 uint64_t tx_seq; 97 98 /* Seen packets, relative to rx_seq. bit(0) will always be 0. */ 99 uint64_t rx_window; 100}; 101 102struct ovpn_kkey { 103 struct ovpn_kkey_dir *encrypt; 104 struct ovpn_kkey_dir *decrypt; 105 uint8_t keyid; 106 uint32_t peerid; 107}; 108 109struct ovpn_keepalive { 110 uint32_t interval; 111 uint32_t timeout; 112}; 113 114struct ovpn_wire_header { 115 uint32_t opcode; /* opcode, key id, peer id */ 116 uint32_t seq; 117 uint8_t auth_tag[16]; 118}; 119 120struct ovpn_peer_counters { 121 uint64_t pkt_in; 122 uint64_t pkt_out; 123 uint64_t bytes_in; 124 uint64_t bytes_out; 125}; 126#define OVPN_PEER_COUNTER_SIZE (sizeof(struct ovpn_peer_counters)/sizeof(uint64_t)) 127 128struct ovpn_notification { 129 enum ovpn_notif_type type; 130 uint32_t peerid; 131 132 /* Delete notification */ 133 enum ovpn_del_reason del_reason; 134 struct ovpn_peer_counters counters; 135}; 136 137struct ovpn_softc; 138 139struct ovpn_kpeer { 140 RB_ENTRY(ovpn_kpeer) tree; 141 int refcount; 142 uint32_t peerid; 143 144 struct ovpn_softc *sc; 145 struct sockaddr_storage local; 146 struct sockaddr_storage remote; 147 148 struct in_addr vpn4; 149 struct in6_addr vpn6; 150 151 struct ovpn_kkey keys[2]; 152 153 enum ovpn_del_reason del_reason; 154 struct ovpn_keepalive keepalive; 155 uint32_t *last_active; 156 struct callout ping_send; 157 struct callout ping_rcv; 158 159 counter_u64_t counters[OVPN_PEER_COUNTER_SIZE]; 160}; 161 162struct ovpn_counters { 163 uint64_t lost_ctrl_pkts_in; 164 uint64_t lost_ctrl_pkts_out; 165 uint64_t lost_data_pkts_in; 166 uint64_t lost_data_pkts_out; 167 uint64_t nomem_data_pkts_in; 168 uint64_t nomem_data_pkts_out; 169 uint64_t received_ctrl_pkts; 170 uint64_t received_data_pkts; 171 uint64_t sent_ctrl_pkts; 172 uint64_t sent_data_pkts; 173 174 uint64_t transport_bytes_sent; 175 uint64_t transport_bytes_received; 176 uint64_t tunnel_bytes_sent; 177 uint64_t tunnel_bytes_received; 178}; 179#define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t)) 180 181RB_HEAD(ovpn_kpeers, ovpn_kpeer); 182 183struct ovpn_softc { 184 int refcount; 185 struct rmlock lock; 186 struct ifnet *ifp; 187 struct socket *so; 188 int peercount; 189 struct ovpn_kpeers peers; 190 191 /* Pending notification */ 192 struct buf_ring *notifring; 193 194 counter_u64_t counters[OVPN_COUNTER_SIZE]; 195 196 struct epoch_context epoch_ctx; 197}; 198 199static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t); 200static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *, 201 const struct sockaddr *, void *); 202static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *, 203 struct ovpn_kpeer *, struct rm_priotracker *); 204static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *); 205static int ovpn_get_af(struct mbuf *); 206static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *); 207static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t); 208static int ovpn_peer_compare(struct ovpn_kpeer *, struct ovpn_kpeer *); 209 210static RB_PROTOTYPE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 211static RB_GENERATE(ovpn_kpeers, ovpn_kpeer, tree, ovpn_peer_compare); 212 213#define OVPN_MTU_MIN 576 214#define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \ 215 sizeof(struct udphdr) - sizeof(struct ovpn_wire_header)) 216 217#define OVPN_OP_DATA_V2 0x09 218#define OVPN_OP_SHIFT 3 219#define OVPN_SEQ_ROTATE 0x80000000 220 221VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner); 222#define V_ovpn_cloner VNET(ovpn_cloner) 223 224#define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \ 225 struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker 226#define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp) 227#define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp) 228#define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock) 229#define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock) 230#define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED) 231#define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED) 232#define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED) 233#define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED) 234 235#define OVPN_COUNTER(sc, name) \ 236 ((sc)->counters[offsetof(struct ovpn_counters, name)/sizeof(uint64_t)]) 237#define OVPN_PEER_COUNTER(peer, name) \ 238 ((peer)->counters[offsetof(struct ovpn_peer_counters, name) / \ 239 sizeof(uint64_t)]) 240 241#define OVPN_COUNTER_ADD(sc, name, val) \ 242 counter_u64_add(OVPN_COUNTER(sc, name), val) 243#define OVPN_PEER_COUNTER_ADD(p, name, val) \ 244 counter_u64_add(OVPN_PEER_COUNTER(p, name), val) 245 246#define TO_IN(x) ((struct sockaddr_in *)(x)) 247#define TO_IN6(x) ((struct sockaddr_in6 *)(x)) 248 249SDT_PROVIDER_DEFINE(if_ovpn); 250SDT_PROBE_DEFINE1(if_ovpn, tx, transmit, start, "struct mbuf *"); 251SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip4, "struct in_addr *", "struct ovpn_kpeer *"); 252SDT_PROBE_DEFINE2(if_ovpn, tx, route, ip6, "struct in6_addr *", "struct ovpn_kpeer *"); 253 254static const char ovpnname[] = "ovpn"; 255static const char ovpngroupname[] = "openvpn"; 256 257static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface"); 258#define MTAG_OVPN_LOOP 0x6f76706e /* ovpn */ 259 260SYSCTL_DECL(_net_link); 261static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 262 "OpenVPN DCO Interface"); 263VNET_DEFINE_STATIC(int, replay_protection) = 0; 264#define V_replay_protection VNET(replay_protection) 265SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW, 266 &VNET_NAME(replay_protection), 0, "Validate sequence numbers"); 267 268VNET_DEFINE_STATIC(int, async_crypto); 269#define V_async_crypto VNET(async_crypto) 270SYSCTL_INT(_net_link_openvpn, OID_AUTO, async_crypto, 271 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_crypto), 0, 272 "Use asynchronous mode to parallelize crypto jobs."); 273 274VNET_DEFINE_STATIC(int, async_netisr_queue); 275#define V_async_netisr_queue VNET(async_netisr_queue) 276SYSCTL_INT(_net_link_openvpn, OID_AUTO, netisr_queue, 277 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(async_netisr_queue), 0, 278 "Use netisr_queue() rather than netisr_dispatch()."); 279 280static int 281ovpn_peer_compare(struct ovpn_kpeer *a, struct ovpn_kpeer *b) 282{ 283 return (a->peerid - b->peerid); 284} 285 286static struct ovpn_kpeer * 287ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid) 288{ 289 struct ovpn_kpeer p; 290 291 OVPN_ASSERT(sc); 292 293 p.peerid = peerid; 294 295 return (RB_FIND(ovpn_kpeers, &sc->peers, &p)); 296} 297 298static struct ovpn_kpeer * 299ovpn_find_only_peer(struct ovpn_softc *sc) 300{ 301 OVPN_ASSERT(sc); 302 303 return (RB_ROOT(&sc->peers)); 304} 305 306static uint16_t 307ovpn_get_port(struct sockaddr_storage *s) 308{ 309 switch (s->ss_family) { 310 case AF_INET: { 311 struct sockaddr_in *in = (struct sockaddr_in *)s; 312 return (in->sin_port); 313 } 314 case AF_INET6: { 315 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s; 316 return (in6->sin6_port); 317 } 318 default: 319 panic("Unsupported address family %d", s->ss_family); 320 } 321} 322 323static int 324ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa) 325{ 326 int af; 327 328 if (! nvlist_exists_number(nvl, "af")) 329 return (EINVAL); 330 if (! nvlist_exists_binary(nvl, "address")) 331 return (EINVAL); 332 if (! nvlist_exists_number(nvl, "port")) 333 return (EINVAL); 334 335 af = nvlist_get_number(nvl, "af"); 336 337 switch (af) { 338#ifdef INET 339 case AF_INET: { 340 struct sockaddr_in *in = (struct sockaddr_in *)sa; 341 size_t len; 342 const void *addr = nvlist_get_binary(nvl, "address", &len); 343 in->sin_family = af; 344 if (len != sizeof(in->sin_addr)) 345 return (EINVAL); 346 347 memcpy(&in->sin_addr, addr, sizeof(in->sin_addr)); 348 in->sin_port = nvlist_get_number(nvl, "port"); 349 break; 350 } 351#endif 352#ifdef INET6 353 case AF_INET6: { 354 struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa; 355 size_t len; 356 const void *addr = nvlist_get_binary(nvl, "address", &len); 357 in6->sin6_family = af; 358 if (len != sizeof(in6->sin6_addr)) 359 return (EINVAL); 360 361 memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr)); 362 in6->sin6_port = nvlist_get_number(nvl, "port"); 363 break; 364 } 365#endif 366 default: 367 return (EINVAL); 368 } 369 370 return (0); 371} 372 373static bool 374ovpn_has_peers(struct ovpn_softc *sc) 375{ 376 OVPN_ASSERT(sc); 377 378 return (sc->peercount > 0); 379} 380 381static void 382ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 383{ 384 bool has_peers; 385 386 OVPN_WASSERT(sc); 387 388 if (sc->so == NULL) 389 return; 390 391 has_peers = ovpn_has_peers(sc); 392 393 /* Only remove the tunnel function if we're releasing the socket for 394 * the last peer. */ 395 if (! has_peers) 396 (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL); 397 398 sorele(sc->so); 399 400 if (! has_peers) 401 sc->so = NULL; 402} 403 404static void 405ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 406{ 407 struct ovpn_notification *n; 408 409 OVPN_WASSERT(sc); 410 411 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT); 412 if (n == NULL) 413 return; 414 415 n->peerid = peer->peerid; 416 n->type = OVPN_NOTIF_DEL_PEER; 417 n->del_reason = peer->del_reason; 418 419 n->counters.pkt_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_in)); 420 n->counters.pkt_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, pkt_out)); 421 n->counters.bytes_in = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_in)); 422 n->counters.bytes_out = counter_u64_fetch(OVPN_PEER_COUNTER(peer, bytes_out)); 423 424 if (buf_ring_enqueue(sc->notifring, n) != 0) { 425 free(n, M_OVPN); 426 } else if (sc->so != NULL) { 427 /* Wake up userspace */ 428 sc->so->so_error = EAGAIN; 429 sorwakeup(sc->so); 430 sowwakeup(sc->so); 431 } 432} 433 434static void 435ovpn_notify_key_rotation(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 436{ 437 struct ovpn_notification *n; 438 439 n = malloc(sizeof(*n), M_OVPN, M_NOWAIT | M_ZERO); 440 if (n == NULL) 441 return; 442 443 n->peerid = peer->peerid; 444 n->type = OVPN_NOTIF_ROTATE_KEY; 445 446 if (buf_ring_enqueue(sc->notifring, n) != 0) { 447 free(n, M_OVPN); 448 } else if (sc->so != NULL) { 449 /* Wake up userspace */ 450 sc->so->so_error = EAGAIN; 451 sorwakeup(sc->so); 452 sowwakeup(sc->so); 453 } 454} 455 456static void 457ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked) 458{ 459 struct ovpn_softc *sc; 460 461 CURVNET_ASSERT_SET(); 462 463 atomic_add_int(&peer->refcount, -1); 464 465 if (atomic_load_int(&peer->refcount) > 0) 466 return; 467 468 sc = peer->sc; 469 470 if (! locked) { 471 OVPN_WLOCK(sc); 472 473 /* Might have changed before we acquired the lock. */ 474 if (atomic_load_int(&peer->refcount) > 0) { 475 OVPN_WUNLOCK(sc); 476 return; 477 } 478 } 479 480 OVPN_ASSERT(sc); 481 482 /* The peer should have been removed from the list already. */ 483 MPASS(ovpn_find_peer(sc, peer->peerid) == NULL); 484 485 ovpn_notify_del_peer(sc, peer); 486 487 for (int i = 0; i < 2; i++) { 488 ovpn_free_kkey_dir(peer->keys[i].encrypt); 489 ovpn_free_kkey_dir(peer->keys[i].decrypt); 490 } 491 492 ovpn_rele_so(sc, peer); 493 494 callout_stop(&peer->ping_send); 495 callout_stop(&peer->ping_rcv); 496 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 497 free(peer, M_OVPN); 498 499 if (! locked) 500 OVPN_WUNLOCK(sc); 501} 502 503static int 504ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl) 505{ 506#ifdef INET6 507 struct epoch_tracker et; 508#endif 509 struct sockaddr_storage remote; 510 struct ovpn_kpeer *peer = NULL; 511 struct file *fp = NULL; 512 struct ovpn_softc *sc = ifp->if_softc; 513 struct thread *td = curthread; 514 struct socket *so = NULL; 515 int fd; 516 uint32_t peerid; 517 int ret = 0; 518 519 if (nvl == NULL) 520 return (EINVAL); 521 522 if (! nvlist_exists_number(nvl, "peerid")) 523 return (EINVAL); 524 525 if (! nvlist_exists_number(nvl, "fd")) 526 return (EINVAL); 527 528 if (! nvlist_exists_nvlist(nvl, "remote")) 529 return (EINVAL); 530 531 peerid = nvlist_get_number(nvl, "peerid"); 532 533 ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"), 534 &remote); 535 if (ret != 0) 536 return (ret); 537 538 fd = nvlist_get_number(nvl, "fd"); 539 540 /* Look up the userspace process and use the fd to find the socket. */ 541 ret = getsock(td, fd, &cap_connect_rights, &fp); 542 if (ret != 0) 543 return (ret); 544 545 so = fp->f_data; 546 547 peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO); 548 peer->peerid = peerid; 549 peer->sc = sc; 550 peer->refcount = 1; 551 peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO); 552 COUNTER_ARRAY_ALLOC(peer->counters, OVPN_PEER_COUNTER_SIZE, M_WAITOK); 553 554 if (nvlist_exists_binary(nvl, "vpn_ipv4")) { 555 size_t len; 556 const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len); 557 if (len != sizeof(peer->vpn4)) { 558 ret = EINVAL; 559 goto error; 560 } 561 memcpy(&peer->vpn4, addr, len); 562 } 563 564 if (nvlist_exists_binary(nvl, "vpn_ipv6")) { 565 size_t len; 566 const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len); 567 if (len != sizeof(peer->vpn6)) { 568 ret = EINVAL; 569 goto error; 570 } 571 memcpy(&peer->vpn6, addr, len); 572 } 573 574 callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK); 575 callout_init_rm(&peer->ping_rcv, &sc->lock, 0); 576 577 peer->local.ss_len = sizeof(peer->local); 578 ret = sosockaddr(so, (struct sockaddr *)&peer->local); 579 if (ret) 580 goto error; 581 582 if (ovpn_get_port(&peer->local) == 0) { 583 ret = EINVAL; 584 goto error; 585 } 586 if (peer->local.ss_family != remote.ss_family) { 587 ret = EINVAL; 588 goto error; 589 } 590 591 memcpy(&peer->remote, &remote, sizeof(remote)); 592 593 if (peer->local.ss_family == AF_INET6 && 594 IN6_IS_ADDR_V4MAPPED(&TO_IN6(&peer->remote)->sin6_addr)) { 595 /* V4 mapped address, so treat this as v4, not v6. */ 596 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->local); 597 in6_sin6_2_sin_in_sock((struct sockaddr *)&peer->remote); 598 } 599 600#ifdef INET6 601 if (peer->local.ss_family == AF_INET6 && 602 IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) { 603 NET_EPOCH_ENTER(et); 604 ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum, 605 &TO_IN6(&peer->remote)->sin6_addr, 606 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL); 607 NET_EPOCH_EXIT(et); 608 if (ret != 0) { 609 goto error; 610 } 611 } 612#endif 613 OVPN_WLOCK(sc); 614 615 /* Disallow peer id re-use. */ 616 if (ovpn_find_peer(sc, peerid) != NULL) { 617 ret = EEXIST; 618 goto error_locked; 619 } 620 621 /* Make sure this is really a UDP socket. */ 622 if (so->so_type != SOCK_DGRAM || so->so_proto->pr_type != SOCK_DGRAM) { 623 ret = EPROTOTYPE; 624 goto error_locked; 625 } 626 627 /* Must be the same socket as for other peers on this interface. */ 628 if (sc->so != NULL && so != sc->so) 629 goto error_locked; 630 631 if (sc->so == NULL) 632 sc->so = so; 633 634 /* Insert the peer into the list. */ 635 RB_INSERT(ovpn_kpeers, &sc->peers, peer); 636 sc->peercount++; 637 soref(sc->so); 638 639 ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc); 640 if (ret == EBUSY) { 641 /* Fine, another peer already set the input function. */ 642 ret = 0; 643 } 644 if (ret != 0) { 645 RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 646 sc->peercount--; 647 goto error_locked; 648 } 649 650 OVPN_WUNLOCK(sc); 651 652 goto done; 653 654error_locked: 655 OVPN_WUNLOCK(sc); 656error: 657 COUNTER_ARRAY_FREE(peer->counters, OVPN_PEER_COUNTER_SIZE); 658 uma_zfree_pcpu(pcpu_zone_4, peer->last_active); 659 free(peer, M_OVPN); 660done: 661 if (fp != NULL) 662 fdrop(fp, td); 663 664 return (ret); 665} 666 667static int 668_ovpn_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer) 669{ 670 struct ovpn_kpeer *tmp __diagused; 671 672 OVPN_WASSERT(sc); 673 CURVNET_ASSERT_SET(); 674 675 MPASS(RB_FIND(ovpn_kpeers, &sc->peers, peer) == peer); 676 677 tmp = RB_REMOVE(ovpn_kpeers, &sc->peers, peer); 678 MPASS(tmp != NULL); 679 680 sc->peercount--; 681 682 ovpn_peer_release_ref(peer, true); 683 684 return (0); 685} 686 687static int 688ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl) 689{ 690 struct ovpn_softc *sc = ifp->if_softc; 691 struct ovpn_kpeer *peer; 692 uint32_t peerid; 693 int ret; 694 695 OVPN_WASSERT(sc); 696 697 if (nvl == NULL) 698 return (EINVAL); 699 700 if (! nvlist_exists_number(nvl, "peerid")) 701 return (EINVAL); 702 703 peerid = nvlist_get_number(nvl, "peerid"); 704 705 peer = ovpn_find_peer(sc, peerid); 706 if (peer == NULL) 707 return (ENOENT); 708 709 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 710 ret = _ovpn_del_peer(sc, peer); 711 712 return (ret); 713} 714 715static int 716ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp, 717 const nvlist_t *nvl) 718{ 719 struct crypto_session_params csp; 720 struct ovpn_kkey_dir *kdir; 721 const char *ciphername; 722 enum ovpn_key_cipher cipher; 723 const void *key, *iv; 724 size_t keylen = 0, ivlen = 0; 725 int error; 726 727 if (! nvlist_exists_string(nvl, "cipher")) 728 return (EINVAL); 729 ciphername = nvlist_get_string(nvl, "cipher"); 730 731 if (strcmp(ciphername, "none") == 0) 732 cipher = OVPN_CIPHER_ALG_NONE; 733 else if (strcmp(ciphername, "AES-256-GCM") == 0 || 734 strcmp(ciphername, "AES-192-GCM") == 0 || 735 strcmp(ciphername, "AES-128-GCM") == 0) 736 cipher = OVPN_CIPHER_ALG_AES_GCM; 737 else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0) 738 cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305; 739 else 740 return (EINVAL); 741 742 if (cipher != OVPN_CIPHER_ALG_NONE) { 743 if (! nvlist_exists_binary(nvl, "key")) 744 return (EINVAL); 745 key = nvlist_get_binary(nvl, "key", &keylen); 746 if (keylen > sizeof(kdir->key)) 747 return (E2BIG); 748 749 if (! nvlist_exists_binary(nvl, "iv")) 750 return (EINVAL); 751 iv = nvlist_get_binary(nvl, "iv", &ivlen); 752 if (ivlen != 8) 753 return (E2BIG); 754 } 755 756 kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN, 757 M_WAITOK | M_ZERO); 758 759 kdir->cipher = cipher; 760 kdir->keylen = keylen; 761 kdir->tx_seq = 1; 762 memcpy(kdir->key, key, keylen); 763 kdir->noncelen = ivlen; 764 memcpy(kdir->nonce, iv, ivlen); 765 766 if (kdir->cipher != OVPN_CIPHER_ALG_NONE) { 767 /* Crypto init */ 768 bzero(&csp, sizeof(csp)); 769 csp.csp_mode = CSP_MODE_AEAD; 770 771 if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305) 772 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305; 773 else 774 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16; 775 776 csp.csp_flags |= CSP_F_SEPARATE_AAD; 777 778 csp.csp_cipher_klen = kdir->keylen; 779 csp.csp_cipher_key = kdir->key; 780 csp.csp_ivlen = 96 / 8; 781 782 error = crypto_newsession(&kdir->cryptoid, &csp, 783 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE); 784 if (error) { 785 free(kdir, M_OVPN); 786 return (error); 787 } 788 } 789 790 mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF); 791 *kdirp = kdir; 792 793 return (0); 794} 795 796static void 797ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir) 798{ 799 if (kdir == NULL) 800 return; 801 802 mtx_destroy(&kdir->replay_mtx); 803 804 crypto_freesession(kdir->cryptoid); 805 free(kdir, M_OVPN); 806} 807 808static int 809ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl) 810{ 811 struct ovpn_softc *sc = ifp->if_softc; 812 struct ovpn_kkey_dir *enc, *dec; 813 struct ovpn_kpeer *peer; 814 int slot, keyid, peerid; 815 int error; 816 817 if (nvl == NULL) 818 return (EINVAL); 819 820 if (! nvlist_exists_number(nvl, "slot")) 821 return (EINVAL); 822 slot = nvlist_get_number(nvl, "slot"); 823 824 if (! nvlist_exists_number(nvl, "keyid")) 825 return (EINVAL); 826 keyid = nvlist_get_number(nvl, "keyid"); 827 828 if (! nvlist_exists_number(nvl, "peerid")) 829 return (EINVAL); 830 peerid = nvlist_get_number(nvl, "peerid"); 831 832 if (slot != OVPN_KEY_SLOT_PRIMARY && 833 slot != OVPN_KEY_SLOT_SECONDARY) 834 return (EINVAL); 835 836 if (! nvlist_exists_nvlist(nvl, "encrypt") || 837 ! nvlist_exists_nvlist(nvl, "decrypt")) 838 return (EINVAL); 839 840 error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt")); 841 if (error) 842 return (error); 843 844 error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt")); 845 if (error) { 846 ovpn_free_kkey_dir(enc); 847 return (error); 848 } 849 850 OVPN_WLOCK(sc); 851 852 peer = ovpn_find_peer(sc, peerid); 853 if (peer == NULL) { 854 ovpn_free_kkey_dir(dec); 855 ovpn_free_kkey_dir(enc); 856 OVPN_WUNLOCK(sc); 857 return (ENOENT); 858 } 859 860 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 861 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 862 863 peer->keys[slot].encrypt = enc; 864 peer->keys[slot].decrypt = dec; 865 866 peer->keys[slot].keyid = keyid; 867 peer->keys[slot].peerid = peerid; 868 869 OVPN_WUNLOCK(sc); 870 871 return (0); 872} 873 874static int 875ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot) 876{ 877 OVPN_ASSERT(sc); 878 879 if (peer->keys[slot].encrypt == NULL) 880 return (ENOLINK); 881 882 if (peer->keys[slot].decrypt == NULL) 883 return (ENOLINK); 884 885 return (0); 886} 887 888static int 889ovpn_start(struct ifnet *ifp) 890{ 891 struct ovpn_softc *sc = ifp->if_softc; 892 893 OVPN_WLOCK(sc); 894 895 ifp->if_flags |= IFF_UP; 896 ifp->if_drv_flags |= IFF_DRV_RUNNING; 897 if_link_state_change(ifp, LINK_STATE_UP); 898 899 OVPN_WUNLOCK(sc); 900 901 return (0); 902} 903 904static int 905ovpn_swap_keys(struct ifnet *ifp, nvlist_t *nvl) 906{ 907 struct ovpn_softc *sc = ifp->if_softc; 908 struct ovpn_kpeer *peer; 909 struct ovpn_kkey tmpkey; 910 int error; 911 912 if (nvl == NULL) 913 return (EINVAL); 914 915 if (! nvlist_exists_number(nvl, "peerid")) 916 return (EINVAL); 917 918 OVPN_WLOCK(sc); 919 920 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 921 if (peer == NULL) { 922 OVPN_WUNLOCK(sc); 923 return (ENOENT); 924 } 925 926 /* Check that we have a second key to swap to. */ 927 error = ovpn_check_key(sc, peer, OVPN_KEY_SLOT_SECONDARY); 928 if (error) { 929 OVPN_WUNLOCK(sc); 930 return (error); 931 } 932 933 tmpkey = peer->keys[0]; 934 peer->keys[0] = peer->keys[1]; 935 peer->keys[1] = tmpkey; 936 937 OVPN_WUNLOCK(sc); 938 939 return (0); 940} 941 942static int 943ovpn_del_key(struct ifnet *ifp, const nvlist_t *nvl) 944{ 945 enum ovpn_key_slot slot; 946 struct ovpn_kpeer *peer; 947 struct ovpn_softc *sc = ifp->if_softc; 948 949 if (nvl == NULL) 950 return (EINVAL); 951 952 if (! nvlist_exists_number(nvl, "peerid")) 953 return (EINVAL); 954 955 if (! nvlist_exists_number(nvl, "slot")) 956 return (EINVAL); 957 slot = nvlist_get_number(nvl, "slot"); 958 959 if (slot != OVPN_KEY_SLOT_PRIMARY && 960 slot != OVPN_KEY_SLOT_SECONDARY) 961 return (EINVAL); 962 963 OVPN_WLOCK(sc); 964 965 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 966 if (peer == NULL) { 967 OVPN_WUNLOCK(sc); 968 return (ENOENT); 969 } 970 971 ovpn_free_kkey_dir(peer->keys[slot].encrypt); 972 ovpn_free_kkey_dir(peer->keys[slot].decrypt); 973 974 peer->keys[slot].encrypt = NULL; 975 peer->keys[slot].decrypt = NULL; 976 977 peer->keys[slot].keyid = 0; 978 peer->keys[slot].peerid = 0; 979 980 OVPN_WUNLOCK(sc); 981 982 return (0); 983} 984 985static void 986ovpn_send_ping(void *arg) 987{ 988 static const uint8_t ping_str[] = { 989 0x2a, 0x18, 0x7b, 0xf3, 0x64, 0x1e, 0xb4, 0xcb, 990 0x07, 0xed, 0x2d, 0x0a, 0x98, 0x1f, 0xc7, 0x48 991 }; 992 993 struct epoch_tracker et; 994 struct ovpn_kpeer *peer = arg; 995 struct ovpn_softc *sc = peer->sc; 996 struct mbuf *m; 997 998 OVPN_RASSERT(sc); 999 1000 /* Ensure we repeat! */ 1001 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1002 ovpn_send_ping, peer); 1003 1004 m = m_get2(sizeof(ping_str), M_NOWAIT, MT_DATA, M_PKTHDR); 1005 if (m == NULL) 1006 return; 1007 1008 m_copyback(m, 0, sizeof(ping_str), ping_str); 1009 m->m_len = m->m_pkthdr.len = sizeof(ping_str); 1010 1011 CURVNET_SET(sc->ifp->if_vnet); 1012 NET_EPOCH_ENTER(et); 1013 (void)ovpn_transmit_to_peer(sc->ifp, m, peer, NULL); 1014 NET_EPOCH_EXIT(et); 1015 CURVNET_RESTORE(); 1016} 1017 1018static void 1019ovpn_timeout(void *arg) 1020{ 1021 struct ovpn_kpeer *peer = arg; 1022 struct ovpn_softc *sc = peer->sc; 1023 uint32_t last, _last_active; 1024 int ret __diagused; 1025 int cpu; 1026 1027 OVPN_WASSERT(sc); 1028 1029 last = 0; 1030 CPU_FOREACH(cpu) { 1031 _last_active = *zpcpu_get_cpu(peer->last_active, cpu); 1032 if (_last_active > last) 1033 last = _last_active; 1034 } 1035 1036 if (last + peer->keepalive.timeout > time_uptime) { 1037 callout_reset(&peer->ping_rcv, 1038 (peer->keepalive.timeout - (time_uptime - last)) * hz, 1039 ovpn_timeout, peer); 1040 return; 1041 } 1042 1043 CURVNET_SET(sc->ifp->if_vnet); 1044 peer->del_reason = OVPN_DEL_REASON_TIMEOUT; 1045 ret = _ovpn_del_peer(sc, peer); 1046 MPASS(ret == 0); 1047 CURVNET_RESTORE(); 1048} 1049 1050static int 1051ovpn_set_peer(struct ifnet *ifp, const nvlist_t *nvl) 1052{ 1053 struct ovpn_softc *sc = ifp->if_softc; 1054 struct ovpn_kpeer *peer; 1055 1056 if (nvl == NULL) 1057 return (EINVAL); 1058 1059 if (! nvlist_exists_number(nvl, "interval") || 1060 ! nvlist_exists_number(nvl, "timeout") || 1061 ! nvlist_exists_number(nvl, "peerid")) 1062 return (EINVAL); 1063 1064 OVPN_WLOCK(sc); 1065 1066 peer = ovpn_find_peer(sc, nvlist_get_number(nvl, "peerid")); 1067 if (peer == NULL) { 1068 OVPN_WUNLOCK(sc); 1069 return (ENOENT); 1070 } 1071 1072 peer->keepalive.interval = nvlist_get_number(nvl, "interval"); 1073 peer->keepalive.timeout = nvlist_get_number(nvl, "timeout"); 1074 1075 if (peer->keepalive.interval > 0) 1076 callout_reset(&peer->ping_send, peer->keepalive.interval * hz, 1077 ovpn_send_ping, peer); 1078 if (peer->keepalive.timeout > 0) 1079 callout_reset(&peer->ping_rcv, peer->keepalive.timeout * hz, 1080 ovpn_timeout, peer); 1081 1082 OVPN_WUNLOCK(sc); 1083 1084 return (0); 1085} 1086 1087static int 1088ovpn_set_ifmode(struct ifnet *ifp, const nvlist_t *nvl) 1089{ 1090 struct ovpn_softc *sc = ifp->if_softc; 1091 int ifmode; 1092 1093 if (nvl == NULL) 1094 return (EINVAL); 1095 1096 if (! nvlist_exists_number(nvl, "ifmode") ) 1097 return (EINVAL); 1098 1099 ifmode = nvlist_get_number(nvl, "ifmode"); 1100 1101 OVPN_WLOCK(sc); 1102 1103 /* deny this if UP */ 1104 if (ifp->if_flags & IFF_UP) { 1105 OVPN_WUNLOCK(sc); 1106 return (EBUSY); 1107 } 1108 1109 switch (ifmode & ~IFF_MULTICAST) { 1110 case IFF_POINTOPOINT: 1111 case IFF_BROADCAST: 1112 ifp->if_flags &= 1113 ~(IFF_BROADCAST|IFF_POINTOPOINT|IFF_MULTICAST); 1114 ifp->if_flags |= ifmode; 1115 break; 1116 default: 1117 OVPN_WUNLOCK(sc); 1118 return (EINVAL); 1119 } 1120 1121 OVPN_WUNLOCK(sc); 1122 1123 return (0); 1124} 1125 1126static int 1127ovpn_ioctl_set(struct ifnet *ifp, struct ifdrv *ifd) 1128{ 1129 struct ovpn_softc *sc = ifp->if_softc; 1130 uint8_t *buf = NULL; 1131 nvlist_t *nvl = NULL; 1132 int ret; 1133 1134 if (ifd->ifd_len != 0) { 1135 if (ifd->ifd_len > OVPN_MAX_REQUEST_SIZE) 1136 return (E2BIG); 1137 1138 buf = malloc(ifd->ifd_len, M_OVPN, M_WAITOK); 1139 1140 ret = copyin(ifd->ifd_data, buf, ifd->ifd_len); 1141 if (ret != 0) { 1142 free(buf, M_OVPN); 1143 return (ret); 1144 } 1145 1146 nvl = nvlist_unpack(buf, ifd->ifd_len, 0); 1147 free(buf, M_OVPN); 1148 if (nvl == NULL) { 1149 return (EINVAL); 1150 } 1151 } 1152 1153 switch (ifd->ifd_cmd) { 1154 case OVPN_NEW_PEER: 1155 ret = ovpn_new_peer(ifp, nvl); 1156 break; 1157 case OVPN_DEL_PEER: 1158 OVPN_WLOCK(sc); 1159 ret = ovpn_del_peer(ifp, nvl); 1160 OVPN_WUNLOCK(sc); 1161 break; 1162 case OVPN_NEW_KEY: 1163 ret = ovpn_set_key(ifp, nvl); 1164 break; 1165 case OVPN_START_VPN: 1166 ret = ovpn_start(ifp); 1167 break; 1168 case OVPN_SWAP_KEYS: 1169 ret = ovpn_swap_keys(ifp, nvl); 1170 break; 1171 case OVPN_DEL_KEY: 1172 ret = ovpn_del_key(ifp, nvl); 1173 break; 1174 case OVPN_SET_PEER: 1175 ret = ovpn_set_peer(ifp, nvl); 1176 break; 1177 case OVPN_SET_IFMODE: 1178 ret = ovpn_set_ifmode(ifp, nvl); 1179 break; 1180 default: 1181 ret = ENOTSUP; 1182 } 1183 1184 nvlist_destroy(nvl); 1185 return (ret); 1186} 1187 1188static int 1189ovpn_add_counters(nvlist_t *parent, const char *name, counter_u64_t in, 1190 counter_u64_t out) 1191{ 1192 nvlist_t *nvl; 1193 1194 nvl = nvlist_create(0); 1195 if (nvl == NULL) 1196 return (ENOMEM); 1197 1198 nvlist_add_number(nvl, "in", counter_u64_fetch(in)); 1199 nvlist_add_number(nvl, "out", counter_u64_fetch(out)); 1200 1201 nvlist_add_nvlist(parent, name, nvl); 1202 1203 nvlist_destroy(nvl); 1204 1205 return (0); 1206} 1207 1208static int 1209ovpn_get_stats(struct ovpn_softc *sc, nvlist_t **onvl) 1210{ 1211 nvlist_t *nvl; 1212 int ret; 1213 1214 nvl = nvlist_create(0); 1215 if (nvl == NULL) 1216 return (ENOMEM); 1217 1218#define OVPN_COUNTER_OUT(name, in, out) \ 1219 do { \ 1220 ret = ovpn_add_counters(nvl, name, OVPN_COUNTER(sc, in), \ 1221 OVPN_COUNTER(sc, out)); \ 1222 if (ret != 0) \ 1223 goto error; \ 1224 } while(0) 1225 1226 OVPN_COUNTER_OUT("lost_ctrl", lost_ctrl_pkts_in, lost_ctrl_pkts_out); 1227 OVPN_COUNTER_OUT("lost_data", lost_data_pkts_in, lost_data_pkts_out); 1228 OVPN_COUNTER_OUT("nomem_data", nomem_data_pkts_in, 1229 nomem_data_pkts_out); 1230 OVPN_COUNTER_OUT("data", received_data_pkts, sent_data_pkts); 1231 OVPN_COUNTER_OUT("ctrl", received_ctrl_pkts, sent_ctrl_pkts); 1232 OVPN_COUNTER_OUT("tunnel", tunnel_bytes_received, 1233 tunnel_bytes_received); 1234 OVPN_COUNTER_OUT("transport", transport_bytes_received, 1235 transport_bytes_received); 1236#undef OVPN_COUNTER_OUT 1237 1238 *onvl = nvl; 1239 1240 return (0); 1241 1242error: 1243 nvlist_destroy(nvl); 1244 return (ret); 1245} 1246 1247static int 1248ovpn_get_peer_stats(struct ovpn_softc *sc, nvlist_t **nvl) 1249{ 1250 struct ovpn_kpeer *peer; 1251 nvlist_t *nvpeer = NULL; 1252 int ret; 1253 1254 OVPN_RLOCK_TRACKER; 1255 1256 *nvl = nvlist_create(0); 1257 if (*nvl == NULL) 1258 return (ENOMEM); 1259 1260#define OVPN_PEER_COUNTER_OUT(name, in, out) \ 1261 do { \ 1262 ret = ovpn_add_counters(nvpeer, name, \ 1263 OVPN_PEER_COUNTER(peer, in), OVPN_PEER_COUNTER(peer, out)); \ 1264 if (ret != 0) \ 1265 goto error; \ 1266 } while(0) 1267 1268 OVPN_RLOCK(sc); 1269 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1270 nvpeer = nvlist_create(0); 1271 if (nvpeer == NULL) { 1272 OVPN_RUNLOCK(sc); 1273 nvlist_destroy(*nvl); 1274 *nvl = NULL; 1275 return (ENOMEM); 1276 } 1277 1278 nvlist_add_number(nvpeer, "peerid", peer->peerid); 1279 1280 OVPN_PEER_COUNTER_OUT("packets", pkt_in, pkt_out); 1281 OVPN_PEER_COUNTER_OUT("bytes", bytes_in, bytes_out); 1282 1283 nvlist_append_nvlist_array(*nvl, "peers", nvpeer); 1284 nvlist_destroy(nvpeer); 1285 } 1286#undef OVPN_PEER_COUNTER_OUT 1287 OVPN_RUNLOCK(sc); 1288 1289 return (0); 1290 1291error: 1292 nvlist_destroy(nvpeer); 1293 nvlist_destroy(*nvl); 1294 *nvl = NULL; 1295 return (ret); 1296} 1297 1298static int 1299ovpn_poll_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1300{ 1301 nvlist_t *nvl; 1302 1303 nvl = nvlist_create(0); 1304 if (nvl == NULL) 1305 return (ENOMEM); 1306 1307 nvlist_add_number(nvl, "pending", buf_ring_count(sc->notifring)); 1308 1309 *onvl = nvl; 1310 1311 return (0); 1312} 1313 1314static void 1315ovpn_notif_add_counters(nvlist_t *parent, struct ovpn_notification *n) 1316{ 1317 nvlist_t *nvl; 1318 1319 nvl = nvlist_create(0); 1320 if (nvl == NULL) 1321 return; 1322 1323 nvlist_add_number(nvl, "in", n->counters.pkt_in); 1324 nvlist_add_number(nvl, "out", n->counters.pkt_out); 1325 1326 nvlist_add_nvlist(parent, "packets", nvl); 1327 nvlist_destroy(nvl); 1328 1329 nvl = nvlist_create(0); 1330 if (nvl == NULL) 1331 return; 1332 1333 nvlist_add_number(nvl, "in", n->counters.bytes_in); 1334 nvlist_add_number(nvl, "out", n->counters.bytes_out); 1335 1336 nvlist_add_nvlist(parent, "bytes", nvl); 1337 nvlist_destroy(nvl); 1338} 1339 1340static int 1341opvn_get_pkt(struct ovpn_softc *sc, nvlist_t **onvl) 1342{ 1343 struct ovpn_notification *n; 1344 nvlist_t *nvl; 1345 1346 /* Check if we have notifications pending. */ 1347 n = buf_ring_dequeue_mc(sc->notifring); 1348 if (n == NULL) 1349 return (ENOENT); 1350 1351 nvl = nvlist_create(0); 1352 if (nvl == NULL) { 1353 free(n, M_OVPN); 1354 return (ENOMEM); 1355 } 1356 nvlist_add_number(nvl, "peerid", n->peerid); 1357 nvlist_add_number(nvl, "notification", n->type); 1358 if (n->type == OVPN_NOTIF_DEL_PEER) { 1359 nvlist_add_number(nvl, "del_reason", n->del_reason); 1360 1361 /* No error handling, because we want to send the notification 1362 * even if we can't attach the counters. */ 1363 ovpn_notif_add_counters(nvl, n); 1364 } 1365 free(n, M_OVPN); 1366 1367 *onvl = nvl; 1368 1369 return (0); 1370} 1371 1372static int 1373ovpn_ioctl_get(struct ifnet *ifp, struct ifdrv *ifd) 1374{ 1375 struct ovpn_softc *sc = ifp->if_softc; 1376 nvlist_t *nvl = NULL; 1377 int error; 1378 1379 switch (ifd->ifd_cmd) { 1380 case OVPN_GET_STATS: 1381 error = ovpn_get_stats(sc, &nvl); 1382 break; 1383 case OVPN_GET_PEER_STATS: 1384 error = ovpn_get_peer_stats(sc, &nvl); 1385 break; 1386 case OVPN_POLL_PKT: 1387 error = ovpn_poll_pkt(sc, &nvl); 1388 break; 1389 case OVPN_GET_PKT: 1390 error = opvn_get_pkt(sc, &nvl); 1391 break; 1392 default: 1393 error = ENOTSUP; 1394 break; 1395 } 1396 1397 if (error == 0) { 1398 void *packed = NULL; 1399 size_t len; 1400 1401 MPASS(nvl != NULL); 1402 1403 packed = nvlist_pack(nvl, &len); 1404 if (! packed) { 1405 nvlist_destroy(nvl); 1406 return (ENOMEM); 1407 } 1408 1409 if (len > ifd->ifd_len) { 1410 free(packed, M_NVLIST); 1411 nvlist_destroy(nvl); 1412 return (ENOSPC); 1413 } 1414 1415 error = copyout(packed, ifd->ifd_data, len); 1416 ifd->ifd_len = len; 1417 1418 free(packed, M_NVLIST); 1419 nvlist_destroy(nvl); 1420 } 1421 1422 return (error); 1423} 1424 1425static int 1426ovpn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1427{ 1428 struct ifdrv *ifd; 1429 int error; 1430 1431 CURVNET_ASSERT_SET(); 1432 1433 switch (cmd) { 1434 case SIOCSDRVSPEC: 1435 case SIOCGDRVSPEC: 1436 error = priv_check(curthread, PRIV_NET_OVPN); 1437 if (error) 1438 return (error); 1439 break; 1440 } 1441 1442 switch (cmd) { 1443 case SIOCSDRVSPEC: 1444 ifd = (struct ifdrv *)data; 1445 error = ovpn_ioctl_set(ifp, ifd); 1446 break; 1447 case SIOCGDRVSPEC: 1448 ifd = (struct ifdrv *)data; 1449 error = ovpn_ioctl_get(ifp, ifd); 1450 break; 1451 case SIOCSIFMTU: { 1452 struct ifreq *ifr = (struct ifreq *)data; 1453 if (ifr->ifr_mtu < OVPN_MTU_MIN || ifr->ifr_mtu > OVPN_MTU_MAX) 1454 return (EINVAL); 1455 1456 ifp->if_mtu = ifr->ifr_mtu; 1457 return (0); 1458 } 1459 case SIOCSIFADDR: 1460 case SIOCADDMULTI: 1461 case SIOCDELMULTI: 1462 case SIOCGIFMTU: 1463 case SIOCSIFFLAGS: 1464 return (0); 1465 default: 1466 error = EINVAL; 1467 } 1468 1469 return (error); 1470} 1471 1472static int 1473ovpn_encrypt_tx_cb(struct cryptop *crp) 1474{ 1475 struct epoch_tracker et; 1476 struct ovpn_kpeer *peer = crp->crp_opaque; 1477 struct ovpn_softc *sc = peer->sc; 1478 struct mbuf *m = crp->crp_buf.cb_mbuf; 1479 int tunnel_len; 1480 int ret; 1481 1482 CURVNET_SET(sc->ifp->if_vnet); 1483 NET_EPOCH_ENTER(et); 1484 1485 if (crp->crp_etype != 0) { 1486 crypto_freereq(crp); 1487 ovpn_peer_release_ref(peer, false); 1488 NET_EPOCH_EXIT(et); 1489 CURVNET_RESTORE(); 1490 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1491 m_freem(m); 1492 return (0); 1493 } 1494 1495 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1496 1497 tunnel_len = m->m_pkthdr.len - sizeof(struct ovpn_wire_header); 1498 ret = ovpn_encap(sc, peer->peerid, m); 1499 if (ret == 0) { 1500 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1501 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1502 } 1503 1504 crypto_freereq(crp); 1505 ovpn_peer_release_ref(peer, false); 1506 1507 NET_EPOCH_EXIT(et); 1508 CURVNET_RESTORE(); 1509 1510 return (0); 1511} 1512 1513static void 1514ovpn_finish_rx(struct ovpn_softc *sc, struct mbuf *m, 1515 struct ovpn_kpeer *peer, struct ovpn_kkey *key, uint32_t seq, 1516 struct rm_priotracker *_ovpn_lock_trackerp) 1517{ 1518 uint32_t af; 1519 1520 OVPN_RASSERT(sc); 1521 NET_EPOCH_ASSERT(); 1522 1523 /* Replay protection. */ 1524 if (V_replay_protection && ! ovpn_check_replay(key->decrypt, seq)) { 1525 OVPN_RUNLOCK(sc); 1526 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1527 m_freem(m); 1528 return; 1529 } 1530 1531 critical_enter(); 1532 *zpcpu_get(peer->last_active) = time_uptime; 1533 critical_exit(); 1534 1535 OVPN_RUNLOCK(sc); 1536 1537 OVPN_COUNTER_ADD(sc, received_data_pkts, 1); 1538 OVPN_COUNTER_ADD(sc, tunnel_bytes_received, m->m_pkthdr.len); 1539 OVPN_PEER_COUNTER_ADD(peer, pkt_in, 1); 1540 OVPN_PEER_COUNTER_ADD(peer, bytes_in, m->m_pkthdr.len); 1541 1542 /* Receive the packet on our interface. */ 1543 m->m_pkthdr.rcvif = sc->ifp; 1544 1545 /* Clear checksum flags in case the real hardware set them. */ 1546 m->m_pkthdr.csum_flags = 0; 1547 1548 /* Clear mbuf tags & flags */ 1549 m_tag_delete_nonpersistent(m); 1550 m_clrprotoflags(m); 1551 1552 /* Ensure we can read the first byte. */ 1553 m = m_pullup(m, 1); 1554 if (m == NULL) { 1555 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 1556 return; 1557 } 1558 1559 /* 1560 * Check for address family, and disregard any control packets (e.g. 1561 * keepalive). 1562 */ 1563 af = ovpn_get_af(m); 1564 if (af != 0) { 1565 BPF_MTAP2(sc->ifp, &af, sizeof(af), m); 1566 if (V_async_netisr_queue) 1567 netisr_queue(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1568 else 1569 netisr_dispatch(af == AF_INET ? NETISR_IP : NETISR_IPV6, m); 1570 } else { 1571 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1572 m_freem(m); 1573 } 1574} 1575 1576static struct ovpn_kkey * 1577ovpn_find_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, 1578 const struct ovpn_wire_header *ohdr) 1579{ 1580 struct ovpn_kkey *key = NULL; 1581 uint8_t keyid; 1582 1583 OVPN_RASSERT(sc); 1584 1585 keyid = (ntohl(ohdr->opcode) >> 24) & 0x07; 1586 1587 if (peer->keys[0].keyid == keyid) 1588 key = &peer->keys[0]; 1589 else if (peer->keys[1].keyid == keyid) 1590 key = &peer->keys[1]; 1591 1592 return (key); 1593} 1594 1595static int 1596ovpn_decrypt_rx_cb(struct cryptop *crp) 1597{ 1598 struct epoch_tracker et; 1599 struct ovpn_softc *sc = crp->crp_opaque; 1600 struct mbuf *m = crp->crp_buf.cb_mbuf; 1601 struct ovpn_kkey *key; 1602 struct ovpn_kpeer *peer; 1603 struct ovpn_wire_header *ohdr; 1604 uint32_t peerid; 1605 1606 OVPN_RLOCK_TRACKER; 1607 1608 OVPN_RLOCK(sc); 1609 1610 MPASS(crp->crp_buf.cb_type == CRYPTO_BUF_MBUF); 1611 1612 if (crp->crp_etype != 0) { 1613 crypto_freereq(crp); 1614 atomic_add_int(&sc->refcount, -1); 1615 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1616 OVPN_RUNLOCK(sc); 1617 m_freem(m); 1618 return (0); 1619 } 1620 1621 CURVNET_SET(sc->ifp->if_vnet); 1622 1623 ohdr = mtodo(m, sizeof(struct udphdr)); 1624 1625 peerid = ntohl(ohdr->opcode) & 0x00ffffff; 1626 peer = ovpn_find_peer(sc, peerid); 1627 if (peer == NULL) { 1628 /* No such peer. Drop packet. */ 1629 crypto_freereq(crp); 1630 atomic_add_int(&sc->refcount, -1); 1631 OVPN_RUNLOCK(sc); 1632 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1633 m_freem(m); 1634 CURVNET_RESTORE(); 1635 return (0); 1636 } 1637 1638 key = ovpn_find_key(sc, peer, ohdr); 1639 if (key == NULL) { 1640 crypto_freereq(crp); 1641 atomic_add_int(&sc->refcount, -1); 1642 /* 1643 * Has this key been removed between us starting the decrypt 1644 * and finishing it? 1645 */ 1646 OVPN_RUNLOCK(sc); 1647 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 1648 m_freem(m); 1649 CURVNET_RESTORE(); 1650 return (0); 1651 } 1652 1653 /* Now remove the outer headers */ 1654 m_adj_decap(m, sizeof(struct udphdr) + 1655 sizeof(struct ovpn_wire_header)); 1656 1657 NET_EPOCH_ENTER(et); 1658 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), _ovpn_lock_trackerp); 1659 NET_EPOCH_EXIT(et); 1660 OVPN_UNLOCK_ASSERT(sc); 1661 1662 CURVNET_RESTORE(); 1663 1664 crypto_freereq(crp); 1665 atomic_add_int(&sc->refcount, -1); 1666 1667 return (0); 1668} 1669 1670static int 1671ovpn_get_af(struct mbuf *m) 1672{ 1673 struct ip *ip; 1674 struct ip6_hdr *ip6; 1675 1676 /* 1677 * We should pullup, but we're only interested in the first byte, so 1678 * that'll always be contiguous. 1679 */ 1680 ip = mtod(m, struct ip *); 1681 if (ip->ip_v == IPVERSION) 1682 return (AF_INET); 1683 1684 ip6 = mtod(m, struct ip6_hdr *); 1685 if ((ip6->ip6_vfc & IPV6_VERSION_MASK) == IPV6_VERSION) 1686 return (AF_INET6); 1687 1688 return (0); 1689} 1690 1691#ifdef INET 1692static struct ovpn_kpeer * 1693ovpn_find_peer_by_ip(struct ovpn_softc *sc, const struct in_addr addr) 1694{ 1695 struct ovpn_kpeer *peer = NULL; 1696 1697 OVPN_ASSERT(sc); 1698 1699 /* TODO: Add a second RB so we can look up by IP. */ 1700 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1701 if (addr.s_addr == peer->vpn4.s_addr) 1702 return (peer); 1703 } 1704 1705 return (peer); 1706} 1707#endif 1708 1709#ifdef INET6 1710static struct ovpn_kpeer * 1711ovpn_find_peer_by_ip6(struct ovpn_softc *sc, const struct in6_addr *addr) 1712{ 1713 struct ovpn_kpeer *peer = NULL; 1714 1715 OVPN_ASSERT(sc); 1716 1717 /* TODO: Add a third RB so we can look up by IPv6 address. */ 1718 RB_FOREACH(peer, ovpn_kpeers, &sc->peers) { 1719 if (memcmp(addr, &peer->vpn6, sizeof(*addr)) == 0) 1720 return (peer); 1721 } 1722 1723 return (peer); 1724} 1725#endif 1726 1727static struct ovpn_kpeer * 1728ovpn_route_peer(struct ovpn_softc *sc, struct mbuf **m0, 1729 const struct sockaddr *dst) 1730{ 1731 struct ovpn_kpeer *peer = NULL; 1732 int af; 1733 1734 NET_EPOCH_ASSERT(); 1735 OVPN_ASSERT(sc); 1736 1737 /* Shortcut if we're a client (or are a server and have only one client). */ 1738 if (sc->peercount == 1) 1739 return (ovpn_find_only_peer(sc)); 1740 1741 if (dst != NULL) 1742 af = dst->sa_family; 1743 else 1744 af = ovpn_get_af(*m0); 1745 1746 switch (af) { 1747#ifdef INET 1748 case AF_INET: { 1749 const struct sockaddr_in *sa = (const struct sockaddr_in *)dst; 1750 struct nhop_object *nh; 1751 const struct in_addr *ip_dst; 1752 1753 if (sa != NULL) { 1754 ip_dst = &sa->sin_addr; 1755 } else { 1756 struct ip *ip; 1757 1758 *m0 = m_pullup(*m0, sizeof(struct ip)); 1759 if (*m0 == NULL) 1760 return (NULL); 1761 ip = mtod(*m0, struct ip *); 1762 ip_dst = &ip->ip_dst; 1763 } 1764 1765 peer = ovpn_find_peer_by_ip(sc, *ip_dst); 1766 SDT_PROBE2(if_ovpn, tx, route, ip4, ip_dst, peer); 1767 if (peer == NULL) { 1768 nh = fib4_lookup(M_GETFIB(*m0), *ip_dst, 0, 1769 NHR_NONE, 0); 1770 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1771 peer = ovpn_find_peer_by_ip(sc, 1772 nh->gw4_sa.sin_addr); 1773 SDT_PROBE2(if_ovpn, tx, route, ip4, 1774 &nh->gw4_sa.sin_addr, peer); 1775 } 1776 } 1777 break; 1778 } 1779#endif 1780#ifdef INET6 1781 case AF_INET6: { 1782 const struct sockaddr_in6 *sa6 = 1783 (const struct sockaddr_in6 *)dst; 1784 struct nhop_object *nh; 1785 const struct in6_addr *ip6_dst; 1786 1787 if (sa6 != NULL) { 1788 ip6_dst = &sa6->sin6_addr; 1789 } else { 1790 struct ip6_hdr *ip6; 1791 1792 *m0 = m_pullup(*m0, sizeof(struct ip6_hdr)); 1793 if (*m0 == NULL) 1794 return (NULL); 1795 ip6 = mtod(*m0, struct ip6_hdr *); 1796 ip6_dst = &ip6->ip6_dst; 1797 } 1798 1799 peer = ovpn_find_peer_by_ip6(sc, ip6_dst); 1800 SDT_PROBE2(if_ovpn, tx, route, ip6, ip6_dst, peer); 1801 if (peer == NULL) { 1802 nh = fib6_lookup(M_GETFIB(*m0), ip6_dst, 0, 1803 NHR_NONE, 0); 1804 if (nh && (nh->nh_flags & NHF_GATEWAY)) { 1805 peer = ovpn_find_peer_by_ip6(sc, 1806 &nh->gw6_sa.sin6_addr); 1807 SDT_PROBE2(if_ovpn, tx, route, ip6, 1808 &nh->gw6_sa.sin6_addr, peer); 1809 } 1810 } 1811 break; 1812 } 1813#endif 1814 } 1815 1816 return (peer); 1817} 1818 1819static int 1820ovpn_transmit(struct ifnet *ifp, struct mbuf *m) 1821{ 1822 return (ifp->if_output(ifp, m, NULL, NULL)); 1823} 1824 1825static int 1826ovpn_transmit_to_peer(struct ifnet *ifp, struct mbuf *m, 1827 struct ovpn_kpeer *peer, struct rm_priotracker *_ovpn_lock_trackerp) 1828{ 1829 struct ovpn_wire_header *ohdr; 1830 struct ovpn_kkey *key; 1831 struct ovpn_softc *sc; 1832 struct cryptop *crp; 1833 uint32_t af, seq; 1834 uint64_t seq64; 1835 size_t len, ovpn_hdr_len; 1836 int tunnel_len; 1837 int ret; 1838 1839 sc = ifp->if_softc; 1840 1841 OVPN_RASSERT(sc); 1842 1843 tunnel_len = m->m_pkthdr.len; 1844 1845 key = &peer->keys[OVPN_KEY_SLOT_PRIMARY]; 1846 if (key->encrypt == NULL) { 1847 if (_ovpn_lock_trackerp != NULL) 1848 OVPN_RUNLOCK(sc); 1849 m_freem(m); 1850 return (ENOLINK); 1851 } 1852 1853 af = ovpn_get_af(m); 1854 /* Don't capture control packets. */ 1855 if (af != 0) 1856 BPF_MTAP2(ifp, &af, sizeof(af), m); 1857 1858 if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_OVPN_LOOP, 3))) { 1859 if (_ovpn_lock_trackerp != NULL) 1860 OVPN_RUNLOCK(sc); 1861 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1862 m_freem(m); 1863 return (ELOOP); 1864 } 1865 1866 len = m->m_pkthdr.len; 1867 MPASS(len <= ifp->if_mtu); 1868 1869 ovpn_hdr_len = sizeof(struct ovpn_wire_header); 1870 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) 1871 ovpn_hdr_len -= 16; /* No auth tag. */ 1872 1873 M_PREPEND(m, ovpn_hdr_len, M_NOWAIT); 1874 if (m == NULL) { 1875 if (_ovpn_lock_trackerp != NULL) 1876 OVPN_RUNLOCK(sc); 1877 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1878 return (ENOBUFS); 1879 } 1880 ohdr = mtod(m, struct ovpn_wire_header *); 1881 ohdr->opcode = (OVPN_OP_DATA_V2 << OVPN_OP_SHIFT) | key->keyid; 1882 ohdr->opcode <<= 24; 1883 ohdr->opcode |= key->peerid; 1884 ohdr->opcode = htonl(ohdr->opcode); 1885 1886 seq64 = atomic_fetchadd_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1); 1887 if (seq64 == OVPN_SEQ_ROTATE) { 1888 ovpn_notify_key_rotation(sc, peer); 1889 } else if (seq64 > UINT32_MAX) { 1890 /* We've wrapped, give up on this packet. */ 1891 if (_ovpn_lock_trackerp != NULL) 1892 OVPN_RUNLOCK(sc); 1893 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1894 1895 /* Let's avoid (very unlikely, but still) wraparounds of the 1896 * 64-bit counter taking us back to 0. */ 1897 atomic_store_64(&peer->keys[OVPN_KEY_SLOT_PRIMARY].encrypt->tx_seq, 1898 UINT32_MAX); 1899 1900 return (ENOBUFS); 1901 } 1902 1903 seq = htonl(seq64 & UINT32_MAX); 1904 ohdr->seq = seq; 1905 1906 OVPN_PEER_COUNTER_ADD(peer, pkt_out, 1); 1907 OVPN_PEER_COUNTER_ADD(peer, bytes_out, len); 1908 1909 if (key->encrypt->cipher == OVPN_CIPHER_ALG_NONE) { 1910 ret = ovpn_encap(sc, peer->peerid, m); 1911 if (_ovpn_lock_trackerp != NULL) 1912 OVPN_RUNLOCK(sc); 1913 if (ret == 0) { 1914 OVPN_COUNTER_ADD(sc, sent_data_pkts, 1); 1915 OVPN_COUNTER_ADD(sc, tunnel_bytes_sent, tunnel_len); 1916 } 1917 return (ret); 1918 } 1919 1920 crp = crypto_getreq(key->encrypt->cryptoid, M_NOWAIT); 1921 if (crp == NULL) { 1922 if (_ovpn_lock_trackerp != NULL) 1923 OVPN_RUNLOCK(sc); 1924 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1925 m_freem(m); 1926 return (ENOBUFS); 1927 } 1928 1929 /* Encryption covers only the payload, not the header. */ 1930 crp->crp_payload_start = sizeof(*ohdr); 1931 crp->crp_payload_length = len; 1932 crp->crp_op = CRYPTO_OP_ENCRYPT; 1933 1934 /* 1935 * AAD data covers the ovpn_wire_header minus the auth 1936 * tag. 1937 */ 1938 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 1939 crp->crp_aad = ohdr; 1940 crp->crp_aad_start = 0; 1941 crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST; 1942 crp->crp_digest_start = offsetof(struct ovpn_wire_header, auth_tag); 1943 1944 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1945 memcpy(crp->crp_iv, &seq, sizeof(seq)); 1946 memcpy(crp->crp_iv + sizeof(seq), key->encrypt->nonce, 1947 key->encrypt->noncelen); 1948 1949 crypto_use_mbuf(crp, m); 1950 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 1951 crp->crp_callback = ovpn_encrypt_tx_cb; 1952 crp->crp_opaque = peer; 1953 1954 atomic_add_int(&peer->refcount, 1); 1955 if (_ovpn_lock_trackerp != NULL) 1956 OVPN_RUNLOCK(sc); 1957 if (V_async_crypto) 1958 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 1959 else 1960 ret = crypto_dispatch(crp); 1961 if (ret) { 1962 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1963 } 1964 1965 return (ret); 1966} 1967 1968/* 1969 * Note: Expects to hold the read lock on entry, and will release it itself. 1970 */ 1971static int 1972ovpn_encap(struct ovpn_softc *sc, uint32_t peerid, struct mbuf *m) 1973{ 1974 struct udphdr *udp; 1975 struct ovpn_kpeer *peer; 1976 int len; 1977 1978 OVPN_RLOCK_TRACKER; 1979 1980 OVPN_RLOCK(sc); 1981 NET_EPOCH_ASSERT(); 1982 1983 peer = ovpn_find_peer(sc, peerid); 1984 if (peer == NULL || sc->ifp->if_link_state != LINK_STATE_UP) { 1985 OVPN_RUNLOCK(sc); 1986 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 1987 m_freem(m); 1988 return (ENETDOWN); 1989 } 1990 1991 len = m->m_pkthdr.len; 1992 1993 M_PREPEND(m, sizeof(struct udphdr), M_NOWAIT); 1994 if (m == NULL) { 1995 OVPN_RUNLOCK(sc); 1996 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 1997 m_freem(m); 1998 return (ENOBUFS); 1999 } 2000 udp = mtod(m, struct udphdr *); 2001 2002 MPASS(peer->local.ss_family == peer->remote.ss_family); 2003 2004 udp->uh_sport = ovpn_get_port(&peer->local); 2005 udp->uh_dport = ovpn_get_port(&peer->remote); 2006 udp->uh_ulen = htons(sizeof(struct udphdr) + len); 2007 2008 switch (peer->remote.ss_family) { 2009#ifdef INET 2010 case AF_INET: { 2011 struct sockaddr_in *in_local = TO_IN(&peer->local); 2012 struct sockaddr_in *in_remote = TO_IN(&peer->remote); 2013 struct ip *ip; 2014 2015 /* 2016 * This requires knowing the source IP, which we don't. Happily 2017 * we're allowed to keep this at 0, and the checksum won't do 2018 * anything the crypto won't already do. 2019 */ 2020 udp->uh_sum = 0; 2021 2022 /* Set the checksum flags so we recalculate checksums. */ 2023 m->m_pkthdr.csum_flags |= CSUM_IP; 2024 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2025 2026 M_PREPEND(m, sizeof(struct ip), M_NOWAIT); 2027 if (m == NULL) { 2028 OVPN_RUNLOCK(sc); 2029 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2030 return (ENOBUFS); 2031 } 2032 ip = mtod(m, struct ip *); 2033 2034 ip->ip_tos = 0; 2035 ip->ip_len = htons(sizeof(struct ip) + sizeof(struct udphdr) + 2036 len); 2037 ip->ip_off = 0; 2038 ip->ip_ttl = V_ip_defttl; 2039 ip->ip_p = IPPROTO_UDP; 2040 ip->ip_sum = 0; 2041 if (in_local->sin_port != 0) 2042 ip->ip_src = in_local->sin_addr; 2043 else 2044 ip->ip_src.s_addr = INADDR_ANY; 2045 ip->ip_dst = in_remote->sin_addr; 2046 2047 OVPN_RUNLOCK(sc); 2048 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2049 2050 return (ip_output(m, NULL, NULL, 0, NULL, NULL)); 2051 } 2052#endif 2053#ifdef INET6 2054 case AF_INET6: { 2055 struct sockaddr_in6 *in6_local = TO_IN6(&peer->local); 2056 struct sockaddr_in6 *in6_remote = TO_IN6(&peer->remote); 2057 struct ip6_hdr *ip6; 2058 2059 M_PREPEND(m, sizeof(struct ip6_hdr), M_NOWAIT); 2060 if (m == NULL) { 2061 OVPN_RUNLOCK(sc); 2062 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2063 return (ENOBUFS); 2064 } 2065 m = m_pullup(m, sizeof(*ip6) + sizeof(*udp)); 2066 if (m == NULL) { 2067 OVPN_RUNLOCK(sc); 2068 OVPN_COUNTER_ADD(sc, nomem_data_pkts_out, 1); 2069 return (ENOBUFS); 2070 } 2071 2072 ip6 = mtod(m, struct ip6_hdr *); 2073 2074 ip6->ip6_vfc = IPV6_VERSION; 2075 ip6->ip6_flow &= ~IPV6_FLOWINFO_MASK; 2076 ip6->ip6_plen = htons(sizeof(*ip6) + sizeof(struct udphdr) + 2077 len); 2078 ip6->ip6_nxt = IPPROTO_UDP; 2079 ip6->ip6_hlim = V_ip6_defhlim; 2080 2081 memcpy(&ip6->ip6_src, &in6_local->sin6_addr, 2082 sizeof(ip6->ip6_src)); 2083 memcpy(&ip6->ip6_dst, &in6_remote->sin6_addr, 2084 sizeof(ip6->ip6_dst)); 2085 2086 udp = mtodo(m, sizeof(*ip6)); 2087 udp->uh_sum = in6_cksum_pseudo(ip6, 2088 m->m_pkthdr.len - sizeof(struct ip6_hdr), 2089 IPPROTO_UDP, 0); 2090 2091 m->m_pkthdr.csum_flags |= CSUM_UDP_IPV6; 2092 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 2093 2094 OVPN_RUNLOCK(sc); 2095 OVPN_COUNTER_ADD(sc, transport_bytes_sent, m->m_pkthdr.len); 2096 2097 return (ip6_output(m, NULL, NULL, IPV6_UNSPECSRC, NULL, NULL, 2098 NULL)); 2099 } 2100#endif 2101 default: 2102 panic("Unsupported address family %d", 2103 peer->remote.ss_family); 2104 } 2105} 2106 2107static int 2108ovpn_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 2109 struct route *ro) 2110{ 2111 struct ovpn_softc *sc; 2112 struct ovpn_kpeer *peer; 2113 2114 OVPN_RLOCK_TRACKER; 2115 2116 sc = ifp->if_softc; 2117 2118 OVPN_RLOCK(sc); 2119 2120 SDT_PROBE1(if_ovpn, tx, transmit, start, m); 2121 2122 if (__predict_false(ifp->if_link_state != LINK_STATE_UP)) { 2123 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2124 OVPN_RUNLOCK(sc); 2125 m_freem(m); 2126 return (ENETDOWN); 2127 } 2128 2129 /** 2130 * Only obey 'dst' (i.e. the gateway) if no route is supplied. 2131 * That's our indication that we're being called through pf's route-to, 2132 * and we should route according to 'dst' instead. We can't do so 2133 * consistently, because the usual openvpn configuration sets the first 2134 * non-server IP in the subnet as the gateway. If we always use that 2135 * one we'd end up routing all traffic to the first client. 2136 * tl;dr: 'ro == NULL' tells us pf is doing a route-to, and then but 2137 * only then, we should treat 'dst' as the destination. */ 2138 peer = ovpn_route_peer(sc, &m, ro == NULL ? dst : NULL); 2139 if (peer == NULL) { 2140 /* No destination. */ 2141 OVPN_COUNTER_ADD(sc, lost_data_pkts_out, 1); 2142 OVPN_RUNLOCK(sc); 2143 m_freem(m); 2144 return (ENETDOWN); 2145 } 2146 2147 return (ovpn_transmit_to_peer(ifp, m, peer, _ovpn_lock_trackerp)); 2148} 2149 2150static bool 2151ovpn_check_replay(struct ovpn_kkey_dir *key, uint32_t seq) 2152{ 2153 uint32_t d; 2154 2155 mtx_lock(&key->replay_mtx); 2156 2157 /* Sequence number must be strictly greater than rx_seq */ 2158 if (seq <= key->rx_seq) { 2159 mtx_unlock(&key->replay_mtx); 2160 return (false); 2161 } 2162 2163 /* Large jump. The packet authenticated okay, so just accept that. */ 2164 if (seq > (key->rx_seq + (sizeof(key->rx_window) * 8))) { 2165 key->rx_seq = seq; 2166 key->rx_window = 0; 2167 mtx_unlock(&key->replay_mtx); 2168 return (true); 2169 } 2170 2171 /* Happy case. */ 2172 if ((seq == key->rx_seq + 1) && key->rx_window == 0) { 2173 key->rx_seq++; 2174 mtx_unlock(&key->replay_mtx); 2175 return (true); 2176 } 2177 2178 d = seq - key->rx_seq - 1; 2179 2180 if (key->rx_window & ((uint64_t)1 << d)) { 2181 /* Dupe! */ 2182 mtx_unlock(&key->replay_mtx); 2183 return (false); 2184 } 2185 2186 key->rx_window |= (uint64_t)1 << d; 2187 2188 while (key->rx_window & 1) { 2189 key->rx_seq++; 2190 key->rx_window >>= 1; 2191 } 2192 2193 mtx_unlock(&key->replay_mtx); 2194 2195 return (true); 2196} 2197 2198static struct ovpn_kpeer * 2199ovpn_peer_from_mbuf(struct ovpn_softc *sc, struct mbuf *m, int off) 2200{ 2201 struct ovpn_wire_header ohdr; 2202 uint32_t peerid; 2203 const size_t hdrlen = sizeof(ohdr) - sizeof(ohdr.auth_tag); 2204 2205 OVPN_RASSERT(sc); 2206 2207 if (m_length(m, NULL) < (off + sizeof(struct udphdr) + hdrlen)) 2208 return (NULL); 2209 2210 m_copydata(m, off + sizeof(struct udphdr), hdrlen, (caddr_t)&ohdr); 2211 2212 peerid = ntohl(ohdr.opcode) & 0x00ffffff; 2213 2214 return (ovpn_find_peer(sc, peerid)); 2215} 2216 2217static bool 2218ovpn_udp_input(struct mbuf *m, int off, struct inpcb *inp, 2219 const struct sockaddr *sa, void *ctx) 2220{ 2221 struct ovpn_softc *sc = ctx; 2222 struct ovpn_wire_header tmphdr; 2223 struct ovpn_wire_header *ohdr; 2224 struct udphdr *uhdr; 2225 struct ovpn_kkey *key; 2226 struct cryptop *crp; 2227 struct ovpn_kpeer *peer; 2228 size_t ohdrlen; 2229 int ret; 2230 uint8_t op; 2231 2232 OVPN_RLOCK_TRACKER; 2233 2234 M_ASSERTPKTHDR(m); 2235 2236 OVPN_COUNTER_ADD(sc, transport_bytes_received, m->m_pkthdr.len - off); 2237 2238 ohdrlen = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2239 2240 OVPN_RLOCK(sc); 2241 2242 peer = ovpn_peer_from_mbuf(sc, m, off); 2243 if (peer == NULL) { 2244 OVPN_RUNLOCK(sc); 2245 return (false); 2246 } 2247 2248 if (m_length(m, NULL) < (off + sizeof(*uhdr) + ohdrlen)) { 2249 /* Short packet. */ 2250 OVPN_RUNLOCK(sc); 2251 return (false); 2252 } 2253 2254 m_copydata(m, off + sizeof(*uhdr), ohdrlen, (caddr_t)&tmphdr); 2255 2256 op = ntohl(tmphdr.opcode) >> 24 >> OVPN_OP_SHIFT; 2257 if (op != OVPN_OP_DATA_V2) { 2258 /* Control packet? */ 2259 OVPN_RUNLOCK(sc); 2260 return (false); 2261 } 2262 2263 m = m_pullup(m, off + sizeof(*uhdr) + ohdrlen); 2264 if (m == NULL) { 2265 OVPN_RUNLOCK(sc); 2266 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2267 return (true); 2268 } 2269 2270 /* 2271 * Simplify things by getting rid of the preceding headers, we don't 2272 * care about them. 2273 */ 2274 m_adj_decap(m, off); 2275 2276 uhdr = mtodo(m, 0); 2277 ohdr = mtodo(m, sizeof(*uhdr)); 2278 2279 key = ovpn_find_key(sc, peer, ohdr); 2280 if (key == NULL || key->decrypt == NULL) { 2281 OVPN_RUNLOCK(sc); 2282 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2283 m_freem(m); 2284 return (true); 2285 } 2286 2287 if (key->decrypt->cipher == OVPN_CIPHER_ALG_NONE) { 2288 /* Now remove the outer headers */ 2289 m_adj_decap(m, sizeof(struct udphdr) + ohdrlen); 2290 2291 ohdr = mtodo(m, sizeof(*uhdr)); 2292 2293 ovpn_finish_rx(sc, m, peer, key, ntohl(ohdr->seq), 2294 _ovpn_lock_trackerp); 2295 OVPN_UNLOCK_ASSERT(sc); 2296 return (true); 2297 } 2298 2299 ohdrlen += sizeof(ohdr->auth_tag); 2300 2301 m = m_pullup(m, sizeof(*uhdr) + ohdrlen); 2302 if (m == NULL) { 2303 OVPN_RUNLOCK(sc); 2304 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2305 return (true); 2306 } 2307 uhdr = mtodo(m, 0); 2308 ohdr = mtodo(m, sizeof(*uhdr)); 2309 2310 /* Decrypt */ 2311 crp = crypto_getreq(key->decrypt->cryptoid, M_NOWAIT); 2312 if (crp == NULL) { 2313 OVPN_COUNTER_ADD(sc, nomem_data_pkts_in, 1); 2314 OVPN_RUNLOCK(sc); 2315 m_freem(m); 2316 return (true); 2317 } 2318 2319 crp->crp_payload_start = sizeof(struct udphdr) + sizeof(*ohdr); 2320 crp->crp_payload_length = ntohs(uhdr->uh_ulen) - 2321 sizeof(*uhdr) - sizeof(*ohdr); 2322 crp->crp_op = CRYPTO_OP_DECRYPT; 2323 2324 /* AAD validation. */ 2325 crp->crp_aad_length = sizeof(*ohdr) - sizeof(ohdr->auth_tag); 2326 crp->crp_aad = ohdr; 2327 crp->crp_aad_start = 0; 2328 crp->crp_op |= CRYPTO_OP_VERIFY_DIGEST; 2329 crp->crp_digest_start = sizeof(struct udphdr) + 2330 offsetof(struct ovpn_wire_header, auth_tag); 2331 2332 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 2333 memcpy(crp->crp_iv, &ohdr->seq, sizeof(ohdr->seq)); 2334 memcpy(crp->crp_iv + sizeof(ohdr->seq), key->decrypt->nonce, 2335 key->decrypt->noncelen); 2336 2337 crypto_use_mbuf(crp, m); 2338 crp->crp_flags |= CRYPTO_F_CBIFSYNC; 2339 crp->crp_callback = ovpn_decrypt_rx_cb; 2340 crp->crp_opaque = sc; 2341 2342 atomic_add_int(&sc->refcount, 1); 2343 OVPN_RUNLOCK(sc); 2344 if (V_async_crypto) 2345 ret = crypto_dispatch_async(crp, CRYPTO_ASYNC_ORDERED); 2346 else 2347 ret = crypto_dispatch(crp); 2348 if (ret != 0) { 2349 OVPN_COUNTER_ADD(sc, lost_data_pkts_in, 1); 2350 } 2351 2352 return (true); 2353} 2354 2355static void 2356ovpn_qflush(struct ifnet *ifp __unused) 2357{ 2358 2359} 2360 2361static void 2362ovpn_flush_rxring(struct ovpn_softc *sc) 2363{ 2364 struct ovpn_notification *n; 2365 2366 OVPN_WASSERT(sc); 2367 2368 while (! buf_ring_empty(sc->notifring)) { 2369 n = buf_ring_dequeue_sc(sc->notifring); 2370 free(n, M_OVPN); 2371 } 2372} 2373 2374#ifdef VIMAGE 2375static void 2376ovpn_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused, 2377 char *unused __unused) 2378{ 2379 struct ovpn_softc *sc = ifp->if_softc; 2380 struct ovpn_kpeer *peer, *tmppeer; 2381 int ret __diagused; 2382 2383 OVPN_WLOCK(sc); 2384 2385 /* Flush keys & configuration. */ 2386 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2387 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2388 ret = _ovpn_del_peer(sc, peer); 2389 MPASS(ret == 0); 2390 } 2391 2392 ovpn_flush_rxring(sc); 2393 2394 OVPN_WUNLOCK(sc); 2395} 2396#endif 2397 2398static int 2399ovpn_clone_match(struct if_clone *ifc, const char *name) 2400{ 2401 /* 2402 * Allow all names that start with 'ovpn', specifically because pfSense 2403 * uses ovpnc1 / ovpns2 2404 */ 2405 return (strncmp(ovpnname, name, strlen(ovpnname)) == 0); 2406} 2407 2408static int 2409ovpn_clone_create(struct if_clone *ifc, char *name, size_t len, 2410 struct ifc_data *ifd, struct ifnet **ifpp) 2411{ 2412 struct ovpn_softc *sc; 2413 struct ifnet *ifp; 2414 char *dp; 2415 int error, unit, wildcard; 2416 2417 /* Try to see if a special unit was requested. */ 2418 error = ifc_name2unit(name, &unit); 2419 if (error != 0) 2420 return (error); 2421 wildcard = (unit < 0); 2422 2423 error = ifc_alloc_unit(ifc, &unit); 2424 if (error != 0) 2425 return (error); 2426 2427 /* 2428 * If no unit had been given, we need to adjust the ifName. 2429 */ 2430 for (dp = name; *dp != '\0'; dp++); 2431 if (wildcard) { 2432 error = snprintf(dp, len - (dp - name), "%d", unit); 2433 if (error > len - (dp - name)) { 2434 /* ifName too long. */ 2435 ifc_free_unit(ifc, unit); 2436 return (ENOSPC); 2437 } 2438 dp += error; 2439 } 2440 2441 /* Make sure it doesn't already exist. */ 2442 if (ifunit(name) != NULL) 2443 return (EEXIST); 2444 2445 sc = malloc(sizeof(struct ovpn_softc), M_OVPN, M_WAITOK | M_ZERO); 2446 sc->ifp = if_alloc(IFT_ENC); 2447 rm_init_flags(&sc->lock, "if_ovpn_lock", RM_RECURSE); 2448 sc->refcount = 0; 2449 2450 sc->notifring = buf_ring_alloc(32, M_OVPN, M_WAITOK, NULL); 2451 2452 COUNTER_ARRAY_ALLOC(sc->counters, OVPN_COUNTER_SIZE, M_WAITOK); 2453 2454 ifp = sc->ifp; 2455 ifp->if_softc = sc; 2456 strlcpy(ifp->if_xname, name, IFNAMSIZ); 2457 ifp->if_dname = ovpngroupname; 2458 ifp->if_dunit = unit; 2459 2460 ifp->if_addrlen = 0; 2461 ifp->if_mtu = 1428; 2462 ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST; 2463 ifp->if_ioctl = ovpn_ioctl; 2464 ifp->if_transmit = ovpn_transmit; 2465 ifp->if_output = ovpn_output; 2466 ifp->if_qflush = ovpn_qflush; 2467#ifdef VIMAGE 2468 ifp->if_reassign = ovpn_reassign; 2469#endif 2470 ifp->if_capabilities |= IFCAP_LINKSTATE; 2471 ifp->if_capenable |= IFCAP_LINKSTATE; 2472 2473 if_attach(ifp); 2474 bpfattach(ifp, DLT_NULL, sizeof(uint32_t)); 2475 *ifpp = ifp; 2476 2477 return (0); 2478} 2479 2480static void 2481ovpn_clone_destroy_cb(struct epoch_context *ctx) 2482{ 2483 struct ovpn_softc *sc; 2484 2485 sc = __containerof(ctx, struct ovpn_softc, epoch_ctx); 2486 2487 MPASS(sc->peercount == 0); 2488 MPASS(RB_EMPTY(&sc->peers)); 2489 2490 COUNTER_ARRAY_FREE(sc->counters, OVPN_COUNTER_SIZE); 2491 2492 if_free(sc->ifp); 2493 free(sc, M_OVPN); 2494} 2495 2496static int 2497ovpn_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags) 2498{ 2499 struct ovpn_softc *sc; 2500 struct ovpn_kpeer *peer, *tmppeer; 2501 int unit; 2502 int ret __diagused; 2503 2504 sc = ifp->if_softc; 2505 unit = ifp->if_dunit; 2506 2507 OVPN_WLOCK(sc); 2508 2509 if (atomic_load_int(&sc->refcount) > 0) { 2510 OVPN_WUNLOCK(sc); 2511 return (EBUSY); 2512 } 2513 2514 RB_FOREACH_SAFE(peer, ovpn_kpeers, &sc->peers, tmppeer) { 2515 peer->del_reason = OVPN_DEL_REASON_REQUESTED; 2516 ret = _ovpn_del_peer(sc, peer); 2517 MPASS(ret == 0); 2518 } 2519 2520 ovpn_flush_rxring(sc); 2521 buf_ring_free(sc->notifring, M_OVPN); 2522 2523 OVPN_WUNLOCK(sc); 2524 2525 bpfdetach(ifp); 2526 if_detach(ifp); 2527 ifp->if_softc = NULL; 2528 2529 NET_EPOCH_CALL(ovpn_clone_destroy_cb, &sc->epoch_ctx); 2530 2531 if (unit != IF_DUNIT_NONE) 2532 ifc_free_unit(ifc, unit); 2533 2534 NET_EPOCH_DRAIN_CALLBACKS(); 2535 2536 return (0); 2537} 2538 2539static void 2540vnet_ovpn_init(const void *unused __unused) 2541{ 2542 struct if_clone_addreq req = { 2543 .match_f = ovpn_clone_match, 2544 .create_f = ovpn_clone_create, 2545 .destroy_f = ovpn_clone_destroy, 2546 }; 2547 V_ovpn_cloner = ifc_attach_cloner(ovpngroupname, &req); 2548} 2549VNET_SYSINIT(vnet_ovpn_init, SI_SUB_PSEUDO, SI_ORDER_ANY, 2550 vnet_ovpn_init, NULL); 2551 2552static void 2553vnet_ovpn_uninit(const void *unused __unused) 2554{ 2555 if_clone_detach(V_ovpn_cloner); 2556} 2557VNET_SYSUNINIT(vnet_ovpn_uninit, SI_SUB_PSEUDO, SI_ORDER_ANY, 2558 vnet_ovpn_uninit, NULL); 2559 2560static int 2561ovpnmodevent(module_t mod, int type, void *data) 2562{ 2563 switch (type) { 2564 case MOD_LOAD: 2565 /* Done in vnet_ovpn_init() */ 2566 break; 2567 case MOD_UNLOAD: 2568 /* Done in vnet_ovpn_uninit() */ 2569 break; 2570 default: 2571 return (EOPNOTSUPP); 2572 } 2573 2574 return (0); 2575} 2576 2577static moduledata_t ovpn_mod = { 2578 "if_ovpn", 2579 ovpnmodevent, 2580 0 2581}; 2582 2583DECLARE_MODULE(if_ovpn, ovpn_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 2584MODULE_VERSION(if_ovpn, 1); 2585