1195699Srwatson/*- 2196019Srwatson * Copyright (c) 2004-2009 University of Zagreb 3196019Srwatson * Copyright (c) 2006-2009 FreeBSD Foundation 4196019Srwatson * All rights reserved. 5196019Srwatson * 6196019Srwatson * This software was developed by the University of Zagreb and the 7196019Srwatson * FreeBSD Foundation under sponsorship by the Stichting NLnet and the 8196019Srwatson * FreeBSD Foundation. 9196019Srwatson * 10195699Srwatson * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> 11195699Srwatson * Copyright (c) 2009 Robert N. M. Watson 12195699Srwatson * All rights reserved. 13195699Srwatson * 14195699Srwatson * Redistribution and use in source and binary forms, with or without 15195699Srwatson * modification, are permitted provided that the following conditions 16195699Srwatson * are met: 17195699Srwatson * 1. Redistributions of source code must retain the above copyright 18195699Srwatson * notice, this list of conditions and the following disclaimer. 19195699Srwatson * 2. Redistributions in binary form must reproduce the above copyright 20195699Srwatson * notice, this list of conditions and the following disclaimer in the 21195699Srwatson * documentation and/or other materials provided with the distribution. 22195699Srwatson * 23195699Srwatson * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24195699Srwatson * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25195699Srwatson * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26195699Srwatson * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27195699Srwatson * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28195699Srwatson * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29195699Srwatson * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30195699Srwatson * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31195699Srwatson * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32195699Srwatson * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33195699Srwatson * SUCH DAMAGE. 34195699Srwatson */ 35195699Srwatson 36195699Srwatson#include <sys/cdefs.h> 37195699Srwatson__FBSDID("$FreeBSD$"); 38195699Srwatson 39196019Srwatson#include "opt_ddb.h" 40203483Szec#include "opt_kdb.h" 41203727Sbz#include "opt_kdtrace.h" 42196019Srwatson 43195699Srwatson#include <sys/param.h> 44203483Szec#include <sys/kdb.h> 45195699Srwatson#include <sys/kernel.h> 46196019Srwatson#include <sys/jail.h> 47203727Sbz#include <sys/sdt.h> 48195699Srwatson#include <sys/systm.h> 49195699Srwatson#include <sys/sysctl.h> 50205345Sbz#include <sys/eventhandler.h> 51195699Srwatson#include <sys/lock.h> 52195699Srwatson#include <sys/malloc.h> 53195699Srwatson#include <sys/proc.h> 54196019Srwatson#include <sys/socket.h> 55195699Srwatson#include <sys/sx.h> 56195699Srwatson#include <sys/sysctl.h> 57195699Srwatson 58205345Sbz#include <machine/stdarg.h> 59205345Sbz 60196019Srwatson#ifdef DDB 61196019Srwatson#include <ddb/ddb.h> 62203729Sbz#include <ddb/db_sym.h> 63196019Srwatson#endif 64196019Srwatson 65196019Srwatson#include <net/if.h> 66196019Srwatson#include <net/if_var.h> 67195699Srwatson#include <net/vnet.h> 68195699Srwatson 69195699Srwatson/*- 70195972Srwatson * This file implements core functions for virtual network stacks: 71195972Srwatson * 72196025Srwatson * - Virtual network stack management functions. 73196019Srwatson * 74196025Srwatson * - Virtual network stack memory allocator, which virtualizes global 75195972Srwatson * variables in the network stack 76195972Srwatson * 77195972Srwatson * - Virtualized SYSINIT's/SYSUNINIT's, which allow network stack subsystems 78195972Srwatson * to register startup/shutdown events to be run for each virtual network 79195972Srwatson * stack instance. 80196019Srwatson */ 81196019Srwatson 82217203SbzFEATURE(vimage, "VIMAGE kernel virtualization"); 83217203Sbz 84249132Smavstatic MALLOC_DEFINE(M_VNET, "vnet", "network stack control block"); 85196019Srwatson 86196019Srwatson/* 87196019Srwatson * The virtual network stack list has two read-write locks, one sleepable and 88196019Srwatson * the other not, so that the list can be stablized and walked in a variety 89196019Srwatson * of network stack contexts. Both must be acquired exclusively to modify 90196025Srwatson * the list, but a read lock of either lock is sufficient to walk the list. 91196019Srwatson */ 92196019Srwatsonstruct rwlock vnet_rwlock; 93196019Srwatsonstruct sx vnet_sxlock; 94196019Srwatson 95196019Srwatson#define VNET_LIST_WLOCK() do { \ 96196019Srwatson sx_xlock(&vnet_sxlock); \ 97196019Srwatson rw_wlock(&vnet_rwlock); \ 98196019Srwatson} while (0) 99196019Srwatson 100196019Srwatson#define VNET_LIST_WUNLOCK() do { \ 101196019Srwatson rw_wunlock(&vnet_rwlock); \ 102196019Srwatson sx_xunlock(&vnet_sxlock); \ 103196019Srwatson} while (0) 104196019Srwatson 105196019Srwatsonstruct vnet_list_head vnet_head; 106196019Srwatsonstruct vnet *vnet0; 107196019Srwatson 108196019Srwatson/* 109195972Srwatson * The virtual network stack allocator provides storage for virtualized 110195972Srwatson * global variables. These variables are defined/declared using the 111195972Srwatson * VNET_DEFINE()/VNET_DECLARE() macros, which place them in the 'set_vnet' 112195972Srwatson * linker set. The details of the implementation are somewhat subtle, but 113195972Srwatson * allow the majority of most network subsystems to maintain 114195699Srwatson * virtualization-agnostic. 115195699Srwatson * 116195699Srwatson * The virtual network stack allocator handles variables in the base kernel 117195699Srwatson * vs. modules in similar but different ways. In both cases, virtualized 118195699Srwatson * global variables are marked as such by being declared to be part of the 119195699Srwatson * vnet linker set. These "master" copies of global variables serve two 120195699Srwatson * functions: 121195699Srwatson * 122195699Srwatson * (1) They contain static initialization or "default" values for global 123195699Srwatson * variables which will be propagated to each virtual network stack 124195699Srwatson * instance when created. As with normal global variables, they default 125195699Srwatson * to zero-filled. 126195699Srwatson * 127195699Srwatson * (2) They act as unique global names by which the variable can be referred 128195699Srwatson * to, regardless of network stack instance. The single global symbol 129195699Srwatson * will be used to calculate the location of a per-virtual instance 130195699Srwatson * variable at run-time. 131195699Srwatson * 132195699Srwatson * Each virtual network stack instance has a complete copy of each 133195699Srwatson * virtualized global variable, stored in a malloc'd block of memory 134195699Srwatson * referred to by vnet->vnet_data_mem. Critical to the design is that each 135195699Srwatson * per-instance memory block is laid out identically to the master block so 136195699Srwatson * that the offset of each global variable is the same across all blocks. To 137195699Srwatson * optimize run-time access, a precalculated 'base' address, 138195699Srwatson * vnet->vnet_data_base, is stored in each vnet, and is the amount that can 139195699Srwatson * be added to the address of a 'master' instance of a variable to get to the 140195699Srwatson * per-vnet instance. 141195699Srwatson * 142195699Srwatson * Virtualized global variables are handled in a similar manner, but as each 143195699Srwatson * module has its own 'set_vnet' linker set, and we want to keep all 144195699Srwatson * virtualized globals togther, we reserve space in the kernel's linker set 145195699Srwatson * for potential module variables using a per-vnet character array, 146195699Srwatson * 'modspace'. The virtual network stack allocator maintains a free list to 147195699Srwatson * track what space in the array is free (all, initially) and as modules are 148195699Srwatson * linked, allocates portions of the space to specific globals. The kernel 149195699Srwatson * module linker queries the virtual network stack allocator and will 150195699Srwatson * bind references of the global to the location during linking. It also 151195699Srwatson * calls into the virtual network stack allocator, once the memory is 152195699Srwatson * initialized, in order to propagate the new static initializations to all 153195699Srwatson * existing virtual network stack instances so that the soon-to-be executing 154195699Srwatson * module will find every network stack instance with proper default values. 155195699Srwatson */ 156195699Srwatson 157195699Srwatson/* 158195699Srwatson * Number of bytes of data in the 'set_vnet' linker set, and hence the total 159195699Srwatson * size of all kernel virtualized global variables, and the malloc(9) type 160195699Srwatson * that will be used to allocate it. 161195699Srwatson */ 162195699Srwatson#define VNET_BYTES (VNET_STOP - VNET_START) 163195699Srwatson 164249132Smavstatic MALLOC_DEFINE(M_VNET_DATA, "vnet_data", "VNET data"); 165195699Srwatson 166195699Srwatson/* 167195699Srwatson * VNET_MODMIN is the minimum number of bytes we will reserve for the sum of 168195699Srwatson * global variables across all loaded modules. As this actually sizes an 169195699Srwatson * array declared as a virtualized global variable in the kernel itself, and 170195699Srwatson * we want the virtualized global variable space to be page-sized, we may 171195699Srwatson * have more space than that in practice. 172195699Srwatson */ 173195699Srwatson#define VNET_MODMIN 8192 174195699Srwatson#define VNET_SIZE roundup2(VNET_BYTES, PAGE_SIZE) 175195699Srwatson#define VNET_MODSIZE (VNET_SIZE - (VNET_BYTES - VNET_MODMIN)) 176195699Srwatson 177195699Srwatson/* 178195699Srwatson * Space to store virtualized global variables from loadable kernel modules, 179195699Srwatson * and the free list to manage it. 180195699Srwatson */ 181215701Sdimstatic VNET_DEFINE(char, modspace[VNET_MODMIN]); 182195699Srwatson 183195837Srwatson/* 184196025Srwatson * Global lists of subsystem constructor and destructors for vnets. They are 185196633Szec * registered via VNET_SYSINIT() and VNET_SYSUNINIT(). Both lists are 186196633Szec * protected by the vnet_sysinit_sxlock global lock. 187195837Srwatson */ 188195837Srwatsonstatic TAILQ_HEAD(vnet_sysinit_head, vnet_sysinit) vnet_constructors = 189195837Srwatson TAILQ_HEAD_INITIALIZER(vnet_constructors); 190195837Srwatsonstatic TAILQ_HEAD(vnet_sysuninit_head, vnet_sysinit) vnet_destructors = 191195837Srwatson TAILQ_HEAD_INITIALIZER(vnet_destructors); 192195837Srwatson 193196633Szecstruct sx vnet_sysinit_sxlock; 194196633Szec 195196633Szec#define VNET_SYSINIT_WLOCK() sx_xlock(&vnet_sysinit_sxlock); 196196633Szec#define VNET_SYSINIT_WUNLOCK() sx_xunlock(&vnet_sysinit_sxlock); 197196633Szec#define VNET_SYSINIT_RLOCK() sx_slock(&vnet_sysinit_sxlock); 198196633Szec#define VNET_SYSINIT_RUNLOCK() sx_sunlock(&vnet_sysinit_sxlock); 199196633Szec 200195699Srwatsonstruct vnet_data_free { 201195699Srwatson uintptr_t vnd_start; 202195699Srwatson int vnd_len; 203195699Srwatson TAILQ_ENTRY(vnet_data_free) vnd_link; 204195699Srwatson}; 205195699Srwatson 206249132Smavstatic MALLOC_DEFINE(M_VNET_DATA_FREE, "vnet_data_free", 207249132Smav "VNET resource accounting"); 208195699Srwatsonstatic TAILQ_HEAD(, vnet_data_free) vnet_data_free_head = 209195699Srwatson TAILQ_HEAD_INITIALIZER(vnet_data_free_head); 210195699Srwatsonstatic struct sx vnet_data_free_lock; 211195699Srwatson 212203727SbzSDT_PROVIDER_DEFINE(vnet); 213262057SavgSDT_PROBE_DEFINE1(vnet, functions, vnet_alloc, entry, "int"); 214262057SavgSDT_PROBE_DEFINE2(vnet, functions, vnet_alloc, alloc, "int", 215211616Srpaulo "struct vnet *"); 216262057SavgSDT_PROBE_DEFINE2(vnet, functions, vnet_alloc, return, 217211616Srpaulo "int", "struct vnet *"); 218262057SavgSDT_PROBE_DEFINE2(vnet, functions, vnet_destroy, entry, 219211616Srpaulo "int", "struct vnet *"); 220262057SavgSDT_PROBE_DEFINE1(vnet, functions, vnet_destroy, return, 221211616Srpaulo "int"); 222203727Sbz 223203729Sbz#ifdef DDB 224203729Sbzstatic void db_show_vnet_print_vs(struct vnet_sysinit *, int); 225203729Sbz#endif 226203729Sbz 227195699Srwatson/* 228196019Srwatson * Allocate a virtual network stack. 229196019Srwatson */ 230196019Srwatsonstruct vnet * 231196019Srwatsonvnet_alloc(void) 232196019Srwatson{ 233196019Srwatson struct vnet *vnet; 234196019Srwatson 235203727Sbz SDT_PROBE1(vnet, functions, vnet_alloc, entry, __LINE__); 236196019Srwatson vnet = malloc(sizeof(struct vnet), M_VNET, M_WAITOK | M_ZERO); 237196019Srwatson vnet->vnet_magic_n = VNET_MAGIC_N; 238203727Sbz SDT_PROBE2(vnet, functions, vnet_alloc, alloc, __LINE__, vnet); 239196019Srwatson 240196024Srwatson /* 241196024Srwatson * Allocate storage for virtualized global variables and copy in 242196024Srwatson * initial values form our 'master' copy. 243196024Srwatson */ 244196024Srwatson vnet->vnet_data_mem = malloc(VNET_SIZE, M_VNET_DATA, M_WAITOK); 245196024Srwatson memcpy(vnet->vnet_data_mem, (void *)VNET_START, VNET_BYTES); 246196024Srwatson 247196024Srwatson /* 248196024Srwatson * All use of vnet-specific data will immediately subtract VNET_START 249196024Srwatson * from the base memory pointer, so pre-calculate that now to avoid 250196024Srwatson * it on each use. 251196024Srwatson */ 252196024Srwatson vnet->vnet_data_base = (uintptr_t)vnet->vnet_data_mem - VNET_START; 253196024Srwatson 254196019Srwatson /* Initialize / attach vnet module instances. */ 255196019Srwatson CURVNET_SET_QUIET(vnet); 256196019Srwatson vnet_sysinit(); 257196019Srwatson CURVNET_RESTORE(); 258196019Srwatson 259196633Szec VNET_LIST_WLOCK(); 260196019Srwatson LIST_INSERT_HEAD(&vnet_head, vnet, vnet_le); 261196019Srwatson VNET_LIST_WUNLOCK(); 262196019Srwatson 263203727Sbz SDT_PROBE2(vnet, functions, vnet_alloc, return, __LINE__, vnet); 264196019Srwatson return (vnet); 265196019Srwatson} 266196019Srwatson 267196019Srwatson/* 268196019Srwatson * Destroy a virtual network stack. 269196019Srwatson */ 270196019Srwatsonvoid 271196019Srwatsonvnet_destroy(struct vnet *vnet) 272196019Srwatson{ 273196019Srwatson struct ifnet *ifp, *nifp; 274196019Srwatson 275203727Sbz SDT_PROBE2(vnet, functions, vnet_destroy, entry, __LINE__, vnet); 276196019Srwatson KASSERT(vnet->vnet_sockcnt == 0, 277196019Srwatson ("%s: vnet still has sockets", __func__)); 278196019Srwatson 279196019Srwatson VNET_LIST_WLOCK(); 280196019Srwatson LIST_REMOVE(vnet, vnet_le); 281196633Szec VNET_LIST_WUNLOCK(); 282196019Srwatson 283196019Srwatson CURVNET_SET_QUIET(vnet); 284196019Srwatson 285196019Srwatson /* Return all inherited interfaces to their parent vnets. */ 286196019Srwatson TAILQ_FOREACH_SAFE(ifp, &V_ifnet, if_link, nifp) { 287196019Srwatson if (ifp->if_home_vnet != ifp->if_vnet) 288196019Srwatson if_vmove(ifp, ifp->if_home_vnet); 289196019Srwatson } 290196019Srwatson 291196019Srwatson vnet_sysuninit(); 292196019Srwatson CURVNET_RESTORE(); 293196019Srwatson 294196024Srwatson /* 295196024Srwatson * Release storage for the virtual network stack instance. 296196024Srwatson */ 297196024Srwatson free(vnet->vnet_data_mem, M_VNET_DATA); 298196024Srwatson vnet->vnet_data_mem = NULL; 299196024Srwatson vnet->vnet_data_base = 0; 300196019Srwatson vnet->vnet_magic_n = 0xdeadbeef; 301196019Srwatson free(vnet, M_VNET); 302203727Sbz SDT_PROBE1(vnet, functions, vnet_destroy, return, __LINE__); 303196019Srwatson} 304196019Srwatson 305196019Srwatson/* 306196019Srwatson * Boot time initialization and allocation of virtual network stacks. 307196019Srwatson */ 308196019Srwatsonstatic void 309196019Srwatsonvnet_init_prelink(void *arg) 310196019Srwatson{ 311196019Srwatson 312196019Srwatson rw_init(&vnet_rwlock, "vnet_rwlock"); 313196019Srwatson sx_init(&vnet_sxlock, "vnet_sxlock"); 314196633Szec sx_init(&vnet_sysinit_sxlock, "vnet_sysinit_sxlock"); 315196019Srwatson LIST_INIT(&vnet_head); 316196019Srwatson} 317196019SrwatsonSYSINIT(vnet_init_prelink, SI_SUB_VNET_PRELINK, SI_ORDER_FIRST, 318196019Srwatson vnet_init_prelink, NULL); 319196019Srwatson 320196019Srwatsonstatic void 321196019Srwatsonvnet0_init(void *arg) 322196019Srwatson{ 323196019Srwatson 324196026Srwatson /* Warn people before take off - in case we crash early. */ 325196026Srwatson printf("WARNING: VIMAGE (virtualized network stack) is a highly " 326196026Srwatson "experimental feature.\n"); 327196026Srwatson 328196019Srwatson /* 329196019Srwatson * We MUST clear curvnet in vi_init_done() before going SMP, 330196019Srwatson * otherwise CURVNET_SET() macros would scream about unnecessary 331196019Srwatson * curvnet recursions. 332196019Srwatson */ 333196019Srwatson curvnet = prison0.pr_vnet = vnet0 = vnet_alloc(); 334196019Srwatson} 335196019SrwatsonSYSINIT(vnet0_init, SI_SUB_VNET, SI_ORDER_FIRST, vnet0_init, NULL); 336196019Srwatson 337196019Srwatsonstatic void 338196019Srwatsonvnet_init_done(void *unused) 339196019Srwatson{ 340196019Srwatson 341196019Srwatson curvnet = NULL; 342196019Srwatson} 343196019Srwatson 344196019SrwatsonSYSINIT(vnet_init_done, SI_SUB_VNET_DONE, SI_ORDER_FIRST, vnet_init_done, 345196019Srwatson NULL); 346196019Srwatson 347196019Srwatson/* 348195699Srwatson * Once on boot, initialize the modspace freelist to entirely cover modspace. 349195699Srwatson */ 350195699Srwatsonstatic void 351195699Srwatsonvnet_data_startup(void *dummy __unused) 352195699Srwatson{ 353195699Srwatson struct vnet_data_free *df; 354195699Srwatson 355195699Srwatson df = malloc(sizeof(*df), M_VNET_DATA_FREE, M_WAITOK | M_ZERO); 356195699Srwatson df->vnd_start = (uintptr_t)&VNET_NAME(modspace); 357208100Sbz df->vnd_len = VNET_MODMIN; 358195699Srwatson TAILQ_INSERT_HEAD(&vnet_data_free_head, df, vnd_link); 359195699Srwatson sx_init(&vnet_data_free_lock, "vnet_data alloc lock"); 360195699Srwatson} 361195699SrwatsonSYSINIT(vnet_data, SI_SUB_KLD, SI_ORDER_FIRST, vnet_data_startup, 0); 362195699Srwatson 363195699Srwatson/* 364195699Srwatson * When a module is loaded and requires storage for a virtualized global 365195699Srwatson * variable, allocate space from the modspace free list. This interface 366195699Srwatson * should be used only by the kernel linker. 367195699Srwatson */ 368195699Srwatsonvoid * 369195699Srwatsonvnet_data_alloc(int size) 370195699Srwatson{ 371195699Srwatson struct vnet_data_free *df; 372195699Srwatson void *s; 373195699Srwatson 374195699Srwatson s = NULL; 375195699Srwatson size = roundup2(size, sizeof(void *)); 376195699Srwatson sx_xlock(&vnet_data_free_lock); 377195699Srwatson TAILQ_FOREACH(df, &vnet_data_free_head, vnd_link) { 378195699Srwatson if (df->vnd_len < size) 379195699Srwatson continue; 380195699Srwatson if (df->vnd_len == size) { 381195699Srwatson s = (void *)df->vnd_start; 382195699Srwatson TAILQ_REMOVE(&vnet_data_free_head, df, vnd_link); 383195699Srwatson free(df, M_VNET_DATA_FREE); 384195699Srwatson break; 385195699Srwatson } 386195699Srwatson s = (void *)df->vnd_start; 387195699Srwatson df->vnd_len -= size; 388195699Srwatson df->vnd_start = df->vnd_start + size; 389195699Srwatson break; 390195699Srwatson } 391195699Srwatson sx_xunlock(&vnet_data_free_lock); 392195699Srwatson 393195699Srwatson return (s); 394195699Srwatson} 395195699Srwatson 396195699Srwatson/* 397195699Srwatson * Free space for a virtualized global variable on module unload. 398195699Srwatson */ 399195699Srwatsonvoid 400195699Srwatsonvnet_data_free(void *start_arg, int size) 401195699Srwatson{ 402195699Srwatson struct vnet_data_free *df; 403195699Srwatson struct vnet_data_free *dn; 404195699Srwatson uintptr_t start; 405195699Srwatson uintptr_t end; 406195699Srwatson 407195699Srwatson size = roundup2(size, sizeof(void *)); 408195699Srwatson start = (uintptr_t)start_arg; 409195699Srwatson end = start + size; 410195699Srwatson /* 411195699Srwatson * Free a region of space and merge it with as many neighbors as 412195699Srwatson * possible. Keeping the list sorted simplifies this operation. 413195699Srwatson */ 414195699Srwatson sx_xlock(&vnet_data_free_lock); 415195699Srwatson TAILQ_FOREACH(df, &vnet_data_free_head, vnd_link) { 416195699Srwatson if (df->vnd_start > end) 417195699Srwatson break; 418195699Srwatson /* 419196025Srwatson * If we expand at the end of an entry we may have to merge 420196025Srwatson * it with the one following it as well. 421195699Srwatson */ 422195699Srwatson if (df->vnd_start + df->vnd_len == start) { 423195699Srwatson df->vnd_len += size; 424195699Srwatson dn = TAILQ_NEXT(df, vnd_link); 425195699Srwatson if (df->vnd_start + df->vnd_len == dn->vnd_start) { 426195699Srwatson df->vnd_len += dn->vnd_len; 427196025Srwatson TAILQ_REMOVE(&vnet_data_free_head, dn, 428196025Srwatson vnd_link); 429195699Srwatson free(dn, M_VNET_DATA_FREE); 430195699Srwatson } 431195699Srwatson sx_xunlock(&vnet_data_free_lock); 432195699Srwatson return; 433195699Srwatson } 434195699Srwatson if (df->vnd_start == end) { 435195699Srwatson df->vnd_start = start; 436195699Srwatson df->vnd_len += size; 437195699Srwatson sx_xunlock(&vnet_data_free_lock); 438195699Srwatson return; 439195699Srwatson } 440195699Srwatson } 441195699Srwatson dn = malloc(sizeof(*df), M_VNET_DATA_FREE, M_WAITOK | M_ZERO); 442195699Srwatson dn->vnd_start = start; 443195699Srwatson dn->vnd_len = size; 444195699Srwatson if (df) 445195699Srwatson TAILQ_INSERT_BEFORE(df, dn, vnd_link); 446195699Srwatson else 447195699Srwatson TAILQ_INSERT_TAIL(&vnet_data_free_head, dn, vnd_link); 448195699Srwatson sx_xunlock(&vnet_data_free_lock); 449195699Srwatson} 450195699Srwatson 451195699Srwatson/* 452195699Srwatson * When a new virtualized global variable has been allocated, propagate its 453195699Srwatson * initial value to each already-allocated virtual network stack instance. 454195699Srwatson */ 455195699Srwatsonvoid 456195699Srwatsonvnet_data_copy(void *start, int size) 457195699Srwatson{ 458196020Srwatson struct vnet *vnet; 459195699Srwatson 460196020Srwatson VNET_LIST_RLOCK(); 461196020Srwatson LIST_FOREACH(vnet, &vnet_head, vnet_le) 462196020Srwatson memcpy((void *)((uintptr_t)vnet->vnet_data_base + 463196020Srwatson (uintptr_t)start), start, size); 464196020Srwatson VNET_LIST_RUNLOCK(); 465195699Srwatson} 466195699Srwatson 467195699Srwatson/* 468195699Srwatson * Variants on sysctl_handle_foo that know how to handle virtualized global 469195699Srwatson * variables: if 'arg1' is a pointer, then we transform it to the local vnet 470195699Srwatson * offset. 471195699Srwatson */ 472195699Srwatsonint 473195699Srwatsonvnet_sysctl_handle_int(SYSCTL_HANDLER_ARGS) 474195699Srwatson{ 475195699Srwatson 476195699Srwatson if (arg1 != NULL) 477195699Srwatson arg1 = (void *)(curvnet->vnet_data_base + (uintptr_t)arg1); 478195699Srwatson return (sysctl_handle_int(oidp, arg1, arg2, req)); 479195699Srwatson} 480195699Srwatson 481195699Srwatsonint 482195699Srwatsonvnet_sysctl_handle_opaque(SYSCTL_HANDLER_ARGS) 483195699Srwatson{ 484195699Srwatson 485195699Srwatson if (arg1 != NULL) 486195699Srwatson arg1 = (void *)(curvnet->vnet_data_base + (uintptr_t)arg1); 487195699Srwatson return (sysctl_handle_opaque(oidp, arg1, arg2, req)); 488195699Srwatson} 489195699Srwatson 490195699Srwatsonint 491195699Srwatsonvnet_sysctl_handle_string(SYSCTL_HANDLER_ARGS) 492195699Srwatson{ 493195699Srwatson 494195699Srwatson if (arg1 != NULL) 495195699Srwatson arg1 = (void *)(curvnet->vnet_data_base + (uintptr_t)arg1); 496195699Srwatson return (sysctl_handle_string(oidp, arg1, arg2, req)); 497195699Srwatson} 498195699Srwatson 499195699Srwatsonint 500195699Srwatsonvnet_sysctl_handle_uint(SYSCTL_HANDLER_ARGS) 501195699Srwatson{ 502195699Srwatson 503195699Srwatson if (arg1 != NULL) 504195699Srwatson arg1 = (void *)(curvnet->vnet_data_base + (uintptr_t)arg1); 505195699Srwatson return (sysctl_handle_int(oidp, arg1, arg2, req)); 506195699Srwatson} 507195837Srwatson 508195837Srwatson/* 509195837Srwatson * Support for special SYSINIT handlers registered via VNET_SYSINIT() 510195837Srwatson * and VNET_SYSUNINIT(). 511195837Srwatson */ 512195837Srwatsonvoid 513195837Srwatsonvnet_register_sysinit(void *arg) 514195837Srwatson{ 515195837Srwatson struct vnet_sysinit *vs, *vs2; 516195837Srwatson struct vnet *vnet; 517195837Srwatson 518195837Srwatson vs = arg; 519195837Srwatson KASSERT(vs->subsystem > SI_SUB_VNET, ("vnet sysinit too early")); 520195837Srwatson 521195837Srwatson /* Add the constructor to the global list of vnet constructors. */ 522196633Szec VNET_SYSINIT_WLOCK(); 523195837Srwatson TAILQ_FOREACH(vs2, &vnet_constructors, link) { 524195837Srwatson if (vs2->subsystem > vs->subsystem) 525195837Srwatson break; 526195837Srwatson if (vs2->subsystem == vs->subsystem && vs2->order > vs->order) 527195837Srwatson break; 528195837Srwatson } 529195837Srwatson if (vs2 != NULL) 530195837Srwatson TAILQ_INSERT_BEFORE(vs2, vs, link); 531195837Srwatson else 532195837Srwatson TAILQ_INSERT_TAIL(&vnet_constructors, vs, link); 533195837Srwatson 534195837Srwatson /* 535195837Srwatson * Invoke the constructor on all the existing vnets when it is 536195837Srwatson * registered. 537195837Srwatson */ 538195837Srwatson VNET_FOREACH(vnet) { 539195837Srwatson CURVNET_SET_QUIET(vnet); 540195837Srwatson vs->func(vs->arg); 541195837Srwatson CURVNET_RESTORE(); 542195837Srwatson } 543196633Szec VNET_SYSINIT_WUNLOCK(); 544195837Srwatson} 545195837Srwatson 546195837Srwatsonvoid 547195837Srwatsonvnet_deregister_sysinit(void *arg) 548195837Srwatson{ 549195837Srwatson struct vnet_sysinit *vs; 550195837Srwatson 551195837Srwatson vs = arg; 552195837Srwatson 553195837Srwatson /* Remove the constructor from the global list of vnet constructors. */ 554196633Szec VNET_SYSINIT_WLOCK(); 555195837Srwatson TAILQ_REMOVE(&vnet_constructors, vs, link); 556196633Szec VNET_SYSINIT_WUNLOCK(); 557195837Srwatson} 558195837Srwatson 559195837Srwatsonvoid 560195837Srwatsonvnet_register_sysuninit(void *arg) 561195837Srwatson{ 562195837Srwatson struct vnet_sysinit *vs, *vs2; 563195837Srwatson 564195837Srwatson vs = arg; 565195837Srwatson 566195837Srwatson /* Add the destructor to the global list of vnet destructors. */ 567196633Szec VNET_SYSINIT_WLOCK(); 568195837Srwatson TAILQ_FOREACH(vs2, &vnet_destructors, link) { 569195837Srwatson if (vs2->subsystem > vs->subsystem) 570195837Srwatson break; 571195837Srwatson if (vs2->subsystem == vs->subsystem && vs2->order > vs->order) 572195837Srwatson break; 573195837Srwatson } 574195837Srwatson if (vs2 != NULL) 575195837Srwatson TAILQ_INSERT_BEFORE(vs2, vs, link); 576195837Srwatson else 577195837Srwatson TAILQ_INSERT_TAIL(&vnet_destructors, vs, link); 578196633Szec VNET_SYSINIT_WUNLOCK(); 579195837Srwatson} 580195837Srwatson 581195837Srwatsonvoid 582195837Srwatsonvnet_deregister_sysuninit(void *arg) 583195837Srwatson{ 584195837Srwatson struct vnet_sysinit *vs; 585195837Srwatson struct vnet *vnet; 586195837Srwatson 587195837Srwatson vs = arg; 588195837Srwatson 589195837Srwatson /* 590195837Srwatson * Invoke the destructor on all the existing vnets when it is 591195837Srwatson * deregistered. 592195837Srwatson */ 593196633Szec VNET_SYSINIT_WLOCK(); 594195837Srwatson VNET_FOREACH(vnet) { 595195837Srwatson CURVNET_SET_QUIET(vnet); 596195837Srwatson vs->func(vs->arg); 597195837Srwatson CURVNET_RESTORE(); 598195837Srwatson } 599195837Srwatson 600195837Srwatson /* Remove the destructor from the global list of vnet destructors. */ 601195837Srwatson TAILQ_REMOVE(&vnet_destructors, vs, link); 602196633Szec VNET_SYSINIT_WUNLOCK(); 603195837Srwatson} 604195837Srwatson 605195837Srwatson/* 606196025Srwatson * Invoke all registered vnet constructors on the current vnet. Used during 607196025Srwatson * vnet construction. The caller is responsible for ensuring the new vnet is 608196633Szec * the current vnet and that the vnet_sysinit_sxlock lock is locked. 609195837Srwatson */ 610195837Srwatsonvoid 611195837Srwatsonvnet_sysinit(void) 612195837Srwatson{ 613195837Srwatson struct vnet_sysinit *vs; 614195837Srwatson 615196633Szec VNET_SYSINIT_RLOCK(); 616195837Srwatson TAILQ_FOREACH(vs, &vnet_constructors, link) { 617195837Srwatson vs->func(vs->arg); 618195837Srwatson } 619196633Szec VNET_SYSINIT_RUNLOCK(); 620195837Srwatson} 621195837Srwatson 622195837Srwatson/* 623196025Srwatson * Invoke all registered vnet destructors on the current vnet. Used during 624196025Srwatson * vnet destruction. The caller is responsible for ensuring the dying vnet 625196633Szec * the current vnet and that the vnet_sysinit_sxlock lock is locked. 626195837Srwatson */ 627195837Srwatsonvoid 628195837Srwatsonvnet_sysuninit(void) 629195837Srwatson{ 630195837Srwatson struct vnet_sysinit *vs; 631195837Srwatson 632196633Szec VNET_SYSINIT_RLOCK(); 633195837Srwatson TAILQ_FOREACH_REVERSE(vs, &vnet_destructors, vnet_sysuninit_head, 634195837Srwatson link) { 635195837Srwatson vs->func(vs->arg); 636195837Srwatson } 637196633Szec VNET_SYSINIT_RUNLOCK(); 638195837Srwatson} 639196019Srwatson 640205345Sbz/* 641205345Sbz * EVENTHANDLER(9) extensions. 642205345Sbz */ 643205345Sbz/* 644205345Sbz * Invoke the eventhandler function originally registered with the possibly 645205345Sbz * registered argument for all virtual network stack instances. 646205345Sbz * 647205345Sbz * This iterator can only be used for eventhandlers that do not take any 648205345Sbz * additional arguments, as we do ignore the variadic arguments from the 649205345Sbz * EVENTHANDLER_INVOKE() call. 650205345Sbz */ 651205345Sbzvoid 652205345Sbzvnet_global_eventhandler_iterator_func(void *arg, ...) 653205345Sbz{ 654205345Sbz VNET_ITERATOR_DECL(vnet_iter); 655205345Sbz struct eventhandler_entry_vimage *v_ee; 656205345Sbz 657205345Sbz /* 658205345Sbz * There is a bug here in that we should actually cast things to 659205345Sbz * (struct eventhandler_entry_ ## name *) but that's not easily 660205345Sbz * possible in here so just re-using the variadic version we 661205345Sbz * defined for the generic vimage case. 662205345Sbz */ 663205345Sbz v_ee = arg; 664205345Sbz VNET_LIST_RLOCK(); 665205345Sbz VNET_FOREACH(vnet_iter) { 666205345Sbz CURVNET_SET(vnet_iter); 667205345Sbz ((vimage_iterator_func_t)v_ee->func)(v_ee->ee_arg); 668205345Sbz CURVNET_RESTORE(); 669205345Sbz } 670205345Sbz VNET_LIST_RUNLOCK(); 671205345Sbz} 672205345Sbz 673203483Szec#ifdef VNET_DEBUG 674203483Szecstruct vnet_recursion { 675203483Szec SLIST_ENTRY(vnet_recursion) vnr_le; 676203483Szec const char *prev_fn; 677203483Szec const char *where_fn; 678203483Szec int where_line; 679203483Szec struct vnet *old_vnet; 680203483Szec struct vnet *new_vnet; 681203483Szec}; 682203483Szec 683203483Szecstatic SLIST_HEAD(, vnet_recursion) vnet_recursions = 684203483Szec SLIST_HEAD_INITIALIZER(vnet_recursions); 685203483Szec 686203483Szecstatic void 687203483Szecvnet_print_recursion(struct vnet_recursion *vnr, int brief) 688203483Szec{ 689203483Szec 690203483Szec if (!brief) 691203483Szec printf("CURVNET_SET() recursion in "); 692203483Szec printf("%s() line %d, prev in %s()", vnr->where_fn, vnr->where_line, 693203483Szec vnr->prev_fn); 694203483Szec if (brief) 695203483Szec printf(", "); 696203483Szec else 697203483Szec printf("\n "); 698203483Szec printf("%p -> %p\n", vnr->old_vnet, vnr->new_vnet); 699203483Szec} 700203483Szec 701203483Szecvoid 702203483Szecvnet_log_recursion(struct vnet *old_vnet, const char *old_fn, int line) 703203483Szec{ 704203483Szec struct vnet_recursion *vnr; 705203483Szec 706203483Szec /* Skip already logged recursion events. */ 707203483Szec SLIST_FOREACH(vnr, &vnet_recursions, vnr_le) 708203483Szec if (vnr->prev_fn == old_fn && 709203483Szec vnr->where_fn == curthread->td_vnet_lpush && 710203483Szec vnr->where_line == line && 711203483Szec (vnr->old_vnet == vnr->new_vnet) == (curvnet == old_vnet)) 712203483Szec return; 713203483Szec 714203483Szec vnr = malloc(sizeof(*vnr), M_VNET, M_NOWAIT | M_ZERO); 715203483Szec if (vnr == NULL) 716203483Szec panic("%s: malloc failed", __func__); 717203483Szec vnr->prev_fn = old_fn; 718203483Szec vnr->where_fn = curthread->td_vnet_lpush; 719203483Szec vnr->where_line = line; 720203483Szec vnr->old_vnet = old_vnet; 721203483Szec vnr->new_vnet = curvnet; 722203483Szec 723203483Szec SLIST_INSERT_HEAD(&vnet_recursions, vnr, vnr_le); 724203483Szec 725203483Szec vnet_print_recursion(vnr, 0); 726203483Szec#ifdef KDB 727203483Szec kdb_backtrace(); 728203483Szec#endif 729203483Szec} 730203483Szec#endif /* VNET_DEBUG */ 731203483Szec 732205345Sbz/* 733205345Sbz * DDB(4). 734205345Sbz */ 735196019Srwatson#ifdef DDB 736196019SrwatsonDB_SHOW_COMMAND(vnets, db_show_vnets) 737196019Srwatson{ 738196019Srwatson VNET_ITERATOR_DECL(vnet_iter); 739196019Srwatson 740196019Srwatson VNET_FOREACH(vnet_iter) { 741196129Sbz db_printf("vnet = %p\n", vnet_iter); 742196129Sbz db_printf(" vnet_magic_n = 0x%x (%s, orig 0x%x)\n", 743196129Sbz vnet_iter->vnet_magic_n, 744196129Sbz (vnet_iter->vnet_magic_n == VNET_MAGIC_N) ? 745196129Sbz "ok" : "mismatch", VNET_MAGIC_N); 746196129Sbz db_printf(" vnet_ifcnt = %u\n", vnet_iter->vnet_ifcnt); 747196129Sbz db_printf(" vnet_sockcnt = %u\n", vnet_iter->vnet_sockcnt); 748196129Sbz db_printf(" vnet_data_mem = %p\n", vnet_iter->vnet_data_mem); 749196129Sbz db_printf(" vnet_data_base = 0x%jx\n", 750196129Sbz (uintmax_t)vnet_iter->vnet_data_base); 751196129Sbz db_printf("\n"); 752196129Sbz if (db_pager_quit) 753196129Sbz break; 754196019Srwatson } 755196019Srwatson} 756203483Szec 757203729Sbzstatic void 758203729Sbzdb_show_vnet_print_vs(struct vnet_sysinit *vs, int ddb) 759203729Sbz{ 760203729Sbz const char *vsname, *funcname; 761203729Sbz c_db_sym_t sym; 762203729Sbz db_expr_t offset; 763203729Sbz 764203729Sbz#define xprint(...) \ 765203729Sbz if (ddb) \ 766203729Sbz db_printf(__VA_ARGS__); \ 767203729Sbz else \ 768203729Sbz printf(__VA_ARGS__) 769203729Sbz 770203729Sbz if (vs == NULL) { 771203729Sbz xprint("%s: no vnet_sysinit * given\n", __func__); 772203729Sbz return; 773203729Sbz } 774203729Sbz 775203729Sbz sym = db_search_symbol((vm_offset_t)vs, DB_STGY_ANY, &offset); 776203729Sbz db_symbol_values(sym, &vsname, NULL); 777203729Sbz sym = db_search_symbol((vm_offset_t)vs->func, DB_STGY_PROC, &offset); 778203729Sbz db_symbol_values(sym, &funcname, NULL); 779203729Sbz xprint("%s(%p)\n", (vsname != NULL) ? vsname : "", vs); 780203729Sbz xprint(" 0x%08x 0x%08x\n", vs->subsystem, vs->order); 781203729Sbz xprint(" %p(%s)(%p)\n", 782203729Sbz vs->func, (funcname != NULL) ? funcname : "", vs->arg); 783203729Sbz#undef xprint 784203729Sbz} 785203729Sbz 786203729SbzDB_SHOW_COMMAND(vnet_sysinit, db_show_vnet_sysinit) 787203729Sbz{ 788203729Sbz struct vnet_sysinit *vs; 789203729Sbz 790203729Sbz db_printf("VNET_SYSINIT vs Name(Ptr)\n"); 791203729Sbz db_printf(" Subsystem Order\n"); 792203729Sbz db_printf(" Function(Name)(Arg)\n"); 793203729Sbz TAILQ_FOREACH(vs, &vnet_constructors, link) { 794203729Sbz db_show_vnet_print_vs(vs, 1); 795203729Sbz if (db_pager_quit) 796203729Sbz break; 797203729Sbz } 798203729Sbz} 799203729Sbz 800203729SbzDB_SHOW_COMMAND(vnet_sysuninit, db_show_vnet_sysuninit) 801203729Sbz{ 802203729Sbz struct vnet_sysinit *vs; 803203729Sbz 804203729Sbz db_printf("VNET_SYSUNINIT vs Name(Ptr)\n"); 805203729Sbz db_printf(" Subsystem Order\n"); 806203729Sbz db_printf(" Function(Name)(Arg)\n"); 807203729Sbz TAILQ_FOREACH_REVERSE(vs, &vnet_destructors, vnet_sysuninit_head, 808203729Sbz link) { 809203729Sbz db_show_vnet_print_vs(vs, 1); 810203729Sbz if (db_pager_quit) 811203729Sbz break; 812203729Sbz } 813203729Sbz} 814203729Sbz 815203483Szec#ifdef VNET_DEBUG 816203483SzecDB_SHOW_COMMAND(vnetrcrs, db_show_vnetrcrs) 817203483Szec{ 818203483Szec struct vnet_recursion *vnr; 819203483Szec 820203483Szec SLIST_FOREACH(vnr, &vnet_recursions, vnr_le) 821203483Szec vnet_print_recursion(vnr, 1); 822203483Szec} 823196019Srwatson#endif 824203483Szec#endif /* DDB */ 825