kern_synch.c revision 314667
1131962Smp/*- 259243Sobrien * Copyright (c) 1982, 1986, 1990, 1991, 1993 359243Sobrien * The Regents of the University of California. All rights reserved. 459243Sobrien * (c) UNIX System Laboratories, Inc. 559243Sobrien * All or some portions of this file are derived from material licensed 659243Sobrien * to the University of California by American Telephone and Telegraph 759243Sobrien * Co. or Unix System Laboratories, Inc. and are reproduced herein with 859243Sobrien * the permission of UNIX System Laboratories, Inc. 959243Sobrien * 1059243Sobrien * Redistribution and use in source and binary forms, with or without 1159243Sobrien * modification, are permitted provided that the following conditions 1259243Sobrien * are met: 1359243Sobrien * 1. Redistributions of source code must retain the above copyright 1459243Sobrien * notice, this list of conditions and the following disclaimer. 1559243Sobrien * 2. Redistributions in binary form must reproduce the above copyright 1659243Sobrien * notice, this list of conditions and the following disclaimer in the 17100616Smp * documentation and/or other materials provided with the distribution. 1859243Sobrien * 4. Neither the name of the University nor the names of its contributors 1959243Sobrien * may be used to endorse or promote products derived from this software 2059243Sobrien * without specific prior written permission. 2159243Sobrien * 2259243Sobrien * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2359243Sobrien * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2459243Sobrien * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2559243Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2659243Sobrien * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2759243Sobrien * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2859243Sobrien * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2959243Sobrien * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3059243Sobrien * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3159243Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3259243Sobrien * SUCH DAMAGE. 3359243Sobrien * 3459243Sobrien * @(#)kern_synch.c 8.9 (Berkeley) 5/19/95 3559243Sobrien */ 36131962Smp 3769408Sache#include <sys/cdefs.h> 3859243Sobrien__FBSDID("$FreeBSD: stable/10/sys/kern/kern_synch.c 314667 2017-03-04 13:03:31Z avg $"); 3969408Sache 40131962Smp#include "opt_kdtrace.h" 4159243Sobrien#include "opt_ktrace.h" 4259243Sobrien#include "opt_sched.h" 4359243Sobrien 4459243Sobrien#include <sys/param.h> 4559243Sobrien#include <sys/systm.h> 4659243Sobrien#include <sys/condvar.h> 4759243Sobrien#include <sys/kdb.h> 4859243Sobrien#include <sys/kernel.h> 4959243Sobrien#include <sys/ktr.h> 5059243Sobrien#include <sys/lock.h> 5159243Sobrien#include <sys/mutex.h> 5269408Sache#include <sys/proc.h> 53131962Smp#include <sys/resourcevar.h> 54131962Smp#include <sys/sched.h> 5569408Sache#include <sys/sdt.h> 5669408Sache#include <sys/signalvar.h> 5769408Sache#include <sys/sleepqueue.h> 5859243Sobrien#include <sys/smp.h> 5959243Sobrien#include <sys/sx.h> 6059243Sobrien#include <sys/sysctl.h> 6159243Sobrien#include <sys/sysproto.h> 6259243Sobrien#include <sys/vmmeter.h> 6359243Sobrien#ifdef KTRACE 6459243Sobrien#include <sys/uio.h> 6559243Sobrien#include <sys/ktrace.h> 6659243Sobrien#endif 6759243Sobrien 6859243Sobrien#include <machine/cpu.h> 6959243Sobrien 7059243Sobrien#ifdef XEN 7159243Sobrien#include <vm/vm.h> 7259243Sobrien#include <vm/vm_param.h> 7359243Sobrien#include <vm/pmap.h> 7459243Sobrien#endif 7559243Sobrien 7659243Sobrien#define KTDSTATE(td) \ 7759243Sobrien (((td)->td_inhibitors & TDI_SLEEPING) != 0 ? "sleep" : \ 7859243Sobrien ((td)->td_inhibitors & TDI_SUSPENDED) != 0 ? "suspended" : \ 7959243Sobrien ((td)->td_inhibitors & TDI_SWAPPED) != 0 ? "swapped" : \ 8059243Sobrien ((td)->td_inhibitors & TDI_LOCK) != 0 ? "blocked" : \ 8159243Sobrien ((td)->td_inhibitors & TDI_IWAIT) != 0 ? "iwait" : "yielding") 8259243Sobrien 8359243Sobrienstatic void synch_setup(void *dummy); 8459243SobrienSYSINIT(synch_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, synch_setup, 8559243Sobrien NULL); 8659243Sobrien 8759243Sobrienint hogticks; 8859243Sobrienstatic uint8_t pause_wchan[MAXCPU]; 8969408Sache 9059243Sobrienstatic struct callout loadav_callout; 9169408Sache 9259243Sobrienstruct loadavg averunnable = 9359243Sobrien { {0, 0, 0}, FSCALE }; /* load average, of runnable procs */ 9459243Sobrien/* 9559243Sobrien * Constants for averages over 1, 5, and 15 minutes 9659243Sobrien * when sampling at 5 second intervals. 9759243Sobrien */ 9859243Sobrienstatic fixpt_t cexp[3] = { 9959243Sobrien 0.9200444146293232 * FSCALE, /* exp(-1/12) */ 10059243Sobrien 0.9834714538216174 * FSCALE, /* exp(-1/60) */ 10159243Sobrien 0.9944598480048967 * FSCALE, /* exp(-1/180) */ 10259243Sobrien}; 10359243Sobrien 10459243Sobrien/* kernel uses `FSCALE', userland (SHOULD) use kern.fscale */ 10559243SobrienSYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE, ""); 10659243Sobrien 10759243Sobrienstatic void loadav(void *arg); 10859243Sobrien 10959243SobrienSDT_PROVIDER_DECLARE(sched); 11059243SobrienSDT_PROBE_DEFINE(sched, , , preempt); 11159243Sobrien 11259243Sobrien/* 11359243Sobrien * These probes reference Solaris features that are not implemented in FreeBSD. 11459243Sobrien * Create the probes anyway for compatibility with existing D scripts; they'll 11559243Sobrien * just never fire. 11659243Sobrien */ 11759243SobrienSDT_PROBE_DEFINE(sched, , , cpucaps__sleep); 11859243SobrienSDT_PROBE_DEFINE(sched, , , cpucaps__wakeup); 11959243SobrienSDT_PROBE_DEFINE(sched, , , schedctl__nopreempt); 12059243SobrienSDT_PROBE_DEFINE(sched, , , schedctl__preempt); 12159243SobrienSDT_PROBE_DEFINE(sched, , , schedctl__yield); 12259243Sobrien 12359243Sobrienstatic void 12459243Sobriensleepinit(void *unused) 12559243Sobrien{ 12659243Sobrien 12759243Sobrien hogticks = (hz / 10) * 2; /* Default only. */ 12859243Sobrien init_sleepqueues(); 12959243Sobrien} 13059243Sobrien 13159243Sobrien/* 13259243Sobrien * vmem tries to lock the sleepq mutexes when free'ing kva, so make sure 13359243Sobrien * it is available. 13459243Sobrien */ 13559243SobrienSYSINIT(sleepinit, SI_SUB_KMEM, SI_ORDER_ANY, sleepinit, 0); 13659243Sobrien 13759243Sobrien/* 13859243Sobrien * General sleep call. Suspends the current thread until a wakeup is 13959243Sobrien * performed on the specified identifier. The thread will then be made 14059243Sobrien * runnable with the specified priority. Sleeps at most sbt units of time 14159243Sobrien * (0 means no timeout). If pri includes the PCATCH flag, let signals 14259243Sobrien * interrupt the sleep, otherwise ignore them while sleeping. Returns 0 if 14359243Sobrien * awakened, EWOULDBLOCK if the timeout expires. If PCATCH is set and a 14459243Sobrien * signal becomes pending, ERESTART is returned if the current system 14559243Sobrien * call should be restarted if possible, and EINTR is returned if the system 14659243Sobrien * call should be interrupted by the signal (return EINTR). 14759243Sobrien * 14859243Sobrien * The lock argument is unlocked before the caller is suspended, and 14959243Sobrien * re-locked before _sleep() returns. If priority includes the PDROP 15059243Sobrien * flag the lock is not re-locked before returning. 15159243Sobrien */ 15259243Sobrienint 15359243Sobrien_sleep(void *ident, struct lock_object *lock, int priority, 15459243Sobrien const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) 15559243Sobrien{ 15659243Sobrien struct thread *td; 15759243Sobrien struct proc *p; 15859243Sobrien struct lock_class *class; 15959243Sobrien uintptr_t lock_state; 16059243Sobrien int catch, pri, rval, sleepq_flags; 16159243Sobrien WITNESS_SAVE_DECL(lock_witness); 16259243Sobrien 16359243Sobrien td = curthread; 16459243Sobrien p = td->td_proc; 16559243Sobrien#ifdef KTRACE 16659243Sobrien if (KTRPOINT(td, KTR_CSW)) 16759243Sobrien ktrcsw(1, 0, wmesg); 16859243Sobrien#endif 16959243Sobrien WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 170100616Smp "Sleeping on \"%s\"", wmesg); 171100616Smp KASSERT(sbt != 0 || mtx_owned(&Giant) || lock != NULL, 172100616Smp ("sleeping without a lock")); 17359243Sobrien KASSERT(p != NULL, ("msleep1")); 17459243Sobrien KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep")); 17559243Sobrien if (priority & PDROP) 17659243Sobrien KASSERT(lock != NULL && lock != &Giant.lock_object, 17759243Sobrien ("PDROP requires a non-Giant lock")); 17859243Sobrien if (lock != NULL) 17959243Sobrien class = LOCK_CLASS(lock); 18059243Sobrien else 18159243Sobrien class = NULL; 18259243Sobrien 18359243Sobrien if (cold || SCHEDULER_STOPPED()) { 18459243Sobrien /* 18559243Sobrien * During autoconfiguration, just return; 18659243Sobrien * don't run any other threads or panic below, 18759243Sobrien * in case this is the idle thread and already asleep. 18859243Sobrien * XXX: this used to do "s = splhigh(); splx(safepri); 18959243Sobrien * splx(s);" to give interrupts a chance, but there is 19059243Sobrien * no way to give interrupts a chance now. 19159243Sobrien */ 19259243Sobrien if (lock != NULL && priority & PDROP) 19359243Sobrien class->lc_unlock(lock); 19459243Sobrien return (0); 19559243Sobrien } 19659243Sobrien catch = priority & PCATCH; 19759243Sobrien pri = priority & PRIMASK; 19859243Sobrien 19959243Sobrien KASSERT(!TD_ON_SLEEPQ(td), ("recursive sleep")); 20059243Sobrien 20159243Sobrien if ((uint8_t *)ident >= &pause_wchan[0] && 20259243Sobrien (uint8_t *)ident <= &pause_wchan[MAXCPU - 1]) 20359243Sobrien sleepq_flags = SLEEPQ_PAUSE; 20459243Sobrien else 20559243Sobrien sleepq_flags = SLEEPQ_SLEEP; 20659243Sobrien if (catch) 20759243Sobrien sleepq_flags |= SLEEPQ_INTERRUPTIBLE; 20859243Sobrien 20959243Sobrien sleepq_lock(ident); 21059243Sobrien CTR5(KTR_PROC, "sleep: thread %ld (pid %ld, %s) on %s (%p)", 21159243Sobrien td->td_tid, p->p_pid, td->td_name, wmesg, ident); 21259243Sobrien 21359243Sobrien if (lock == &Giant.lock_object) 21459243Sobrien mtx_assert(&Giant, MA_OWNED); 21559243Sobrien DROP_GIANT(); 21659243Sobrien if (lock != NULL && lock != &Giant.lock_object && 21759243Sobrien !(class->lc_flags & LC_SLEEPABLE)) { 21859243Sobrien WITNESS_SAVE(lock, lock_witness); 21959243Sobrien lock_state = class->lc_unlock(lock); 22059243Sobrien } else 22159243Sobrien /* GCC needs to follow the Yellow Brick Road */ 22259243Sobrien lock_state = -1; 22359243Sobrien 22459243Sobrien /* 22559243Sobrien * We put ourselves on the sleep queue and start our timeout 22659243Sobrien * before calling thread_suspend_check, as we could stop there, 22759243Sobrien * and a wakeup or a SIGCONT (or both) could occur while we were 22859243Sobrien * stopped without resuming us. Thus, we must be ready for sleep 22959243Sobrien * when cursig() is called. If the wakeup happens while we're 23059243Sobrien * stopped, then td will no longer be on a sleep queue upon 23159243Sobrien * return from cursig(). 23259243Sobrien */ 23359243Sobrien sleepq_add(ident, lock, wmesg, sleepq_flags, 0); 23459243Sobrien if (sbt != 0) 23559243Sobrien sleepq_set_timeout_sbt(ident, sbt, pr, flags); 23659243Sobrien if (lock != NULL && class->lc_flags & LC_SLEEPABLE) { 23759243Sobrien sleepq_release(ident); 23859243Sobrien WITNESS_SAVE(lock, lock_witness); 23959243Sobrien lock_state = class->lc_unlock(lock); 24059243Sobrien sleepq_lock(ident); 24159243Sobrien } 24259243Sobrien if (sbt != 0 && catch) 24369408Sache rval = sleepq_timedwait_sig(ident, pri); 24459243Sobrien else if (sbt != 0) 24559243Sobrien rval = sleepq_timedwait(ident, pri); 24659243Sobrien else if (catch) 24759243Sobrien rval = sleepq_wait_sig(ident, pri); 24859243Sobrien else { 24959243Sobrien sleepq_wait(ident, pri); 25059243Sobrien rval = 0; 25159243Sobrien } 25259243Sobrien#ifdef KTRACE 25369408Sache if (KTRPOINT(td, KTR_CSW)) 25459243Sobrien ktrcsw(0, 0, wmesg); 25569408Sache#endif 25659243Sobrien PICKUP_GIANT(); 25759243Sobrien if (lock != NULL && lock != &Giant.lock_object && !(priority & PDROP)) { 25859243Sobrien class->lc_lock(lock, lock_state); 25959243Sobrien WITNESS_RESTORE(lock, lock_witness); 26059243Sobrien } 26159243Sobrien return (rval); 26259243Sobrien} 26359243Sobrien 26459243Sobrienint 26559243Sobrienmsleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg, 26659243Sobrien sbintime_t sbt, sbintime_t pr, int flags) 26759243Sobrien{ 26859243Sobrien struct thread *td; 26959243Sobrien struct proc *p; 27059243Sobrien int rval; 27159243Sobrien WITNESS_SAVE_DECL(mtx); 27259243Sobrien 27359243Sobrien td = curthread; 27459243Sobrien p = td->td_proc; 27559243Sobrien KASSERT(mtx != NULL, ("sleeping without a mutex")); 27659243Sobrien KASSERT(p != NULL, ("msleep1")); 27759243Sobrien KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep")); 27859243Sobrien 27959243Sobrien if (cold || SCHEDULER_STOPPED()) { 28059243Sobrien /* 28159243Sobrien * During autoconfiguration, just return; 28259243Sobrien * don't run any other threads or panic below, 28359243Sobrien * in case this is the idle thread and already asleep. 28459243Sobrien * XXX: this used to do "s = splhigh(); splx(safepri); 28559243Sobrien * splx(s);" to give interrupts a chance, but there is 28659243Sobrien * no way to give interrupts a chance now. 28759243Sobrien */ 28859243Sobrien return (0); 28959243Sobrien } 29059243Sobrien 29159243Sobrien sleepq_lock(ident); 29259243Sobrien CTR5(KTR_PROC, "msleep_spin: thread %ld (pid %ld, %s) on %s (%p)", 29359243Sobrien td->td_tid, p->p_pid, td->td_name, wmesg, ident); 29459243Sobrien 29559243Sobrien DROP_GIANT(); 29659243Sobrien mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED); 29759243Sobrien WITNESS_SAVE(&mtx->lock_object, mtx); 29859243Sobrien mtx_unlock_spin(mtx); 29959243Sobrien 30059243Sobrien /* 30159243Sobrien * We put ourselves on the sleep queue and start our timeout. 30259243Sobrien */ 30359243Sobrien sleepq_add(ident, &mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0); 30459243Sobrien if (sbt != 0) 30559243Sobrien sleepq_set_timeout_sbt(ident, sbt, pr, flags); 30659243Sobrien 30759243Sobrien /* 30859243Sobrien * Can't call ktrace with any spin locks held so it can lock the 30959243Sobrien * ktrace_mtx lock, and WITNESS_WARN considers it an error to hold 31059243Sobrien * any spin lock. Thus, we have to drop the sleepq spin lock while 31159243Sobrien * we handle those requests. This is safe since we have placed our 31259243Sobrien * thread on the sleep queue already. 31359243Sobrien */ 31459243Sobrien#ifdef KTRACE 31559243Sobrien if (KTRPOINT(td, KTR_CSW)) { 31659243Sobrien sleepq_release(ident); 31759243Sobrien ktrcsw(1, 0, wmesg); 31859243Sobrien sleepq_lock(ident); 31959243Sobrien } 32059243Sobrien#endif 32159243Sobrien#ifdef WITNESS 32259243Sobrien sleepq_release(ident); 32359243Sobrien WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "Sleeping on \"%s\"", 32459243Sobrien wmesg); 32559243Sobrien sleepq_lock(ident); 32659243Sobrien#endif 32759243Sobrien if (sbt != 0) 32859243Sobrien rval = sleepq_timedwait(ident, 0); 32959243Sobrien else { 33059243Sobrien sleepq_wait(ident, 0); 33159243Sobrien rval = 0; 33259243Sobrien } 33359243Sobrien#ifdef KTRACE 33459243Sobrien if (KTRPOINT(td, KTR_CSW)) 33559243Sobrien ktrcsw(0, 0, wmesg); 33659243Sobrien#endif 33759243Sobrien PICKUP_GIANT(); 33859243Sobrien mtx_lock_spin(mtx); 33959243Sobrien WITNESS_RESTORE(&mtx->lock_object, mtx); 34059243Sobrien return (rval); 34159243Sobrien} 34259243Sobrien 34359243Sobrien/* 34459243Sobrien * pause() delays the calling thread by the given number of system ticks. 34559243Sobrien * During cold bootup, pause() uses the DELAY() function instead of 34659243Sobrien * the tsleep() function to do the waiting. The "timo" argument must be 34759243Sobrien * greater than or equal to zero. A "timo" value of zero is equivalent 34859243Sobrien * to a "timo" value of one. 34959243Sobrien */ 35059243Sobrienint 35159243Sobrienpause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags) 35259243Sobrien{ 35359243Sobrien KASSERT(sbt >= 0, ("pause: timeout must be >= 0")); 35459243Sobrien 35559243Sobrien /* silently convert invalid timeouts */ 35659243Sobrien if (sbt == 0) 35759243Sobrien sbt = tick_sbt; 35859243Sobrien 35959243Sobrien if (cold || kdb_active || SCHEDULER_STOPPED()) { 36059243Sobrien /* 36159243Sobrien * We delay one second at a time to avoid overflowing the 36259243Sobrien * system specific DELAY() function(s): 36359243Sobrien */ 36459243Sobrien while (sbt >= SBT_1S) { 36559243Sobrien DELAY(1000000); 36659243Sobrien sbt -= SBT_1S; 36759243Sobrien } 36859243Sobrien /* Do the delay remainder, if any */ 36959243Sobrien sbt = (sbt + SBT_1US - 1) / SBT_1US; 37059243Sobrien if (sbt > 0) 37159243Sobrien DELAY(sbt); 37259243Sobrien return (0); 37359243Sobrien } 37459243Sobrien return (_sleep(&pause_wchan[curcpu], NULL, 0, wmesg, sbt, pr, flags)); 37559243Sobrien} 37659243Sobrien 37759243Sobrien/* 37859243Sobrien * Make all threads sleeping on the specified identifier runnable. 37959243Sobrien */ 38059243Sobrienvoid 38159243Sobrienwakeup(void *ident) 38259243Sobrien{ 38359243Sobrien int wakeup_swapper; 38459243Sobrien 38559243Sobrien sleepq_lock(ident); 38659243Sobrien wakeup_swapper = sleepq_broadcast(ident, SLEEPQ_SLEEP, 0, 0); 38759243Sobrien sleepq_release(ident); 38859243Sobrien if (wakeup_swapper) { 38959243Sobrien KASSERT(ident != &proc0, 39059243Sobrien ("wakeup and wakeup_swapper and proc0")); 39159243Sobrien kick_proc0(); 39259243Sobrien } 39359243Sobrien} 39459243Sobrien 39559243Sobrien/* 39659243Sobrien * Make a thread sleeping on the specified identifier runnable. 39759243Sobrien * May wake more than one thread if a target thread is currently 39859243Sobrien * swapped out. 39959243Sobrien */ 40059243Sobrienvoid 40159243Sobrienwakeup_one(void *ident) 40259243Sobrien{ 40359243Sobrien int wakeup_swapper; 40459243Sobrien 40559243Sobrien sleepq_lock(ident); 40659243Sobrien wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); 40759243Sobrien sleepq_release(ident); 40859243Sobrien if (wakeup_swapper) 40959243Sobrien kick_proc0(); 41059243Sobrien} 41159243Sobrien 41259243Sobrienstatic void 41359243Sobrienkdb_switch(void) 41459243Sobrien{ 41559243Sobrien thread_unlock(curthread); 41659243Sobrien kdb_backtrace(); 41759243Sobrien kdb_reenter(); 41859243Sobrien panic("%s: did not reenter debugger", __func__); 41959243Sobrien} 42059243Sobrien 42159243Sobrien/* 42259243Sobrien * The machine independent parts of context switching. 42359243Sobrien */ 42459243Sobrienvoid 42559243Sobrienmi_switch(int flags, struct thread *newtd) 42659243Sobrien{ 42759243Sobrien uint64_t runtime, new_switchtime; 42859243Sobrien struct thread *td; 42959243Sobrien 43059243Sobrien td = curthread; /* XXX */ 43159243Sobrien THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED); 43259243Sobrien KASSERT(!TD_ON_RUNQ(td), ("mi_switch: called by old code")); 43359243Sobrien#ifdef INVARIANTS 43459243Sobrien if (!TD_ON_LOCK(td) && !TD_IS_RUNNING(td)) 43559243Sobrien mtx_assert(&Giant, MA_NOTOWNED); 43659243Sobrien#endif 43759243Sobrien KASSERT(td->td_critnest == 1 || panicstr, 43859243Sobrien ("mi_switch: switch in a critical section")); 43959243Sobrien KASSERT((flags & (SW_INVOL | SW_VOL)) != 0, 44059243Sobrien ("mi_switch: switch must be voluntary or involuntary")); 44159243Sobrien KASSERT(newtd != curthread, ("mi_switch: preempting back to ourself")); 44259243Sobrien 44359243Sobrien /* 44459243Sobrien * Don't perform context switches from the debugger. 44559243Sobrien */ 44659243Sobrien if (kdb_active) 44759243Sobrien kdb_switch(); 44859243Sobrien if (SCHEDULER_STOPPED()) 44959243Sobrien return; 45059243Sobrien if (flags & SW_VOL) { 45159243Sobrien td->td_ru.ru_nvcsw++; 45259243Sobrien td->td_swvoltick = ticks; 45359243Sobrien } else 45459243Sobrien td->td_ru.ru_nivcsw++; 45559243Sobrien#ifdef SCHED_STATS 45659243Sobrien SCHED_STAT_INC(sched_switch_stats[flags & SW_TYPE_MASK]); 45759243Sobrien#endif 45859243Sobrien /* 45959243Sobrien * Compute the amount of time during which the current 46059243Sobrien * thread was running, and add that to its total so far. 46159243Sobrien */ 46259243Sobrien new_switchtime = cpu_ticks(); 46359243Sobrien runtime = new_switchtime - PCPU_GET(switchtime); 46459243Sobrien td->td_runtime += runtime; 46559243Sobrien td->td_incruntime += runtime; 46659243Sobrien PCPU_SET(switchtime, new_switchtime); 46759243Sobrien td->td_generation++; /* bump preempt-detect counter */ 46859243Sobrien PCPU_INC(cnt.v_swtch); 46959243Sobrien PCPU_SET(switchticks, ticks); 47059243Sobrien CTR4(KTR_PROC, "mi_switch: old thread %ld (td_sched %p, pid %ld, %s)", 47159243Sobrien td->td_tid, td->td_sched, td->td_proc->p_pid, td->td_name); 47259243Sobrien#if (KTR_COMPILE & KTR_SCHED) != 0 47359243Sobrien if (TD_IS_IDLETHREAD(td)) 47459243Sobrien KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "idle", 47559243Sobrien "prio:%d", td->td_priority); 47659243Sobrien else 47759243Sobrien KTR_STATE3(KTR_SCHED, "thread", sched_tdname(td), KTDSTATE(td), 47859243Sobrien "prio:%d", td->td_priority, "wmesg:\"%s\"", td->td_wmesg, 47959243Sobrien "lockname:\"%s\"", td->td_lockname); 48059243Sobrien#endif 48159243Sobrien SDT_PROBE0(sched, , , preempt); 48259243Sobrien#ifdef XEN 48359243Sobrien PT_UPDATES_FLUSH(); 48459243Sobrien#endif 48559243Sobrien sched_switch(td, newtd, flags); 48659243Sobrien KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "running", 48759243Sobrien "prio:%d", td->td_priority); 48859243Sobrien 48959243Sobrien CTR4(KTR_PROC, "mi_switch: new thread %ld (td_sched %p, pid %ld, %s)", 49059243Sobrien td->td_tid, td->td_sched, td->td_proc->p_pid, td->td_name); 49159243Sobrien 49259243Sobrien /* 49359243Sobrien * If the last thread was exiting, finish cleaning it up. 49459243Sobrien */ 49559243Sobrien if ((td = PCPU_GET(deadthread))) { 49659243Sobrien PCPU_SET(deadthread, NULL); 49759243Sobrien thread_stash(td); 49859243Sobrien } 49959243Sobrien} 50059243Sobrien 50159243Sobrien/* 50259243Sobrien * Change thread state to be runnable, placing it on the run queue if 50359243Sobrien * it is in memory. If it is swapped out, return true so our caller 50459243Sobrien * will know to awaken the swapper. 50559243Sobrien */ 50659243Sobrienint 50759243Sobriensetrunnable(struct thread *td) 50859243Sobrien{ 50959243Sobrien 51059243Sobrien THREAD_LOCK_ASSERT(td, MA_OWNED); 51159243Sobrien KASSERT(td->td_proc->p_state != PRS_ZOMBIE, 51269408Sache ("setrunnable: pid %d is a zombie", td->td_proc->p_pid)); 51359243Sobrien switch (td->td_state) { 51459243Sobrien case TDS_RUNNING: 51559243Sobrien case TDS_RUNQ: 51659243Sobrien return (0); 51759243Sobrien case TDS_INHIBITED: 51859243Sobrien /* 51959243Sobrien * If we are only inhibited because we are swapped out 52059243Sobrien * then arange to swap in this process. Otherwise just return. 52159243Sobrien */ 52259243Sobrien if (td->td_inhibitors != TDI_SWAPPED) 52359243Sobrien return (0); 52459243Sobrien /* FALLTHROUGH */ 52559243Sobrien case TDS_CAN_RUN: 52659243Sobrien break; 52759243Sobrien default: 52859243Sobrien printf("state is 0x%x", td->td_state); 52959243Sobrien panic("setrunnable(2)"); 53059243Sobrien } 53159243Sobrien if ((td->td_flags & TDF_INMEM) == 0) { 53259243Sobrien if ((td->td_flags & TDF_SWAPINREQ) == 0) { 53359243Sobrien td->td_flags |= TDF_SWAPINREQ; 53459243Sobrien return (1); 53559243Sobrien } 53659243Sobrien } else 53769408Sache sched_wakeup(td); 53869408Sache return (0); 53959243Sobrien} 54059243Sobrien 54159243Sobrien/* 54259243Sobrien * Compute a tenex style load average of a quantity on 54359243Sobrien * 1, 5 and 15 minute intervals. 54459243Sobrien */ 54559243Sobrienstatic void 54659243Sobrienloadav(void *arg) 54759243Sobrien{ 54859243Sobrien int i, nrun; 54959243Sobrien struct loadavg *avg; 55059243Sobrien 55159243Sobrien nrun = sched_load(); 55259243Sobrien avg = &averunnable; 55359243Sobrien 55459243Sobrien for (i = 0; i < 3; i++) 55559243Sobrien avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + 55659243Sobrien nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; 55759243Sobrien 55859243Sobrien /* 55959243Sobrien * Schedule the next update to occur after 5 seconds, but add a 56059243Sobrien * random variation to avoid synchronisation with processes that 56159243Sobrien * run at regular intervals. 56259243Sobrien */ 56359243Sobrien callout_reset_sbt(&loadav_callout, 56469408Sache SBT_1US * (4000000 + (int)(random() % 2000001)), SBT_1US, 56559243Sobrien loadav, NULL, C_DIRECT_EXEC | C_PREL(32)); 56659243Sobrien} 56759243Sobrien 56859243Sobrien/* ARGSUSED */ 56959243Sobrienstatic void 57059243Sobriensynch_setup(void *dummy) 57159243Sobrien{ 57259243Sobrien callout_init(&loadav_callout, 1); 57359243Sobrien 57459243Sobrien /* Kick off timeout driven events by calling first time. */ 57559243Sobrien loadav(NULL); 57669408Sache} 57759243Sobrien 57859243Sobrienint 57959243Sobrienshould_yield(void) 58059243Sobrien{ 58159243Sobrien 58259243Sobrien return ((u_int)ticks - (u_int)curthread->td_swvoltick >= hogticks); 58359243Sobrien} 58459243Sobrien 58559243Sobrienvoid 58659243Sobrienmaybe_yield(void) 58759243Sobrien{ 58859243Sobrien 58959243Sobrien if (should_yield()) 59059243Sobrien kern_yield(PRI_USER); 59159243Sobrien} 59259243Sobrien 59359243Sobrienvoid 59459243Sobrienkern_yield(int prio) 59559243Sobrien{ 59659243Sobrien struct thread *td; 59759243Sobrien 59859243Sobrien td = curthread; 59959243Sobrien DROP_GIANT(); 60059243Sobrien thread_lock(td); 60159243Sobrien if (prio == PRI_USER) 60259243Sobrien prio = td->td_user_pri; 60359243Sobrien if (prio >= 0) 60459243Sobrien sched_prio(td, prio); 60559243Sobrien mi_switch(SW_VOL | SWT_RELINQUISH, NULL); 60659243Sobrien thread_unlock(td); 60759243Sobrien PICKUP_GIANT(); 60859243Sobrien} 60959243Sobrien 61059243Sobrien/* 61159243Sobrien * General purpose yield system call. 61259243Sobrien */ 61359243Sobrienint 61459243Sobriensys_yield(struct thread *td, struct yield_args *uap) 61559243Sobrien{ 61659243Sobrien 61759243Sobrien thread_lock(td); 61859243Sobrien if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE) 61959243Sobrien sched_prio(td, PRI_MAX_TIMESHARE); 62059243Sobrien mi_switch(SW_VOL | SWT_RELINQUISH, NULL); 62159243Sobrien thread_unlock(td); 62259243Sobrien td->td_retval[0] = 0; 62359243Sobrien return (0); 62459243Sobrien} 62559243Sobrien