171088Sjasone/*- 271088Sjasone * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 371088Sjasone * All rights reserved. 471088Sjasone * 571088Sjasone * Redistribution and use in source and binary forms, with or without 671088Sjasone * modification, are permitted provided that the following conditions 771088Sjasone * are met: 871088Sjasone * 1. Redistributions of source code must retain the above copyright 971088Sjasone * notice, this list of conditions and the following disclaimer. 1071088Sjasone * 2. Redistributions in binary form must reproduce the above copyright 1171088Sjasone * notice, this list of conditions and the following disclaimer in the 1271088Sjasone * documentation and/or other materials provided with the distribution. 1371088Sjasone * 1471088Sjasone * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1571088Sjasone * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1671088Sjasone * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1771088Sjasone * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1871088Sjasone * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1971088Sjasone * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2071088Sjasone * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2171088Sjasone * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2271088Sjasone * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2371088Sjasone * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2471088Sjasone * SUCH DAMAGE. 2571088Sjasone */ 2671088Sjasone 27116182Sobrien#include <sys/cdefs.h> 28116182Sobrien__FBSDID("$FreeBSD$"); 29116182Sobrien 3071088Sjasone#include "opt_ktrace.h" 3171088Sjasone 3271088Sjasone#include <sys/param.h> 3371088Sjasone#include <sys/systm.h> 3476166Smarkm#include <sys/lock.h> 3576166Smarkm#include <sys/mutex.h> 3671088Sjasone#include <sys/proc.h> 3771088Sjasone#include <sys/kernel.h> 3871088Sjasone#include <sys/ktr.h> 3971088Sjasone#include <sys/condvar.h> 40109862Sjeff#include <sys/sched.h> 4171088Sjasone#include <sys/signalvar.h> 42126326Sjhb#include <sys/sleepqueue.h> 4371088Sjasone#include <sys/resourcevar.h> 4471088Sjasone#ifdef KTRACE 4571088Sjasone#include <sys/uio.h> 4671088Sjasone#include <sys/ktrace.h> 4771088Sjasone#endif 4871088Sjasone 4971088Sjasone/* 5071088Sjasone * Common sanity checks for cv_wait* functions. 5171088Sjasone */ 52167789Sjhb#define CV_ASSERT(cvp, lock, td) do { \ 53240475Sattilio KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 54103216Sjulian KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 5587594Sobrien KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 56167789Sjhb KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 5771088Sjasone} while (0) 5871088Sjasone 5971088Sjasone/* 6071088Sjasone * Initialize a condition variable. Must be called before use. 6171088Sjasone */ 6271088Sjasonevoid 6371088Sjasonecv_init(struct cv *cvp, const char *desc) 6471088Sjasone{ 6571088Sjasone 6671088Sjasone cvp->cv_description = desc; 67127954Sjhb cvp->cv_waiters = 0; 6871088Sjasone} 6971088Sjasone 7071088Sjasone/* 7171088Sjasone * Destroy a condition variable. The condition variable must be re-initialized 7271088Sjasone * in order to be re-used. 7371088Sjasone */ 7471088Sjasonevoid 7571088Sjasonecv_destroy(struct cv *cvp) 7671088Sjasone{ 77126326Sjhb#ifdef INVARIANTS 78136445Sjhb struct sleepqueue *sq; 7971088Sjasone 80136445Sjhb sleepq_lock(cvp); 81126326Sjhb sq = sleepq_lookup(cvp); 82126326Sjhb sleepq_release(cvp); 83126326Sjhb KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 84126326Sjhb#endif 8571088Sjasone} 8671088Sjasone 8771088Sjasone/* 8883366Sjulian * Wait on a condition variable. The current thread is placed on the condition 8971088Sjasone * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 9083366Sjulian * condition variable will resume the thread. The mutex is released before 9171088Sjasone * sleeping and will be held on return. It is recommended that the mutex be 9271088Sjasone * held when cv_signal or cv_broadcast are called. 9371088Sjasone */ 9471088Sjasonevoid 95167789Sjhb_cv_wait(struct cv *cvp, struct lock_object *lock) 9671088Sjasone{ 97167789Sjhb WITNESS_SAVE_DECL(lock_witness); 98167789Sjhb struct lock_class *class; 99167786Sjhb struct thread *td; 100255745Sdavide uintptr_t lock_state; 10171088Sjasone 102167786Sjhb td = curthread; 103181394Sjhb lock_state = 0; 104167786Sjhb#ifdef KTRACE 105167786Sjhb if (KTRPOINT(td, KTR_CSW)) 106234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 107167786Sjhb#endif 108167789Sjhb CV_ASSERT(cvp, lock, td); 109167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 110167786Sjhb "Waiting on \"%s\"", cvp->cv_description); 111167789Sjhb class = LOCK_CLASS(lock); 112153321Srodrigc 113153321Srodrigc if (cold || panicstr) { 114153321Srodrigc /* 115153321Srodrigc * During autoconfiguration, just give interrupts 116153321Srodrigc * a chance, then just return. Don't run any other 117153321Srodrigc * thread or panic below, in case this is the idle 118153321Srodrigc * process and already asleep. 119153321Srodrigc */ 120153321Srodrigc return; 121153321Srodrigc } 122153321Srodrigc 123167786Sjhb sleepq_lock(cvp); 124167786Sjhb 125167786Sjhb cvp->cv_waiters++; 126183352Sjhb if (lock == &Giant.lock_object) 127183352Sjhb mtx_assert(&Giant, MA_OWNED); 128167786Sjhb DROP_GIANT(); 129167786Sjhb 130167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 131181394Sjhb if (lock != &Giant.lock_object) { 132181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 133181394Sjhb sleepq_release(cvp); 134183352Sjhb WITNESS_SAVE(lock, lock_witness); 135181394Sjhb lock_state = class->lc_unlock(lock); 136181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 137181394Sjhb sleepq_lock(cvp); 138181394Sjhb } 139177085Sjeff sleepq_wait(cvp, 0); 140167786Sjhb 141167786Sjhb#ifdef KTRACE 142167786Sjhb if (KTRPOINT(td, KTR_CSW)) 143234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 144167786Sjhb#endif 145167786Sjhb PICKUP_GIANT(); 146181394Sjhb if (lock != &Giant.lock_object) { 147181394Sjhb class->lc_lock(lock, lock_state); 148181394Sjhb WITNESS_RESTORE(lock, lock_witness); 149181394Sjhb } 150153321Srodrigc} 151153321Srodrigc 152153321Srodrigc/* 153153321Srodrigc * Wait on a condition variable. This function differs from cv_wait by 154153321Srodrigc * not aquiring the mutex after condition variable was signaled. 155153321Srodrigc */ 156153321Srodrigcvoid 157167789Sjhb_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 158153321Srodrigc{ 159167789Sjhb struct lock_class *class; 160153321Srodrigc struct thread *td; 161153321Srodrigc 16283366Sjulian td = curthread; 16371088Sjasone#ifdef KTRACE 16497995Sjhb if (KTRPOINT(td, KTR_CSW)) 165234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 16671088Sjasone#endif 167167789Sjhb CV_ASSERT(cvp, lock, td); 168167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 169111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 170181394Sjhb KASSERT(lock != &Giant.lock_object, 171181394Sjhb ("cv_wait_unlock cannot be used with Giant")); 172167789Sjhb class = LOCK_CLASS(lock); 17371088Sjasone 174126326Sjhb if (cold || panicstr) { 17571088Sjasone /* 176100209Sgallatin * During autoconfiguration, just give interrupts 177100209Sgallatin * a chance, then just return. Don't run any other 178100209Sgallatin * thread or panic below, in case this is the idle 179100209Sgallatin * process and already asleep. 18071088Sjasone */ 181167789Sjhb class->lc_unlock(lock); 18271088Sjasone return; 18371088Sjasone } 18495322Shsu 185136445Sjhb sleepq_lock(cvp); 18695322Shsu 187127954Sjhb cvp->cv_waiters++; 18888900Sjhb DROP_GIANT(); 18971088Sjasone 190167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 191169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 192169392Sjhb sleepq_release(cvp); 193169392Sjhb class->lc_unlock(lock); 194169392Sjhb if (class->lc_flags & LC_SLEEPABLE) 195169392Sjhb sleepq_lock(cvp); 196177085Sjeff sleepq_wait(cvp, 0); 19771088Sjasone 198167786Sjhb#ifdef KTRACE 199167786Sjhb if (KTRPOINT(td, KTR_CSW)) 200234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 201167786Sjhb#endif 20271088Sjasone PICKUP_GIANT(); 20371088Sjasone} 20471088Sjasone 20571088Sjasone/* 20671088Sjasone * Wait on a condition variable, allowing interruption by signals. Return 0 if 20783366Sjulian * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 20871088Sjasone * a signal was caught. If ERESTART is returned the system call should be 20971088Sjasone * restarted if possible. 21071088Sjasone */ 21171088Sjasoneint 212167789Sjhb_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 21371088Sjasone{ 214167789Sjhb WITNESS_SAVE_DECL(lock_witness); 215167789Sjhb struct lock_class *class; 21683366Sjulian struct thread *td; 217255745Sdavide uintptr_t lock_state; 218255745Sdavide int rval; 21971088Sjasone 22083366Sjulian td = curthread; 221181394Sjhb lock_state = 0; 22271088Sjasone#ifdef KTRACE 22397995Sjhb if (KTRPOINT(td, KTR_CSW)) 224234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 22571088Sjasone#endif 226167789Sjhb CV_ASSERT(cvp, lock, td); 227167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 228111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 229167789Sjhb class = LOCK_CLASS(lock); 23071088Sjasone 23171088Sjasone if (cold || panicstr) { 23271088Sjasone /* 23371088Sjasone * After a panic, or during autoconfiguration, just give 23471088Sjasone * interrupts a chance, then just return; don't run any other 23571088Sjasone * procs or panic below, in case this is the idle process and 23671088Sjasone * already asleep. 23771088Sjasone */ 238133440Sjhb return (0); 23971088Sjasone } 24095322Shsu 241136445Sjhb sleepq_lock(cvp); 24295322Shsu 243127954Sjhb cvp->cv_waiters++; 244183352Sjhb if (lock == &Giant.lock_object) 245183352Sjhb mtx_assert(&Giant, MA_OWNED); 24688900Sjhb DROP_GIANT(); 24771088Sjasone 248167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 249165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 250181394Sjhb if (lock != &Giant.lock_object) { 251181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 252181394Sjhb sleepq_release(cvp); 253183352Sjhb WITNESS_SAVE(lock, lock_witness); 254181394Sjhb lock_state = class->lc_unlock(lock); 255181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 256181394Sjhb sleepq_lock(cvp); 257181394Sjhb } 258177085Sjeff rval = sleepq_wait_sig(cvp, 0); 25971088Sjasone 26071088Sjasone#ifdef KTRACE 26197995Sjhb if (KTRPOINT(td, KTR_CSW)) 262234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 26371088Sjasone#endif 26497995Sjhb PICKUP_GIANT(); 265181394Sjhb if (lock != &Giant.lock_object) { 266181394Sjhb class->lc_lock(lock, lock_state); 267181394Sjhb WITNESS_RESTORE(lock, lock_witness); 268181394Sjhb } 26971088Sjasone 27071088Sjasone return (rval); 27171088Sjasone} 27271088Sjasone 27371088Sjasone/* 274247785Sdavide * Wait on a condition variable for (at most) the value specified in sbt 275247785Sdavide * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 276247785Sdavide * EWOULDBLOCK if the timeout expires. 27771088Sjasone */ 27871088Sjasoneint 279247785Sdavide_cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 280247785Sdavide sbintime_t pr, int flags) 28171088Sjasone{ 282167789Sjhb WITNESS_SAVE_DECL(lock_witness); 283167789Sjhb struct lock_class *class; 28483366Sjulian struct thread *td; 285167789Sjhb int lock_state, rval; 28671088Sjasone 28783366Sjulian td = curthread; 288181394Sjhb lock_state = 0; 28971088Sjasone#ifdef KTRACE 29097995Sjhb if (KTRPOINT(td, KTR_CSW)) 291234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 29271088Sjasone#endif 293167789Sjhb CV_ASSERT(cvp, lock, td); 294167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 295111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 296167789Sjhb class = LOCK_CLASS(lock); 29771088Sjasone 29871088Sjasone if (cold || panicstr) { 29971088Sjasone /* 30071088Sjasone * After a panic, or during autoconfiguration, just give 30171088Sjasone * interrupts a chance, then just return; don't run any other 30283366Sjulian * thread or panic below, in case this is the idle process and 30371088Sjasone * already asleep. 30471088Sjasone */ 30571088Sjasone return 0; 30671088Sjasone } 30795322Shsu 308136445Sjhb sleepq_lock(cvp); 30995322Shsu 310127954Sjhb cvp->cv_waiters++; 311183352Sjhb if (lock == &Giant.lock_object) 312183352Sjhb mtx_assert(&Giant, MA_OWNED); 31388900Sjhb DROP_GIANT(); 31471088Sjasone 315167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 316247785Sdavide sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 317181394Sjhb if (lock != &Giant.lock_object) { 318181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 319181394Sjhb sleepq_release(cvp); 320183352Sjhb WITNESS_SAVE(lock, lock_witness); 321181394Sjhb lock_state = class->lc_unlock(lock); 322181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 323181394Sjhb sleepq_lock(cvp); 324181394Sjhb } 325177085Sjeff rval = sleepq_timedwait(cvp, 0); 32671088Sjasone 32771088Sjasone#ifdef KTRACE 32897995Sjhb if (KTRPOINT(td, KTR_CSW)) 329234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 33071088Sjasone#endif 33171088Sjasone PICKUP_GIANT(); 332181394Sjhb if (lock != &Giant.lock_object) { 333181394Sjhb class->lc_lock(lock, lock_state); 334181394Sjhb WITNESS_RESTORE(lock, lock_witness); 335181394Sjhb } 33671088Sjasone 33771088Sjasone return (rval); 33871088Sjasone} 33971088Sjasone 34071088Sjasone/* 341247785Sdavide * Wait on a condition variable for (at most) the value specified in sbt 342247785Sdavide * argument, allowing interruption by signals. 343247785Sdavide * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 344247785Sdavide * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 345247785Sdavide * was caught. 34671088Sjasone */ 34771088Sjasoneint 348247785Sdavide_cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 349247785Sdavide sbintime_t sbt, sbintime_t pr, int flags) 35071088Sjasone{ 351167789Sjhb WITNESS_SAVE_DECL(lock_witness); 352167789Sjhb struct lock_class *class; 35383366Sjulian struct thread *td; 354167789Sjhb int lock_state, rval; 35571088Sjasone 35683366Sjulian td = curthread; 357181394Sjhb lock_state = 0; 35871088Sjasone#ifdef KTRACE 35997995Sjhb if (KTRPOINT(td, KTR_CSW)) 360234494Sjhb ktrcsw(1, 0, cv_wmesg(cvp)); 36171088Sjasone#endif 362167789Sjhb CV_ASSERT(cvp, lock, td); 363167789Sjhb WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 364111883Sjhb "Waiting on \"%s\"", cvp->cv_description); 365167789Sjhb class = LOCK_CLASS(lock); 36671088Sjasone 36771088Sjasone if (cold || panicstr) { 36871088Sjasone /* 36971088Sjasone * After a panic, or during autoconfiguration, just give 37071088Sjasone * interrupts a chance, then just return; don't run any other 37183366Sjulian * thread or panic below, in case this is the idle process and 37271088Sjasone * already asleep. 37371088Sjasone */ 37471088Sjasone return 0; 37571088Sjasone } 37695322Shsu 377136445Sjhb sleepq_lock(cvp); 37895322Shsu 379127954Sjhb cvp->cv_waiters++; 380183352Sjhb if (lock == &Giant.lock_object) 381183352Sjhb mtx_assert(&Giant, MA_OWNED); 38288900Sjhb DROP_GIANT(); 38371088Sjasone 384167789Sjhb sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 385165272Skmacy SLEEPQ_INTERRUPTIBLE, 0); 386247785Sdavide sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 387181394Sjhb if (lock != &Giant.lock_object) { 388181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 389181394Sjhb sleepq_release(cvp); 390183352Sjhb WITNESS_SAVE(lock, lock_witness); 391181394Sjhb lock_state = class->lc_unlock(lock); 392181394Sjhb if (class->lc_flags & LC_SLEEPABLE) 393181394Sjhb sleepq_lock(cvp); 394181394Sjhb } 395177085Sjeff rval = sleepq_timedwait_sig(cvp, 0); 39671088Sjasone 39771088Sjasone#ifdef KTRACE 39897995Sjhb if (KTRPOINT(td, KTR_CSW)) 399234494Sjhb ktrcsw(0, 0, cv_wmesg(cvp)); 40071088Sjasone#endif 40197995Sjhb PICKUP_GIANT(); 402181394Sjhb if (lock != &Giant.lock_object) { 403181394Sjhb class->lc_lock(lock, lock_state); 404181394Sjhb WITNESS_RESTORE(lock, lock_witness); 405181394Sjhb } 40671088Sjasone 40771088Sjasone return (rval); 40871088Sjasone} 40971088Sjasone 41071088Sjasone/* 41183366Sjulian * Signal a condition variable, wakes up one waiting thread. Will also wakeup 41271088Sjasone * the swapper if the process is not in memory, so that it can bring the 41383366Sjulian * sleeping process in. Note that this may also result in additional threads 41471088Sjasone * being made runnable. Should be called with the same mutex as was passed to 41571088Sjasone * cv_wait held. 41671088Sjasone */ 41771088Sjasonevoid 41871088Sjasonecv_signal(struct cv *cvp) 41971088Sjasone{ 420181334Sjhb int wakeup_swapper; 42171088Sjasone 422181334Sjhb wakeup_swapper = 0; 423136445Sjhb sleepq_lock(cvp); 424127954Sjhb if (cvp->cv_waiters > 0) { 425127954Sjhb cvp->cv_waiters--; 426181334Sjhb wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 0); 427170294Sjeff } 428170294Sjeff sleepq_release(cvp); 429181334Sjhb if (wakeup_swapper) 430181334Sjhb kick_proc0(); 43171088Sjasone} 43271088Sjasone 43371088Sjasone/* 43483366Sjulian * Broadcast a signal to a condition variable. Wakes up all waiting threads. 43571088Sjasone * Should be called with the same mutex as was passed to cv_wait held. 43671088Sjasone */ 43771088Sjasonevoid 438122352Stanimuracv_broadcastpri(struct cv *cvp, int pri) 43971088Sjasone{ 440181334Sjhb int wakeup_swapper; 441181334Sjhb 442177085Sjeff /* 443177085Sjeff * XXX sleepq_broadcast pri argument changed from -1 meaning 444177085Sjeff * no pri to 0 meaning no pri. 445177085Sjeff */ 446181334Sjhb wakeup_swapper = 0; 447177085Sjeff if (pri == -1) 448177085Sjeff pri = 0; 449136445Sjhb sleepq_lock(cvp); 450127954Sjhb if (cvp->cv_waiters > 0) { 451127954Sjhb cvp->cv_waiters = 0; 452181334Sjhb wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 453177085Sjeff } 454177085Sjeff sleepq_release(cvp); 455181334Sjhb if (wakeup_swapper) 456181334Sjhb kick_proc0(); 45771088Sjasone} 458