1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD$"); 31 32#include "opt_ktrace.h" 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/limits.h> 37#include <sys/lock.h> 38#include <sys/mutex.h> 39#include <sys/proc.h> 40#include <sys/kernel.h> 41#include <sys/ktr.h> 42#include <sys/condvar.h> 43#include <sys/sched.h> 44#include <sys/signalvar.h> 45#include <sys/sleepqueue.h> 46#include <sys/resourcevar.h> 47#ifdef KTRACE 48#include <sys/uio.h> 49#include <sys/ktrace.h> 50#endif 51 52/* 53 * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound, 54 * cv_signal must manually check the wait queue for threads. 55 */ 56#define CV_WAITERS_BOUND INT_MAX 57 58#define CV_WAITERS_INC(cvp) do { \ 59 if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \ 60 (cvp)->cv_waiters++; \ 61} while (0) 62 63/* 64 * Common sanity checks for cv_wait* functions. 65 */ 66#define CV_ASSERT(cvp, lock, td) do { \ 67 KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 68 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 69 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 70 KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 71} while (0) 72 73/* 74 * Initialize a condition variable. Must be called before use. 75 */ 76void 77cv_init(struct cv *cvp, const char *desc) 78{ 79 80 cvp->cv_description = desc; 81 cvp->cv_waiters = 0; 82} 83 84/* 85 * Destroy a condition variable. The condition variable must be re-initialized 86 * in order to be re-used. 87 */ 88void 89cv_destroy(struct cv *cvp) 90{ 91#ifdef INVARIANTS 92 struct sleepqueue *sq; 93 94 sleepq_lock(cvp); 95 sq = sleepq_lookup(cvp); 96 sleepq_release(cvp); 97 KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 98#endif 99} 100 101/* 102 * Wait on a condition variable. The current thread is placed on the condition 103 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 104 * condition variable will resume the thread. The mutex is released before 105 * sleeping and will be held on return. It is recommended that the mutex be 106 * held when cv_signal or cv_broadcast are called. 107 */ 108void 109_cv_wait(struct cv *cvp, struct lock_object *lock) 110{ 111 WITNESS_SAVE_DECL(lock_witness); 112 struct lock_class *class; 113 struct thread *td; 114 uintptr_t lock_state; 115 116 td = curthread; 117 lock_state = 0; 118#ifdef KTRACE 119 if (KTRPOINT(td, KTR_CSW)) 120 ktrcsw(1, 0, cv_wmesg(cvp)); 121#endif 122 CV_ASSERT(cvp, lock, td); 123 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 124 "Waiting on \"%s\"", cvp->cv_description); 125 class = LOCK_CLASS(lock); 126 127 if (SCHEDULER_STOPPED_TD(td)) 128 return; 129 130 sleepq_lock(cvp); 131 132 CV_WAITERS_INC(cvp); 133 if (lock == &Giant.lock_object) 134 mtx_assert(&Giant, MA_OWNED); 135 DROP_GIANT(); 136 137 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 138 if (lock != &Giant.lock_object) { 139 if (class->lc_flags & LC_SLEEPABLE) 140 sleepq_release(cvp); 141 WITNESS_SAVE(lock, lock_witness); 142 lock_state = class->lc_unlock(lock); 143 if (class->lc_flags & LC_SLEEPABLE) 144 sleepq_lock(cvp); 145 } 146 sleepq_wait(cvp, 0); 147 148#ifdef KTRACE 149 if (KTRPOINT(td, KTR_CSW)) 150 ktrcsw(0, 0, cv_wmesg(cvp)); 151#endif 152 PICKUP_GIANT(); 153 if (lock != &Giant.lock_object) { 154 class->lc_lock(lock, lock_state); 155 WITNESS_RESTORE(lock, lock_witness); 156 } 157} 158 159/* 160 * Wait on a condition variable. This function differs from cv_wait by 161 * not acquiring the mutex after condition variable was signaled. 162 */ 163void 164_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 165{ 166 struct lock_class *class; 167 struct thread *td; 168 169 td = curthread; 170#ifdef KTRACE 171 if (KTRPOINT(td, KTR_CSW)) 172 ktrcsw(1, 0, cv_wmesg(cvp)); 173#endif 174 CV_ASSERT(cvp, lock, td); 175 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 176 "Waiting on \"%s\"", cvp->cv_description); 177 KASSERT(lock != &Giant.lock_object, 178 ("cv_wait_unlock cannot be used with Giant")); 179 class = LOCK_CLASS(lock); 180 181 if (SCHEDULER_STOPPED_TD(td)) { 182 class->lc_unlock(lock); 183 return; 184 } 185 186 sleepq_lock(cvp); 187 188 CV_WAITERS_INC(cvp); 189 DROP_GIANT(); 190 191 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 192 if (class->lc_flags & LC_SLEEPABLE) 193 sleepq_release(cvp); 194 class->lc_unlock(lock); 195 if (class->lc_flags & LC_SLEEPABLE) 196 sleepq_lock(cvp); 197 sleepq_wait(cvp, 0); 198 199#ifdef KTRACE 200 if (KTRPOINT(td, KTR_CSW)) 201 ktrcsw(0, 0, cv_wmesg(cvp)); 202#endif 203 PICKUP_GIANT(); 204} 205 206/* 207 * Wait on a condition variable, allowing interruption by signals. Return 0 if 208 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 209 * a signal was caught. If ERESTART is returned the system call should be 210 * restarted if possible. 211 */ 212int 213_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 214{ 215 WITNESS_SAVE_DECL(lock_witness); 216 struct lock_class *class; 217 struct thread *td; 218 uintptr_t lock_state; 219 int rval; 220 221 td = curthread; 222 lock_state = 0; 223#ifdef KTRACE 224 if (KTRPOINT(td, KTR_CSW)) 225 ktrcsw(1, 0, cv_wmesg(cvp)); 226#endif 227 CV_ASSERT(cvp, lock, td); 228 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 229 "Waiting on \"%s\"", cvp->cv_description); 230 class = LOCK_CLASS(lock); 231 232 if (SCHEDULER_STOPPED_TD(td)) 233 return (0); 234 235 sleepq_lock(cvp); 236 237 CV_WAITERS_INC(cvp); 238 if (lock == &Giant.lock_object) 239 mtx_assert(&Giant, MA_OWNED); 240 DROP_GIANT(); 241 242 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 243 SLEEPQ_INTERRUPTIBLE, 0); 244 if (lock != &Giant.lock_object) { 245 if (class->lc_flags & LC_SLEEPABLE) 246 sleepq_release(cvp); 247 WITNESS_SAVE(lock, lock_witness); 248 lock_state = class->lc_unlock(lock); 249 if (class->lc_flags & LC_SLEEPABLE) 250 sleepq_lock(cvp); 251 } 252 rval = sleepq_wait_sig(cvp, 0); 253 254#ifdef KTRACE 255 if (KTRPOINT(td, KTR_CSW)) 256 ktrcsw(0, 0, cv_wmesg(cvp)); 257#endif 258 PICKUP_GIANT(); 259 if (lock != &Giant.lock_object) { 260 class->lc_lock(lock, lock_state); 261 WITNESS_RESTORE(lock, lock_witness); 262 } 263 264 return (rval); 265} 266 267/* 268 * Wait on a condition variable for (at most) the value specified in sbt 269 * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 270 * EWOULDBLOCK if the timeout expires. 271 */ 272int 273_cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 274 sbintime_t pr, int flags) 275{ 276 WITNESS_SAVE_DECL(lock_witness); 277 struct lock_class *class; 278 struct thread *td; 279 int lock_state, rval; 280 281 td = curthread; 282 lock_state = 0; 283#ifdef KTRACE 284 if (KTRPOINT(td, KTR_CSW)) 285 ktrcsw(1, 0, cv_wmesg(cvp)); 286#endif 287 CV_ASSERT(cvp, lock, td); 288 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 289 "Waiting on \"%s\"", cvp->cv_description); 290 class = LOCK_CLASS(lock); 291 292 if (SCHEDULER_STOPPED_TD(td)) 293 return (0); 294 295 sleepq_lock(cvp); 296 297 CV_WAITERS_INC(cvp); 298 if (lock == &Giant.lock_object) 299 mtx_assert(&Giant, MA_OWNED); 300 DROP_GIANT(); 301 302 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 303 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 304 if (lock != &Giant.lock_object) { 305 if (class->lc_flags & LC_SLEEPABLE) 306 sleepq_release(cvp); 307 WITNESS_SAVE(lock, lock_witness); 308 lock_state = class->lc_unlock(lock); 309 if (class->lc_flags & LC_SLEEPABLE) 310 sleepq_lock(cvp); 311 } 312 rval = sleepq_timedwait(cvp, 0); 313 314#ifdef KTRACE 315 if (KTRPOINT(td, KTR_CSW)) 316 ktrcsw(0, 0, cv_wmesg(cvp)); 317#endif 318 PICKUP_GIANT(); 319 if (lock != &Giant.lock_object) { 320 class->lc_lock(lock, lock_state); 321 WITNESS_RESTORE(lock, lock_witness); 322 } 323 324 return (rval); 325} 326 327/* 328 * Wait on a condition variable for (at most) the value specified in sbt 329 * argument, allowing interruption by signals. 330 * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 331 * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 332 * was caught. 333 */ 334int 335_cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 336 sbintime_t sbt, sbintime_t pr, int flags) 337{ 338 WITNESS_SAVE_DECL(lock_witness); 339 struct lock_class *class; 340 struct thread *td; 341 int lock_state, rval; 342 343 td = curthread; 344 lock_state = 0; 345#ifdef KTRACE 346 if (KTRPOINT(td, KTR_CSW)) 347 ktrcsw(1, 0, cv_wmesg(cvp)); 348#endif 349 CV_ASSERT(cvp, lock, td); 350 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 351 "Waiting on \"%s\"", cvp->cv_description); 352 class = LOCK_CLASS(lock); 353 354 if (SCHEDULER_STOPPED_TD(td)) 355 return (0); 356 357 sleepq_lock(cvp); 358 359 CV_WAITERS_INC(cvp); 360 if (lock == &Giant.lock_object) 361 mtx_assert(&Giant, MA_OWNED); 362 DROP_GIANT(); 363 364 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 365 SLEEPQ_INTERRUPTIBLE, 0); 366 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 367 if (lock != &Giant.lock_object) { 368 if (class->lc_flags & LC_SLEEPABLE) 369 sleepq_release(cvp); 370 WITNESS_SAVE(lock, lock_witness); 371 lock_state = class->lc_unlock(lock); 372 if (class->lc_flags & LC_SLEEPABLE) 373 sleepq_lock(cvp); 374 } 375 rval = sleepq_timedwait_sig(cvp, 0); 376 377#ifdef KTRACE 378 if (KTRPOINT(td, KTR_CSW)) 379 ktrcsw(0, 0, cv_wmesg(cvp)); 380#endif 381 PICKUP_GIANT(); 382 if (lock != &Giant.lock_object) { 383 class->lc_lock(lock, lock_state); 384 WITNESS_RESTORE(lock, lock_witness); 385 } 386 387 return (rval); 388} 389 390/* 391 * Signal a condition variable, wakes up one waiting thread. Will also wakeup 392 * the swapper if the process is not in memory, so that it can bring the 393 * sleeping process in. Note that this may also result in additional threads 394 * being made runnable. Should be called with the same mutex as was passed to 395 * cv_wait held. 396 */ 397void 398cv_signal(struct cv *cvp) 399{ 400 int wakeup_swapper; 401 402 if (cvp->cv_waiters == 0) 403 return; 404 wakeup_swapper = 0; 405 sleepq_lock(cvp); 406 if (cvp->cv_waiters > 0) { 407 if (cvp->cv_waiters == CV_WAITERS_BOUND && 408 sleepq_lookup(cvp) == NULL) { 409 cvp->cv_waiters = 0; 410 } else { 411 if (cvp->cv_waiters < CV_WAITERS_BOUND) 412 cvp->cv_waiters--; 413 wakeup_swapper = sleepq_signal(cvp, SLEEPQ_CONDVAR, 0, 414 0); 415 } 416 } 417 sleepq_release(cvp); 418 if (wakeup_swapper) 419 kick_proc0(); 420} 421 422/* 423 * Broadcast a signal to a condition variable. Wakes up all waiting threads. 424 * Should be called with the same mutex as was passed to cv_wait held. 425 */ 426void 427cv_broadcastpri(struct cv *cvp, int pri) 428{ 429 int wakeup_swapper; 430 431 if (cvp->cv_waiters == 0) 432 return; 433 /* 434 * XXX sleepq_broadcast pri argument changed from -1 meaning 435 * no pri to 0 meaning no pri. 436 */ 437 wakeup_swapper = 0; 438 if (pri == -1) 439 pri = 0; 440 sleepq_lock(cvp); 441 if (cvp->cv_waiters > 0) { 442 cvp->cv_waiters = 0; 443 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 444 } 445 sleepq_release(cvp); 446 if (wakeup_swapper) 447 kick_proc0(); 448} 449