1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2000 Jake Burkholder <jake@freebsd.org>. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30#include "opt_ktrace.h" 31 32#include <sys/param.h> 33#include <sys/systm.h> 34#include <sys/limits.h> 35#include <sys/lock.h> 36#include <sys/mutex.h> 37#include <sys/proc.h> 38#include <sys/kernel.h> 39#include <sys/ktr.h> 40#include <sys/ktrace.h> 41#include <sys/condvar.h> 42#include <sys/sched.h> 43#include <sys/signalvar.h> 44#include <sys/sleepqueue.h> 45#include <sys/resourcevar.h> 46#ifdef KTRACE 47#include <sys/uio.h> 48#include <sys/user.h> 49#endif 50 51/* 52 * A bound below which cv_waiters is valid. Once cv_waiters reaches this bound, 53 * cv_signal must manually check the wait queue for threads. 54 */ 55#define CV_WAITERS_BOUND INT_MAX 56 57#define CV_WAITERS_INC(cvp) do { \ 58 if ((cvp)->cv_waiters < CV_WAITERS_BOUND) \ 59 (cvp)->cv_waiters++; \ 60} while (0) 61 62/* 63 * Common sanity checks for cv_wait* functions. 64 */ 65#define CV_ASSERT(cvp, lock, td) do { \ 66 KASSERT((td) != NULL, ("%s: td NULL", __func__)); \ 67 KASSERT(TD_IS_RUNNING(td), ("%s: not TDS_RUNNING", __func__)); \ 68 KASSERT((cvp) != NULL, ("%s: cvp NULL", __func__)); \ 69 KASSERT((lock) != NULL, ("%s: lock NULL", __func__)); \ 70} while (0) 71 72/* 73 * Initialize a condition variable. Must be called before use. 74 */ 75void 76cv_init(struct cv *cvp, const char *desc) 77{ 78 79 cvp->cv_description = desc; 80 cvp->cv_waiters = 0; 81} 82 83/* 84 * Destroy a condition variable. The condition variable must be re-initialized 85 * in order to be re-used. 86 */ 87void 88cv_destroy(struct cv *cvp) 89{ 90#ifdef INVARIANTS 91 struct sleepqueue *sq; 92 93 sleepq_lock(cvp); 94 sq = sleepq_lookup(cvp); 95 sleepq_release(cvp); 96 KASSERT(sq == NULL, ("%s: associated sleep queue non-empty", __func__)); 97#endif 98} 99 100/* 101 * Wait on a condition variable. The current thread is placed on the condition 102 * variable's wait queue and suspended. A cv_signal or cv_broadcast on the same 103 * condition variable will resume the thread. The mutex is released before 104 * sleeping and will be held on return. It is recommended that the mutex be 105 * held when cv_signal or cv_broadcast are called. 106 */ 107void 108_cv_wait(struct cv *cvp, struct lock_object *lock) 109{ 110 WITNESS_SAVE_DECL(lock_witness); 111#ifdef KTRACE 112 char wmesg[WMESGLEN + 1]; 113#endif 114 struct lock_class *class; 115 struct thread *td __ktrace_used; 116 uintptr_t lock_state; 117 118 td = curthread; 119 CV_ASSERT(cvp, lock, td); 120 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 121 "Waiting on \"%s\"", cvp->cv_description); 122 123 if (SCHEDULER_STOPPED()) 124 return; 125 126#ifdef KTRACE 127 if (KTRPOINT(td, KTR_CSW)) { 128 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 129 ktrcsw(1, 0, wmesg); 130 } else { 131 wmesg[0] = '\0'; 132 } 133#endif 134 135 class = LOCK_CLASS(lock); 136 lock_state = 0; 137 sleepq_lock(cvp); 138 139 CV_WAITERS_INC(cvp); 140 if (lock == &Giant.lock_object) 141 mtx_assert(&Giant, MA_OWNED); 142 DROP_GIANT(); 143 144 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 145 if (lock != &Giant.lock_object) { 146 if (class->lc_flags & LC_SLEEPABLE) 147 sleepq_release(cvp); 148 WITNESS_SAVE(lock, lock_witness); 149 lock_state = class->lc_unlock(lock); 150 if (class->lc_flags & LC_SLEEPABLE) 151 sleepq_lock(cvp); 152 } 153 sleepq_wait(cvp, 0); 154 155#ifdef KTRACE 156 if (KTRPOINT(td, KTR_CSW)) 157 ktrcsw(0, 0, wmesg); 158#endif 159 PICKUP_GIANT(); 160 if (lock != &Giant.lock_object) { 161 class->lc_lock(lock, lock_state); 162 WITNESS_RESTORE(lock, lock_witness); 163 } 164} 165 166/* 167 * Wait on a condition variable. This function differs from cv_wait by 168 * not acquiring the mutex after condition variable was signaled. 169 */ 170void 171_cv_wait_unlock(struct cv *cvp, struct lock_object *lock) 172{ 173#ifdef KTRACE 174 char wmesg[WMESGLEN + 1]; 175#endif 176 struct lock_class *class; 177 struct thread *td __ktrace_used; 178 179 td = curthread; 180 CV_ASSERT(cvp, lock, td); 181 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 182 "Waiting on \"%s\"", cvp->cv_description); 183 KASSERT(lock != &Giant.lock_object, 184 ("cv_wait_unlock cannot be used with Giant")); 185 class = LOCK_CLASS(lock); 186 187 if (SCHEDULER_STOPPED()) { 188 class->lc_unlock(lock); 189 return; 190 } 191 192#ifdef KTRACE 193 if (KTRPOINT(td, KTR_CSW)) { 194 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 195 ktrcsw(1, 0, wmesg); 196 } else { 197 wmesg[0] = '\0'; 198 } 199#endif 200 201 sleepq_lock(cvp); 202 203 CV_WAITERS_INC(cvp); 204 DROP_GIANT(); 205 206 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 207 if (class->lc_flags & LC_SLEEPABLE) 208 sleepq_release(cvp); 209 class->lc_unlock(lock); 210 if (class->lc_flags & LC_SLEEPABLE) 211 sleepq_lock(cvp); 212 sleepq_wait(cvp, 0); 213 214#ifdef KTRACE 215 if (KTRPOINT(td, KTR_CSW)) 216 ktrcsw(0, 0, wmesg); 217#endif 218 PICKUP_GIANT(); 219} 220 221/* 222 * Wait on a condition variable, allowing interruption by signals. Return 0 if 223 * the thread was resumed with cv_signal or cv_broadcast, EINTR or ERESTART if 224 * a signal was caught. If ERESTART is returned the system call should be 225 * restarted if possible. 226 */ 227int 228_cv_wait_sig(struct cv *cvp, struct lock_object *lock) 229{ 230 WITNESS_SAVE_DECL(lock_witness); 231#ifdef KTRACE 232 char wmesg[WMESGLEN + 1]; 233#endif 234 struct lock_class *class; 235 struct thread *td __ktrace_used; 236 uintptr_t lock_state; 237 int rval; 238 239 td = curthread; 240 CV_ASSERT(cvp, lock, td); 241 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 242 "Waiting on \"%s\"", cvp->cv_description); 243 244 if (SCHEDULER_STOPPED()) 245 return (0); 246 247#ifdef KTRACE 248 if (KTRPOINT(td, KTR_CSW)) { 249 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 250 ktrcsw(1, 0, wmesg); 251 } else { 252 wmesg[0] = '\0'; 253 } 254#endif 255 256 class = LOCK_CLASS(lock); 257 lock_state = 0; 258 sleepq_lock(cvp); 259 260 CV_WAITERS_INC(cvp); 261 if (lock == &Giant.lock_object) 262 mtx_assert(&Giant, MA_OWNED); 263 DROP_GIANT(); 264 265 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 266 SLEEPQ_INTERRUPTIBLE, 0); 267 if (lock != &Giant.lock_object) { 268 if (class->lc_flags & LC_SLEEPABLE) 269 sleepq_release(cvp); 270 WITNESS_SAVE(lock, lock_witness); 271 lock_state = class->lc_unlock(lock); 272 if (class->lc_flags & LC_SLEEPABLE) 273 sleepq_lock(cvp); 274 } 275 rval = sleepq_wait_sig(cvp, 0); 276 277#ifdef KTRACE 278 if (KTRPOINT(td, KTR_CSW)) 279 ktrcsw(0, 0, wmesg); 280#endif 281 PICKUP_GIANT(); 282 if (lock != &Giant.lock_object) { 283 class->lc_lock(lock, lock_state); 284 WITNESS_RESTORE(lock, lock_witness); 285 } 286 287 return (rval); 288} 289 290/* 291 * Wait on a condition variable for (at most) the value specified in sbt 292 * argument. Returns 0 if the process was resumed by cv_signal or cv_broadcast, 293 * EWOULDBLOCK if the timeout expires. 294 */ 295int 296_cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt, 297 sbintime_t pr, int flags) 298{ 299 WITNESS_SAVE_DECL(lock_witness); 300#ifdef KTRACE 301 char wmesg[WMESGLEN + 1]; 302#endif 303 struct lock_class *class; 304 struct thread *td __ktrace_used; 305 int lock_state, rval; 306 307 td = curthread; 308 CV_ASSERT(cvp, lock, td); 309 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 310 "Waiting on \"%s\"", cvp->cv_description); 311 312 if (SCHEDULER_STOPPED()) 313 return (0); 314 315#ifdef KTRACE 316 if (KTRPOINT(td, KTR_CSW)) { 317 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 318 ktrcsw(1, 0, wmesg); 319 } else { 320 wmesg[0] = '\0'; 321 } 322#endif 323 324 class = LOCK_CLASS(lock); 325 lock_state = 0; 326 sleepq_lock(cvp); 327 328 CV_WAITERS_INC(cvp); 329 if (lock == &Giant.lock_object) 330 mtx_assert(&Giant, MA_OWNED); 331 DROP_GIANT(); 332 333 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0); 334 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 335 if (lock != &Giant.lock_object) { 336 if (class->lc_flags & LC_SLEEPABLE) 337 sleepq_release(cvp); 338 WITNESS_SAVE(lock, lock_witness); 339 lock_state = class->lc_unlock(lock); 340 if (class->lc_flags & LC_SLEEPABLE) 341 sleepq_lock(cvp); 342 } 343 rval = sleepq_timedwait(cvp, 0); 344 345#ifdef KTRACE 346 if (KTRPOINT(td, KTR_CSW)) 347 ktrcsw(0, 0, wmesg); 348#endif 349 PICKUP_GIANT(); 350 if (lock != &Giant.lock_object) { 351 class->lc_lock(lock, lock_state); 352 WITNESS_RESTORE(lock, lock_witness); 353 } 354 355 return (rval); 356} 357 358/* 359 * Wait on a condition variable for (at most) the value specified in sbt 360 * argument, allowing interruption by signals. 361 * Returns 0 if the thread was resumed by cv_signal or cv_broadcast, 362 * EWOULDBLOCK if the timeout expires, and EINTR or ERESTART if a signal 363 * was caught. 364 */ 365int 366_cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock, 367 sbintime_t sbt, sbintime_t pr, int flags) 368{ 369 WITNESS_SAVE_DECL(lock_witness); 370#ifdef KTRACE 371 char wmesg[WMESGLEN + 1]; 372#endif 373 struct lock_class *class; 374 struct thread *td __ktrace_used; 375 int lock_state, rval; 376 377 td = curthread; 378 CV_ASSERT(cvp, lock, td); 379 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock, 380 "Waiting on \"%s\"", cvp->cv_description); 381 382 if (SCHEDULER_STOPPED()) 383 return (0); 384 385#ifdef KTRACE 386 if (KTRPOINT(td, KTR_CSW)) { 387 strlcpy(wmesg, cv_wmesg(cvp), sizeof(wmesg)); 388 ktrcsw(1, 0, wmesg); 389 } else { 390 wmesg[0] = '\0'; 391 } 392#endif 393 394 class = LOCK_CLASS(lock); 395 lock_state = 0; 396 sleepq_lock(cvp); 397 398 CV_WAITERS_INC(cvp); 399 if (lock == &Giant.lock_object) 400 mtx_assert(&Giant, MA_OWNED); 401 DROP_GIANT(); 402 403 sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR | 404 SLEEPQ_INTERRUPTIBLE, 0); 405 sleepq_set_timeout_sbt(cvp, sbt, pr, flags); 406 if (lock != &Giant.lock_object) { 407 if (class->lc_flags & LC_SLEEPABLE) 408 sleepq_release(cvp); 409 WITNESS_SAVE(lock, lock_witness); 410 lock_state = class->lc_unlock(lock); 411 if (class->lc_flags & LC_SLEEPABLE) 412 sleepq_lock(cvp); 413 } 414 rval = sleepq_timedwait_sig(cvp, 0); 415 416#ifdef KTRACE 417 if (KTRPOINT(td, KTR_CSW)) 418 ktrcsw(0, 0, wmesg); 419#endif 420 PICKUP_GIANT(); 421 if (lock != &Giant.lock_object) { 422 class->lc_lock(lock, lock_state); 423 WITNESS_RESTORE(lock, lock_witness); 424 } 425 426 return (rval); 427} 428 429/* 430 * Signal a condition variable, wakes up one waiting thread. Will also wakeup 431 * the swapper if the process is not in memory, so that it can bring the 432 * sleeping process in. Note that this may also result in additional threads 433 * being made runnable. Should be called with the same mutex as was passed to 434 * cv_wait held. 435 */ 436void 437cv_signal(struct cv *cvp) 438{ 439 440 if (cvp->cv_waiters == 0) 441 return; 442 sleepq_lock(cvp); 443 if (cvp->cv_waiters == 0) { 444 sleepq_release(cvp); 445 return; 446 } 447 if (cvp->cv_waiters == CV_WAITERS_BOUND && sleepq_lookup(cvp) == NULL) { 448 cvp->cv_waiters = 0; 449 sleepq_release(cvp); 450 } else { 451 if (cvp->cv_waiters < CV_WAITERS_BOUND) 452 cvp->cv_waiters--; 453 if (sleepq_signal(cvp, SLEEPQ_CONDVAR | SLEEPQ_DROP, 0, 0)) 454 kick_proc0(); 455 } 456} 457 458/* 459 * Broadcast a signal to a condition variable. Wakes up all waiting threads. 460 * Should be called with the same mutex as was passed to cv_wait held. 461 */ 462void 463cv_broadcastpri(struct cv *cvp, int pri) 464{ 465 int wakeup_swapper; 466 467 if (cvp->cv_waiters == 0) 468 return; 469 /* 470 * XXX sleepq_broadcast pri argument changed from -1 meaning 471 * no pri to 0 meaning no pri. 472 */ 473 wakeup_swapper = 0; 474 if (pri == -1) 475 pri = 0; 476 sleepq_lock(cvp); 477 if (cvp->cv_waiters > 0) { 478 cvp->cv_waiters = 0; 479 wakeup_swapper = sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0); 480 } 481 sleepq_release(cvp); 482 if (wakeup_swapper) 483 kick_proc0(); 484} 485