1/*- 2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org> 3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice(s), this list of conditions and the following disclaimer as 11 * the first lines of this file unmodified other than the possible 12 * addition of one or more copyright notices. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice(s), this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 18 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 21 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 27 * DAMAGE. 28 */ 29 30/* 31 * Shared/exclusive locks. This implementation attempts to ensure 32 * deterministic lock granting behavior, so that slocks and xlocks are 33 * interleaved. 34 * 35 * Priority propagation will not generally raise the priority of lock holders, 36 * so should not be relied upon in combination with sx locks. 37 */ 38 39#include "opt_ddb.h" 40#include "opt_hwpmc_hooks.h" 41#include "opt_kdtrace.h" 42#include "opt_no_adaptive_sx.h" 43 44#include <sys/cdefs.h> 45__FBSDID("$FreeBSD: stable/10/sys/kern/kern_sx.c 323870 2017-09-21 19:24:11Z marius $"); 46 47#include <sys/param.h> 48#include <sys/systm.h> 49#include <sys/kdb.h> 50#include <sys/kernel.h> 51#include <sys/ktr.h> 52#include <sys/lock.h> 53#include <sys/mutex.h> 54#include <sys/proc.h> 55#include <sys/sched.h> 56#include <sys/sleepqueue.h> 57#include <sys/sx.h> 58#include <sys/smp.h> 59#include <sys/sysctl.h> 60 61#if defined(SMP) && !defined(NO_ADAPTIVE_SX) 62#include <machine/cpu.h> 63#endif 64 65#ifdef DDB 66#include <ddb/ddb.h> 67#endif 68 69#if defined(SMP) && !defined(NO_ADAPTIVE_SX) 70#define ADAPTIVE_SX 71#endif 72 73CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE); 74 75#ifdef HWPMC_HOOKS 76#include <sys/pmckern.h> 77PMC_SOFT_DECLARE( , , lock, failed); 78#endif 79 80/* Handy macros for sleep queues. */ 81#define SQ_EXCLUSIVE_QUEUE 0 82#define SQ_SHARED_QUEUE 1 83 84/* 85 * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file. We 86 * drop Giant anytime we have to sleep or if we adaptively spin. 87 */ 88#define GIANT_DECLARE \ 89 int _giantcnt = 0; \ 90 WITNESS_SAVE_DECL(Giant) \ 91 92#define GIANT_SAVE() do { \ 93 if (mtx_owned(&Giant)) { \ 94 WITNESS_SAVE(&Giant.lock_object, Giant); \ 95 while (mtx_owned(&Giant)) { \ 96 _giantcnt++; \ 97 mtx_unlock(&Giant); \ 98 } \ 99 } \ 100} while (0) 101 102#define GIANT_RESTORE() do { \ 103 if (_giantcnt > 0) { \ 104 mtx_assert(&Giant, MA_NOTOWNED); \ 105 while (_giantcnt--) \ 106 mtx_lock(&Giant); \ 107 WITNESS_RESTORE(&Giant.lock_object, Giant); \ 108 } \ 109} while (0) 110 111/* 112 * Returns true if an exclusive lock is recursed. It assumes 113 * curthread currently has an exclusive lock. 114 */ 115#define sx_recurse lock_object.lo_data 116#define sx_recursed(sx) ((sx)->sx_recurse != 0) 117 118static void assert_sx(const struct lock_object *lock, int what); 119#ifdef DDB 120static void db_show_sx(const struct lock_object *lock); 121#endif 122static void lock_sx(struct lock_object *lock, uintptr_t how); 123#ifdef KDTRACE_HOOKS 124static int owner_sx(const struct lock_object *lock, struct thread **owner); 125#endif 126static uintptr_t unlock_sx(struct lock_object *lock); 127 128struct lock_class lock_class_sx = { 129 .lc_name = "sx", 130 .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE, 131 .lc_assert = assert_sx, 132#ifdef DDB 133 .lc_ddb_show = db_show_sx, 134#endif 135 .lc_lock = lock_sx, 136 .lc_unlock = unlock_sx, 137#ifdef KDTRACE_HOOKS 138 .lc_owner = owner_sx, 139#endif 140}; 141 142#ifndef INVARIANTS 143#define _sx_assert(sx, what, file, line) 144#endif 145 146#ifdef ADAPTIVE_SX 147static u_int asx_retries = 10; 148static u_int asx_loops = 10000; 149static SYSCTL_NODE(_debug, OID_AUTO, sx, CTLFLAG_RD, NULL, "sxlock debugging"); 150SYSCTL_UINT(_debug_sx, OID_AUTO, retries, CTLFLAG_RW, &asx_retries, 0, ""); 151SYSCTL_UINT(_debug_sx, OID_AUTO, loops, CTLFLAG_RW, &asx_loops, 0, ""); 152 153static struct lock_delay_config sx_delay = { 154 .initial = 1000, 155 .step = 500, 156 .min = 100, 157 .max = 5000, 158}; 159 160SYSCTL_INT(_debug_sx, OID_AUTO, delay_initial, CTLFLAG_RW, &sx_delay.initial, 161 0, ""); 162SYSCTL_INT(_debug_sx, OID_AUTO, delay_step, CTLFLAG_RW, &sx_delay.step, 163 0, ""); 164SYSCTL_INT(_debug_sx, OID_AUTO, delay_min, CTLFLAG_RW, &sx_delay.min, 165 0, ""); 166SYSCTL_INT(_debug_sx, OID_AUTO, delay_max, CTLFLAG_RW, &sx_delay.max, 167 0, ""); 168 169static void 170sx_delay_sysinit(void *dummy) 171{ 172 173 sx_delay.initial = mp_ncpus * 25; 174 sx_delay.step = (mp_ncpus * 25) / 2; 175 sx_delay.min = mp_ncpus * 5; 176 sx_delay.max = mp_ncpus * 25 * 10; 177} 178LOCK_DELAY_SYSINIT(sx_delay_sysinit); 179#endif 180 181void 182assert_sx(const struct lock_object *lock, int what) 183{ 184 185 sx_assert((const struct sx *)lock, what); 186} 187 188void 189lock_sx(struct lock_object *lock, uintptr_t how) 190{ 191 struct sx *sx; 192 193 sx = (struct sx *)lock; 194 if (how) 195 sx_slock(sx); 196 else 197 sx_xlock(sx); 198} 199 200uintptr_t 201unlock_sx(struct lock_object *lock) 202{ 203 struct sx *sx; 204 205 sx = (struct sx *)lock; 206 sx_assert(sx, SA_LOCKED | SA_NOTRECURSED); 207 if (sx_xlocked(sx)) { 208 sx_xunlock(sx); 209 return (0); 210 } else { 211 sx_sunlock(sx); 212 return (1); 213 } 214} 215 216#ifdef KDTRACE_HOOKS 217int 218owner_sx(const struct lock_object *lock, struct thread **owner) 219{ 220 const struct sx *sx = (const struct sx *)lock; 221 uintptr_t x = sx->sx_lock; 222 223 *owner = (struct thread *)SX_OWNER(x); 224 return ((x & SX_LOCK_SHARED) != 0 ? (SX_SHARERS(x) != 0) : 225 (*owner != NULL)); 226} 227#endif 228 229void 230sx_sysinit(void *arg) 231{ 232 struct sx_args *sargs = arg; 233 234 sx_init_flags(sargs->sa_sx, sargs->sa_desc, sargs->sa_flags); 235} 236 237void 238sx_init_flags(struct sx *sx, const char *description, int opts) 239{ 240 int flags; 241 242 MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK | 243 SX_NOPROFILE | SX_NOADAPTIVE | SX_NEW)) == 0); 244 ASSERT_ATOMIC_LOAD_PTR(sx->sx_lock, 245 ("%s: sx_lock not aligned for %s: %p", __func__, description, 246 &sx->sx_lock)); 247 248 flags = LO_SLEEPABLE | LO_UPGRADABLE; 249 if (opts & SX_DUPOK) 250 flags |= LO_DUPOK; 251 if (opts & SX_NOPROFILE) 252 flags |= LO_NOPROFILE; 253 if (!(opts & SX_NOWITNESS)) 254 flags |= LO_WITNESS; 255 if (opts & SX_RECURSE) 256 flags |= LO_RECURSABLE; 257 if (opts & SX_QUIET) 258 flags |= LO_QUIET; 259 if (opts & SX_NEW) 260 flags |= LO_NEW; 261 262 flags |= opts & SX_NOADAPTIVE; 263 lock_init(&sx->lock_object, &lock_class_sx, description, NULL, flags); 264 sx->sx_lock = SX_LOCK_UNLOCKED; 265 sx->sx_recurse = 0; 266} 267 268void 269sx_destroy(struct sx *sx) 270{ 271 272 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held")); 273 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed")); 274 sx->sx_lock = SX_LOCK_DESTROYED; 275 lock_destroy(&sx->lock_object); 276} 277 278int 279_sx_slock(struct sx *sx, int opts, const char *file, int line) 280{ 281 int error = 0; 282 283 if (SCHEDULER_STOPPED()) 284 return (0); 285 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 286 ("sx_slock() by idle thread %p on sx %s @ %s:%d", 287 curthread, sx->lock_object.lo_name, file, line)); 288 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 289 ("sx_slock() of destroyed sx @ %s:%d", file, line)); 290 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line, NULL); 291 error = __sx_slock(sx, opts, file, line); 292 if (!error) { 293 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line); 294 WITNESS_LOCK(&sx->lock_object, 0, file, line); 295 curthread->td_locks++; 296 } 297 298 return (error); 299} 300 301int 302sx_try_slock_(struct sx *sx, const char *file, int line) 303{ 304 uintptr_t x; 305 306 if (SCHEDULER_STOPPED()) 307 return (1); 308 309 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 310 ("sx_try_slock() by idle thread %p on sx %s @ %s:%d", 311 curthread, sx->lock_object.lo_name, file, line)); 312 313 for (;;) { 314 x = sx->sx_lock; 315 KASSERT(x != SX_LOCK_DESTROYED, 316 ("sx_try_slock() of destroyed sx @ %s:%d", file, line)); 317 if (!(x & SX_LOCK_SHARED)) 318 break; 319 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER)) { 320 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line); 321 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line); 322 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, 323 sx, 0, 0, file, line); 324 curthread->td_locks++; 325 return (1); 326 } 327 } 328 329 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line); 330 return (0); 331} 332 333int 334_sx_xlock(struct sx *sx, int opts, const char *file, int line) 335{ 336 int error = 0; 337 338 if (SCHEDULER_STOPPED()) 339 return (0); 340 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 341 ("sx_xlock() by idle thread %p on sx %s @ %s:%d", 342 curthread, sx->lock_object.lo_name, file, line)); 343 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 344 ("sx_xlock() of destroyed sx @ %s:%d", file, line)); 345 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 346 line, NULL); 347 error = __sx_xlock(sx, curthread, opts, file, line); 348 if (!error) { 349 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, 350 file, line); 351 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 352 curthread->td_locks++; 353 } 354 355 return (error); 356} 357 358int 359sx_try_xlock_(struct sx *sx, const char *file, int line) 360{ 361 int rval; 362 363 if (SCHEDULER_STOPPED()) 364 return (1); 365 366 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 367 ("sx_try_xlock() by idle thread %p on sx %s @ %s:%d", 368 curthread, sx->lock_object.lo_name, file, line)); 369 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 370 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line)); 371 372 if (sx_xlocked(sx) && 373 (sx->lock_object.lo_flags & LO_RECURSABLE) != 0) { 374 sx->sx_recurse++; 375 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 376 rval = 1; 377 } else 378 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, 379 (uintptr_t)curthread); 380 LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line); 381 if (rval) { 382 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 383 file, line); 384 if (!sx_recursed(sx)) 385 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, 386 sx, 0, 0, file, line); 387 curthread->td_locks++; 388 } 389 390 return (rval); 391} 392 393void 394_sx_sunlock(struct sx *sx, const char *file, int line) 395{ 396 397 if (SCHEDULER_STOPPED()) 398 return; 399 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 400 ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); 401 _sx_assert(sx, SA_SLOCKED, file, line); 402 WITNESS_UNLOCK(&sx->lock_object, 0, file, line); 403 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); 404 __sx_sunlock(sx, file, line); 405 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx); 406 curthread->td_locks--; 407} 408 409void 410_sx_xunlock(struct sx *sx, const char *file, int line) 411{ 412 413 if (SCHEDULER_STOPPED()) 414 return; 415 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 416 ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); 417 _sx_assert(sx, SA_XLOCKED, file, line); 418 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); 419 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, 420 line); 421 if (!sx_recursed(sx)) 422 LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_XUNLOCK_RELEASE, sx); 423 __sx_xunlock(sx, curthread, file, line); 424 curthread->td_locks--; 425} 426 427/* 428 * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. 429 * This will only succeed if this thread holds a single shared lock. 430 * Return 1 if if the upgrade succeed, 0 otherwise. 431 */ 432int 433sx_try_upgrade_(struct sx *sx, const char *file, int line) 434{ 435 uintptr_t x; 436 int success; 437 438 if (SCHEDULER_STOPPED()) 439 return (1); 440 441 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 442 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); 443 _sx_assert(sx, SA_SLOCKED, file, line); 444 445 /* 446 * Try to switch from one shared lock to an exclusive lock. We need 447 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that 448 * we will wake up the exclusive waiters when we drop the lock. 449 */ 450 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; 451 success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, 452 (uintptr_t)curthread | x); 453 LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); 454 if (success) { 455 WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 456 file, line); 457 LOCKSTAT_RECORD0(LS_SX_TRYUPGRADE_UPGRADE, sx); 458 } 459 return (success); 460} 461 462/* 463 * Downgrade an unrecursed exclusive lock into a single shared lock. 464 */ 465void 466sx_downgrade_(struct sx *sx, const char *file, int line) 467{ 468 uintptr_t x; 469 int wakeup_swapper; 470 471 if (SCHEDULER_STOPPED()) 472 return; 473 474 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, 475 ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); 476 _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); 477#ifndef INVARIANTS 478 if (sx_recursed(sx)) 479 panic("downgrade of a recursed lock"); 480#endif 481 482 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); 483 484 /* 485 * Try to switch from an exclusive lock with no shared waiters 486 * to one sharer with no shared waiters. If there are 487 * exclusive waiters, we don't need to lock the sleep queue so 488 * long as we preserve the flag. We do one quick try and if 489 * that fails we grab the sleepq lock to keep the flags from 490 * changing and do it the slow way. 491 * 492 * We have to lock the sleep queue if there are shared waiters 493 * so we can wake them up. 494 */ 495 x = sx->sx_lock; 496 if (!(x & SX_LOCK_SHARED_WAITERS) && 497 atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | 498 (x & SX_LOCK_EXCLUSIVE_WAITERS))) { 499 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 500 return; 501 } 502 503 /* 504 * Lock the sleep queue so we can read the waiters bits 505 * without any races and wakeup any shared waiters. 506 */ 507 sleepq_lock(&sx->lock_object); 508 509 /* 510 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single 511 * shared lock. If there are any shared waiters, wake them up. 512 */ 513 wakeup_swapper = 0; 514 x = sx->sx_lock; 515 atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | 516 (x & SX_LOCK_EXCLUSIVE_WAITERS)); 517 if (x & SX_LOCK_SHARED_WAITERS) 518 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 519 0, SQ_SHARED_QUEUE); 520 sleepq_release(&sx->lock_object); 521 522 LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); 523 LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx); 524 525 if (wakeup_swapper) 526 kick_proc0(); 527} 528 529/* 530 * This function represents the so-called 'hard case' for sx_xlock 531 * operation. All 'easy case' failures are redirected to this. Note 532 * that ideally this would be a static function, but it needs to be 533 * accessible from at least sx.h. 534 */ 535int 536_sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file, 537 int line) 538{ 539 GIANT_DECLARE; 540#ifdef ADAPTIVE_SX 541 volatile struct thread *owner; 542 u_int i, spintries = 0; 543#endif 544 uintptr_t x; 545#ifdef LOCK_PROFILING 546 uint64_t waittime = 0; 547 int contested = 0; 548#endif 549 int error = 0; 550#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) 551 struct lock_delay_arg lda; 552#endif 553#ifdef KDTRACE_HOOKS 554 uintptr_t state; 555 u_int sleep_cnt = 0; 556 int64_t sleep_time = 0; 557 int64_t all_time = 0; 558#endif 559 560 if (SCHEDULER_STOPPED()) 561 return (0); 562 563#if defined(ADAPTIVE_SX) 564 lock_delay_arg_init(&lda, &sx_delay); 565#elif defined(KDTRACE_HOOKS) 566 lock_delay_arg_init(&lda, NULL); 567#endif 568 569 /* If we already hold an exclusive lock, then recurse. */ 570 if (sx_xlocked(sx)) { 571 KASSERT((sx->lock_object.lo_flags & LO_RECURSABLE) != 0, 572 ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n", 573 sx->lock_object.lo_name, file, line)); 574 sx->sx_recurse++; 575 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 576 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 577 CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx); 578 return (0); 579 } 580 581 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 582 CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, 583 sx->lock_object.lo_name, (void *)sx->sx_lock, file, line); 584 585#ifdef KDTRACE_HOOKS 586 all_time -= lockstat_nsecs(&sx->lock_object); 587 state = sx->sx_lock; 588#endif 589 for (;;) { 590 if (sx->sx_lock == SX_LOCK_UNLOCKED && 591 atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) 592 break; 593#ifdef KDTRACE_HOOKS 594 lda.spin_cnt++; 595#endif 596#ifdef HWPMC_HOOKS 597 PMC_SOFT_CALL( , , lock, failed); 598#endif 599 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 600 &waittime); 601#ifdef ADAPTIVE_SX 602 /* 603 * If the lock is write locked and the owner is 604 * running on another CPU, spin until the owner stops 605 * running or the state of the lock changes. 606 */ 607 x = sx->sx_lock; 608 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 609 if ((x & SX_LOCK_SHARED) == 0) { 610 x = SX_OWNER(x); 611 owner = (struct thread *)x; 612 if (TD_IS_RUNNING(owner)) { 613 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 614 CTR3(KTR_LOCK, 615 "%s: spinning on %p held by %p", 616 __func__, sx, owner); 617 KTR_STATE1(KTR_SCHED, "thread", 618 sched_tdname(curthread), "spinning", 619 "lockname:\"%s\"", 620 sx->lock_object.lo_name); 621 GIANT_SAVE(); 622 while (SX_OWNER(sx->sx_lock) == x && 623 TD_IS_RUNNING(owner)) 624 lock_delay(&lda); 625 KTR_STATE0(KTR_SCHED, "thread", 626 sched_tdname(curthread), "running"); 627 continue; 628 } 629 } else if (SX_SHARERS(x) && spintries < asx_retries) { 630 KTR_STATE1(KTR_SCHED, "thread", 631 sched_tdname(curthread), "spinning", 632 "lockname:\"%s\"", sx->lock_object.lo_name); 633 GIANT_SAVE(); 634 spintries++; 635 for (i = 0; i < asx_loops; i++) { 636 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 637 CTR4(KTR_LOCK, 638 "%s: shared spinning on %p with %u and %u", 639 __func__, sx, spintries, i); 640 x = sx->sx_lock; 641 if ((x & SX_LOCK_SHARED) == 0 || 642 SX_SHARERS(x) == 0) 643 break; 644 cpu_spinwait(); 645#ifdef KDTRACE_HOOKS 646 lda.spin_cnt++; 647#endif 648 } 649 KTR_STATE0(KTR_SCHED, "thread", 650 sched_tdname(curthread), "running"); 651 if (i != asx_loops) 652 continue; 653 } 654 } 655#endif 656 657 sleepq_lock(&sx->lock_object); 658 x = sx->sx_lock; 659 660 /* 661 * If the lock was released while spinning on the 662 * sleep queue chain lock, try again. 663 */ 664 if (x == SX_LOCK_UNLOCKED) { 665 sleepq_release(&sx->lock_object); 666 continue; 667 } 668 669#ifdef ADAPTIVE_SX 670 /* 671 * The current lock owner might have started executing 672 * on another CPU (or the lock could have changed 673 * owners) while we were waiting on the sleep queue 674 * chain lock. If so, drop the sleep queue lock and try 675 * again. 676 */ 677 if (!(x & SX_LOCK_SHARED) && 678 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 679 owner = (struct thread *)SX_OWNER(x); 680 if (TD_IS_RUNNING(owner)) { 681 sleepq_release(&sx->lock_object); 682 continue; 683 } 684 } 685#endif 686 687 /* 688 * If an exclusive lock was released with both shared 689 * and exclusive waiters and a shared waiter hasn't 690 * woken up and acquired the lock yet, sx_lock will be 691 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS. 692 * If we see that value, try to acquire it once. Note 693 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS 694 * as there are other exclusive waiters still. If we 695 * fail, restart the loop. 696 */ 697 if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) { 698 if (atomic_cmpset_acq_ptr(&sx->sx_lock, 699 SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS, 700 tid | SX_LOCK_EXCLUSIVE_WAITERS)) { 701 sleepq_release(&sx->lock_object); 702 CTR2(KTR_LOCK, "%s: %p claimed by new writer", 703 __func__, sx); 704 break; 705 } 706 sleepq_release(&sx->lock_object); 707 continue; 708 } 709 710 /* 711 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS. If we fail, 712 * than loop back and retry. 713 */ 714 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 715 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 716 x | SX_LOCK_EXCLUSIVE_WAITERS)) { 717 sleepq_release(&sx->lock_object); 718 continue; 719 } 720 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 721 CTR2(KTR_LOCK, "%s: %p set excl waiters flag", 722 __func__, sx); 723 } 724 725 /* 726 * Since we have been unable to acquire the exclusive 727 * lock and the exclusive waiters flag is set, we have 728 * to sleep. 729 */ 730 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 731 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 732 __func__, sx); 733 734#ifdef KDTRACE_HOOKS 735 sleep_time -= lockstat_nsecs(&sx->lock_object); 736#endif 737 GIANT_SAVE(); 738 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 739 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 740 SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE); 741 if (!(opts & SX_INTERRUPTIBLE)) 742 sleepq_wait(&sx->lock_object, 0); 743 else 744 error = sleepq_wait_sig(&sx->lock_object, 0); 745#ifdef KDTRACE_HOOKS 746 sleep_time += lockstat_nsecs(&sx->lock_object); 747 sleep_cnt++; 748#endif 749 if (error) { 750 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 751 CTR2(KTR_LOCK, 752 "%s: interruptible sleep by %p suspended by signal", 753 __func__, sx); 754 break; 755 } 756 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 757 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 758 __func__, sx); 759 } 760#ifdef KDTRACE_HOOKS 761 all_time += lockstat_nsecs(&sx->lock_object); 762 if (sleep_time) 763 LOCKSTAT_RECORD4(LS_SX_XLOCK_BLOCK, sx, sleep_time, 764 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 765 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 766 if (lda.spin_cnt > sleep_cnt) 767 LOCKSTAT_RECORD4(LS_SX_XLOCK_SPIN, sx, all_time - sleep_time, 768 LOCKSTAT_WRITER, (state & SX_LOCK_SHARED) == 0, 769 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 770#endif 771 if (!error) 772 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_XLOCK_ACQUIRE, sx, 773 contested, waittime, file, line); 774 GIANT_RESTORE(); 775 return (error); 776} 777 778/* 779 * This function represents the so-called 'hard case' for sx_xunlock 780 * operation. All 'easy case' failures are redirected to this. Note 781 * that ideally this would be a static function, but it needs to be 782 * accessible from at least sx.h. 783 */ 784void 785_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) 786{ 787 uintptr_t x; 788 int queue, wakeup_swapper; 789 790 if (SCHEDULER_STOPPED()) 791 return; 792 793 MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); 794 795 /* If the lock is recursed, then unrecurse one level. */ 796 if (sx_xlocked(sx) && sx_recursed(sx)) { 797 if ((--sx->sx_recurse) == 0) 798 atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); 799 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 800 CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); 801 return; 802 } 803 MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | 804 SX_LOCK_EXCLUSIVE_WAITERS)); 805 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 806 CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); 807 808 sleepq_lock(&sx->lock_object); 809 x = SX_LOCK_UNLOCKED; 810 811 /* 812 * The wake up algorithm here is quite simple and probably not 813 * ideal. It gives precedence to shared waiters if they are 814 * present. For this condition, we have to preserve the 815 * state of the exclusive waiters flag. 816 * If interruptible sleeps left the shared queue empty avoid a 817 * starvation for the threads sleeping on the exclusive queue by giving 818 * them precedence and cleaning up the shared waiters bit anyway. 819 */ 820 if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 && 821 sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) { 822 queue = SQ_SHARED_QUEUE; 823 x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); 824 } else 825 queue = SQ_EXCLUSIVE_QUEUE; 826 827 /* Wake up all the waiters for the specific queue. */ 828 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 829 CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", 830 __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : 831 "exclusive"); 832 atomic_store_rel_ptr(&sx->sx_lock, x); 833 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, 834 queue); 835 sleepq_release(&sx->lock_object); 836 if (wakeup_swapper) 837 kick_proc0(); 838} 839 840/* 841 * This function represents the so-called 'hard case' for sx_slock 842 * operation. All 'easy case' failures are redirected to this. Note 843 * that ideally this would be a static function, but it needs to be 844 * accessible from at least sx.h. 845 */ 846int 847_sx_slock_hard(struct sx *sx, int opts, const char *file, int line) 848{ 849 GIANT_DECLARE; 850#ifdef ADAPTIVE_SX 851 volatile struct thread *owner; 852#endif 853#ifdef LOCK_PROFILING 854 uint64_t waittime = 0; 855 int contested = 0; 856#endif 857 uintptr_t x; 858 int error = 0; 859#if defined(ADAPTIVE_SX) || defined(KDTRACE_HOOKS) 860 struct lock_delay_arg lda; 861#endif 862#ifdef KDTRACE_HOOKS 863 uintptr_t state; 864 u_int sleep_cnt = 0; 865 int64_t sleep_time = 0; 866 int64_t all_time = 0; 867#endif 868 869 if (SCHEDULER_STOPPED()) 870 return (0); 871 872#if defined(ADAPTIVE_SX) 873 lock_delay_arg_init(&lda, &sx_delay); 874#elif defined(KDTRACE_HOOKS) 875 lock_delay_arg_init(&lda, NULL); 876#endif 877#ifdef KDTRACE_HOOKS 878 state = sx->sx_lock; 879 all_time -= lockstat_nsecs(&sx->lock_object); 880#endif 881 882 /* 883 * As with rwlocks, we don't make any attempt to try to block 884 * shared locks once there is an exclusive waiter. 885 */ 886 for (;;) { 887#ifdef KDTRACE_HOOKS 888 lda.spin_cnt++; 889#endif 890 x = sx->sx_lock; 891 892 /* 893 * If no other thread has an exclusive lock then try to bump up 894 * the count of sharers. Since we have to preserve the state 895 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the 896 * shared lock loop back and retry. 897 */ 898 if (x & SX_LOCK_SHARED) { 899 MPASS(!(x & SX_LOCK_SHARED_WAITERS)); 900 if (atomic_cmpset_acq_ptr(&sx->sx_lock, x, 901 x + SX_ONE_SHARER)) { 902 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 903 CTR4(KTR_LOCK, 904 "%s: %p succeed %p -> %p", __func__, 905 sx, (void *)x, 906 (void *)(x + SX_ONE_SHARER)); 907 break; 908 } 909 continue; 910 } 911#ifdef HWPMC_HOOKS 912 PMC_SOFT_CALL( , , lock, failed); 913#endif 914 lock_profile_obtain_lock_failed(&sx->lock_object, &contested, 915 &waittime); 916 917#ifdef ADAPTIVE_SX 918 /* 919 * If the owner is running on another CPU, spin until 920 * the owner stops running or the state of the lock 921 * changes. 922 */ 923 if ((sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 924 x = SX_OWNER(x); 925 owner = (struct thread *)x; 926 if (TD_IS_RUNNING(owner)) { 927 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 928 CTR3(KTR_LOCK, 929 "%s: spinning on %p held by %p", 930 __func__, sx, owner); 931 KTR_STATE1(KTR_SCHED, "thread", 932 sched_tdname(curthread), "spinning", 933 "lockname:\"%s\"", sx->lock_object.lo_name); 934 GIANT_SAVE(); 935 while (SX_OWNER(sx->sx_lock) == x && 936 TD_IS_RUNNING(owner)) 937 lock_delay(&lda); 938 KTR_STATE0(KTR_SCHED, "thread", 939 sched_tdname(curthread), "running"); 940 continue; 941 } 942 } 943#endif 944 945 /* 946 * Some other thread already has an exclusive lock, so 947 * start the process of blocking. 948 */ 949 sleepq_lock(&sx->lock_object); 950 x = sx->sx_lock; 951 952 /* 953 * The lock could have been released while we spun. 954 * In this case loop back and retry. 955 */ 956 if (x & SX_LOCK_SHARED) { 957 sleepq_release(&sx->lock_object); 958 continue; 959 } 960 961#ifdef ADAPTIVE_SX 962 /* 963 * If the owner is running on another CPU, spin until 964 * the owner stops running or the state of the lock 965 * changes. 966 */ 967 if (!(x & SX_LOCK_SHARED) && 968 (sx->lock_object.lo_flags & SX_NOADAPTIVE) == 0) { 969 owner = (struct thread *)SX_OWNER(x); 970 if (TD_IS_RUNNING(owner)) { 971 sleepq_release(&sx->lock_object); 972 continue; 973 } 974 } 975#endif 976 977 /* 978 * Try to set the SX_LOCK_SHARED_WAITERS flag. If we 979 * fail to set it drop the sleep queue lock and loop 980 * back. 981 */ 982 if (!(x & SX_LOCK_SHARED_WAITERS)) { 983 if (!atomic_cmpset_ptr(&sx->sx_lock, x, 984 x | SX_LOCK_SHARED_WAITERS)) { 985 sleepq_release(&sx->lock_object); 986 continue; 987 } 988 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 989 CTR2(KTR_LOCK, "%s: %p set shared waiters flag", 990 __func__, sx); 991 } 992 993 /* 994 * Since we have been unable to acquire the shared lock, 995 * we have to sleep. 996 */ 997 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 998 CTR2(KTR_LOCK, "%s: %p blocking on sleep queue", 999 __func__, sx); 1000 1001#ifdef KDTRACE_HOOKS 1002 sleep_time -= lockstat_nsecs(&sx->lock_object); 1003#endif 1004 GIANT_SAVE(); 1005 sleepq_add(&sx->lock_object, NULL, sx->lock_object.lo_name, 1006 SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ? 1007 SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE); 1008 if (!(opts & SX_INTERRUPTIBLE)) 1009 sleepq_wait(&sx->lock_object, 0); 1010 else 1011 error = sleepq_wait_sig(&sx->lock_object, 0); 1012#ifdef KDTRACE_HOOKS 1013 sleep_time += lockstat_nsecs(&sx->lock_object); 1014 sleep_cnt++; 1015#endif 1016 if (error) { 1017 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1018 CTR2(KTR_LOCK, 1019 "%s: interruptible sleep by %p suspended by signal", 1020 __func__, sx); 1021 break; 1022 } 1023 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1024 CTR2(KTR_LOCK, "%s: %p resuming from sleep queue", 1025 __func__, sx); 1026 } 1027#ifdef KDTRACE_HOOKS 1028 all_time += lockstat_nsecs(&sx->lock_object); 1029 if (sleep_time) 1030 LOCKSTAT_RECORD4(LS_SX_SLOCK_BLOCK, sx, sleep_time, 1031 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 1032 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 1033 if (lda.spin_cnt > sleep_cnt) 1034 LOCKSTAT_RECORD4(LS_SX_SLOCK_SPIN, sx, all_time - sleep_time, 1035 LOCKSTAT_READER, (state & SX_LOCK_SHARED) == 0, 1036 (state & SX_LOCK_SHARED) == 0 ? 0 : SX_SHARERS(state)); 1037#endif 1038 if (error == 0) 1039 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_SX_SLOCK_ACQUIRE, sx, 1040 contested, waittime, file, line); 1041 GIANT_RESTORE(); 1042 return (error); 1043} 1044 1045/* 1046 * This function represents the so-called 'hard case' for sx_sunlock 1047 * operation. All 'easy case' failures are redirected to this. Note 1048 * that ideally this would be a static function, but it needs to be 1049 * accessible from at least sx.h. 1050 */ 1051void 1052_sx_sunlock_hard(struct sx *sx, const char *file, int line) 1053{ 1054 uintptr_t x; 1055 int wakeup_swapper; 1056 1057 if (SCHEDULER_STOPPED()) 1058 return; 1059 1060 for (;;) { 1061 x = sx->sx_lock; 1062 1063 /* 1064 * We should never have sharers while at least one thread 1065 * holds a shared lock. 1066 */ 1067 KASSERT(!(x & SX_LOCK_SHARED_WAITERS), 1068 ("%s: waiting sharers", __func__)); 1069 1070 /* 1071 * See if there is more than one shared lock held. If 1072 * so, just drop one and return. 1073 */ 1074 if (SX_SHARERS(x) > 1) { 1075 if (atomic_cmpset_rel_ptr(&sx->sx_lock, x, 1076 x - SX_ONE_SHARER)) { 1077 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1078 CTR4(KTR_LOCK, 1079 "%s: %p succeeded %p -> %p", 1080 __func__, sx, (void *)x, 1081 (void *)(x - SX_ONE_SHARER)); 1082 break; 1083 } 1084 continue; 1085 } 1086 1087 /* 1088 * If there aren't any waiters for an exclusive lock, 1089 * then try to drop it quickly. 1090 */ 1091 if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) { 1092 MPASS(x == SX_SHARERS_LOCK(1)); 1093 if (atomic_cmpset_rel_ptr(&sx->sx_lock, 1094 SX_SHARERS_LOCK(1), SX_LOCK_UNLOCKED)) { 1095 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1096 CTR2(KTR_LOCK, "%s: %p last succeeded", 1097 __func__, sx); 1098 break; 1099 } 1100 continue; 1101 } 1102 1103 /* 1104 * At this point, there should just be one sharer with 1105 * exclusive waiters. 1106 */ 1107 MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS)); 1108 1109 sleepq_lock(&sx->lock_object); 1110 1111 /* 1112 * Wake up semantic here is quite simple: 1113 * Just wake up all the exclusive waiters. 1114 * Note that the state of the lock could have changed, 1115 * so if it fails loop back and retry. 1116 */ 1117 if (!atomic_cmpset_rel_ptr(&sx->sx_lock, 1118 SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS, 1119 SX_LOCK_UNLOCKED)) { 1120 sleepq_release(&sx->lock_object); 1121 continue; 1122 } 1123 if (LOCK_LOG_TEST(&sx->lock_object, 0)) 1124 CTR2(KTR_LOCK, "%s: %p waking up all thread on" 1125 "exclusive queue", __func__, sx); 1126 wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 1127 0, SQ_EXCLUSIVE_QUEUE); 1128 sleepq_release(&sx->lock_object); 1129 if (wakeup_swapper) 1130 kick_proc0(); 1131 break; 1132 } 1133} 1134 1135#ifdef INVARIANT_SUPPORT 1136#ifndef INVARIANTS 1137#undef _sx_assert 1138#endif 1139 1140/* 1141 * In the non-WITNESS case, sx_assert() can only detect that at least 1142 * *some* thread owns an slock, but it cannot guarantee that *this* 1143 * thread owns an slock. 1144 */ 1145void 1146_sx_assert(const struct sx *sx, int what, const char *file, int line) 1147{ 1148#ifndef WITNESS 1149 int slocked = 0; 1150#endif 1151 1152 if (panicstr != NULL) 1153 return; 1154 switch (what) { 1155 case SA_SLOCKED: 1156 case SA_SLOCKED | SA_NOTRECURSED: 1157 case SA_SLOCKED | SA_RECURSED: 1158#ifndef WITNESS 1159 slocked = 1; 1160 /* FALLTHROUGH */ 1161#endif 1162 case SA_LOCKED: 1163 case SA_LOCKED | SA_NOTRECURSED: 1164 case SA_LOCKED | SA_RECURSED: 1165#ifdef WITNESS 1166 witness_assert(&sx->lock_object, what, file, line); 1167#else 1168 /* 1169 * If some other thread has an exclusive lock or we 1170 * have one and are asserting a shared lock, fail. 1171 * Also, if no one has a lock at all, fail. 1172 */ 1173 if (sx->sx_lock == SX_LOCK_UNLOCKED || 1174 (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked || 1175 sx_xholder(sx) != curthread))) 1176 panic("Lock %s not %slocked @ %s:%d\n", 1177 sx->lock_object.lo_name, slocked ? "share " : "", 1178 file, line); 1179 1180 if (!(sx->sx_lock & SX_LOCK_SHARED)) { 1181 if (sx_recursed(sx)) { 1182 if (what & SA_NOTRECURSED) 1183 panic("Lock %s recursed @ %s:%d\n", 1184 sx->lock_object.lo_name, file, 1185 line); 1186 } else if (what & SA_RECURSED) 1187 panic("Lock %s not recursed @ %s:%d\n", 1188 sx->lock_object.lo_name, file, line); 1189 } 1190#endif 1191 break; 1192 case SA_XLOCKED: 1193 case SA_XLOCKED | SA_NOTRECURSED: 1194 case SA_XLOCKED | SA_RECURSED: 1195 if (sx_xholder(sx) != curthread) 1196 panic("Lock %s not exclusively locked @ %s:%d\n", 1197 sx->lock_object.lo_name, file, line); 1198 if (sx_recursed(sx)) { 1199 if (what & SA_NOTRECURSED) 1200 panic("Lock %s recursed @ %s:%d\n", 1201 sx->lock_object.lo_name, file, line); 1202 } else if (what & SA_RECURSED) 1203 panic("Lock %s not recursed @ %s:%d\n", 1204 sx->lock_object.lo_name, file, line); 1205 break; 1206 case SA_UNLOCKED: 1207#ifdef WITNESS 1208 witness_assert(&sx->lock_object, what, file, line); 1209#else 1210 /* 1211 * If we hold an exclusve lock fail. We can't 1212 * reliably check to see if we hold a shared lock or 1213 * not. 1214 */ 1215 if (sx_xholder(sx) == curthread) 1216 panic("Lock %s exclusively locked @ %s:%d\n", 1217 sx->lock_object.lo_name, file, line); 1218#endif 1219 break; 1220 default: 1221 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 1222 line); 1223 } 1224} 1225#endif /* INVARIANT_SUPPORT */ 1226 1227#ifdef DDB 1228static void 1229db_show_sx(const struct lock_object *lock) 1230{ 1231 struct thread *td; 1232 const struct sx *sx; 1233 1234 sx = (const struct sx *)lock; 1235 1236 db_printf(" state: "); 1237 if (sx->sx_lock == SX_LOCK_UNLOCKED) 1238 db_printf("UNLOCKED\n"); 1239 else if (sx->sx_lock == SX_LOCK_DESTROYED) { 1240 db_printf("DESTROYED\n"); 1241 return; 1242 } else if (sx->sx_lock & SX_LOCK_SHARED) 1243 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock)); 1244 else { 1245 td = sx_xholder(sx); 1246 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td, 1247 td->td_tid, td->td_proc->p_pid, td->td_name); 1248 if (sx_recursed(sx)) 1249 db_printf(" recursed: %d\n", sx->sx_recurse); 1250 } 1251 1252 db_printf(" waiters: "); 1253 switch(sx->sx_lock & 1254 (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) { 1255 case SX_LOCK_SHARED_WAITERS: 1256 db_printf("shared\n"); 1257 break; 1258 case SX_LOCK_EXCLUSIVE_WAITERS: 1259 db_printf("exclusive\n"); 1260 break; 1261 case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS: 1262 db_printf("exclusive and shared\n"); 1263 break; 1264 default: 1265 db_printf("none\n"); 1266 } 1267} 1268 1269/* 1270 * Check to see if a thread that is blocked on a sleep queue is actually 1271 * blocked on an sx lock. If so, output some details and return true. 1272 * If the lock has an exclusive owner, return that in *ownerp. 1273 */ 1274int 1275sx_chain(struct thread *td, struct thread **ownerp) 1276{ 1277 struct sx *sx; 1278 1279 /* 1280 * Check to see if this thread is blocked on an sx lock. 1281 * First, we check the lock class. If that is ok, then we 1282 * compare the lock name against the wait message. 1283 */ 1284 sx = td->td_wchan; 1285 if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx || 1286 sx->lock_object.lo_name != td->td_wmesg) 1287 return (0); 1288 1289 /* We think we have an sx lock, so output some details. */ 1290 db_printf("blocked on sx \"%s\" ", td->td_wmesg); 1291 *ownerp = sx_xholder(sx); 1292 if (sx->sx_lock & SX_LOCK_SHARED) 1293 db_printf("SLOCK (count %ju)\n", 1294 (uintmax_t)SX_SHARERS(sx->sx_lock)); 1295 else 1296 db_printf("XLOCK\n"); 1297 return (1); 1298} 1299#endif 1300