kern_mutex.c revision 340270
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> 37__FBSDID("$FreeBSD: stable/11/sys/kern/kern_mutex.c 340270 2018-11-08 22:42:55Z jhb $"); 38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_hwpmc_hooks.h" 42#include "opt_sched.h" 43 44#include <sys/param.h> 45#include <sys/systm.h> 46#include <sys/bus.h> 47#include <sys/conf.h> 48#include <sys/kdb.h> 49#include <sys/kernel.h> 50#include <sys/ktr.h> 51#include <sys/lock.h> 52#include <sys/malloc.h> 53#include <sys/mutex.h> 54#include <sys/proc.h> 55#include <sys/resourcevar.h> 56#include <sys/sched.h> 57#include <sys/sbuf.h> 58#include <sys/smp.h> 59#include <sys/sysctl.h> 60#include <sys/turnstile.h> 61#include <sys/vmmeter.h> 62#include <sys/lock_profile.h> 63 64#include <machine/atomic.h> 65#include <machine/bus.h> 66#include <machine/cpu.h> 67 68#include <ddb/ddb.h> 69 70#include <fs/devfs/devfs_int.h> 71 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74 75#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 76#define ADAPTIVE_MUTEXES 77#endif 78 79#ifdef HWPMC_HOOKS 80#include <sys/pmckern.h> 81PMC_SOFT_DEFINE( , , lock, failed); 82#endif 83 84/* 85 * Return the mutex address when the lock cookie address is provided. 86 * This functionality assumes that struct mtx* have a member named mtx_lock. 87 */ 88#define mtxlock2mtx(c) (__containerof(c, struct mtx, mtx_lock)) 89 90/* 91 * Internal utility macros. 92 */ 93#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 94 95#define mtx_destroyed(m) ((m)->mtx_lock == MTX_DESTROYED) 96 97static void assert_mtx(const struct lock_object *lock, int what); 98#ifdef DDB 99static void db_show_mtx(const struct lock_object *lock); 100#endif 101static void lock_mtx(struct lock_object *lock, uintptr_t how); 102static void lock_spin(struct lock_object *lock, uintptr_t how); 103#ifdef KDTRACE_HOOKS 104static int owner_mtx(const struct lock_object *lock, 105 struct thread **owner); 106#endif 107static uintptr_t unlock_mtx(struct lock_object *lock); 108static uintptr_t unlock_spin(struct lock_object *lock); 109 110/* 111 * Lock classes for sleep and spin mutexes. 112 */ 113struct lock_class lock_class_mtx_sleep = { 114 .lc_name = "sleep mutex", 115 .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 116 .lc_assert = assert_mtx, 117#ifdef DDB 118 .lc_ddb_show = db_show_mtx, 119#endif 120 .lc_lock = lock_mtx, 121 .lc_unlock = unlock_mtx, 122#ifdef KDTRACE_HOOKS 123 .lc_owner = owner_mtx, 124#endif 125}; 126struct lock_class lock_class_mtx_spin = { 127 .lc_name = "spin mutex", 128 .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 129 .lc_assert = assert_mtx, 130#ifdef DDB 131 .lc_ddb_show = db_show_mtx, 132#endif 133 .lc_lock = lock_spin, 134 .lc_unlock = unlock_spin, 135#ifdef KDTRACE_HOOKS 136 .lc_owner = owner_mtx, 137#endif 138}; 139 140#ifdef ADAPTIVE_MUTEXES 141static SYSCTL_NODE(_debug, OID_AUTO, mtx, CTLFLAG_RD, NULL, "mtx debugging"); 142 143static struct lock_delay_config __read_frequently mtx_delay; 144 145SYSCTL_INT(_debug_mtx, OID_AUTO, delay_base, CTLFLAG_RW, &mtx_delay.base, 146 0, ""); 147SYSCTL_INT(_debug_mtx, OID_AUTO, delay_max, CTLFLAG_RW, &mtx_delay.max, 148 0, ""); 149 150LOCK_DELAY_SYSINIT_DEFAULT(mtx_delay); 151#endif 152 153static SYSCTL_NODE(_debug, OID_AUTO, mtx_spin, CTLFLAG_RD, NULL, 154 "mtx spin debugging"); 155 156static struct lock_delay_config __read_frequently mtx_spin_delay; 157 158SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_base, CTLFLAG_RW, 159 &mtx_spin_delay.base, 0, ""); 160SYSCTL_INT(_debug_mtx_spin, OID_AUTO, delay_max, CTLFLAG_RW, 161 &mtx_spin_delay.max, 0, ""); 162 163LOCK_DELAY_SYSINIT_DEFAULT(mtx_spin_delay); 164 165/* 166 * System-wide mutexes 167 */ 168struct mtx blocked_lock; 169struct mtx Giant; 170 171static void _mtx_lock_indefinite_check(struct mtx *, struct lock_delay_arg *); 172 173void 174assert_mtx(const struct lock_object *lock, int what) 175{ 176 177 mtx_assert((const struct mtx *)lock, what); 178} 179 180void 181lock_mtx(struct lock_object *lock, uintptr_t how) 182{ 183 184 mtx_lock((struct mtx *)lock); 185} 186 187void 188lock_spin(struct lock_object *lock, uintptr_t how) 189{ 190 191 panic("spin locks can only use msleep_spin"); 192} 193 194uintptr_t 195unlock_mtx(struct lock_object *lock) 196{ 197 struct mtx *m; 198 199 m = (struct mtx *)lock; 200 mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 201 mtx_unlock(m); 202 return (0); 203} 204 205uintptr_t 206unlock_spin(struct lock_object *lock) 207{ 208 209 panic("spin locks can only use msleep_spin"); 210} 211 212#ifdef KDTRACE_HOOKS 213int 214owner_mtx(const struct lock_object *lock, struct thread **owner) 215{ 216 const struct mtx *m; 217 uintptr_t x; 218 219 m = (const struct mtx *)lock; 220 x = m->mtx_lock; 221 *owner = (struct thread *)(x & ~MTX_FLAGMASK); 222 return (*owner != NULL); 223} 224#endif 225 226/* 227 * Function versions of the inlined __mtx_* macros. These are used by 228 * modules and can also be called from assembly language if needed. 229 */ 230void 231__mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, int line) 232{ 233 struct mtx *m; 234 uintptr_t tid, v; 235 236 m = mtxlock2mtx(c); 237 238 KASSERT(kdb_active != 0 || SCHEDULER_STOPPED() || 239 !TD_IS_IDLETHREAD(curthread), 240 ("mtx_lock() by idle thread %p on sleep mutex %s @ %s:%d", 241 curthread, m->lock_object.lo_name, file, line)); 242 KASSERT(m->mtx_lock != MTX_DESTROYED, 243 ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 244 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 245 ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 246 file, line)); 247 WITNESS_CHECKORDER(&m->lock_object, (opts & ~MTX_RECURSE) | 248 LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 249 250 tid = (uintptr_t)curthread; 251 v = MTX_UNOWNED; 252 if (!_mtx_obtain_lock_fetch(m, &v, tid)) 253 _mtx_lock_sleep(m, v, opts, file, line); 254 else 255 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, 256 m, 0, 0, file, line); 257 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 258 line); 259 WITNESS_LOCK(&m->lock_object, (opts & ~MTX_RECURSE) | LOP_EXCLUSIVE, 260 file, line); 261 TD_LOCKS_INC(curthread); 262} 263 264void 265__mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, int line) 266{ 267 struct mtx *m; 268 269 m = mtxlock2mtx(c); 270 271 KASSERT(m->mtx_lock != MTX_DESTROYED, 272 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 273 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 274 ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 275 file, line)); 276 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 277 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 278 line); 279 mtx_assert(m, MA_OWNED); 280 281#ifdef LOCK_PROFILING 282 __mtx_unlock_sleep(c, (uintptr_t)curthread, opts, file, line); 283#else 284 __mtx_unlock(m, curthread, opts, file, line); 285#endif 286 TD_LOCKS_DEC(curthread); 287} 288 289void 290__mtx_lock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 291 int line) 292{ 293 struct mtx *m; 294#ifdef SMP 295 uintptr_t tid, v; 296#endif 297 298 m = mtxlock2mtx(c); 299 300 KASSERT(m->mtx_lock != MTX_DESTROYED, 301 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 302 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 303 ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 304 m->lock_object.lo_name, file, line)); 305 if (mtx_owned(m)) 306 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 307 (opts & MTX_RECURSE) != 0, 308 ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", 309 m->lock_object.lo_name, file, line)); 310 opts &= ~MTX_RECURSE; 311 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 312 file, line, NULL); 313#ifdef SMP 314 spinlock_enter(); 315 tid = (uintptr_t)curthread; 316 v = MTX_UNOWNED; 317 if (!_mtx_obtain_lock_fetch(m, &v, tid)) 318 _mtx_lock_spin(m, v, opts, file, line); 319 else 320 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, 321 m, 0, 0, file, line); 322#else 323 __mtx_lock_spin(m, curthread, opts, file, line); 324#endif 325 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 326 line); 327 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 328} 329 330int 331__mtx_trylock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 332 int line) 333{ 334 struct mtx *m; 335 336 if (SCHEDULER_STOPPED()) 337 return (1); 338 339 m = mtxlock2mtx(c); 340 341 KASSERT(m->mtx_lock != MTX_DESTROYED, 342 ("mtx_trylock_spin() of destroyed mutex @ %s:%d", file, line)); 343 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 344 ("mtx_trylock_spin() of sleep mutex %s @ %s:%d", 345 m->lock_object.lo_name, file, line)); 346 KASSERT((opts & MTX_RECURSE) == 0, 347 ("mtx_trylock_spin: unsupp. opt MTX_RECURSE on mutex %s @ %s:%d\n", 348 m->lock_object.lo_name, file, line)); 349 if (__mtx_trylock_spin(m, curthread, opts, file, line)) { 350 LOCK_LOG_TRY("LOCK", &m->lock_object, opts, 1, file, line); 351 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 352 return (1); 353 } 354 LOCK_LOG_TRY("LOCK", &m->lock_object, opts, 0, file, line); 355 return (0); 356} 357 358void 359__mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 360 int line) 361{ 362 struct mtx *m; 363 364 m = mtxlock2mtx(c); 365 366 KASSERT(m->mtx_lock != MTX_DESTROYED, 367 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 368 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 369 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 370 m->lock_object.lo_name, file, line)); 371 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 372 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 373 line); 374 mtx_assert(m, MA_OWNED); 375 376 __mtx_unlock_spin(m); 377} 378 379/* 380 * The important part of mtx_trylock{,_flags}() 381 * Tries to acquire lock `m.' If this function is called on a mutex that 382 * is already owned, it will recursively acquire the lock. 383 */ 384int 385_mtx_trylock_flags_int(struct mtx *m, int opts LOCK_FILE_LINE_ARG_DEF) 386{ 387 struct thread *td; 388 uintptr_t tid, v; 389#ifdef LOCK_PROFILING 390 uint64_t waittime = 0; 391 int contested = 0; 392#endif 393 int rval; 394 bool recursed; 395 396 td = curthread; 397 tid = (uintptr_t)td; 398 if (SCHEDULER_STOPPED_TD(td)) 399 return (1); 400 401 KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(td), 402 ("mtx_trylock() by idle thread %p on sleep mutex %s @ %s:%d", 403 curthread, m->lock_object.lo_name, file, line)); 404 KASSERT(m->mtx_lock != MTX_DESTROYED, 405 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 406 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 407 ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 408 file, line)); 409 410 rval = 1; 411 recursed = false; 412 v = MTX_UNOWNED; 413 for (;;) { 414 if (_mtx_obtain_lock_fetch(m, &v, tid)) 415 break; 416 if (v == MTX_UNOWNED) 417 continue; 418 if (v == tid && 419 ((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 420 (opts & MTX_RECURSE) != 0)) { 421 m->mtx_recurse++; 422 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 423 recursed = true; 424 break; 425 } 426 rval = 0; 427 break; 428 } 429 430 opts &= ~MTX_RECURSE; 431 432 LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line); 433 if (rval) { 434 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 435 file, line); 436 TD_LOCKS_INC(curthread); 437 if (!recursed) 438 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, 439 m, contested, waittime, file, line); 440 } 441 442 return (rval); 443} 444 445int 446_mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, int line) 447{ 448 struct mtx *m; 449 450 m = mtxlock2mtx(c); 451 return (_mtx_trylock_flags_int(m, opts LOCK_FILE_LINE_ARG)); 452} 453 454/* 455 * __mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 456 * 457 * We call this if the lock is either contested (i.e. we need to go to 458 * sleep waiting for it), or if we need to recurse on it. 459 */ 460#if LOCK_DEBUG > 0 461void 462__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v, int opts, const char *file, 463 int line) 464#else 465void 466__mtx_lock_sleep(volatile uintptr_t *c, uintptr_t v) 467#endif 468{ 469 struct thread *td; 470 struct mtx *m; 471 struct turnstile *ts; 472 uintptr_t tid; 473 struct thread *owner; 474#ifdef LOCK_PROFILING 475 int contested = 0; 476 uint64_t waittime = 0; 477#endif 478#if defined(ADAPTIVE_MUTEXES) || defined(KDTRACE_HOOKS) 479 struct lock_delay_arg lda; 480#endif 481#ifdef KDTRACE_HOOKS 482 u_int sleep_cnt = 0; 483 int64_t sleep_time = 0; 484 int64_t all_time = 0; 485#endif 486#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 487 int doing_lockprof; 488#endif 489 490 td = curthread; 491 tid = (uintptr_t)td; 492 m = mtxlock2mtx(c); 493 494#ifdef KDTRACE_HOOKS 495 if (LOCKSTAT_PROFILE_ENABLED(adaptive__acquire)) { 496 while (v == MTX_UNOWNED) { 497 if (_mtx_obtain_lock_fetch(m, &v, tid)) 498 goto out_lockstat; 499 } 500 doing_lockprof = 1; 501 all_time -= lockstat_nsecs(&m->lock_object); 502 } 503#endif 504#ifdef LOCK_PROFILING 505 doing_lockprof = 1; 506#endif 507 508 if (SCHEDULER_STOPPED_TD(td)) 509 return; 510 511#if defined(ADAPTIVE_MUTEXES) 512 lock_delay_arg_init(&lda, &mtx_delay); 513#elif defined(KDTRACE_HOOKS) 514 lock_delay_arg_init(&lda, NULL); 515#endif 516 517 if (__predict_false(v == MTX_UNOWNED)) 518 v = MTX_READ_VALUE(m); 519 520 if (__predict_false(lv_mtx_owner(v) == td)) { 521 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0 || 522 (opts & MTX_RECURSE) != 0, 523 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 524 m->lock_object.lo_name, file, line)); 525#if LOCK_DEBUG > 0 526 opts &= ~MTX_RECURSE; 527#endif 528 m->mtx_recurse++; 529 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 530 if (LOCK_LOG_TEST(&m->lock_object, opts)) 531 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 532 return; 533 } 534#if LOCK_DEBUG > 0 535 opts &= ~MTX_RECURSE; 536#endif 537 538#ifdef HWPMC_HOOKS 539 PMC_SOFT_CALL( , , lock, failed); 540#endif 541 lock_profile_obtain_lock_failed(&m->lock_object, 542 &contested, &waittime); 543 if (LOCK_LOG_TEST(&m->lock_object, opts)) 544 CTR4(KTR_LOCK, 545 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 546 m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 547 548 for (;;) { 549 if (v == MTX_UNOWNED) { 550 if (_mtx_obtain_lock_fetch(m, &v, tid)) 551 break; 552 continue; 553 } 554#ifdef KDTRACE_HOOKS 555 lda.spin_cnt++; 556#endif 557#ifdef ADAPTIVE_MUTEXES 558 /* 559 * If the owner is running on another CPU, spin until the 560 * owner stops running or the state of the lock changes. 561 */ 562 owner = lv_mtx_owner(v); 563 if (TD_IS_RUNNING(owner)) { 564 if (LOCK_LOG_TEST(&m->lock_object, 0)) 565 CTR3(KTR_LOCK, 566 "%s: spinning on %p held by %p", 567 __func__, m, owner); 568 KTR_STATE1(KTR_SCHED, "thread", 569 sched_tdname((struct thread *)tid), 570 "spinning", "lockname:\"%s\"", 571 m->lock_object.lo_name); 572 do { 573 lock_delay(&lda); 574 v = MTX_READ_VALUE(m); 575 owner = lv_mtx_owner(v); 576 } while (v != MTX_UNOWNED && TD_IS_RUNNING(owner)); 577 KTR_STATE0(KTR_SCHED, "thread", 578 sched_tdname((struct thread *)tid), 579 "running"); 580 continue; 581 } 582#endif 583 584 ts = turnstile_trywait(&m->lock_object); 585 v = MTX_READ_VALUE(m); 586retry_turnstile: 587 588 /* 589 * Check if the lock has been released while spinning for 590 * the turnstile chain lock. 591 */ 592 if (v == MTX_UNOWNED) { 593 turnstile_cancel(ts); 594 continue; 595 } 596 597#ifdef ADAPTIVE_MUTEXES 598 /* 599 * The current lock owner might have started executing 600 * on another CPU (or the lock could have changed 601 * owners) while we were waiting on the turnstile 602 * chain lock. If so, drop the turnstile lock and try 603 * again. 604 */ 605 owner = lv_mtx_owner(v); 606 if (TD_IS_RUNNING(owner)) { 607 turnstile_cancel(ts); 608 continue; 609 } 610#endif 611 612 /* 613 * If the mutex isn't already contested and a failure occurs 614 * setting the contested bit, the mutex was either released 615 * or the state of the MTX_RECURSED bit changed. 616 */ 617 if ((v & MTX_CONTESTED) == 0 && 618 !atomic_fcmpset_ptr(&m->mtx_lock, &v, v | MTX_CONTESTED)) { 619 goto retry_turnstile; 620 } 621 622 /* 623 * We definitely must sleep for this lock. 624 */ 625 mtx_assert(m, MA_NOTOWNED); 626 627 /* 628 * Block on the turnstile. 629 */ 630#ifdef KDTRACE_HOOKS 631 sleep_time -= lockstat_nsecs(&m->lock_object); 632#endif 633#ifndef ADAPTIVE_MUTEXES 634 owner = mtx_owner(m); 635#endif 636 MPASS(owner == mtx_owner(m)); 637 turnstile_wait(ts, owner, TS_EXCLUSIVE_QUEUE); 638#ifdef KDTRACE_HOOKS 639 sleep_time += lockstat_nsecs(&m->lock_object); 640 sleep_cnt++; 641#endif 642 v = MTX_READ_VALUE(m); 643 } 644#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 645 if (__predict_true(!doing_lockprof)) 646 return; 647#endif 648#ifdef KDTRACE_HOOKS 649 all_time += lockstat_nsecs(&m->lock_object); 650 if (sleep_time) 651 LOCKSTAT_RECORD1(adaptive__block, m, sleep_time); 652 653 /* 654 * Only record the loops spinning and not sleeping. 655 */ 656 if (lda.spin_cnt > sleep_cnt) 657 LOCKSTAT_RECORD1(adaptive__spin, m, all_time - sleep_time); 658out_lockstat: 659#endif 660 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire, m, contested, 661 waittime, file, line); 662} 663 664#ifdef SMP 665/* 666 * _mtx_lock_spin_cookie: the tougher part of acquiring an MTX_SPIN lock. 667 * 668 * This is only called if we need to actually spin for the lock. Recursion 669 * is handled inline. 670 */ 671#if LOCK_DEBUG > 0 672void 673_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t v, int opts, 674 const char *file, int line) 675#else 676void 677_mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t v) 678#endif 679{ 680 struct mtx *m; 681 struct lock_delay_arg lda; 682 uintptr_t tid; 683#ifdef LOCK_PROFILING 684 int contested = 0; 685 uint64_t waittime = 0; 686#endif 687#ifdef KDTRACE_HOOKS 688 int64_t spin_time = 0; 689#endif 690#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 691 int doing_lockprof; 692#endif 693 694 tid = (uintptr_t)curthread; 695 m = mtxlock2mtx(c); 696 697#ifdef KDTRACE_HOOKS 698 if (LOCKSTAT_PROFILE_ENABLED(adaptive__acquire)) { 699 while (v == MTX_UNOWNED) { 700 if (_mtx_obtain_lock_fetch(m, &v, tid)) 701 goto out_lockstat; 702 } 703 doing_lockprof = 1; 704 spin_time -= lockstat_nsecs(&m->lock_object); 705 } 706#endif 707#ifdef LOCK_PROFILING 708 doing_lockprof = 1; 709#endif 710 711 if (__predict_false(v == MTX_UNOWNED)) 712 v = MTX_READ_VALUE(m); 713 714 if (__predict_false(v == tid)) { 715 m->mtx_recurse++; 716 return; 717 } 718 719 if (SCHEDULER_STOPPED()) 720 return; 721 722 lock_delay_arg_init(&lda, &mtx_spin_delay); 723 724 if (LOCK_LOG_TEST(&m->lock_object, opts)) 725 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 726 KTR_STATE1(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 727 "spinning", "lockname:\"%s\"", m->lock_object.lo_name); 728 729#ifdef HWPMC_HOOKS 730 PMC_SOFT_CALL( , , lock, failed); 731#endif 732 lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); 733 734 for (;;) { 735 if (v == MTX_UNOWNED) { 736 if (_mtx_obtain_lock_fetch(m, &v, tid)) 737 break; 738 continue; 739 } 740 /* Give interrupts a chance while we spin. */ 741 spinlock_exit(); 742 do { 743 if (__predict_true(lda.spin_cnt < 10000000)) { 744 lock_delay(&lda); 745 } else { 746 _mtx_lock_indefinite_check(m, &lda); 747 } 748 v = MTX_READ_VALUE(m); 749 } while (v != MTX_UNOWNED); 750 spinlock_enter(); 751 } 752 753 if (LOCK_LOG_TEST(&m->lock_object, opts)) 754 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 755 KTR_STATE0(KTR_SCHED, "thread", sched_tdname((struct thread *)tid), 756 "running"); 757 758#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 759 if (__predict_true(!doing_lockprof)) 760 return; 761#endif 762#ifdef KDTRACE_HOOKS 763 spin_time += lockstat_nsecs(&m->lock_object); 764 if (lda.spin_cnt != 0) 765 LOCKSTAT_RECORD1(spin__spin, m, spin_time); 766out_lockstat: 767#endif 768 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m, 769 contested, waittime, file, line); 770} 771#endif /* SMP */ 772 773#ifdef INVARIANTS 774static void 775thread_lock_validate(struct mtx *m, int opts, const char *file, int line) 776{ 777 778 KASSERT(m->mtx_lock != MTX_DESTROYED, 779 ("thread_lock() of destroyed mutex @ %s:%d", file, line)); 780 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 781 ("thread_lock() of sleep mutex %s @ %s:%d", 782 m->lock_object.lo_name, file, line)); 783 if (mtx_owned(m)) 784 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 785 ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", 786 m->lock_object.lo_name, file, line)); 787 WITNESS_CHECKORDER(&m->lock_object, 788 opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); 789} 790#else 791#define thread_lock_validate(m, opts, file, line) do { } while (0) 792#endif 793 794#ifndef LOCK_PROFILING 795#if LOCK_DEBUG > 0 796void 797_thread_lock(struct thread *td, int opts, const char *file, int line) 798#else 799void 800_thread_lock(struct thread *td) 801#endif 802{ 803 struct mtx *m; 804 uintptr_t tid, v; 805 806 tid = (uintptr_t)curthread; 807 808 if (__predict_false(LOCKSTAT_PROFILE_ENABLED(spin__acquire))) 809 goto slowpath_noirq; 810 spinlock_enter(); 811 m = td->td_lock; 812 thread_lock_validate(m, 0, file, line); 813 v = MTX_READ_VALUE(m); 814 if (__predict_true(v == MTX_UNOWNED)) { 815 if (__predict_false(!_mtx_obtain_lock(m, tid))) 816 goto slowpath_unlocked; 817 } else if (v == tid) { 818 m->mtx_recurse++; 819 } else 820 goto slowpath_unlocked; 821 if (__predict_true(m == td->td_lock)) { 822 WITNESS_LOCK(&m->lock_object, LOP_EXCLUSIVE, file, line); 823 return; 824 } 825 MPASS(m->mtx_recurse == 0); 826 _mtx_release_lock_quick(m); 827slowpath_unlocked: 828 spinlock_exit(); 829slowpath_noirq: 830#if LOCK_DEBUG > 0 831 thread_lock_flags_(td, opts, file, line); 832#else 833 thread_lock_flags_(td, 0, 0, 0); 834#endif 835} 836#endif 837 838void 839thread_lock_flags_(struct thread *td, int opts, const char *file, int line) 840{ 841 struct mtx *m; 842 uintptr_t tid, v; 843 struct lock_delay_arg lda; 844#ifdef LOCK_PROFILING 845 int contested = 0; 846 uint64_t waittime = 0; 847#endif 848#ifdef KDTRACE_HOOKS 849 int64_t spin_time = 0; 850#endif 851#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 852 int doing_lockprof = 1; 853#endif 854 855 tid = (uintptr_t)curthread; 856 857 if (SCHEDULER_STOPPED()) { 858 /* 859 * Ensure that spinlock sections are balanced even when the 860 * scheduler is stopped, since we may otherwise inadvertently 861 * re-enable interrupts while dumping core. 862 */ 863 spinlock_enter(); 864 return; 865 } 866 867 lock_delay_arg_init(&lda, &mtx_spin_delay); 868 869#ifdef HWPMC_HOOKS 870 PMC_SOFT_CALL( , , lock, failed); 871#endif 872 873#ifdef LOCK_PROFILING 874 doing_lockprof = 1; 875#elif defined(KDTRACE_HOOKS) 876 doing_lockprof = lockstat_enabled; 877 if (__predict_false(doing_lockprof)) 878 spin_time -= lockstat_nsecs(&td->td_lock->lock_object); 879#endif 880 spinlock_enter(); 881 882 for (;;) { 883retry: 884 m = td->td_lock; 885 thread_lock_validate(m, opts, file, line); 886 v = MTX_READ_VALUE(m); 887 for (;;) { 888 if (v == MTX_UNOWNED) { 889 if (_mtx_obtain_lock_fetch(m, &v, tid)) 890 break; 891 continue; 892 } 893 if (v == tid) { 894 m->mtx_recurse++; 895 MPASS(m == td->td_lock); 896 break; 897 } 898 lock_profile_obtain_lock_failed(&m->lock_object, 899 &contested, &waittime); 900 /* Give interrupts a chance while we spin. */ 901 spinlock_exit(); 902 do { 903 if (__predict_true(lda.spin_cnt < 10000000)) { 904 lock_delay(&lda); 905 } else { 906 _mtx_lock_indefinite_check(m, &lda); 907 } 908 if (m != td->td_lock) { 909 spinlock_enter(); 910 goto retry; 911 } 912 v = MTX_READ_VALUE(m); 913 } while (v != MTX_UNOWNED); 914 spinlock_enter(); 915 } 916 if (m == td->td_lock) 917 break; 918 MPASS(m->mtx_recurse == 0); 919 _mtx_release_lock_quick(m); 920 } 921 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 922 line); 923 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 924 925#if defined(KDTRACE_HOOKS) || defined(LOCK_PROFILING) 926 if (__predict_true(!doing_lockprof)) 927 return; 928#endif 929#ifdef KDTRACE_HOOKS 930 spin_time += lockstat_nsecs(&m->lock_object); 931#endif 932 if (m->mtx_recurse == 0) 933 LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(spin__acquire, m, 934 contested, waittime, file, line); 935#ifdef KDTRACE_HOOKS 936 if (lda.spin_cnt != 0) 937 LOCKSTAT_RECORD1(thread__spin, m, spin_time); 938#endif 939} 940 941struct mtx * 942thread_lock_block(struct thread *td) 943{ 944 struct mtx *lock; 945 946 THREAD_LOCK_ASSERT(td, MA_OWNED); 947 lock = td->td_lock; 948 td->td_lock = &blocked_lock; 949 mtx_unlock_spin(lock); 950 951 return (lock); 952} 953 954void 955thread_lock_unblock(struct thread *td, struct mtx *new) 956{ 957 mtx_assert(new, MA_OWNED); 958 MPASS(td->td_lock == &blocked_lock); 959 atomic_store_rel_ptr((volatile void *)&td->td_lock, (uintptr_t)new); 960} 961 962void 963thread_lock_set(struct thread *td, struct mtx *new) 964{ 965 struct mtx *lock; 966 967 mtx_assert(new, MA_OWNED); 968 THREAD_LOCK_ASSERT(td, MA_OWNED); 969 lock = td->td_lock; 970 td->td_lock = new; 971 mtx_unlock_spin(lock); 972} 973 974/* 975 * __mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 976 * 977 * We are only called here if the lock is recursed, contested (i.e. we 978 * need to wake up a blocked thread) or lockstat probe is active. 979 */ 980#if LOCK_DEBUG > 0 981void 982__mtx_unlock_sleep(volatile uintptr_t *c, uintptr_t v, int opts, 983 const char *file, int line) 984#else 985void 986__mtx_unlock_sleep(volatile uintptr_t *c, uintptr_t v) 987#endif 988{ 989 struct mtx *m; 990 struct turnstile *ts; 991 uintptr_t tid; 992 993 if (SCHEDULER_STOPPED()) 994 return; 995 996 tid = (uintptr_t)curthread; 997 m = mtxlock2mtx(c); 998 999 if (__predict_false(v == tid)) 1000 v = MTX_READ_VALUE(m); 1001 1002 if (__predict_false(v & MTX_RECURSED)) { 1003 if (--(m->mtx_recurse) == 0) 1004 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 1005 if (LOCK_LOG_TEST(&m->lock_object, opts)) 1006 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 1007 return; 1008 } 1009 1010 LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, m); 1011 if (v == tid && _mtx_release_lock(m, tid)) 1012 return; 1013 1014 /* 1015 * We have to lock the chain before the turnstile so this turnstile 1016 * can be removed from the hash list if it is empty. 1017 */ 1018 turnstile_chain_lock(&m->lock_object); 1019 _mtx_release_lock_quick(m); 1020 ts = turnstile_lookup(&m->lock_object); 1021 MPASS(ts != NULL); 1022 if (LOCK_LOG_TEST(&m->lock_object, opts)) 1023 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 1024 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 1025 1026 /* 1027 * This turnstile is now no longer associated with the mutex. We can 1028 * unlock the chain lock so a new turnstile may take it's place. 1029 */ 1030 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 1031 turnstile_chain_unlock(&m->lock_object); 1032} 1033 1034/* 1035 * All the unlocking of MTX_SPIN locks is done inline. 1036 * See the __mtx_unlock_spin() macro for the details. 1037 */ 1038 1039/* 1040 * The backing function for the INVARIANTS-enabled mtx_assert() 1041 */ 1042#ifdef INVARIANT_SUPPORT 1043void 1044__mtx_assert(const volatile uintptr_t *c, int what, const char *file, int line) 1045{ 1046 const struct mtx *m; 1047 1048 if (panicstr != NULL || dumping || SCHEDULER_STOPPED()) 1049 return; 1050 1051 m = mtxlock2mtx(c); 1052 1053 switch (what) { 1054 case MA_OWNED: 1055 case MA_OWNED | MA_RECURSED: 1056 case MA_OWNED | MA_NOTRECURSED: 1057 if (!mtx_owned(m)) 1058 panic("mutex %s not owned at %s:%d", 1059 m->lock_object.lo_name, file, line); 1060 if (mtx_recursed(m)) { 1061 if ((what & MA_NOTRECURSED) != 0) 1062 panic("mutex %s recursed at %s:%d", 1063 m->lock_object.lo_name, file, line); 1064 } else if ((what & MA_RECURSED) != 0) { 1065 panic("mutex %s unrecursed at %s:%d", 1066 m->lock_object.lo_name, file, line); 1067 } 1068 break; 1069 case MA_NOTOWNED: 1070 if (mtx_owned(m)) 1071 panic("mutex %s owned at %s:%d", 1072 m->lock_object.lo_name, file, line); 1073 break; 1074 default: 1075 panic("unknown mtx_assert at %s:%d", file, line); 1076 } 1077} 1078#endif 1079 1080/* 1081 * General init routine used by the MTX_SYSINIT() macro. 1082 */ 1083void 1084mtx_sysinit(void *arg) 1085{ 1086 struct mtx_args *margs = arg; 1087 1088 mtx_init((struct mtx *)margs->ma_mtx, margs->ma_desc, NULL, 1089 margs->ma_opts); 1090} 1091 1092/* 1093 * Mutex initialization routine; initialize lock `m' of type contained in 1094 * `opts' with options contained in `opts' and name `name.' The optional 1095 * lock type `type' is used as a general lock category name for use with 1096 * witness. 1097 */ 1098void 1099_mtx_init(volatile uintptr_t *c, const char *name, const char *type, int opts) 1100{ 1101 struct mtx *m; 1102 struct lock_class *class; 1103 int flags; 1104 1105 m = mtxlock2mtx(c); 1106 1107 MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 1108 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE | MTX_NEW)) == 0); 1109 ASSERT_ATOMIC_LOAD_PTR(m->mtx_lock, 1110 ("%s: mtx_lock not aligned for %s: %p", __func__, name, 1111 &m->mtx_lock)); 1112 1113 /* Determine lock class and lock flags. */ 1114 if (opts & MTX_SPIN) 1115 class = &lock_class_mtx_spin; 1116 else 1117 class = &lock_class_mtx_sleep; 1118 flags = 0; 1119 if (opts & MTX_QUIET) 1120 flags |= LO_QUIET; 1121 if (opts & MTX_RECURSE) 1122 flags |= LO_RECURSABLE; 1123 if ((opts & MTX_NOWITNESS) == 0) 1124 flags |= LO_WITNESS; 1125 if (opts & MTX_DUPOK) 1126 flags |= LO_DUPOK; 1127 if (opts & MTX_NOPROFILE) 1128 flags |= LO_NOPROFILE; 1129 if (opts & MTX_NEW) 1130 flags |= LO_NEW; 1131 1132 /* Initialize mutex. */ 1133 lock_init(&m->lock_object, class, name, type, flags); 1134 1135 m->mtx_lock = MTX_UNOWNED; 1136 m->mtx_recurse = 0; 1137} 1138 1139/* 1140 * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 1141 * passed in as a flag here because if the corresponding mtx_init() was 1142 * called with MTX_QUIET set, then it will already be set in the mutex's 1143 * flags. 1144 */ 1145void 1146_mtx_destroy(volatile uintptr_t *c) 1147{ 1148 struct mtx *m; 1149 1150 m = mtxlock2mtx(c); 1151 1152 if (!mtx_owned(m)) 1153 MPASS(mtx_unowned(m)); 1154 else { 1155 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 1156 1157 /* Perform the non-mtx related part of mtx_unlock_spin(). */ 1158 if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) 1159 spinlock_exit(); 1160 else 1161 TD_LOCKS_DEC(curthread); 1162 1163 lock_profile_release_lock(&m->lock_object); 1164 /* Tell witness this isn't locked to make it happy. */ 1165 WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__, 1166 __LINE__); 1167 } 1168 1169 m->mtx_lock = MTX_DESTROYED; 1170 lock_destroy(&m->lock_object); 1171} 1172 1173/* 1174 * Intialize the mutex code and system mutexes. This is called from the MD 1175 * startup code prior to mi_startup(). The per-CPU data space needs to be 1176 * setup before this is called. 1177 */ 1178void 1179mutex_init(void) 1180{ 1181 1182 /* Setup turnstiles so that sleep mutexes work. */ 1183 init_turnstiles(); 1184 1185 /* 1186 * Initialize mutexes. 1187 */ 1188 mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 1189 mtx_init(&blocked_lock, "blocked lock", NULL, MTX_SPIN); 1190 blocked_lock.mtx_lock = 0xdeadc0de; /* Always blocked. */ 1191 mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 1192 mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN); 1193 mtx_init(&proc0.p_statmtx, "pstatl", NULL, MTX_SPIN); 1194 mtx_init(&proc0.p_itimmtx, "pitiml", NULL, MTX_SPIN); 1195 mtx_init(&proc0.p_profmtx, "pprofl", NULL, MTX_SPIN); 1196 mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 1197 mtx_lock(&Giant); 1198} 1199 1200static void __noinline 1201_mtx_lock_indefinite_check(struct mtx *m, struct lock_delay_arg *ldap) 1202{ 1203 struct thread *td; 1204 1205 ldap->spin_cnt++; 1206 if (ldap->spin_cnt < 60000000 || kdb_active || panicstr != NULL) 1207 cpu_lock_delay(); 1208 else { 1209 td = mtx_owner(m); 1210 1211 /* If the mutex is unlocked, try again. */ 1212 if (td == NULL) 1213 return; 1214 1215 printf( "spin lock %p (%s) held by %p (tid %d) too long\n", 1216 m, m->lock_object.lo_name, td, td->td_tid); 1217#ifdef WITNESS 1218 witness_display_spinlock(&m->lock_object, td, printf); 1219#endif 1220 panic("spin lock held too long"); 1221 } 1222 cpu_spinwait(); 1223} 1224 1225#ifdef DDB 1226void 1227db_show_mtx(const struct lock_object *lock) 1228{ 1229 struct thread *td; 1230 const struct mtx *m; 1231 1232 m = (const struct mtx *)lock; 1233 1234 db_printf(" flags: {"); 1235 if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 1236 db_printf("SPIN"); 1237 else 1238 db_printf("DEF"); 1239 if (m->lock_object.lo_flags & LO_RECURSABLE) 1240 db_printf(", RECURSE"); 1241 if (m->lock_object.lo_flags & LO_DUPOK) 1242 db_printf(", DUPOK"); 1243 db_printf("}\n"); 1244 db_printf(" state: {"); 1245 if (mtx_unowned(m)) 1246 db_printf("UNOWNED"); 1247 else if (mtx_destroyed(m)) 1248 db_printf("DESTROYED"); 1249 else { 1250 db_printf("OWNED"); 1251 if (m->mtx_lock & MTX_CONTESTED) 1252 db_printf(", CONTESTED"); 1253 if (m->mtx_lock & MTX_RECURSED) 1254 db_printf(", RECURSED"); 1255 } 1256 db_printf("}\n"); 1257 if (!mtx_unowned(m) && !mtx_destroyed(m)) { 1258 td = mtx_owner(m); 1259 db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 1260 td->td_tid, td->td_proc->p_pid, td->td_name); 1261 if (mtx_recursed(m)) 1262 db_printf(" recursed: %d\n", m->mtx_recurse); 1263 } 1264} 1265#endif 1266