kern_mutex.c revision 167801
1/*- 2 * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. Berkeley Software Design Inc's name may not be used to endorse or 13 * promote products derived from this software without specific prior 14 * written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29 * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30 */ 31 32/* 33 * Machine independent bits of mutex implementation. 34 */ 35 36#include <sys/cdefs.h> 37__FBSDID("$FreeBSD: head/sys/kern/kern_mutex.c 167801 2007-03-22 16:09:23Z jhb $"); 38 39#include "opt_adaptive_mutexes.h" 40#include "opt_ddb.h" 41#include "opt_global.h" 42#include "opt_mutex_wake_all.h" 43#include "opt_sched.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/bus.h> 48#include <sys/conf.h> 49#include <sys/kdb.h> 50#include <sys/kernel.h> 51#include <sys/ktr.h> 52#include <sys/lock.h> 53#include <sys/malloc.h> 54#include <sys/mutex.h> 55#include <sys/proc.h> 56#include <sys/resourcevar.h> 57#include <sys/sched.h> 58#include <sys/sbuf.h> 59#include <sys/sysctl.h> 60#include <sys/turnstile.h> 61#include <sys/vmmeter.h> 62#include <sys/lock_profile.h> 63 64#include <machine/atomic.h> 65#include <machine/bus.h> 66#include <machine/cpu.h> 67 68#include <ddb/ddb.h> 69 70#include <fs/devfs/devfs_int.h> 71 72#include <vm/vm.h> 73#include <vm/vm_extern.h> 74 75/* 76 * Force MUTEX_WAKE_ALL for now. 77 * single thread wakeup needs fixes to avoid race conditions with 78 * priority inheritance. 79 */ 80#ifndef MUTEX_WAKE_ALL 81#define MUTEX_WAKE_ALL 82#endif 83 84#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) 85#define ADAPTIVE_MUTEXES 86#endif 87 88/* 89 * Internal utility macros. 90 */ 91#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 92 93#define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) 94 95#ifdef DDB 96static void db_show_mtx(struct lock_object *lock); 97#endif 98static void lock_mtx(struct lock_object *lock, int how); 99static void lock_spin(struct lock_object *lock, int how); 100static int unlock_mtx(struct lock_object *lock); 101static int unlock_spin(struct lock_object *lock); 102 103/* 104 * Lock classes for sleep and spin mutexes. 105 */ 106struct lock_class lock_class_mtx_sleep = { 107 .lc_name = "sleep mutex", 108 .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 109#ifdef DDB 110 .lc_ddb_show = db_show_mtx, 111#endif 112 .lc_lock = lock_mtx, 113 .lc_unlock = unlock_mtx, 114}; 115struct lock_class lock_class_mtx_spin = { 116 .lc_name = "spin mutex", 117 .lc_flags = LC_SPINLOCK | LC_RECURSABLE, 118#ifdef DDB 119 .lc_ddb_show = db_show_mtx, 120#endif 121 .lc_lock = lock_spin, 122 .lc_unlock = unlock_spin, 123}; 124 125/* 126 * System-wide mutexes 127 */ 128struct mtx sched_lock; 129struct mtx Giant; 130 131#ifdef LOCK_PROFILING 132static inline void lock_profile_init(void) 133{ 134 int i; 135 /* Initialize the mutex profiling locks */ 136 for (i = 0; i < LPROF_LOCK_SIZE; i++) { 137 mtx_init(&lprof_locks[i], "mprof lock", 138 NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE); 139 } 140} 141#else 142static inline void lock_profile_init(void) {;} 143#endif 144 145void 146lock_mtx(struct lock_object *lock, int how) 147{ 148 149 mtx_lock((struct mtx *)lock); 150} 151 152void 153lock_spin(struct lock_object *lock, int how) 154{ 155 156 panic("spin locks can only use msleep_spin"); 157} 158 159int 160unlock_mtx(struct lock_object *lock) 161{ 162 struct mtx *m; 163 164 m = (struct mtx *)lock; 165 mtx_assert(m, MA_OWNED | MA_NOTRECURSED); 166 mtx_unlock(m); 167 return (0); 168} 169 170int 171unlock_spin(struct lock_object *lock) 172{ 173 174 panic("spin locks can only use msleep_spin"); 175} 176 177/* 178 * Function versions of the inlined __mtx_* macros. These are used by 179 * modules and can also be called from assembly language if needed. 180 */ 181void 182_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 183{ 184 185 MPASS(curthread != NULL); 186 KASSERT(m->mtx_lock != MTX_DESTROYED, 187 ("mtx_lock() of destroyed mutex @ %s:%d", file, line)); 188 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 189 ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 190 file, line)); 191 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 192 file, line); 193 194 _get_sleep_lock(m, curthread, opts, file, line); 195 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 196 line); 197 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 198 curthread->td_locks++; 199} 200 201void 202_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 203{ 204 MPASS(curthread != NULL); 205 KASSERT(m->mtx_lock != MTX_DESTROYED, 206 ("mtx_unlock() of destroyed mutex @ %s:%d", file, line)); 207 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 208 ("mtx_unlock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 209 file, line)); 210 curthread->td_locks--; 211 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 212 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 213 line); 214 mtx_assert(m, MA_OWNED); 215 216 lock_profile_release_lock(&m->lock_object); 217 _rel_sleep_lock(m, curthread, opts, file, line); 218} 219 220void 221_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 222{ 223 224 MPASS(curthread != NULL); 225 KASSERT(m->mtx_lock != MTX_DESTROYED, 226 ("mtx_lock_spin() of destroyed mutex @ %s:%d", file, line)); 227 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 228 ("mtx_lock_spin() of sleep mutex %s @ %s:%d", 229 m->lock_object.lo_name, file, line)); 230 WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, 231 file, line); 232 _get_spin_lock(m, curthread, opts, file, line); 233 LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, 234 line); 235 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 236} 237 238void 239_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 240{ 241 242 MPASS(curthread != NULL); 243 KASSERT(m->mtx_lock != MTX_DESTROYED, 244 ("mtx_unlock_spin() of destroyed mutex @ %s:%d", file, line)); 245 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin, 246 ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", 247 m->lock_object.lo_name, file, line)); 248 WITNESS_UNLOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); 249 LOCK_LOG_LOCK("UNLOCK", &m->lock_object, opts, m->mtx_recurse, file, 250 line); 251 mtx_assert(m, MA_OWNED); 252 253 lock_profile_release_lock(&m->lock_object); 254 _rel_spin_lock(m); 255} 256 257/* 258 * The important part of mtx_trylock{,_flags}() 259 * Tries to acquire lock `m.' If this function is called on a mutex that 260 * is already owned, it will recursively acquire the lock. 261 */ 262int 263_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 264{ 265 int rval, contested = 0; 266 uint64_t waittime = 0; 267 268 MPASS(curthread != NULL); 269 KASSERT(m->mtx_lock != MTX_DESTROYED, 270 ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); 271 KASSERT(LOCK_CLASS(&m->lock_object) == &lock_class_mtx_sleep, 272 ("mtx_trylock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, 273 file, line)); 274 275 if (mtx_owned(m) && (m->lock_object.lo_flags & LO_RECURSABLE) != 0) { 276 m->mtx_recurse++; 277 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 278 rval = 1; 279 } else 280 rval = _obtain_lock(m, (uintptr_t)curthread); 281 282 LOCK_LOG_TRY("LOCK", &m->lock_object, opts, rval, file, line); 283 if (rval) { 284 WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 285 file, line); 286 curthread->td_locks++; 287 if (m->mtx_recurse == 0) 288 lock_profile_obtain_lock_success(&m->lock_object, contested, 289 waittime, file, line); 290 291 } 292 293 return (rval); 294} 295 296/* 297 * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 298 * 299 * We call this if the lock is either contested (i.e. we need to go to 300 * sleep waiting for it), or if we need to recurse on it. 301 */ 302void 303_mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, 304 int line) 305{ 306#ifdef ADAPTIVE_MUTEXES 307 volatile struct thread *owner; 308#endif 309#ifdef KTR 310 int cont_logged = 0; 311#endif 312 uintptr_t v; 313 314 if (mtx_owned(m)) { 315 KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, 316 ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", 317 m->lock_object.lo_name, file, line)); 318 m->mtx_recurse++; 319 atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 320 if (LOCK_LOG_TEST(&m->lock_object, opts)) 321 CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 322 return; 323 } 324 325 if (LOCK_LOG_TEST(&m->lock_object, opts)) 326 CTR4(KTR_LOCK, 327 "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 328 m->lock_object.lo_name, (void *)m->mtx_lock, file, line); 329 330 while (!_obtain_lock(m, tid)) { 331 turnstile_lock(&m->lock_object); 332 v = m->mtx_lock; 333 334 /* 335 * Check if the lock has been released while spinning for 336 * the turnstile chain lock. 337 */ 338 if (v == MTX_UNOWNED) { 339 turnstile_release(&m->lock_object); 340 cpu_spinwait(); 341 continue; 342 } 343 344#ifdef MUTEX_WAKE_ALL 345 MPASS(v != MTX_CONTESTED); 346#else 347 /* 348 * The mutex was marked contested on release. This means that 349 * there are other threads blocked on it. Grab ownership of 350 * it and propagate its priority to the current thread if 351 * necessary. 352 */ 353 if (v == MTX_CONTESTED) { 354 m->mtx_lock = tid | MTX_CONTESTED; 355 turnstile_claim(&m->lock_object); 356 break; 357 } 358#endif 359 360 /* 361 * If the mutex isn't already contested and a failure occurs 362 * setting the contested bit, the mutex was either released 363 * or the state of the MTX_RECURSED bit changed. 364 */ 365 if ((v & MTX_CONTESTED) == 0 && 366 !atomic_cmpset_ptr(&m->mtx_lock, v, v | MTX_CONTESTED)) { 367 turnstile_release(&m->lock_object); 368 cpu_spinwait(); 369 continue; 370 } 371 372#ifdef ADAPTIVE_MUTEXES 373 /* 374 * If the current owner of the lock is executing on another 375 * CPU, spin instead of blocking. 376 */ 377 owner = (struct thread *)(v & ~MTX_FLAGMASK); 378#ifdef ADAPTIVE_GIANT 379 if (TD_IS_RUNNING(owner)) 380#else 381 if (m != &Giant && TD_IS_RUNNING(owner)) 382#endif 383 { 384 turnstile_release(&m->lock_object); 385 while (mtx_owner(m) == owner && TD_IS_RUNNING(owner)) { 386 cpu_spinwait(); 387 } 388 continue; 389 } 390#endif /* ADAPTIVE_MUTEXES */ 391 392 /* 393 * We definitely must sleep for this lock. 394 */ 395 mtx_assert(m, MA_NOTOWNED); 396 397#ifdef KTR 398 if (!cont_logged) { 399 CTR6(KTR_CONTENTION, 400 "contention: %p at %s:%d wants %s, taken by %s:%d", 401 (void *)tid, file, line, m->lock_object.lo_name, 402 WITNESS_FILE(&m->lock_object), 403 WITNESS_LINE(&m->lock_object)); 404 cont_logged = 1; 405 } 406#endif 407 408 /* 409 * Block on the turnstile. 410 */ 411 turnstile_wait(&m->lock_object, mtx_owner(m), 412 TS_EXCLUSIVE_QUEUE); 413 } 414#ifdef KTR 415 if (cont_logged) { 416 CTR4(KTR_CONTENTION, 417 "contention end: %s acquired by %p at %s:%d", 418 m->lock_object.lo_name, (void *)tid, file, line); 419 } 420#endif 421 return; 422} 423 424#ifdef SMP 425/* 426 * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 427 * 428 * This is only called if we need to actually spin for the lock. Recursion 429 * is handled inline. 430 */ 431void 432_mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, 433 int line) 434{ 435 int i = 0; 436 struct thread *td; 437 438 if (LOCK_LOG_TEST(&m->lock_object, opts)) 439 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 440 441 while (!_obtain_lock(m, tid)) { 442 443 /* Give interrupts a chance while we spin. */ 444 spinlock_exit(); 445 while (m->mtx_lock != MTX_UNOWNED) { 446 if (i++ < 10000000) { 447 cpu_spinwait(); 448 continue; 449 } 450 if (i < 60000000 || kdb_active || panicstr != NULL) 451 DELAY(1); 452 else { 453 td = mtx_owner(m); 454 455 /* If the mutex is unlocked, try again. */ 456 if (td == NULL) 457 continue; 458 printf( 459 "spin lock %p (%s) held by %p (tid %d) too long\n", 460 m, m->lock_object.lo_name, td, td->td_tid); 461#ifdef WITNESS 462 witness_display_spinlock(&m->lock_object, td); 463#endif 464 panic("spin lock held too long"); 465 } 466 cpu_spinwait(); 467 } 468 spinlock_enter(); 469 } 470 471 if (LOCK_LOG_TEST(&m->lock_object, opts)) 472 CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 473 474 return; 475} 476#endif /* SMP */ 477 478/* 479 * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 480 * 481 * We are only called here if the lock is recursed or contested (i.e. we 482 * need to wake up a blocked thread). 483 */ 484void 485_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 486{ 487 struct turnstile *ts; 488#ifndef PREEMPTION 489 struct thread *td, *td1; 490#endif 491 492 if (mtx_recursed(m)) { 493 if (--(m->mtx_recurse) == 0) 494 atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 495 if (LOCK_LOG_TEST(&m->lock_object, opts)) 496 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 497 return; 498 } 499 500 turnstile_lock(&m->lock_object); 501 ts = turnstile_lookup(&m->lock_object); 502 if (LOCK_LOG_TEST(&m->lock_object, opts)) 503 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 504 505#ifdef ADAPTIVE_MUTEXES 506 if (ts == NULL) { 507 _release_lock_quick(m); 508 if (LOCK_LOG_TEST(&m->lock_object, opts)) 509 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); 510 turnstile_release(&m->lock_object); 511 return; 512 } 513#else 514 MPASS(ts != NULL); 515#endif 516#ifndef PREEMPTION 517 /* XXX */ 518 td1 = turnstile_head(ts, TS_EXCLUSIVE_QUEUE); 519#endif 520#ifdef MUTEX_WAKE_ALL 521 turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); 522 _release_lock_quick(m); 523#else 524 if (turnstile_signal(ts, TS_EXCLUSIVE_QUEUE)) { 525 _release_lock_quick(m); 526 if (LOCK_LOG_TEST(&m->lock_object, opts)) 527 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 528 } else { 529 m->mtx_lock = MTX_CONTESTED; 530 if (LOCK_LOG_TEST(&m->lock_object, opts)) 531 CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p still contested", 532 m); 533 } 534#endif 535 turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 536 537#ifndef PREEMPTION 538 /* 539 * XXX: This is just a hack until preemption is done. However, 540 * once preemption is done we need to either wrap the 541 * turnstile_signal() and release of the actual lock in an 542 * extra critical section or change the preemption code to 543 * always just set a flag and never do instant-preempts. 544 */ 545 td = curthread; 546 if (td->td_critnest > 0 || td1->td_priority >= td->td_priority) 547 return; 548 mtx_lock_spin(&sched_lock); 549 if (!TD_IS_RUNNING(td1)) { 550#ifdef notyet 551 if (td->td_ithd != NULL) { 552 struct ithd *it = td->td_ithd; 553 554 if (it->it_interrupted) { 555 if (LOCK_LOG_TEST(&m->lock_object, opts)) 556 CTR2(KTR_LOCK, 557 "_mtx_unlock_sleep: %p interrupted %p", 558 it, it->it_interrupted); 559 intr_thd_fixup(it); 560 } 561 } 562#endif 563 if (LOCK_LOG_TEST(&m->lock_object, opts)) 564 CTR2(KTR_LOCK, 565 "_mtx_unlock_sleep: %p switching out lock=%p", m, 566 (void *)m->mtx_lock); 567 568 mi_switch(SW_INVOL, NULL); 569 if (LOCK_LOG_TEST(&m->lock_object, opts)) 570 CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 571 m, (void *)m->mtx_lock); 572 } 573 mtx_unlock_spin(&sched_lock); 574#endif 575 576 return; 577} 578 579/* 580 * All the unlocking of MTX_SPIN locks is done inline. 581 * See the _rel_spin_lock() macro for the details. 582 */ 583 584/* 585 * The backing function for the INVARIANTS-enabled mtx_assert() 586 */ 587#ifdef INVARIANT_SUPPORT 588void 589_mtx_assert(struct mtx *m, int what, const char *file, int line) 590{ 591 592 if (panicstr != NULL || dumping) 593 return; 594 switch (what) { 595 case MA_OWNED: 596 case MA_OWNED | MA_RECURSED: 597 case MA_OWNED | MA_NOTRECURSED: 598 if (!mtx_owned(m)) 599 panic("mutex %s not owned at %s:%d", 600 m->lock_object.lo_name, file, line); 601 if (mtx_recursed(m)) { 602 if ((what & MA_NOTRECURSED) != 0) 603 panic("mutex %s recursed at %s:%d", 604 m->lock_object.lo_name, file, line); 605 } else if ((what & MA_RECURSED) != 0) { 606 panic("mutex %s unrecursed at %s:%d", 607 m->lock_object.lo_name, file, line); 608 } 609 break; 610 case MA_NOTOWNED: 611 if (mtx_owned(m)) 612 panic("mutex %s owned at %s:%d", 613 m->lock_object.lo_name, file, line); 614 break; 615 default: 616 panic("unknown mtx_assert at %s:%d", file, line); 617 } 618} 619#endif 620 621/* 622 * The MUTEX_DEBUG-enabled mtx_validate() 623 * 624 * Most of these checks have been moved off into the LO_INITIALIZED flag 625 * maintained by the witness code. 626 */ 627#ifdef MUTEX_DEBUG 628 629void mtx_validate(struct mtx *); 630 631void 632mtx_validate(struct mtx *m) 633{ 634 635/* 636 * XXX: When kernacc() does not require Giant we can reenable this check 637 */ 638#ifdef notyet 639 /* 640 * Can't call kernacc() from early init386(), especially when 641 * initializing Giant mutex, because some stuff in kernacc() 642 * requires Giant itself. 643 */ 644 if (!cold) 645 if (!kernacc((caddr_t)m, sizeof(m), 646 VM_PROT_READ | VM_PROT_WRITE)) 647 panic("Can't read and write to mutex %p", m); 648#endif 649} 650#endif 651 652/* 653 * General init routine used by the MTX_SYSINIT() macro. 654 */ 655void 656mtx_sysinit(void *arg) 657{ 658 struct mtx_args *margs = arg; 659 660 mtx_init(margs->ma_mtx, margs->ma_desc, NULL, margs->ma_opts); 661} 662 663/* 664 * Mutex initialization routine; initialize lock `m' of type contained in 665 * `opts' with options contained in `opts' and name `name.' The optional 666 * lock type `type' is used as a general lock category name for use with 667 * witness. 668 */ 669void 670mtx_init(struct mtx *m, const char *name, const char *type, int opts) 671{ 672 struct lock_class *class; 673 int flags; 674 675 MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 676 MTX_NOWITNESS | MTX_DUPOK | MTX_NOPROFILE)) == 0); 677 678#ifdef MUTEX_DEBUG 679 /* Diagnostic and error correction */ 680 mtx_validate(m); 681#endif 682 683 /* Determine lock class and lock flags. */ 684 if (opts & MTX_SPIN) 685 class = &lock_class_mtx_spin; 686 else 687 class = &lock_class_mtx_sleep; 688 flags = 0; 689 if (opts & MTX_QUIET) 690 flags |= LO_QUIET; 691 if (opts & MTX_RECURSE) 692 flags |= LO_RECURSABLE; 693 if ((opts & MTX_NOWITNESS) == 0) 694 flags |= LO_WITNESS; 695 if (opts & MTX_DUPOK) 696 flags |= LO_DUPOK; 697 if (opts & MTX_NOPROFILE) 698 flags |= LO_NOPROFILE; 699 700 /* Initialize mutex. */ 701 m->mtx_lock = MTX_UNOWNED; 702 m->mtx_recurse = 0; 703 704 lock_profile_object_init(&m->lock_object, class, name); 705 lock_init(&m->lock_object, class, name, type, flags); 706} 707 708/* 709 * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 710 * passed in as a flag here because if the corresponding mtx_init() was 711 * called with MTX_QUIET set, then it will already be set in the mutex's 712 * flags. 713 */ 714void 715mtx_destroy(struct mtx *m) 716{ 717 718 if (!mtx_owned(m)) 719 MPASS(mtx_unowned(m)); 720 else { 721 MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 722 723 /* Perform the non-mtx related part of mtx_unlock_spin(). */ 724 if (LOCK_CLASS(&m->lock_object) == &lock_class_mtx_spin) 725 spinlock_exit(); 726 else 727 curthread->td_locks--; 728 729 /* Tell witness this isn't locked to make it happy. */ 730 WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__, 731 __LINE__); 732 } 733 734 m->mtx_lock = MTX_DESTROYED; 735 lock_profile_object_destroy(&m->lock_object); 736 lock_destroy(&m->lock_object); 737} 738 739/* 740 * Intialize the mutex code and system mutexes. This is called from the MD 741 * startup code prior to mi_startup(). The per-CPU data space needs to be 742 * setup before this is called. 743 */ 744void 745mutex_init(void) 746{ 747 748 /* Setup turnstiles so that sleep mutexes work. */ 749 init_turnstiles(); 750 751 /* 752 * Initialize mutexes. 753 */ 754 mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE); 755 mtx_init(&sched_lock, "sched lock", NULL, MTX_SPIN | MTX_RECURSE); 756 mtx_init(&proc0.p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); 757 mtx_init(&devmtx, "cdev", NULL, MTX_DEF); 758 mtx_lock(&Giant); 759 760 lock_profile_init(); 761} 762 763#ifdef DDB 764void 765db_show_mtx(struct lock_object *lock) 766{ 767 struct thread *td; 768 struct mtx *m; 769 770 m = (struct mtx *)lock; 771 772 db_printf(" flags: {"); 773 if (LOCK_CLASS(lock) == &lock_class_mtx_spin) 774 db_printf("SPIN"); 775 else 776 db_printf("DEF"); 777 if (m->lock_object.lo_flags & LO_RECURSABLE) 778 db_printf(", RECURSE"); 779 if (m->lock_object.lo_flags & LO_DUPOK) 780 db_printf(", DUPOK"); 781 db_printf("}\n"); 782 db_printf(" state: {"); 783 if (mtx_unowned(m)) 784 db_printf("UNOWNED"); 785 else { 786 db_printf("OWNED"); 787 if (m->mtx_lock & MTX_CONTESTED) 788 db_printf(", CONTESTED"); 789 if (m->mtx_lock & MTX_RECURSED) 790 db_printf(", RECURSED"); 791 } 792 db_printf("}\n"); 793 if (!mtx_unowned(m)) { 794 td = mtx_owner(m); 795 db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, 796 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); 797 if (mtx_recursed(m)) 798 db_printf(" recursed: %d\n", m->mtx_recurse); 799 } 800} 801#endif 802