kern_mutex.c revision 83679
1183724Ssos/*- 2230132Suqs * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3183724Ssos * 4183724Ssos * Redistribution and use in source and binary forms, with or without 5183724Ssos * modification, are permitted provided that the following conditions 6183724Ssos * are met: 7183724Ssos * 1. Redistributions of source code must retain the above copyright 8183724Ssos * notice, this list of conditions and the following disclaimer. 9183724Ssos * 2. Redistributions in binary form must reproduce the above copyright 10183724Ssos * notice, this list of conditions and the following disclaimer in the 11183724Ssos * documentation and/or other materials provided with the distribution. 12183724Ssos * 3. Berkeley Software Design Inc's name may not be used to endorse or 13183724Ssos * promote products derived from this software without specific prior 14183724Ssos * written permission. 15183724Ssos * 16183724Ssos * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17183724Ssos * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18183724Ssos * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19183724Ssos * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20183724Ssos * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21183724Ssos * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22183724Ssos * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23183724Ssos * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24183724Ssos * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25183724Ssos * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26183724Ssos * SUCH DAMAGE. 27183724Ssos * 28183724Ssos * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29183724Ssos * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30183724Ssos * $FreeBSD: head/sys/kern/kern_mutex.c 83679 2001-09-19 22:52:59Z jhb $ 31183724Ssos */ 32183724Ssos 33183724Ssos/* 34183724Ssos * Machine independent bits of mutex implementation and implementation of 35183724Ssos * `witness' structure & related debugging routines. 36183724Ssos */ 37183724Ssos 38183724Ssos/* 39183724Ssos * Main Entry: witness 40183724Ssos * Pronunciation: 'wit-n&s 41183724Ssos * Function: noun 42183724Ssos * Etymology: Middle English witnesse, from Old English witnes knowledge, 43183724Ssos * testimony, witness, from 2wit 44183724Ssos * Date: before 12th century 45183724Ssos * 1 : attestation of a fact or event : TESTIMONY 46183724Ssos * 2 : one that gives evidence; specifically : one who testifies in 47183724Ssos * a cause or before a judicial tribunal 48183724Ssos * 3 : one asked to be present at a transaction so as to be able to 49183724Ssos * testify to its having taken place 50183724Ssos * 4 : one who has personal knowledge of something 51183724Ssos * 5 a : something serving as evidence or proof : SIGN 52183724Ssos * b : public affirmation by word or example of usually 53183724Ssos * religious faith or conviction <the heroic witness to divine 54199322Smav * life -- Pilot> 55199322Smav * 6 capitalized : a member of the Jehovah's Witnesses 56199322Smav */ 57199322Smav 58199322Smav#include "opt_ddb.h" 59199322Smav 60183724Ssos#include <sys/param.h> 61183724Ssos#include <sys/bus.h> 62183724Ssos#include <sys/kernel.h> 63183724Ssos#include <sys/lock.h> 64183724Ssos#include <sys/malloc.h> 65183724Ssos#include <sys/mutex.h> 66190581Smav#include <sys/proc.h> 67183724Ssos#include <sys/resourcevar.h> 68183724Ssos#include <sys/sysctl.h> 69183724Ssos#include <sys/systm.h> 70188769Smav#include <sys/vmmeter.h> 71190581Smav#include <sys/ktr.h> 72190581Smav 73190581Smav#include <machine/atomic.h> 74190581Smav#include <machine/bus.h> 75190581Smav#include <machine/clock.h> 76183724Ssos#include <machine/cpu.h> 77183724Ssos 78183724Ssos#include <ddb/ddb.h> 79183724Ssos 80183724Ssos#include <vm/vm.h> 81183724Ssos#include <vm/vm_extern.h> 82183724Ssos 83183724Ssos/* 84183724Ssos * Internal utility macros. 85183724Ssos */ 86183724Ssos#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 87186296Smav 88186296Smav#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 89186296Smav : (struct thread *)((m)->mtx_lock & MTX_FLAGMASK)) 90183724Ssos 91183724Ssos#define SET_PRIO(td, pri) (td)->td_ksegrp->kg_pri.pri_level = (pri) 92191601Sjkim 93191600Sjkim/* 94183724Ssos * Lock classes for sleep and spin mutexes. 95183724Ssos */ 96183724Ssosstruct lock_class lock_class_mtx_sleep = { 97183724Ssos "sleep mutex", 98183724Ssos LC_SLEEPLOCK | LC_RECURSABLE 99183724Ssos}; 100183724Ssosstruct lock_class lock_class_mtx_spin = { 101183724Ssos "spin mutex", 102191600Sjkim LC_SPINLOCK | LC_RECURSABLE 103183724Ssos}; 104183724Ssos 105199322Smav/* 106199322Smav * Prototypes for non-exported routines. 107199322Smav */ 108199322Smavstatic void propagate_priority(struct thread *); 109199322Smav 110199322Smavstatic void 111199322Smavpropagate_priority(struct thread *td) 112199322Smav{ 113199322Smav struct ksegrp *kg = td->td_ksegrp; 114199322Smav int pri = kg->kg_pri.pri_level; 115199322Smav struct mtx *m = td->td_blocked; 116199322Smav 117199322Smav mtx_assert(&sched_lock, MA_OWNED); 118199322Smav for (;;) { 119199322Smav struct thread *td1; 120199322Smav 121199322Smav td = mtx_owner(m); 122199322Smav 123199322Smav if (td == NULL) { 124199322Smav /* 125199322Smav * This really isn't quite right. Really 126199322Smav * ought to bump priority of thread that 127199322Smav * next acquires the mutex. 128199322Smav */ 129199322Smav MPASS(m->mtx_lock == MTX_CONTESTED); 130199322Smav return; 131199322Smav } 132199322Smav kg = td->td_ksegrp; 133199322Smav 134199322Smav MPASS(td->td_proc->p_magic == P_MAGIC); 135199322Smav KASSERT(td->td_proc->p_stat != SSLEEP, ("sleeping thread owns a mutex")); 136199322Smav if (kg->kg_pri.pri_level <= pri) /* lower is higher priority */ 137199322Smav return; 138199322Smav 139199322Smav /* 140199322Smav * Bump this thread's priority. 141199322Smav */ 142199322Smav SET_PRIO(td, pri); 143199322Smav 144199322Smav /* 145199322Smav * If lock holder is actually running, just bump priority. 146199322Smav */ 147199322Smav /* XXXKSE this test is not sufficient */ 148183724Ssos if (td->td_kse && (td->td_kse->ke_oncpu != NOCPU)) { 149183724Ssos MPASS(td->td_proc->p_stat == SRUN 150183724Ssos || td->td_proc->p_stat == SZOMB 151183724Ssos || td->td_proc->p_stat == SSTOP); 152193277Smav return; 153193277Smav } 154183724Ssos 155183724Ssos#ifndef SMP 156183724Ssos /* 157183724Ssos * For UP, we check to see if td is curthread (this shouldn't 158183724Ssos * ever happen however as it would mean we are in a deadlock.) 159183724Ssos */ 160183724Ssos KASSERT(td != curthread, ("Deadlock detected")); 161183724Ssos#endif 162183724Ssos 163183724Ssos /* 164186250Smav * If on run queue move to new run queue, and quit. 165186250Smav * XXXKSE this gets a lot more complicated under threads 166183724Ssos * but try anyhow. 167186250Smav */ 168183724Ssos if (td->td_proc->p_stat == SRUN) { 169183724Ssos MPASS(td->td_blocked == NULL); 170183724Ssos remrunqueue(td); 171183724Ssos setrunqueue(td); 172183724Ssos return; 173186250Smav } 174186250Smav 175186250Smav /* 176186250Smav * If we aren't blocked on a mutex, we should be. 177183724Ssos */ 178183724Ssos KASSERT(td->td_proc->p_stat == SMTX, ( 179188694Smav "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 180199322Smav td->td_proc->p_pid, td->td_proc->p_comm, td->td_proc->p_stat, 181191674Smav m->mtx_object.lo_name)); 182232380Smav 183199322Smav /* 184232380Smav * Pick up the mutex that td is blocked on. 185199322Smav */ 186232380Smav m = td->td_blocked; 187232380Smav MPASS(m != NULL); 188199322Smav 189183724Ssos /* 190183724Ssos * Check if the thread needs to be moved up on 191188765Smav * the blocked chain 192188769Smav */ 193190581Smav if (td == TAILQ_FIRST(&m->mtx_blocked)) { 194190581Smav continue; 195183724Ssos } 196200171Smav 197183724Ssos td1 = TAILQ_PREV(td, threadqueue, td_blkq); 198183724Ssos if (td1->td_ksegrp->kg_pri.pri_level <= pri) { 199183724Ssos continue; 200193277Smav } 201193277Smav 202193277Smav /* 203193277Smav * Remove thread from blocked chain and determine where 204193277Smav * it should be moved up to. Since we know that td1 has 205193277Smav * a lower priority than td, we know that at least one 206193277Smav * thread in the chain has a lower priority and that 207193277Smav * td1 will thus not be NULL after the loop. 208193277Smav */ 209193277Smav TAILQ_REMOVE(&m->mtx_blocked, td, td_blkq); 210193277Smav TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) { 211193277Smav MPASS(td1->td_proc->p_magic == P_MAGIC); 212193277Smav if (td1->td_ksegrp->kg_pri.pri_level > pri) 213193277Smav break; 214193277Smav } 215193277Smav 216193277Smav MPASS(td1 != NULL); 217193277Smav TAILQ_INSERT_BEFORE(td1, td, td_blkq); 218193277Smav CTR4(KTR_LOCK, 219193277Smav "propagate_priority: p %p moved before %p on [%p] %s", 220193277Smav td, td1, m, m->mtx_object.lo_name); 221193277Smav } 222193277Smav} 223193277Smav 224193277Smav/* 225193277Smav * Function versions of the inlined __mtx_* macros. These are used by 226193277Smav * modules and can also be called from assembly language if needed. 227193277Smav */ 228193277Smavvoid 229193277Smav_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 230193277Smav{ 231193277Smav 232193277Smav __mtx_lock_flags(m, opts, file, line); 233193277Smav} 234193277Smav 235193277Smavvoid 236193277Smav_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 237193277Smav{ 238193277Smav 239183724Ssos __mtx_unlock_flags(m, opts, file, line); 240183724Ssos} 241199322Smav 242183724Ssosvoid 243183724Ssos_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 244183724Ssos{ 245188621Smav 246183724Ssos __mtx_lock_spin_flags(m, opts, file, line); 247183724Ssos} 248183724Ssos 249183724Ssosvoid 250183724Ssos_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 251188621Smav{ 252188621Smav 253188621Smav __mtx_unlock_spin_flags(m, opts, file, line); 254188621Smav} 255188621Smav 256188621Smav/* 257188621Smav * The important part of mtx_trylock{,_flags}() 258183724Ssos * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 259183724Ssos * if we're called, it's because we know we don't already own this lock. 260183724Ssos */ 261183724Ssosint 262183724Ssos_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 263183724Ssos{ 264183724Ssos int rval; 265183724Ssos 266183724Ssos MPASS(curthread != NULL); 267183724Ssos 268183724Ssos /* 269183724Ssos * _mtx_trylock does not accept MTX_NOSWITCH option. 270183724Ssos */ 271183724Ssos KASSERT((opts & MTX_NOSWITCH) == 0, 272183724Ssos ("mtx_trylock() called with invalid option flag(s) %d", opts)); 273183724Ssos 274183724Ssos rval = _obtain_lock(m, curthread); 275183724Ssos 276183724Ssos LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 277183724Ssos if (rval) { 278183724Ssos /* 279183724Ssos * We do not handle recursion in _mtx_trylock; see the 280183724Ssos * note at the top of the routine. 281183724Ssos */ 282183724Ssos KASSERT(!mtx_recursed(m), 283183724Ssos ("mtx_trylock() called on a recursed mutex")); 284183724Ssos WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 285183724Ssos file, line); 286199322Smav } 287188765Smav 288183724Ssos return (rval); 289183724Ssos} 290183724Ssos 291183724Ssos/* 292183724Ssos * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 293188765Smav * 294188765Smav * We call this if the lock is either contested (i.e. we need to go to 295183724Ssos * sleep waiting for it), or if we need to recurse on it. 296183724Ssos */ 297183724Ssosvoid 298183724Ssos_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 299183724Ssos{ 300183724Ssos struct thread *td = curthread; 301183724Ssos struct ksegrp *kg = td->td_ksegrp; 302183724Ssos 303183724Ssos if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)td) { 304183724Ssos m->mtx_recurse++; 305183724Ssos atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 306183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 307183724Ssos CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 308183724Ssos return; 309183724Ssos } 310183724Ssos 311183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 312200171Smav CTR4(KTR_LOCK, 313200171Smav "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 314183724Ssos m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 315190581Smav 316183724Ssos while (!_obtain_lock(m, td)) { 317183724Ssos uintptr_t v; 318183724Ssos struct thread *td1; 319199322Smav 320188769Smav mtx_lock_spin(&sched_lock); 321188769Smav /* 322219337Smarius * Check if the lock has been released while spinning for 323219337Smarius * the sched_lock. 324219337Smarius */ 325219337Smarius if ((v = m->mtx_lock) == MTX_UNOWNED) { 326219337Smarius mtx_unlock_spin(&sched_lock); 327190581Smav continue; 328190581Smav } 329190581Smav 330190581Smav /* 331190581Smav * The mutex was marked contested on release. This means that 332199322Smav * there are threads blocked on it. 333190581Smav */ 334190581Smav if (v == MTX_CONTESTED) { 335188877Smav td1 = TAILQ_FIRST(&m->mtx_blocked); 336188877Smav MPASS(td1 != NULL); 337188877Smav m->mtx_lock = (uintptr_t)td | MTX_CONTESTED; 338188769Smav 339188877Smav if (td1->td_ksegrp->kg_pri.pri_level < kg->kg_pri.pri_level) 340188877Smav SET_PRIO(td, td1->td_ksegrp->kg_pri.pri_level); 341188877Smav mtx_unlock_spin(&sched_lock); 342190581Smav return; 343190581Smav } 344188877Smav 345188877Smav /* 346188877Smav * If the mutex isn't already contested and a failure occurs 347188877Smav * setting the contested bit, the mutex was either released 348188877Smav * or the state of the MTX_RECURSED bit changed. 349188877Smav */ 350188877Smav if ((v & MTX_CONTESTED) == 0 && 351188877Smav !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 352188877Smav (void *)(v | MTX_CONTESTED))) { 353188877Smav mtx_unlock_spin(&sched_lock); 354188769Smav continue; 355188769Smav } 356188769Smav 357199322Smav /* 358190581Smav * We deffinately must sleep for this lock. 359190581Smav */ 360190581Smav mtx_assert(m, MA_NOTOWNED); 361190581Smav 362190581Smav#ifdef notyet 363190581Smav /* 364190581Smav * If we're borrowing an interrupted thread's VM context, we 365190581Smav * must clean up before going to sleep. 366190581Smav */ 367190581Smav if (td->td_ithd != NULL) { 368190581Smav struct ithd *it = td->td_ithd; 369190581Smav 370190581Smav if (it->it_interrupted) { 371190581Smav if (LOCK_LOG_TEST(&m->mtx_object, opts)) 372190581Smav CTR2(KTR_LOCK, 373190581Smav "_mtx_lock_sleep: %p interrupted %p", 374190581Smav it, it->it_interrupted); 375190581Smav intr_thd_fixup(it); 376190581Smav } 377190581Smav } 378190581Smav#endif 379191674Smav 380191674Smav /* 381191674Smav * Put us on the list of threads blocked on this mutex. 382190581Smav */ 383190581Smav if (TAILQ_EMPTY(&m->mtx_blocked)) { 384190581Smav td1 = (struct thread *)(m->mtx_lock & MTX_FLAGMASK); 385190581Smav LIST_INSERT_HEAD(&td1->td_contested, m, mtx_contested); 386190581Smav TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); 387190581Smav } else { 388183724Ssos TAILQ_FOREACH(td1, &m->mtx_blocked, td_blkq) 389183724Ssos if (td1->td_ksegrp->kg_pri.pri_level > kg->kg_pri.pri_level) 390183724Ssos break; 391183724Ssos if (td1) 392183724Ssos TAILQ_INSERT_BEFORE(td1, td, td_blkq); 393183724Ssos else 394183724Ssos TAILQ_INSERT_TAIL(&m->mtx_blocked, td, td_blkq); 395183724Ssos } 396183724Ssos 397183724Ssos /* 398183724Ssos * Save who we're blocked on. 399183724Ssos */ 400183724Ssos td->td_blocked = m; 401183724Ssos td->td_mtxname = m->mtx_object.lo_name; 402183724Ssos td->td_proc->p_stat = SMTX; 403183724Ssos propagate_priority(td); 404183724Ssos 405188658Smav if (LOCK_LOG_TEST(&m->mtx_object, opts)) 406183724Ssos CTR3(KTR_LOCK, 407183724Ssos "_mtx_lock_sleep: p %p blocked on [%p] %s", td, m, 408183724Ssos m->mtx_object.lo_name); 409214016Smav 410183724Ssos td->td_proc->p_stats->p_ru.ru_nvcsw++; 411183724Ssos mi_switch(); 412183724Ssos 413183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 414183724Ssos CTR3(KTR_LOCK, 415183724Ssos "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 416183724Ssos td, m, m->mtx_object.lo_name); 417183724Ssos 418183724Ssos mtx_unlock_spin(&sched_lock); 419183724Ssos } 420183724Ssos 421183724Ssos return; 422183724Ssos} 423183724Ssos 424183724Ssos/* 425183724Ssos * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 426183724Ssos * 427183724Ssos * This is only called if we need to actually spin for the lock. Recursion 428183724Ssos * is handled inline. 429183724Ssos */ 430183724Ssosvoid 431183724Ssos_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, 432183724Ssos int line) 433183724Ssos{ 434183724Ssos int i = 0; 435183724Ssos 436183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 437183724Ssos CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 438183724Ssos 439183724Ssos for (;;) { 440183724Ssos if (_obtain_lock(m, curthread)) 441183724Ssos break; 442183724Ssos 443183724Ssos /* Give interrupts a chance while we spin. */ 444183724Ssos critical_exit(mtx_crit); 445183724Ssos while (m->mtx_lock != MTX_UNOWNED) { 446183724Ssos if (i++ < 1000000) 447183724Ssos continue; 448183724Ssos if (i++ < 6000000) 449198717Smav DELAY(1); 450183724Ssos#ifdef DDB 451183724Ssos else if (!db_active) 452183724Ssos#else 453183724Ssos else 454198717Smav#endif 455183724Ssos panic("spin lock %s held by %p for > 5 seconds", 456183724Ssos m->mtx_object.lo_name, (void *)m->mtx_lock); 457183724Ssos } 458183724Ssos mtx_crit = critical_enter(); 459183724Ssos } 460198717Smav 461183724Ssos m->mtx_savecrit = mtx_crit; 462183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 463183724Ssos CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 464198717Smav 465183724Ssos return; 466183724Ssos} 467183724Ssos 468183724Ssos/* 469183724Ssos * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 470183724Ssos * 471183724Ssos * We are only called here if the lock is recursed or contested (i.e. we 472198717Smav * need to wake up a blocked thread). 473183724Ssos */ 474183724Ssosvoid 475183724Ssos_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 476183724Ssos{ 477183724Ssos struct thread *td, *td1; 478183724Ssos struct mtx *m1; 479183724Ssos int pri; 480198717Smav struct ksegrp *kg; 481183724Ssos 482183724Ssos td = curthread; 483183724Ssos kg = td->td_ksegrp; 484183724Ssos 485183724Ssos if (mtx_recursed(m)) { 486183724Ssos if (--(m->mtx_recurse) == 0) 487183724Ssos atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 488183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 489198717Smav CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 490183724Ssos return; 491183724Ssos } 492183724Ssos 493183724Ssos mtx_lock_spin(&sched_lock); 494183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 495183724Ssos CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 496183724Ssos 497183724Ssos td1 = TAILQ_FIRST(&m->mtx_blocked); 498183724Ssos MPASS(td->td_proc->p_magic == P_MAGIC); 499183724Ssos MPASS(td1->td_proc->p_magic == P_MAGIC); 500183724Ssos 501219337Smarius TAILQ_REMOVE(&m->mtx_blocked, td1, td_blkq); 502219337Smarius 503219337Smarius if (TAILQ_EMPTY(&m->mtx_blocked)) { 504183724Ssos LIST_REMOVE(m, mtx_contested); 505198717Smav _release_lock_quick(m); 506183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 507183724Ssos CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 508183724Ssos } else 509183724Ssos atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 510183724Ssos 511183724Ssos pri = PRI_MAX; 512183724Ssos LIST_FOREACH(m1, &td->td_contested, mtx_contested) { 513183724Ssos int cp = TAILQ_FIRST(&m1->mtx_blocked)->td_ksegrp->kg_pri.pri_level; 514183724Ssos if (cp < pri) 515183724Ssos pri = cp; 516183724Ssos } 517183724Ssos 518183724Ssos if (pri > kg->kg_pri.pri_native) 519183724Ssos pri = kg->kg_pri.pri_native; 520183724Ssos SET_PRIO(td, pri); 521183724Ssos 522183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 523183724Ssos CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 524183724Ssos m, td1); 525183724Ssos 526183724Ssos td1->td_blocked = NULL; 527183724Ssos td1->td_proc->p_stat = SRUN; 528183724Ssos setrunqueue(td1); 529183724Ssos 530183724Ssos if ((opts & MTX_NOSWITCH) == 0 && td1->td_ksegrp->kg_pri.pri_level < pri) { 531183724Ssos#ifdef notyet 532183724Ssos if (td->td_ithd != NULL) { 533183724Ssos struct ithd *it = td->td_ithd; 534183724Ssos 535183724Ssos if (it->it_interrupted) { 536183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 537198717Smav CTR2(KTR_LOCK, 538183724Ssos "_mtx_unlock_sleep: %p interrupted %p", 539183724Ssos it, it->it_interrupted); 540183724Ssos intr_thd_fixup(it); 541183724Ssos } 542183724Ssos } 543183724Ssos#endif 544183724Ssos setrunqueue(td); 545183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 546219337Smarius CTR2(KTR_LOCK, 547219337Smarius "_mtx_unlock_sleep: %p switching out lock=%p", m, 548219337Smarius (void *)m->mtx_lock); 549183724Ssos 550183724Ssos td->td_proc->p_stats->p_ru.ru_nivcsw++; 551183724Ssos mi_switch(); 552183724Ssos if (LOCK_LOG_TEST(&m->mtx_object, opts)) 553183724Ssos CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 554183724Ssos m, (void *)m->mtx_lock); 555183724Ssos } 556183724Ssos 557242156Smav mtx_unlock_spin(&sched_lock); 558242156Smav 559242156Smav return; 560242156Smav} 561183724Ssos 562183724Ssos/* 563183724Ssos * All the unlocking of MTX_SPIN locks is done inline. 564183724Ssos * See the _rel_spin_lock() macro for the details. 565183724Ssos */ 566198717Smav 567183724Ssos/* 568183724Ssos * The backing function for the INVARIANTS-enabled mtx_assert() 569183724Ssos */ 570183724Ssos#ifdef INVARIANT_SUPPORT 571183724Ssosvoid 572183724Ssos_mtx_assert(struct mtx *m, int what, const char *file, int line) 573183724Ssos{ 574183724Ssos 575183724Ssos if (panicstr != NULL) 576198717Smav return; 577213092Smav switch (what) { 578183724Ssos case MA_OWNED: 579183724Ssos case MA_OWNED | MA_RECURSED: 580183724Ssos case MA_OWNED | MA_NOTRECURSED: 581183724Ssos if (!mtx_owned(m)) 582183724Ssos panic("mutex %s not owned at %s:%d", 583183724Ssos m->mtx_object.lo_name, file, line); 584183724Ssos if (mtx_recursed(m)) { 585183724Ssos if ((what & MA_NOTRECURSED) != 0) 586183724Ssos panic("mutex %s recursed at %s:%d", 587183724Ssos m->mtx_object.lo_name, file, line); 588183724Ssos } else if ((what & MA_RECURSED) != 0) { 589183724Ssos panic("mutex %s unrecursed at %s:%d", 590183724Ssos m->mtx_object.lo_name, file, line); 591183724Ssos } 592183724Ssos break; 593183724Ssos case MA_NOTOWNED: 594183724Ssos if (mtx_owned(m)) 595183724Ssos panic("mutex %s owned at %s:%d", 596183724Ssos m->mtx_object.lo_name, file, line); 597183724Ssos break; 598183724Ssos default: 599183724Ssos panic("unknown mtx_assert at %s:%d", file, line); 600183724Ssos } 601183724Ssos} 602183724Ssos#endif 603183724Ssos 604219337Smarius/* 605219337Smarius * The MUTEX_DEBUG-enabled mtx_validate() 606219337Smarius * 607183724Ssos * Most of these checks have been moved off into the LO_INITIALIZED flag 608183724Ssos * maintained by the witness code. 609183724Ssos */ 610183724Ssos#ifdef MUTEX_DEBUG 611183724Ssos 612183724Ssosvoid mtx_validate __P((struct mtx *)); 613183724Ssos 614183724Ssosvoid 615183724Ssosmtx_validate(struct mtx *m) 616183724Ssos{ 617219337Smarius 618219337Smarius/* 619219337Smarius * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 620183724Ssos * we can re-enable the kernacc() checks. 621183724Ssos */ 622190581Smav#ifndef __alpha__ 623183724Ssos /* 624188918Smav * Can't call kernacc() from early init386(), especially when 625188918Smav * initializing Giant mutex, because some stuff in kernacc() 626188918Smav * requires Giant itself. 627188918Smav */ 628188918Smav if (!cold) 629183724Ssos if (!kernacc((caddr_t)m, sizeof(m), 630188918Smav VM_PROT_READ | VM_PROT_WRITE)) 631183724Ssos panic("Can't read and write to mutex %p", m); 632183724Ssos#endif 633183724Ssos} 634183724Ssos#endif 635183724Ssos 636183724Ssos/* 637183724Ssos * Mutex initialization routine; initialize lock `m' of type contained in 638183724Ssos * `opts' with options contained in `opts' and description `description.' 639183724Ssos */ 640183724Ssosvoid 641183724Ssosmtx_init(struct mtx *m, const char *description, int opts) 642183724Ssos{ 643214016Smav struct lock_object *lock; 644214016Smav 645214016Smav MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 646214016Smav MTX_SLEEPABLE | MTX_NOWITNESS)) == 0); 647214016Smav 648214016Smav#ifdef MUTEX_DEBUG 649214016Smav /* Diagnostic and error correction */ 650214016Smav mtx_validate(m); 651214016Smav#endif 652214016Smav 653214016Smav bzero(m, sizeof(*m)); 654214016Smav lock = &m->mtx_object; 655214016Smav if (opts & MTX_SPIN) 656214016Smav lock->lo_class = &lock_class_mtx_spin; 657214016Smav else 658214016Smav lock->lo_class = &lock_class_mtx_sleep; 659214016Smav lock->lo_name = description; 660214016Smav if (opts & MTX_QUIET) 661214016Smav lock->lo_flags = LO_QUIET; 662183724Ssos if (opts & MTX_RECURSE) 663183724Ssos lock->lo_flags |= LO_RECURSABLE; 664183724Ssos if (opts & MTX_SLEEPABLE) 665183724Ssos lock->lo_flags |= LO_SLEEPABLE; 666183724Ssos if ((opts & MTX_NOWITNESS) == 0) 667183724Ssos lock->lo_flags |= LO_WITNESS; 668183724Ssos 669183724Ssos m->mtx_lock = MTX_UNOWNED; 670183724Ssos TAILQ_INIT(&m->mtx_blocked); 671183724Ssos 672183724Ssos LOCK_LOG_INIT(lock, opts); 673183724Ssos 674183724Ssos WITNESS_INIT(lock); 675183724Ssos} 676183724Ssos 677183724Ssos/* 678183724Ssos * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 679183724Ssos * passed in as a flag here because if the corresponding mtx_init() was 680183724Ssos * called with MTX_QUIET set, then it will already be set in the mutex's 681183724Ssos * flags. 682183724Ssos */ 683183724Ssosvoid 684183724Ssosmtx_destroy(struct mtx *m) 685183724Ssos{ 686183724Ssos 687183724Ssos LOCK_LOG_DESTROY(&m->mtx_object, 0); 688214016Smav 689214016Smav if (!mtx_owned(m)) 690214016Smav MPASS(mtx_unowned(m)); 691214016Smav else { 692214016Smav MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 693214016Smav 694214016Smav /* Tell witness this isn't locked to make it happy. */ 695214016Smav WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE | LOP_NOSWITCH, 696214016Smav __FILE__, __LINE__); 697214016Smav } 698214016Smav 699214016Smav WITNESS_DESTROY(&m->mtx_object); 700214016Smav} 701214016Smav