kern_mutex.c revision 78766
1139804Simp/*- 2185435Sbz * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved. 3185435Sbz * 4191673Sjamie * Redistribution and use in source and binary forms, with or without 5185435Sbz * modification, are permitted provided that the following conditions 6190466Sjamie * are met: 7185404Sbz * 1. Redistributions of source code must retain the above copyright 8185404Sbz * notice, this list of conditions and the following disclaimer. 9185404Sbz * 2. Redistributions in binary form must reproduce the above copyright 10185404Sbz * notice, this list of conditions and the following disclaimer in the 11185404Sbz * documentation and/or other materials provided with the distribution. 12185404Sbz * 3. Berkeley Software Design Inc's name may not be used to endorse or 13185404Sbz * promote products derived from this software without specific prior 14185404Sbz * written permission. 15185404Sbz * 16185404Sbz * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 17185404Sbz * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18185404Sbz * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19185404Sbz * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 20185404Sbz * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21185404Sbz * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22185404Sbz * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23185404Sbz * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24185404Sbz * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25185404Sbz * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26185404Sbz * SUCH DAMAGE. 2746197Sphk * 2846155Sphk * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ 29116182Sobrien * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ 30116182Sobrien * $FreeBSD: head/sys/kern/kern_mutex.c 78766 2001-06-25 18:29:32Z jhb $ 31116182Sobrien */ 32193066Sjamie 33185435Sbz/* 34185435Sbz * Machine independent bits of mutex implementation and implementation of 35185435Sbz * `witness' structure & related debugging routines. 36131177Spjd */ 3746155Sphk 3846155Sphk/* 3946155Sphk * Main Entry: witness 4046155Sphk * Pronunciation: 'wit-n&s 4146155Sphk * Function: noun 4246155Sphk * Etymology: Middle English witnesse, from Old English witnes knowledge, 4346155Sphk * testimony, witness, from 2wit 44192895Sjamie * Date: before 12th century 45164032Srwatson * 1 : attestation of a fact or event : TESTIMONY 4646155Sphk * 2 : one that gives evidence; specifically : one who testifies in 47124882Srwatson * a cause or before a judicial tribunal 48177785Skib * 3 : one asked to be present at a transaction so as to be able to 4946155Sphk * testify to its having taken place 5087275Srwatson * 4 : one who has personal knowledge of something 5187275Srwatson * 5 a : something serving as evidence or proof : SIGN 52168401Spjd * b : public affirmation by word or example of usually 53193066Sjamie * religious faith or conviction <the heroic witness to divine 54113275Smike * life -- Pilot> 55147185Spjd * 6 capitalized : a member of the Jehovah's Witnesses 56113275Smike */ 5746155Sphk 58113275Smike#include "opt_ddb.h" 5957163Srwatson 60113275Smike#include <sys/param.h> 61181803Sbz#include <sys/bus.h> 6246155Sphk#include <sys/kernel.h> 6346155Sphk#include <sys/lock.h> 64185435Sbz#include <sys/malloc.h> 65185435Sbz#include <sys/mutex.h> 66185435Sbz#include <sys/proc.h> 67185435Sbz#include <sys/resourcevar.h> 68185435Sbz#include <sys/sysctl.h> 69185435Sbz#include <sys/systm.h> 7046155Sphk#include <sys/vmmeter.h> 71163606Srwatson#include <sys/ktr.h> 72163606Srwatson 7346155Sphk#include <machine/atomic.h> 7446155Sphk#include <machine/bus.h> 75192895Sjamie#include <machine/clock.h> 76192895Sjamie#include <machine/cpu.h> 77192895Sjamie 78192895Sjamie#include <ddb/ddb.h> 79192895Sjamie 80192895Sjamie#include <vm/vm.h> 81192895Sjamie#include <vm/vm_extern.h> 82192895Sjamie 83194118Sjamie/* 84192895Sjamie * Internal utility macros. 85193066Sjamie */ 86192895Sjamie#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) 87192895Sjamie 88192895Sjamie#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ 8957163Srwatson : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) 90192895Sjamie 91168401Spjd#define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) 92191673Sjamie 93191673Sjamie/* 94179881Sdelphij * Lock classes for sleep and spin mutexes. 95113275Smike */ 96191673Sjamiestruct lock_class lock_class_mtx_sleep = { 97190466Sjamie "sleep mutex", 98191673Sjamie LC_SLEEPLOCK | LC_RECURSABLE 99192895Sjamie}; 100192895Sjamiestruct lock_class lock_class_mtx_spin = { 101185435Sbz "spin mutex", 102190466Sjamie LC_SPINLOCK | LC_RECURSABLE 103192895Sjamie}; 104185435Sbz 105185435Sbz/* 106190466Sjamie * Prototypes for non-exported routines. 107192895Sjamie */ 108185435Sbzstatic void propagate_priority(struct proc *); 109113275Smike 110191673Sjamiestatic void 111191673Sjamiepropagate_priority(struct proc *p) 112191673Sjamie{ 113191673Sjamie int pri = p->p_pri.pri_level; 114191673Sjamie struct mtx *m = p->p_blocked; 115191673Sjamie 116113275Smike mtx_assert(&sched_lock, MA_OWNED); 117192895Sjamie for (;;) { 118192895Sjamie struct proc *p1; 119192895Sjamie 120192895Sjamie p = mtx_owner(m); 121192895Sjamie 122193066Sjamie if (p == NULL) { 123185435Sbz /* 124193066Sjamie * This really isn't quite right. Really 125192895Sjamie * ought to bump priority of process that 126192895Sjamie * next acquires the mutex. 127192895Sjamie */ 128192895Sjamie MPASS(m->mtx_lock == MTX_CONTESTED); 129194251Sjamie return; 130194251Sjamie } 131194251Sjamie 132192895Sjamie MPASS(p->p_magic == P_MAGIC); 133192895Sjamie KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); 134192895Sjamie if (p->p_pri.pri_level <= pri) 135192895Sjamie return; 136193066Sjamie 137192895Sjamie /* 138193066Sjamie * Bump this process' priority. 139192895Sjamie */ 140192895Sjamie SET_PRIO(p, pri); 141192895Sjamie 142192895Sjamie /* 143194251Sjamie * If lock holder is actually running, just bump priority. 144194251Sjamie */ 145194251Sjamie if (p->p_oncpu != NOCPU) { 146192895Sjamie MPASS(p->p_stat == SRUN || p->p_stat == SZOMB || p->p_stat == SSTOP); 147192895Sjamie return; 148192895Sjamie } 149192895Sjamie 150192895Sjamie#ifndef SMP 151192895Sjamie /* 152192895Sjamie * For UP, we check to see if p is curproc (this shouldn't 153192895Sjamie * ever happen however as it would mean we are in a deadlock.) 154192895Sjamie */ 155192895Sjamie KASSERT(p != curproc, ("Deadlock detected")); 156192895Sjamie#endif 157192895Sjamie 158192895Sjamie /* 159192895Sjamie * If on run queue move to new run queue, and 160192895Sjamie * quit. 161192895Sjamie */ 162192895Sjamie if (p->p_stat == SRUN) { 163192895Sjamie MPASS(p->p_blocked == NULL); 164192895Sjamie remrunqueue(p); 165192895Sjamie setrunqueue(p); 166192895Sjamie return; 167192895Sjamie } 168192895Sjamie 169192895Sjamie /* 170192895Sjamie * If we aren't blocked on a mutex, we should be. 171192895Sjamie */ 172192895Sjamie KASSERT(p->p_stat == SMTX, ( 173192895Sjamie "process %d(%s):%d holds %s but isn't blocked on a mutex\n", 174193865Sjamie p->p_pid, p->p_comm, p->p_stat, 175192895Sjamie m->mtx_object.lo_name)); 176192895Sjamie 177192895Sjamie /* 178185435Sbz * Pick up the mutex that p is blocked on. 179185435Sbz */ 180185435Sbz m = p->p_blocked; 181185435Sbz MPASS(m != NULL); 182185435Sbz 183185435Sbz /* 184185435Sbz * Check if the proc needs to be moved up on 185185435Sbz * the blocked chain 186185435Sbz */ 187185435Sbz if (p == TAILQ_FIRST(&m->mtx_blocked)) { 188185435Sbz continue; 189185435Sbz } 190185435Sbz 191185435Sbz p1 = TAILQ_PREV(p, procqueue, p_procq); 192185435Sbz if (p1->p_pri.pri_level <= pri) { 193185435Sbz continue; 194185435Sbz } 195185435Sbz 196185435Sbz /* 197185435Sbz * Remove proc from blocked chain and determine where 198185435Sbz * it should be moved up to. Since we know that p1 has 199185435Sbz * a lower priority than p, we know that at least one 200185435Sbz * process in the chain has a lower priority and that 201185435Sbz * p1 will thus not be NULL after the loop. 202185435Sbz */ 203185435Sbz TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); 204185435Sbz TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { 205185435Sbz MPASS(p1->p_magic == P_MAGIC); 206185435Sbz if (p1->p_pri.pri_level > pri) 207185435Sbz break; 208185435Sbz } 209185435Sbz 210185435Sbz MPASS(p1 != NULL); 211185435Sbz TAILQ_INSERT_BEFORE(p1, p, p_procq); 212185435Sbz CTR4(KTR_LOCK, 213185435Sbz "propagate_priority: p %p moved before %p on [%p] %s", 214185435Sbz p, p1, m, m->mtx_object.lo_name); 215190466Sjamie } 216185435Sbz} 217185435Sbz 218185435Sbz/* 219185435Sbz * Function versions of the inlined __mtx_* macros. These are used by 220185435Sbz * modules and can also be called from assembly language if needed. 221185435Sbz */ 222185435Sbzvoid 223185435Sbz_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) 224185435Sbz{ 225191673Sjamie 226191673Sjamie __mtx_lock_flags(m, opts, file, line); 227191673Sjamie} 228191673Sjamie 229191673Sjamievoid 230191673Sjamie_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) 231191673Sjamie{ 232185435Sbz 233191673Sjamie __mtx_unlock_flags(m, opts, file, line); 234191673Sjamie} 235192895Sjamie 236185435Sbzvoid 237191673Sjamie_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) 238191673Sjamie{ 239191673Sjamie 240185435Sbz __mtx_lock_spin_flags(m, opts, file, line); 241191673Sjamie} 242191673Sjamie 243191673Sjamievoid 244191673Sjamie_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) 245185435Sbz{ 246192895Sjamie 247192895Sjamie __mtx_unlock_spin_flags(m, opts, file, line); 248191673Sjamie} 249191673Sjamie 250191673Sjamie/* 251192895Sjamie * The important part of mtx_trylock{,_flags}() 252192895Sjamie * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that 253192895Sjamie * if we're called, it's because we know we don't already own this lock. 254192895Sjamie */ 255191673Sjamieint 256191673Sjamie_mtx_trylock(struct mtx *m, int opts, const char *file, int line) 257191673Sjamie{ 258191673Sjamie int rval; 259185435Sbz 260191673Sjamie MPASS(curproc != NULL); 261191673Sjamie 262185435Sbz /* 263191673Sjamie * _mtx_trylock does not accept MTX_NOSWITCH option. 264185435Sbz */ 265191673Sjamie KASSERT((opts & MTX_NOSWITCH) == 0, 266191673Sjamie ("mtx_trylock() called with invalid option flag(s) %d", opts)); 267191673Sjamie 268191673Sjamie rval = _obtain_lock(m, curproc); 269191673Sjamie 270192895Sjamie LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); 271192895Sjamie if (rval) { 272192895Sjamie /* 273192895Sjamie * We do not handle recursion in _mtx_trylock; see the 274192895Sjamie * note at the top of the routine. 275192895Sjamie */ 276192895Sjamie KASSERT(!mtx_recursed(m), 277192895Sjamie ("mtx_trylock() called on a recursed mutex")); 278192895Sjamie WITNESS_LOCK(&m->mtx_object, opts | LOP_EXCLUSIVE | LOP_TRYLOCK, 279192895Sjamie file, line); 280192895Sjamie } 281192895Sjamie 282193865Sjamie return (rval); 283193865Sjamie} 284193865Sjamie 285193865Sjamie/* 286193865Sjamie * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. 287193865Sjamie * 288193865Sjamie * We call this if the lock is either contested (i.e. we need to go to 289193865Sjamie * sleep waiting for it), or if we need to recurse on it. 290193865Sjamie */ 291192895Sjamievoid 292192895Sjamie_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) 293185435Sbz{ 294193865Sjamie struct proc *p = curproc; 295192895Sjamie 296192895Sjamie if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { 297192895Sjamie m->mtx_recurse++; 298192895Sjamie atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); 299192895Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 300192895Sjamie CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); 301192895Sjamie return; 302192895Sjamie } 303192895Sjamie 304192895Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 305192895Sjamie CTR4(KTR_LOCK, 306192895Sjamie "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", 307192895Sjamie m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); 308192895Sjamie 309192895Sjamie while (!_obtain_lock(m, p)) { 310192895Sjamie uintptr_t v; 311192895Sjamie struct proc *p1; 312192895Sjamie 313192895Sjamie mtx_lock_spin(&sched_lock); 314192895Sjamie /* 315192895Sjamie * Check if the lock has been released while spinning for 316192895Sjamie * the sched_lock. 317192895Sjamie */ 318192895Sjamie if ((v = m->mtx_lock) == MTX_UNOWNED) { 319192895Sjamie mtx_unlock_spin(&sched_lock); 320192895Sjamie continue; 321192895Sjamie } 322192895Sjamie 323192895Sjamie /* 324192895Sjamie * The mutex was marked contested on release. This means that 325192895Sjamie * there are processes blocked on it. 326192895Sjamie */ 327192895Sjamie if (v == MTX_CONTESTED) { 328192895Sjamie p1 = TAILQ_FIRST(&m->mtx_blocked); 329192895Sjamie MPASS(p1 != NULL); 330192895Sjamie m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; 331192895Sjamie 332192895Sjamie if (p1->p_pri.pri_level < p->p_pri.pri_level) 333192895Sjamie SET_PRIO(p, p1->p_pri.pri_level); 334192895Sjamie mtx_unlock_spin(&sched_lock); 335192895Sjamie return; 336192895Sjamie } 337192895Sjamie 338191673Sjamie /* 339192895Sjamie * If the mutex isn't already contested and a failure occurs 340192895Sjamie * setting the contested bit, the mutex was either released 341191673Sjamie * or the state of the MTX_RECURSED bit changed. 342191673Sjamie */ 343192895Sjamie if ((v & MTX_CONTESTED) == 0 && 344192895Sjamie !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, 345192895Sjamie (void *)(v | MTX_CONTESTED))) { 346191673Sjamie mtx_unlock_spin(&sched_lock); 347192895Sjamie continue; 348192895Sjamie } 349191673Sjamie 350192895Sjamie /* 351192895Sjamie * We deffinately must sleep for this lock. 352192895Sjamie */ 353191673Sjamie mtx_assert(m, MA_NOTOWNED); 354192895Sjamie 355191673Sjamie#ifdef notyet 356191673Sjamie /* 357191673Sjamie * If we're borrowing an interrupted thread's VM context, we 358192895Sjamie * must clean up before going to sleep. 359191673Sjamie */ 360192895Sjamie if (p->p_ithd != NULL) { 361191673Sjamie struct ithd *it = p->p_ithd; 362191673Sjamie 363192895Sjamie if (it->it_interrupted) { 364192895Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 365192895Sjamie CTR2(KTR_LOCK, 366192895Sjamie "_mtx_lock_sleep: %p interrupted %p", 367192895Sjamie it, it->it_interrupted); 368192895Sjamie intr_thd_fixup(it); 369192895Sjamie } 370192895Sjamie } 371192895Sjamie#endif 372192895Sjamie 373192895Sjamie /* 374192895Sjamie * Put us on the list of threads blocked on this mutex. 375192895Sjamie */ 376192895Sjamie if (TAILQ_EMPTY(&m->mtx_blocked)) { 377192895Sjamie p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); 378192895Sjamie LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); 379192895Sjamie TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 380192895Sjamie } else { 381192895Sjamie TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) 382192895Sjamie if (p1->p_pri.pri_level > p->p_pri.pri_level) 383192895Sjamie break; 384192895Sjamie if (p1) 385192895Sjamie TAILQ_INSERT_BEFORE(p1, p, p_procq); 386192895Sjamie else 387192895Sjamie TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); 388192895Sjamie } 389192895Sjamie 390192895Sjamie /* 391192895Sjamie * Save who we're blocked on. 392191673Sjamie */ 393191673Sjamie p->p_blocked = m; 394191673Sjamie p->p_mtxname = m->mtx_object.lo_name; 395191673Sjamie p->p_stat = SMTX; 396192895Sjamie propagate_priority(p); 397192895Sjamie 398191673Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 399192895Sjamie CTR3(KTR_LOCK, 400192895Sjamie "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, 401192895Sjamie m->mtx_object.lo_name); 402192895Sjamie 403192895Sjamie p->p_stats->p_ru.ru_nvcsw++; 404192895Sjamie mi_switch(); 405192895Sjamie 406192895Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 407192895Sjamie CTR3(KTR_LOCK, 408191673Sjamie "_mtx_lock_sleep: p %p free from blocked on [%p] %s", 409191673Sjamie p, m, m->mtx_object.lo_name); 410191673Sjamie 411191673Sjamie mtx_unlock_spin(&sched_lock); 412192895Sjamie } 413192895Sjamie 414185435Sbz return; 415185435Sbz} 416192895Sjamie 417192895Sjamie/* 418192895Sjamie * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. 419192895Sjamie * 420192895Sjamie * This is only called if we need to actually spin for the lock. Recursion 421192895Sjamie * is handled inline. 422192895Sjamie */ 423192895Sjamievoid 424192895Sjamie_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, 425192895Sjamie int line) 426192895Sjamie{ 427185435Sbz int i = 0; 428192895Sjamie 429192895Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 430191673Sjamie CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); 431191673Sjamie 432191673Sjamie for (;;) { 433185435Sbz if (_obtain_lock(m, curproc)) 434185435Sbz break; 435192895Sjamie 436191673Sjamie /* Give interrupts a chance while we spin. */ 437191673Sjamie critical_exit(mtx_crit); 438191673Sjamie while (m->mtx_lock != MTX_UNOWNED) { 439191673Sjamie if (i++ < 1000000) 440191673Sjamie continue; 441191673Sjamie if (i++ < 6000000) 442191673Sjamie DELAY(1); 443191673Sjamie#ifdef DDB 444191673Sjamie else if (!db_active) 445185435Sbz#else 446191673Sjamie else 447191673Sjamie#endif 448191673Sjamie panic("spin lock %s held by %p for > 5 seconds", 449191673Sjamie m->mtx_object.lo_name, (void *)m->mtx_lock); 450191673Sjamie } 451191673Sjamie mtx_crit = critical_enter(); 452191673Sjamie } 453191673Sjamie 454191673Sjamie m->mtx_savecrit = mtx_crit; 455191673Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 456191673Sjamie CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); 457191673Sjamie 458191673Sjamie return; 459191673Sjamie} 460191673Sjamie 461191673Sjamie/* 462191673Sjamie * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. 463191673Sjamie * 464191673Sjamie * We are only called here if the lock is recursed or contested (i.e. we 465185435Sbz * need to wake up a blocked thread). 466190466Sjamie */ 467185435Sbzvoid 468185435Sbz_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) 469185435Sbz{ 470185435Sbz struct proc *p, *p1; 471191673Sjamie struct mtx *m1; 472191673Sjamie int pri; 473192895Sjamie 474191673Sjamie p = curproc; 475193066Sjamie 476192895Sjamie if (mtx_recursed(m)) { 477191673Sjamie if (--(m->mtx_recurse) == 0) 478192895Sjamie atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); 479193066Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 480192895Sjamie CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); 481192895Sjamie return; 482193066Sjamie } 483191673Sjamie 484191673Sjamie mtx_lock_spin(&sched_lock); 485192895Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 486191673Sjamie CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); 487191673Sjamie 488192895Sjamie p1 = TAILQ_FIRST(&m->mtx_blocked); 489191673Sjamie MPASS(p->p_magic == P_MAGIC); 490191673Sjamie MPASS(p1->p_magic == P_MAGIC); 491192895Sjamie 492191673Sjamie TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); 493191673Sjamie 494192895Sjamie if (TAILQ_EMPTY(&m->mtx_blocked)) { 495191673Sjamie LIST_REMOVE(m, mtx_contested); 496185435Sbz _release_lock_quick(m); 497191673Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 498191673Sjamie CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); 499191673Sjamie } else 500191673Sjamie atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); 501191673Sjamie 502192895Sjamie pri = PRI_MAX; 503192895Sjamie LIST_FOREACH(m1, &p->p_contested, mtx_contested) { 504192895Sjamie int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level; 505191673Sjamie if (cp < pri) 506191673Sjamie pri = cp; 507191673Sjamie } 508185435Sbz 509191673Sjamie if (pri > p->p_pri.pri_native) 510191673Sjamie pri = p->p_pri.pri_native; 511191673Sjamie SET_PRIO(p, pri); 512191673Sjamie 513185435Sbz if (LOCK_LOG_TEST(&m->mtx_object, opts)) 514191673Sjamie CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", 515191673Sjamie m, p1); 516191673Sjamie 517185435Sbz p1->p_blocked = NULL; 518191673Sjamie p1->p_stat = SRUN; 519191673Sjamie setrunqueue(p1); 520191673Sjamie 521185435Sbz if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) { 522192895Sjamie#ifdef notyet 523185435Sbz if (p->p_ithd != NULL) { 524185435Sbz struct ithd *it = p->p_ithd; 525185435Sbz 526192895Sjamie if (it->it_interrupted) { 527185435Sbz if (LOCK_LOG_TEST(&m->mtx_object, opts)) 528185435Sbz CTR2(KTR_LOCK, 529191673Sjamie "_mtx_unlock_sleep: %p interrupted %p", 530192895Sjamie it, it->it_interrupted); 531192895Sjamie intr_thd_fixup(it); 532192895Sjamie } 533191673Sjamie } 534191673Sjamie#endif 535191673Sjamie setrunqueue(p); 536191673Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 537191673Sjamie CTR2(KTR_LOCK, 538191673Sjamie "_mtx_unlock_sleep: %p switching out lock=%p", m, 539191673Sjamie (void *)m->mtx_lock); 540191673Sjamie 541191673Sjamie p->p_stats->p_ru.ru_nivcsw++; 542191673Sjamie mi_switch(); 543191673Sjamie if (LOCK_LOG_TEST(&m->mtx_object, opts)) 544191673Sjamie CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", 545191673Sjamie m, (void *)m->mtx_lock); 546191673Sjamie } 547192895Sjamie 548192895Sjamie mtx_unlock_spin(&sched_lock); 549192895Sjamie 550192895Sjamie return; 551192895Sjamie} 552192895Sjamie 553192895Sjamie/* 554192895Sjamie * All the unlocking of MTX_SPIN locks is done inline. 555191673Sjamie * See the _rel_spin_lock() macro for the details. 556192895Sjamie */ 557192895Sjamie 558192895Sjamie/* 559192895Sjamie * The backing function for the INVARIANTS-enabled mtx_assert() 560192895Sjamie */ 561192895Sjamie#ifdef INVARIANT_SUPPORT 562192895Sjamievoid 563191673Sjamie_mtx_assert(struct mtx *m, int what, const char *file, int line) 564191673Sjamie{ 565191673Sjamie switch (what) { 566191673Sjamie case MA_OWNED: 567191673Sjamie case MA_OWNED | MA_RECURSED: 568191673Sjamie case MA_OWNED | MA_NOTRECURSED: 569191673Sjamie if (!mtx_owned(m)) 570194251Sjamie panic("mutex %s not owned at %s:%d", 571194251Sjamie m->mtx_object.lo_name, file, line); 572194251Sjamie if (mtx_recursed(m)) { 573194251Sjamie if ((what & MA_NOTRECURSED) != 0) 574194251Sjamie panic("mutex %s recursed at %s:%d", 575194251Sjamie m->mtx_object.lo_name, file, line); 576194251Sjamie } else if ((what & MA_RECURSED) != 0) { 577191673Sjamie panic("mutex %s unrecursed at %s:%d", 578192895Sjamie m->mtx_object.lo_name, file, line); 579192895Sjamie } 580192895Sjamie break; 581192895Sjamie case MA_NOTOWNED: 582192895Sjamie if (mtx_owned(m)) 583192895Sjamie panic("mutex %s owned at %s:%d", 584192895Sjamie m->mtx_object.lo_name, file, line); 585192895Sjamie break; 586191673Sjamie default: 587191673Sjamie panic("unknown mtx_assert at %s:%d", file, line); 588191673Sjamie } 589191673Sjamie} 590191673Sjamie#endif 591191673Sjamie 592191673Sjamie/* 593191673Sjamie * The MUTEX_DEBUG-enabled mtx_validate() 594191673Sjamie * 595191673Sjamie * Most of these checks have been moved off into the LO_INITIALIZED flag 596191673Sjamie * maintained by the witness code. 597191673Sjamie */ 598191673Sjamie#ifdef MUTEX_DEBUG 599191673Sjamie 600191673Sjamievoid mtx_validate __P((struct mtx *)); 601191673Sjamie 602191673Sjamievoid 603191673Sjamiemtx_validate(struct mtx *m) 604191673Sjamie{ 605191673Sjamie 606191673Sjamie/* 607191673Sjamie * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly 608193066Sjamie * we can re-enable the kernacc() checks. 609193066Sjamie */ 610191673Sjamie#ifndef __alpha__ 611191673Sjamie if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE)) 612191673Sjamie panic("Can't read and write to mutex %p", m); 613191673Sjamie#endif 614191673Sjamie} 615191673Sjamie#endif 616191673Sjamie 617191673Sjamie/* 618191673Sjamie * Mutex initialization routine; initialize lock `m' of type contained in 619191673Sjamie * `opts' with options contained in `opts' and description `description.' 620193066Sjamie */ 621193066Sjamievoid 622193066Sjamiemtx_init(struct mtx *m, const char *description, int opts) 623193066Sjamie{ 624193066Sjamie struct lock_object *lock; 625193066Sjamie 626193066Sjamie MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | 627193066Sjamie MTX_SLEEPABLE | MTX_NOWITNESS)) == 0); 628193066Sjamie 629193066Sjamie#ifdef MUTEX_DEBUG 630193066Sjamie /* Diagnostic and error correction */ 631193066Sjamie mtx_validate(m); 632193066Sjamie#endif 633193066Sjamie 634193066Sjamie bzero(m, sizeof(*m)); 635193066Sjamie lock = &m->mtx_object; 636193066Sjamie if (opts & MTX_SPIN) 637193066Sjamie lock->lo_class = &lock_class_mtx_spin; 638193066Sjamie else 639193066Sjamie lock->lo_class = &lock_class_mtx_sleep; 640193066Sjamie lock->lo_name = description; 641193066Sjamie if (opts & MTX_QUIET) 642193066Sjamie lock->lo_flags = LO_QUIET; 643193066Sjamie if (opts & MTX_RECURSE) 644193066Sjamie lock->lo_flags |= LO_RECURSABLE; 645193066Sjamie if (opts & MTX_SLEEPABLE) 646193066Sjamie lock->lo_flags |= LO_SLEEPABLE; 647193066Sjamie if ((opts & MTX_NOWITNESS) == 0) 648193066Sjamie lock->lo_flags |= LO_WITNESS; 649193066Sjamie 650193066Sjamie m->mtx_lock = MTX_UNOWNED; 651193066Sjamie TAILQ_INIT(&m->mtx_blocked); 652193066Sjamie 653193066Sjamie LOCK_LOG_INIT(lock, opts); 654193066Sjamie 655193066Sjamie WITNESS_INIT(lock); 656193066Sjamie} 657193066Sjamie 658193066Sjamie/* 659193066Sjamie * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be 660193066Sjamie * passed in as a flag here because if the corresponding mtx_init() was 661193066Sjamie * called with MTX_QUIET set, then it will already be set in the mutex's 662193066Sjamie * flags. 663193066Sjamie */ 664193066Sjamievoid 665193066Sjamiemtx_destroy(struct mtx *m) 666193066Sjamie{ 667193066Sjamie 668193066Sjamie LOCK_LOG_DESTROY(&m->mtx_object, 0); 669193066Sjamie 670193066Sjamie if (!mtx_owned(m)) 671193066Sjamie MPASS(mtx_unowned(m)); 672193066Sjamie else { 673193066Sjamie MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); 674193066Sjamie 675192895Sjamie /* Tell witness this isn't locked to make it happy. */ 676185435Sbz WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE | LOP_NOSWITCH, 677191673Sjamie __FILE__, __LINE__); 678191673Sjamie } 679191673Sjamie 680191673Sjamie WITNESS_DESTROY(&m->mtx_object); 681191673Sjamie} 682191673Sjamie