kern_rmlock.c revision 253047
11556Srgrimes/*- 21556Srgrimes * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org> 31556Srgrimes * All rights reserved. 41556Srgrimes * 51556Srgrimes * Redistribution and use in source and binary forms, with or without 61556Srgrimes * modification, are permitted provided that the following conditions 71556Srgrimes * are met: 81556Srgrimes * 1. Redistributions of source code must retain the above copyright 91556Srgrimes * notice, this list of conditions and the following disclaimer. 101556Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 111556Srgrimes * notice, this list of conditions and the following disclaimer in the 121556Srgrimes * documentation and/or other materials provided with the distribution. 131556Srgrimes * 3. Neither the name of the author nor the names of any co-contributors 141556Srgrimes * may be used to endorse or promote products derived from this software 151556Srgrimes * without specific prior written permission. 161556Srgrimes * 171556Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 181556Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 191556Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 201556Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 211556Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 221556Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 231556Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 241556Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 251556Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 261556Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 271556Srgrimes * SUCH DAMAGE. 281556Srgrimes */ 291556Srgrimes 301556Srgrimes/* 311556Srgrimes * Machine independent bits of reader/writer lock implementation. 32127499Sgad */ 33127499Sgad 34127499Sgad#include <sys/cdefs.h> 35127499Sgad__FBSDID("$FreeBSD: head/sys/kern/kern_rmlock.c 253047 2013-07-08 21:17:20Z jhb $"); 36127499Sgad 37127499Sgad#include "opt_ddb.h" 38127499Sgad#include "opt_kdtrace.h" 391556Srgrimes 401556Srgrimes#include <sys/param.h> 411556Srgrimes#include <sys/systm.h> 4290143Smarkm 431556Srgrimes#include <sys/kernel.h> 441556Srgrimes#include <sys/kdb.h> 451556Srgrimes#include <sys/ktr.h> 461556Srgrimes#include <sys/lock.h> 4790143Smarkm#include <sys/mutex.h> 481556Srgrimes#include <sys/proc.h> 4936049Scharnier#include <sys/rmlock.h> 5090143Smarkm#include <sys/sched.h> 5136049Scharnier#include <sys/smp.h> 52110391Scharnier#include <sys/turnstile.h> 5399110Sobrien#include <sys/lock_profile.h> 5499110Sobrien#include <machine/cpu.h> 551556Srgrimes 561556Srgrimes#ifdef DDB 573296Sdg#include <ddb/ddb.h> 581556Srgrimes#endif 591556Srgrimes 601556Srgrimes/* 611556Srgrimes * A cookie to mark destroyed rmlocks. This is stored in the head of 621556Srgrimes * rm_activeReaders. 631556Srgrimes */ 64127149Sgad#define RM_DESTROYED ((void *)0xdead) 651556Srgrimes 66127499Sgad#define rm_destroyed(rm) \ 671556Srgrimes (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED) 6813514Smpp 6973367Sache#define RMPF_ONQUEUE 1 701556Srgrimes#define RMPF_SIGNAL 2 7190143Smarkm 72127499Sgad#ifndef INVARIANTS 731556Srgrimes#define _rm_assert(c, what, file, line) 741556Srgrimes#endif 751556Srgrimes 761556Srgrimesstatic void assert_rm(const struct lock_object *lock, int what); 771556Srgrimes#ifdef DDB 781556Srgrimesstatic void db_show_rm(const struct lock_object *lock); 791556Srgrimes#endif 80127499Sgadstatic void lock_rm(struct lock_object *lock, int how); 81127499Sgad#ifdef KDTRACE_HOOKS 8266377Sbrianstatic int owner_rm(const struct lock_object *lock, struct thread **owner); 8390143Smarkm#endif 8490143Smarkmstatic int unlock_rm(struct lock_object *lock); 851556Srgrimes 861556Srgrimesstruct lock_class lock_class_rm = { 8719068Speter .lc_name = "rm", 88127499Sgad .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 891556Srgrimes .lc_assert = assert_rm, 901556Srgrimes#ifdef DDB 911556Srgrimes .lc_ddb_show = db_show_rm, 921556Srgrimes#endif 931556Srgrimes .lc_lock = lock_rm, 9497966Sjmallett .lc_unlock = unlock_rm, 9597966Sjmallett#ifdef KDTRACE_HOOKS 96127499Sgad .lc_owner = owner_rm, 97127499Sgad#endif 98127499Sgad}; 99127499Sgad 100127499Sgadstruct lock_class lock_class_rm_sleepable = { 101127499Sgad .lc_name = "sleepable rm", 102127499Sgad .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE, 103127499Sgad .lc_assert = assert_rm, 104127499Sgad#ifdef DDB 105127499Sgad .lc_ddb_show = db_show_rm, 106127499Sgad#endif 107127499Sgad .lc_lock = lock_rm, 108127499Sgad .lc_unlock = unlock_rm, 109127499Sgad#ifdef KDTRACE_HOOKS 110127499Sgad .lc_owner = owner_rm, 111127499Sgad#endif 112127499Sgad}; 113127499Sgad 1141556Srgrimesstatic void 115127507Sgadassert_rm(const struct lock_object *lock, int what) 116127507Sgad{ 117127507Sgad 118127507Sgad rm_assert((const struct rmlock *)lock, what); 11931552Sdyson} 120127507Sgad 121127507Sgad/* 12231552Sdyson * These do not support read locks because it would be hard to make 1231556Srgrimes * the tracker work correctly with the current lock_class API as you 12490143Smarkm * would need to have the tracker pointer available when calling 1251556Srgrimes * rm_rlock() in lock_rm(). 126127499Sgad */ 127127499Sgadstatic void 128127499Sgadlock_rm(struct lock_object *lock, int how) 129127499Sgad{ 130127499Sgad struct rmlock *rm; 131127536Sgad 132127499Sgad rm = (struct rmlock *)lock; 133127536Sgad if (how) 134127536Sgad rm_wlock(rm); 135127499Sgad#ifdef INVARIANTS 136127499Sgad else 137127536Sgad panic("lock_rm called in read mode"); 138127536Sgad#endif 139127536Sgad} 140127536Sgad 141127536Sgadstatic int 142127536Sgadunlock_rm(struct lock_object *lock) 143127499Sgad{ 14497875Sjmallett struct rmlock *rm; 14597875Sjmallett 14697875Sjmallett rm = (struct rmlock *)lock; 14790143Smarkm rm_wunlock(rm); 14897875Sjmallett return (1); 14997875Sjmallett} 15097875Sjmallett 151105831Srwatson#ifdef KDTRACE_HOOKS 1521556Srgrimesstatic int 15390143Smarkmowner_rm(const struct lock_object *lock, struct thread **owner) 1541556Srgrimes{ 155127507Sgad const struct rmlock *rm; 15698494Ssobomax struct lock_class *lc; 1571556Srgrimes 15890110Simp rm = (const struct rmlock *)lock; 1591556Srgrimes lc = LOCK_CLASS(&rm->rm_wlock_object); 160127499Sgad return (lc->lc_owner(&rm->rm_wlock_object, owner)); 161127499Sgad} 1621556Srgrimes#endif 1631556Srgrimes 1641556Srgrimesstatic struct mtx rm_spinlock; 165127499Sgad 166127499SgadMTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN); 167127499Sgad 16897804Stjr/* 16990143Smarkm * Add or remove tracker from per-cpu list. 17098494Ssobomax * 1711556Srgrimes * The per-cpu list can be traversed at any time in forward direction from an 17211809Sache * interrupt on the *local* cpu. 17397966Sjmallett */ 17497966Sjmallettstatic void inline 17511809Sacherm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker) 17697804Stjr{ 17797804Stjr struct rm_queue *next; 17897804Stjr 1791556Srgrimes /* Initialize all tracker pointers */ 1801556Srgrimes tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue; 1811556Srgrimes next = pc->pc_rm_queue.rmq_next; 1821556Srgrimes tracker->rmp_cpuQueue.rmq_next = next; 1831556Srgrimes 1841556Srgrimes /* rmq_prev is not used during froward traversal. */ 1851556Srgrimes next->rmq_prev = &tracker->rmp_cpuQueue; 18698494Ssobomax 18798494Ssobomax /* Update pointer to first element. */ 18898494Ssobomax pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue; 18998494Ssobomax} 19098494Ssobomax 19198494Ssobomax/* 19298494Ssobomax * Return a count of the number of trackers the thread 'td' already 19398494Ssobomax * has on this CPU for the lock 'rm'. 19498494Ssobomax */ 19598494Ssobomaxstatic int 19698494Ssobomaxrm_trackers_present(const struct pcpu *pc, const struct rmlock *rm, 19798494Ssobomax const struct thread *td) 19898494Ssobomax{ 19998494Ssobomax struct rm_queue *queue; 20098494Ssobomax struct rm_priotracker *tracker; 20198494Ssobomax int count; 20298494Ssobomax 20398494Ssobomax count = 0; 20498494Ssobomax for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 2051556Srgrimes queue = queue->rmq_next) { 206127499Sgad tracker = (struct rm_priotracker *)queue; 207127499Sgad if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td)) 208127499Sgad count++; 209127499Sgad } 210127499Sgad return (count); 211127499Sgad} 212127499Sgad 213127499Sgadstatic void inline 214127499Sgadrm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker) 21537317Sphk{ 216127499Sgad struct rm_queue *next, *prev; 21789909Sru 218116265Sscottl next = tracker->rmp_cpuQueue.rmq_next; 21998494Ssobomax prev = tracker->rmp_cpuQueue.rmq_prev; 2201556Srgrimes 221127499Sgad /* Not used during forward traversal. */ 222127499Sgad next->rmq_prev = prev; 223127499Sgad 224127499Sgad /* Remove from list. */ 225127499Sgad prev->rmq_next = next; 226127499Sgad} 227127499Sgad 228127499Sgadstatic void 229127499Sgadrm_cleanIPI(void *arg) 2301556Srgrimes{ 231127499Sgad struct pcpu *pc; 2321556Srgrimes struct rmlock *rm = arg; 2331556Srgrimes struct rm_priotracker *tracker; 23419068Speter struct rm_queue *queue; 23519068Speter pc = pcpu_find(curcpu); 23619068Speter 23719068Speter for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 23819068Speter queue = queue->rmq_next) { 23919068Speter tracker = (struct rm_priotracker *)queue; 2401556Srgrimes if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) { 2411556Srgrimes tracker->rmp_flags = RMPF_ONQUEUE; 2421556Srgrimes mtx_lock_spin(&rm_spinlock); 243127506Sgad LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 244127506Sgad rmp_qentry); 245127506Sgad mtx_unlock_spin(&rm_spinlock); 246127506Sgad } 247127506Sgad } 248127506Sgad} 249127499Sgad 250127499Sgadvoid 251127499Sgadrm_init_flags(struct rmlock *rm, const char *name, int opts) 252127499Sgad{ 253127499Sgad struct lock_class *lc; 254127499Sgad int liflags; 255127499Sgad 256127499Sgad liflags = 0; 2571556Srgrimes if (!(opts & RM_NOWITNESS)) 258127499Sgad liflags |= LO_WITNESS; 259127499Sgad if (opts & RM_RECURSE) 260127499Sgad liflags |= LO_RECURSABLE; 261127499Sgad rm->rm_writecpus = all_cpus; 262127499Sgad LIST_INIT(&rm->rm_activeReaders); 263127499Sgad if (opts & RM_SLEEPABLE) { 264127499Sgad liflags |= LO_SLEEPABLE; 2651556Srgrimes lc = &lock_class_rm_sleepable; 266127499Sgad sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS); 267116265Sscottl } else { 268126127Sdeischen lc = &lock_class_rm; 269116265Sscottl mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS); 2701556Srgrimes } 2711556Srgrimes lock_init(&rm->lock_object, lc, name, NULL, liflags); 2721556Srgrimes} 2731556Srgrimes 274109502Sjmallettvoid 27590143Smarkmrm_init(struct rmlock *rm, const char *name) 2761556Srgrimes{ 2771556Srgrimes 2781556Srgrimes rm_init_flags(rm, name, 0); 2791556Srgrimes} 2801556Srgrimes 2811556Srgrimesvoid 282109502Sjmallettrm_destroy(struct rmlock *rm) 28390143Smarkm{ 2841556Srgrimes 2851556Srgrimes rm_assert(rm, RA_UNLOCKED); 2861556Srgrimes LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED; 2871556Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 28837317Sphk sx_destroy(&rm->rm_lock_sx); 2891556Srgrimes else 2901556Srgrimes mtx_destroy(&rm->rm_lock_mtx); 2911556Srgrimes lock_destroy(&rm->lock_object); 2921556Srgrimes} 2931556Srgrimes 2941556Srgrimesint 29537317Sphkrm_wowned(const struct rmlock *rm) 2961556Srgrimes{ 2971556Srgrimes 298109502Sjmallett if (rm->lock_object.lo_flags & LO_SLEEPABLE) 299109502Sjmallett return (sx_xlocked(&rm->rm_lock_sx)); 300109502Sjmallett else 3011556Srgrimes return (mtx_owned(&rm->rm_lock_mtx)); 30290143Smarkm} 3031556Srgrimes 3041556Srgrimesvoid 305109502Sjmallettrm_sysinit(void *arg) 30690143Smarkm{ 3071556Srgrimes struct rm_args *args = arg; 3081556Srgrimes 309127499Sgad rm_init(args->ra_rm, args->ra_desc); 310127499Sgad} 311127499Sgad 312127499Sgadvoid 313127499Sgadrm_sysinit_flags(void *arg) 314127499Sgad{ 315127499Sgad struct rm_args_flags *args = arg; 316127499Sgad 3171556Srgrimes rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts); 318127499Sgad} 319127499Sgad 320127499Sgadstatic int 321127499Sgad_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 322127499Sgad{ 323127499Sgad struct pcpu *pc; 324127499Sgad 325127499Sgad critical_enter(); 326127499Sgad pc = pcpu_find(curcpu); 3271556Srgrimes 3281556Srgrimes /* Check if we just need to do a proper critical_exit. */ 3291556Srgrimes if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { 3301556Srgrimes critical_exit(); 3311556Srgrimes return (1); 3321556Srgrimes } 333127499Sgad 334127499Sgad /* Remove our tracker from the per-cpu list. */ 335127499Sgad rm_tracker_remove(pc, tracker); 336127499Sgad 337127499Sgad /* Check to see if the IPI granted us the lock after all. */ 338127499Sgad if (tracker->rmp_flags) { 339127499Sgad /* Just add back tracker - we hold the lock. */ 340127499Sgad rm_tracker_add(pc, tracker); 341127499Sgad critical_exit(); 342127499Sgad return (1); 3431556Srgrimes } 3441556Srgrimes 3451556Srgrimes /* 3461556Srgrimes * We allow readers to aquire a lock even if a writer is blocked if 347127499Sgad * the lock is recursive and the reader already holds the lock. 348127499Sgad */ 349127499Sgad if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) { 350127499Sgad /* 3511556Srgrimes * Just grant the lock if this thread already has a tracker 35213020Speter * for this lock on the per-cpu queue. 353127499Sgad */ 354127499Sgad if (rm_trackers_present(pc, rm, curthread) != 0) { 355127499Sgad mtx_lock_spin(&rm_spinlock); 356127499Sgad LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 35713020Speter rmp_qentry); 3581556Srgrimes tracker->rmp_flags = RMPF_ONQUEUE; 359109502Sjmallett mtx_unlock_spin(&rm_spinlock); 3601556Srgrimes rm_tracker_add(pc, tracker); 36190143Smarkm critical_exit(); 3621556Srgrimes return (1); 3631556Srgrimes } 3641556Srgrimes } 365109502Sjmallett 3661556Srgrimes sched_unpin(); 36790143Smarkm critical_exit(); 3681556Srgrimes 3691556Srgrimes if (trylock) { 3701556Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) { 3711556Srgrimes if (!sx_try_xlock(&rm->rm_lock_sx)) 3721556Srgrimes return (0); 3731556Srgrimes } else { 3741556Srgrimes if (!mtx_trylock(&rm->rm_lock_mtx)) 3751556Srgrimes return (0); 3761556Srgrimes } 377127499Sgad } else { 378127499Sgad if (rm->lock_object.lo_flags & LO_SLEEPABLE) 379127499Sgad sx_xlock(&rm->rm_lock_sx); 380127499Sgad else 381127499Sgad mtx_lock(&rm->rm_lock_mtx); 382127499Sgad } 383127499Sgad 384127499Sgad critical_enter(); 385127499Sgad pc = pcpu_find(curcpu); 386127499Sgad CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); 387127499Sgad rm_tracker_add(pc, tracker); 388127499Sgad sched_pin(); 389127499Sgad critical_exit(); 390127499Sgad 3911556Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 392127499Sgad sx_xunlock(&rm->rm_lock_sx); 3931556Srgrimes else 39486922Sgreen mtx_unlock(&rm->rm_lock_mtx); 395109502Sjmallett 39686922Sgreen return (1); 39786922Sgreen} 3981556Srgrimes 3991556Srgrimesint 4001556Srgrimes_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 4011556Srgrimes{ 4021556Srgrimes struct thread *td = curthread; 4031556Srgrimes struct pcpu *pc; 4041556Srgrimes 405127499Sgad if (SCHEDULER_STOPPED()) 406127499Sgad return (1); 407127499Sgad 408127499Sgad tracker->rmp_flags = 0; 409127499Sgad tracker->rmp_thread = td; 410127499Sgad tracker->rmp_rmlock = rm; 4111556Srgrimes 4121556Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 4131556Srgrimes THREAD_NO_SLEEPING(); 4141556Srgrimes 4151556Srgrimes td->td_critnest++; /* critical_enter(); */ 4161556Srgrimes 4171556Srgrimes __compiler_membar(); 4181556Srgrimes 4191556Srgrimes pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 4201556Srgrimes 4211556Srgrimes rm_tracker_add(pc, tracker); 4221556Srgrimes 4231556Srgrimes sched_pin(); 42437317Sphk 4251556Srgrimes __compiler_membar(); 42637317Sphk 42737317Sphk td->td_critnest--; 4281556Srgrimes 42989909Sru /* 4301556Srgrimes * Fast path to combine two common conditions into a single 4311556Srgrimes * conditional jump. 4321556Srgrimes */ 43390143Smarkm if (0 == (td->td_owepreempt | 434109502Sjmallett CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus))) 4351556Srgrimes return (1); 436127499Sgad 437127499Sgad /* We do not have a read token and need to acquire one. */ 438127499Sgad return _rm_rlock_hard(rm, tracker, trylock); 43997877Sjmallett} 440127499Sgad 441127499Sgadstatic void 442127499Sgad_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker) 44366377Sbrian{ 4441556Srgrimes 4451556Srgrimes if (td->td_owepreempt) { 4461556Srgrimes td->td_critnest++; 44753170Skris critical_exit(); 4481556Srgrimes } 4491556Srgrimes 450127499Sgad if (!tracker->rmp_flags) 4511556Srgrimes return; 452127499Sgad 453127499Sgad mtx_lock_spin(&rm_spinlock); 454127499Sgad LIST_REMOVE(tracker, rmp_qentry); 455127499Sgad 456127499Sgad if (tracker->rmp_flags & RMPF_SIGNAL) { 4571556Srgrimes struct rmlock *rm; 458127499Sgad struct turnstile *ts; 459127499Sgad 460127499Sgad rm = tracker->rmp_rmlock; 461127499Sgad 462127499Sgad turnstile_chain_lock(&rm->lock_object); 463127499Sgad mtx_unlock_spin(&rm_spinlock); 464127499Sgad 465127499Sgad ts = turnstile_lookup(&rm->lock_object); 466127499Sgad 467127499Sgad turnstile_signal(ts, TS_EXCLUSIVE_QUEUE); 468127499Sgad turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 469127499Sgad turnstile_chain_unlock(&rm->lock_object); 470127499Sgad } else 471127499Sgad mtx_unlock_spin(&rm_spinlock); 472127499Sgad} 473127499Sgad 474127499Sgadvoid 475127499Sgad_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker) 476127499Sgad{ 477127499Sgad struct pcpu *pc; 478127499Sgad struct thread *td = tracker->rmp_thread; 479127499Sgad 480127499Sgad if (SCHEDULER_STOPPED()) 481127499Sgad return; 482127499Sgad 483127499Sgad td->td_critnest++; /* critical_enter(); */ 484127499Sgad pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 485127499Sgad rm_tracker_remove(pc, tracker); 486127499Sgad td->td_critnest--; 487127499Sgad sched_unpin(); 488127499Sgad 489127499Sgad if (rm->lock_object.lo_flags & LO_SLEEPABLE) 490127499Sgad THREAD_SLEEPING_OK(); 491127499Sgad 4921556Srgrimes if (0 == (td->td_owepreempt | tracker->rmp_flags)) 493126127Sdeischen return; 4941556Srgrimes 4951556Srgrimes _rm_unlock_hard(td, tracker); 4961556Srgrimes} 497127499Sgad 498127149Sgadvoid 499127513Sgad_rm_wlock(struct rmlock *rm) 5001556Srgrimes{ 501127499Sgad struct rm_priotracker *prio; 502127149Sgad struct turnstile *ts; 503127149Sgad cpuset_t readcpus; 504127149Sgad 505127149Sgad if (SCHEDULER_STOPPED()) 506127499Sgad return; 507127499Sgad 508127499Sgad if (rm->lock_object.lo_flags & LO_SLEEPABLE) 509127499Sgad sx_xlock(&rm->rm_lock_sx); 510127499Sgad else 511127499Sgad mtx_lock(&rm->rm_lock_mtx); 512127499Sgad 513127499Sgad if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) { 514127499Sgad /* Get all read tokens back */ 515127499Sgad readcpus = all_cpus; 516127499Sgad CPU_NAND(&readcpus, &rm->rm_writecpus); 517127499Sgad rm->rm_writecpus = all_cpus; 518127499Sgad 519127499Sgad /* 520127499Sgad * Assumes rm->rm_writecpus update is visible on other CPUs 521127499Sgad * before rm_cleanIPI is called. 522127499Sgad */ 523127499Sgad#ifdef SMP 524127499Sgad smp_rendezvous_cpus(readcpus, 525127499Sgad smp_no_rendevous_barrier, 526127499Sgad rm_cleanIPI, 527127499Sgad smp_no_rendevous_barrier, 528127499Sgad rm); 529127499Sgad 530127499Sgad#else 531127499Sgad rm_cleanIPI(rm); 532127499Sgad#endif 533127499Sgad 534127499Sgad mtx_lock_spin(&rm_spinlock); 535127499Sgad while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { 536127499Sgad ts = turnstile_trywait(&rm->lock_object); 537127499Sgad prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL; 538127499Sgad mtx_unlock_spin(&rm_spinlock); 539127499Sgad turnstile_wait(ts, prio->rmp_thread, 540127499Sgad TS_EXCLUSIVE_QUEUE); 541127499Sgad mtx_lock_spin(&rm_spinlock); 542127499Sgad } 543127499Sgad mtx_unlock_spin(&rm_spinlock); 544127499Sgad } 545127499Sgad} 546127499Sgad 547127499Sgadvoid 548127499Sgad_rm_wunlock(struct rmlock *rm) 549127499Sgad{ 550127499Sgad 551127499Sgad if (rm->lock_object.lo_flags & LO_SLEEPABLE) 552127499Sgad sx_xunlock(&rm->rm_lock_sx); 553127499Sgad else 554127499Sgad mtx_unlock(&rm->rm_lock_mtx); 555127499Sgad} 556127499Sgad 557127499Sgad#ifdef LOCK_DEBUG 558127499Sgad 559127499Sgadvoid 560127499Sgad_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 561127499Sgad{ 562127499Sgad 563127499Sgad if (SCHEDULER_STOPPED()) 564127499Sgad return; 565127499Sgad 566127149Sgad KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 567127499Sgad ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d", 568127499Sgad curthread, rm->lock_object.lo_name, file, line)); 569127499Sgad KASSERT(!rm_destroyed(rm), 570127149Sgad ("rm_wlock() of destroyed rmlock @ %s:%d", file, line)); 5711556Srgrimes _rm_assert(rm, RA_UNLOCKED, file, line); 57225271Sjkh 57325271Sjkh WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, 57425271Sjkh file, line, NULL); 5751556Srgrimes 5761556Srgrimes _rm_wlock(rm); 5771556Srgrimes 5781556Srgrimes LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line); 579127499Sgad 58062803Swill WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 581127499Sgad 5821556Srgrimes curthread->td_locks++; 5831556Srgrimes 5841556Srgrimes} 585127499Sgad 5861556Srgrimesvoid 587127499Sgad_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 5881556Srgrimes{ 589127499Sgad 5901556Srgrimes if (SCHEDULER_STOPPED()) 5911556Srgrimes return; 5921556Srgrimes 5931556Srgrimes KASSERT(!rm_destroyed(rm), 5941556Srgrimes ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line)); 5951556Srgrimes _rm_assert(rm, RA_WLOCKED, file, line); 5961556Srgrimes WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 5971556Srgrimes LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); 5981556Srgrimes _rm_wunlock(rm); 5991556Srgrimes curthread->td_locks--; 6001556Srgrimes} 6011556Srgrimes 602127499Sgadint 603127499Sgad_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 604127499Sgad int trylock, const char *file, int line) 605127499Sgad{ 606127499Sgad 607127499Sgad if (SCHEDULER_STOPPED()) 608127499Sgad return (1); 60966377Sbrian 6101556Srgrimes#ifdef INVARIANTS 6111556Srgrimes if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) { 6121556Srgrimes critical_enter(); 613127499Sgad KASSERT(rm_trackers_present(pcpu_find(curcpu), rm, 614127499Sgad curthread) == 0, 615127499Sgad ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n", 616127499Sgad rm->lock_object.lo_name, file, line)); 617127499Sgad critical_exit(); 618127499Sgad } 619127499Sgad#endif 620127499Sgad KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 621127499Sgad ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d", 622127499Sgad curthread, rm->lock_object.lo_name, file, line)); 623127499Sgad KASSERT(!rm_destroyed(rm), 624127499Sgad ("rm_rlock() of destroyed rmlock @ %s:%d", file, line)); 625127499Sgad if (!trylock) { 626127499Sgad KASSERT(!rm_wowned(rm), 627127499Sgad ("rm_rlock: wlock already held for %s @ %s:%d", 628127499Sgad rm->lock_object.lo_name, file, line)); 629127499Sgad WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, 630127499Sgad NULL); 631127499Sgad } 632127499Sgad 633127499Sgad if (_rm_rlock(rm, tracker, trylock)) { 634127499Sgad if (trylock) 635127499Sgad LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file, 636127499Sgad line); 637127499Sgad else 638127499Sgad LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, 639127499Sgad line); 640127499Sgad WITNESS_LOCK(&rm->lock_object, 0, file, line); 641127499Sgad 642127499Sgad curthread->td_locks++; 643127499Sgad 644127499Sgad return (1); 645127499Sgad } else if (trylock) 646127499Sgad LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line); 647127499Sgad 648127499Sgad return (0); 649127499Sgad} 650127499Sgad 651127499Sgadvoid 652127499Sgad_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 653127499Sgad const char *file, int line) 654127499Sgad{ 655127499Sgad 656127499Sgad if (SCHEDULER_STOPPED()) 657127499Sgad return; 658127499Sgad 659127499Sgad KASSERT(!rm_destroyed(rm), 660127499Sgad ("rm_runlock() of destroyed rmlock @ %s:%d", file, line)); 661127149Sgad _rm_assert(rm, RA_RLOCKED, file, line); 662127499Sgad WITNESS_UNLOCK(&rm->lock_object, 0, file, line); 663127499Sgad LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); 664127149Sgad _rm_runlock(rm, tracker); 665127149Sgad curthread->td_locks--; 666127499Sgad} 667127149Sgad 668127499Sgad#else 669127499Sgad 670127499Sgad/* 671127499Sgad * Just strip out file and line arguments if no lock debugging is enabled in 672127499Sgad * the kernel - we are called from a kernel module. 673127499Sgad */ 674127499Sgadvoid 675127499Sgad_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 676127499Sgad{ 677127499Sgad 678127499Sgad _rm_wlock(rm); 679127149Sgad} 680127499Sgad 681127499Sgadvoid 682127499Sgad_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 683127149Sgad{ 684127149Sgad 685127149Sgad _rm_wunlock(rm); 686127499Sgad} 687127499Sgad 688127499Sgadint 689127499Sgad_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 690127499Sgad int trylock, const char *file, int line) 691127499Sgad{ 692127149Sgad 693127499Sgad return _rm_rlock(rm, tracker, trylock); 694127499Sgad} 695127499Sgad 696127499Sgadvoid 697127499Sgad_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 698127499Sgad const char *file, int line) 699127499Sgad{ 700127499Sgad 701127499Sgad _rm_runlock(rm, tracker); 702127499Sgad} 703127499Sgad 704127499Sgad#endif 705127499Sgad 706127499Sgad#ifdef INVARIANT_SUPPORT 707127499Sgad#ifndef INVARIANTS 708127499Sgad#undef _rm_assert 709127499Sgad#endif 710127499Sgad 711127499Sgad/* 712127499Sgad * Note that this does not need to use witness_assert() for read lock 713127499Sgad * assertions since an exact count of read locks held by this thread 714127499Sgad * is computable. 715127499Sgad */ 716127499Sgadvoid 717127499Sgad_rm_assert(const struct rmlock *rm, int what, const char *file, int line) 718127499Sgad{ 719127499Sgad int count; 720127499Sgad 721127499Sgad if (panicstr != NULL) 722127499Sgad return; 723127499Sgad switch (what) { 724127499Sgad case RA_LOCKED: 725127149Sgad case RA_LOCKED | RA_RECURSED: 726127149Sgad case RA_LOCKED | RA_NOTRECURSED: 727127499Sgad case RA_RLOCKED: 728127499Sgad case RA_RLOCKED | RA_RECURSED: 72966377Sbrian case RA_RLOCKED | RA_NOTRECURSED: 73066377Sbrian /* 731127499Sgad * Handle the write-locked case. Unlike other 732127499Sgad * primitives, writers can never recurse. 73366377Sbrian */ 734127499Sgad if (rm_wowned(rm)) { 735127499Sgad if (what & RA_RLOCKED) 736127499Sgad panic("Lock %s exclusively locked @ %s:%d\n", 737127499Sgad rm->lock_object.lo_name, file, line); 738127499Sgad if (what & RA_RECURSED) 739127499Sgad panic("Lock %s not recursed @ %s:%d\n", 740127499Sgad rm->lock_object.lo_name, file, line); 741127499Sgad break; 74266377Sbrian } 743127499Sgad 744127499Sgad critical_enter(); 745127499Sgad count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 746127499Sgad critical_exit(); 747127499Sgad 748127499Sgad if (count == 0) 749127499Sgad panic("Lock %s not %slocked @ %s:%d\n", 750127499Sgad rm->lock_object.lo_name, (what & RA_RLOCKED) ? 751127499Sgad "read " : "", file, line); 752127499Sgad if (count > 1) { 753127499Sgad if (what & RA_NOTRECURSED) 754127499Sgad panic("Lock %s recursed @ %s:%d\n", 755127499Sgad rm->lock_object.lo_name, file, line); 75666377Sbrian } else if (what & RA_RECURSED) 757127499Sgad panic("Lock %s not recursed @ %s:%d\n", 758127499Sgad rm->lock_object.lo_name, file, line); 759127509Sgad break; 760127509Sgad case RA_WLOCKED: 761127509Sgad if (!rm_wowned(rm)) 762127509Sgad panic("Lock %s not exclusively locked @ %s:%d\n", 763127509Sgad rm->lock_object.lo_name, file, line); 764127509Sgad break; 765127499Sgad case RA_UNLOCKED: 766127499Sgad if (rm_wowned(rm)) 767127499Sgad panic("Lock %s exclusively locked @ %s:%d\n", 768127499Sgad rm->lock_object.lo_name, file, line); 769127499Sgad 770127499Sgad critical_enter(); 771127499Sgad count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 772127499Sgad critical_exit(); 773127499Sgad 774127499Sgad if (count != 0) 775127499Sgad panic("Lock %s read locked @ %s:%d\n", 776127499Sgad rm->lock_object.lo_name, file, line); 777127499Sgad break; 778127499Sgad default: 779127499Sgad panic("Unknown rm lock assertion: %d @ %s:%d", what, file, 780127499Sgad line); 781127499Sgad } 782127499Sgad} 783127499Sgad#endif /* INVARIANT_SUPPORT */ 784127499Sgad 785127499Sgad#ifdef DDB 786127499Sgadstatic void 787127499Sgadprint_tracker(struct rm_priotracker *tr) 788127499Sgad{ 789127499Sgad struct thread *td; 790127499Sgad 791127499Sgad td = tr->rmp_thread; 792127499Sgad db_printf(" thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid, 793127499Sgad td->td_proc->p_pid, td->td_name); 794127499Sgad if (tr->rmp_flags & RMPF_ONQUEUE) { 79566377Sbrian db_printf("ONQUEUE"); 796127499Sgad if (tr->rmp_flags & RMPF_SIGNAL) 797127499Sgad db_printf(",SIGNAL"); 798127499Sgad } else 799127499Sgad db_printf("0"); 800127499Sgad db_printf("}\n"); 801127499Sgad} 802127499Sgad 803127499Sgadstatic void 804127499Sgaddb_show_rm(const struct lock_object *lock) 805127499Sgad{ 806127499Sgad struct rm_priotracker *tr; 807127499Sgad struct rm_queue *queue; 808127499Sgad const struct rmlock *rm; 809127499Sgad struct lock_class *lc; 810127499Sgad struct pcpu *pc; 811127499Sgad 812127499Sgad rm = (const struct rmlock *)lock; 813127499Sgad db_printf(" writecpus: "); 814127499Sgad ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus)); 815127499Sgad db_printf("\n"); 816127499Sgad db_printf(" per-CPU readers:\n"); 817127499Sgad STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 818127499Sgad for (queue = pc->pc_rm_queue.rmq_next; 819127499Sgad queue != &pc->pc_rm_queue; queue = queue->rmq_next) { 820127499Sgad tr = (struct rm_priotracker *)queue; 821127499Sgad if (tr->rmp_rmlock == rm) 822127499Sgad print_tracker(tr); 823127499Sgad } 824127499Sgad db_printf(" active readers:\n"); 825127499Sgad LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry) 826127499Sgad print_tracker(tr); 827127499Sgad lc = LOCK_CLASS(&rm->rm_wlock_object); 828127499Sgad db_printf("Backing write-lock (%s):\n", lc->lc_name); 829127499Sgad lc->lc_ddb_show(&rm->rm_wlock_object); 830127499Sgad} 83166377Sbrian#endif 832127499Sgad