kern_rmlock.c revision 255745
11558Srgrimes/*- 21558Srgrimes * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org> 31558Srgrimes * All rights reserved. 41558Srgrimes * 51558Srgrimes * Redistribution and use in source and binary forms, with or without 61558Srgrimes * modification, are permitted provided that the following conditions 71558Srgrimes * are met: 81558Srgrimes * 1. Redistributions of source code must retain the above copyright 91558Srgrimes * notice, this list of conditions and the following disclaimer. 101558Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 111558Srgrimes * notice, this list of conditions and the following disclaimer in the 121558Srgrimes * documentation and/or other materials provided with the distribution. 131558Srgrimes * 3. Neither the name of the author nor the names of any co-contributors 141558Srgrimes * may be used to endorse or promote products derived from this software 151558Srgrimes * without specific prior written permission. 161558Srgrimes * 171558Srgrimes * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 181558Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 191558Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 201558Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 211558Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 221558Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 231558Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 241558Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 251558Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 261558Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 271558Srgrimes * SUCH DAMAGE. 281558Srgrimes */ 291558Srgrimes 301558Srgrimes/* 311558Srgrimes * Machine independent bits of reader/writer lock implementation. 321558Srgrimes */ 331558Srgrimes 341558Srgrimes#include <sys/cdefs.h> 351558Srgrimes__FBSDID("$FreeBSD: head/sys/kern/kern_rmlock.c 255745 2013-09-20 23:06:21Z davide $"); 361558Srgrimes 371558Srgrimes#include "opt_ddb.h" 381558Srgrimes#include "opt_kdtrace.h" 391558Srgrimes 401558Srgrimes#include <sys/param.h> 412999Swollman#include <sys/systm.h> 421558Srgrimes 431558Srgrimes#include <sys/kernel.h> 4423681Speter#include <sys/kdb.h> 452999Swollman#include <sys/ktr.h> 4637003Sjoerg#include <sys/lock.h> 472999Swollman#include <sys/mutex.h> 481558Srgrimes#include <sys/proc.h> 491558Srgrimes#include <sys/rmlock.h> 501558Srgrimes#include <sys/sched.h> 511558Srgrimes#include <sys/smp.h> 521558Srgrimes#include <sys/turnstile.h> 531558Srgrimes#include <sys/lock_profile.h> 541558Srgrimes#include <machine/cpu.h> 551558Srgrimes 561558Srgrimes#ifdef DDB 5724330Sguido#include <ddb/ddb.h> 581558Srgrimes#endif 591558Srgrimes 601558Srgrimes/* 611558Srgrimes * A cookie to mark destroyed rmlocks. This is stored in the head of 621558Srgrimes * rm_activeReaders. 631558Srgrimes */ 641558Srgrimes#define RM_DESTROYED ((void *)0xdead) 651558Srgrimes 669336Sdfr#define rm_destroyed(rm) \ 6724330Sguido (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED) 6823681Speter 6923681Speter#define RMPF_ONQUEUE 1 7023681Speter#define RMPF_SIGNAL 2 711558Srgrimes 721558Srgrimes#ifndef INVARIANTS 731558Srgrimes#define _rm_assert(c, what, file, line) 741558Srgrimes#endif 751558Srgrimes 761558Srgrimesstatic void assert_rm(const struct lock_object *lock, int what); 771558Srgrimes#ifdef DDB 781558Srgrimesstatic void db_show_rm(const struct lock_object *lock); 791558Srgrimes#endif 801558Srgrimesstatic void lock_rm(struct lock_object *lock, uintptr_t how); 811558Srgrimes#ifdef KDTRACE_HOOKS 821558Srgrimesstatic int owner_rm(const struct lock_object *lock, struct thread **owner); 831558Srgrimes#endif 841558Srgrimesstatic uintptr_t unlock_rm(struct lock_object *lock); 851558Srgrimes 861558Srgrimesstruct lock_class lock_class_rm = { 871558Srgrimes .lc_name = "rm", 881558Srgrimes .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 891558Srgrimes .lc_assert = assert_rm, 901558Srgrimes#ifdef DDB 911558Srgrimes .lc_ddb_show = db_show_rm, 921558Srgrimes#endif 931558Srgrimes .lc_lock = lock_rm, 941558Srgrimes .lc_unlock = unlock_rm, 951558Srgrimes#ifdef KDTRACE_HOOKS 961558Srgrimes .lc_owner = owner_rm, 971558Srgrimes#endif 981558Srgrimes}; 991558Srgrimes 1001558Srgrimesstruct lock_class lock_class_rm_sleepable = { 1011558Srgrimes .lc_name = "sleepable rm", 1021558Srgrimes .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE, 1031558Srgrimes .lc_assert = assert_rm, 1041558Srgrimes#ifdef DDB 1051558Srgrimes .lc_ddb_show = db_show_rm, 1061558Srgrimes#endif 1071558Srgrimes .lc_lock = lock_rm, 1089336Sdfr .lc_unlock = unlock_rm, 1099336Sdfr#ifdef KDTRACE_HOOKS 1101558Srgrimes .lc_owner = owner_rm, 1111558Srgrimes#endif 1121558Srgrimes}; 1131558Srgrimes 1141558Srgrimesstatic void 1151558Srgrimesassert_rm(const struct lock_object *lock, int what) 1161558Srgrimes{ 1171558Srgrimes 11827447Sdfr rm_assert((const struct rmlock *)lock, what); 1191558Srgrimes} 1201558Srgrimes 1211558Srgrimesstatic void 1221558Srgrimeslock_rm(struct lock_object *lock, uintptr_t how) 1231558Srgrimes{ 1241558Srgrimes struct rmlock *rm; 1251558Srgrimes struct rm_priotracker *tracker; 1261558Srgrimes 1271558Srgrimes rm = (struct rmlock *)lock; 1281558Srgrimes if (how == 0) 1291558Srgrimes rm_wlock(rm); 1301558Srgrimes else { 1311558Srgrimes tracker = (struct rm_priotracker *)how; 1321558Srgrimes rm_rlock(rm, tracker); 1331558Srgrimes } 1341558Srgrimes} 1351558Srgrimes 1361558Srgrimesstatic uintptr_t 1371558Srgrimesunlock_rm(struct lock_object *lock) 1381558Srgrimes{ 1391558Srgrimes struct thread *td; 1401558Srgrimes struct pcpu *pc; 1411558Srgrimes struct rmlock *rm; 1421558Srgrimes struct rm_queue *queue; 1431558Srgrimes struct rm_priotracker *tracker; 1441558Srgrimes uintptr_t how; 1451558Srgrimes 1461558Srgrimes rm = (struct rmlock *)lock; 1477401Swpaul tracker = NULL; 1481558Srgrimes how = 0; 1491558Srgrimes rm_assert(rm, RA_LOCKED | RA_NOTRECURSED); 1509336Sdfr if (rm_wowned(rm)) 1511558Srgrimes rm_wunlock(rm); 1521558Srgrimes else { 1531558Srgrimes /* 1541558Srgrimes * Find the right rm_priotracker structure for curthread. 1559336Sdfr * The guarantee about its uniqueness is given by the fact 1569336Sdfr * we already asserted the lock wasn't recursively acquired. 1579336Sdfr */ 1589336Sdfr critical_enter(); 1599336Sdfr td = curthread; 1609336Sdfr pc = pcpu_find(curcpu); 1611558Srgrimes for (queue = pc->pc_rm_queue.rmq_next; 1621558Srgrimes queue != &pc->pc_rm_queue; queue = queue->rmq_next) { 1631558Srgrimes tracker = (struct rm_priotracker *)queue; 1649336Sdfr if ((tracker->rmp_rmlock == rm) && 1651558Srgrimes (tracker->rmp_thread == td)) { 1661558Srgrimes how = (uintptr_t)tracker; 1671558Srgrimes break; 1689336Sdfr } 1691558Srgrimes } 1701558Srgrimes KASSERT(tracker != NULL, 1711558Srgrimes ("rm_priotracker is non-NULL when lock held in read mode")); 1729336Sdfr critical_exit(); 1731558Srgrimes rm_runlock(rm, tracker); 1741558Srgrimes } 1751558Srgrimes return (how); 1761558Srgrimes} 1771558Srgrimes 1781558Srgrimes#ifdef KDTRACE_HOOKS 1791558Srgrimesstatic int 1801558Srgrimesowner_rm(const struct lock_object *lock, struct thread **owner) 1811558Srgrimes{ 1827401Swpaul const struct rmlock *rm; 1839336Sdfr struct lock_class *lc; 1841558Srgrimes 1851558Srgrimes rm = (const struct rmlock *)lock; 1861558Srgrimes lc = LOCK_CLASS(&rm->rm_wlock_object); 1871558Srgrimes return (lc->lc_owner(&rm->rm_wlock_object, owner)); 1881558Srgrimes} 1891558Srgrimes#endif 1901558Srgrimes 1911558Srgrimesstatic struct mtx rm_spinlock; 1921558Srgrimes 1931558SrgrimesMTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN); 1941558Srgrimes 1951558Srgrimes/* 1961558Srgrimes * Add or remove tracker from per-cpu list. 1971558Srgrimes * 1981558Srgrimes * The per-cpu list can be traversed at any time in forward direction from an 1991558Srgrimes * interrupt on the *local* cpu. 2001558Srgrimes */ 2011558Srgrimesstatic void inline 2029336Sdfrrm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker) 2031558Srgrimes{ 2041558Srgrimes struct rm_queue *next; 2051558Srgrimes 2061558Srgrimes /* Initialize all tracker pointers */ 2071558Srgrimes tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue; 2081558Srgrimes next = pc->pc_rm_queue.rmq_next; 2091558Srgrimes tracker->rmp_cpuQueue.rmq_next = next; 2101558Srgrimes 2111558Srgrimes /* rmq_prev is not used during froward traversal. */ 2121558Srgrimes next->rmq_prev = &tracker->rmp_cpuQueue; 2131558Srgrimes 2141558Srgrimes /* Update pointer to first element. */ 2151558Srgrimes pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue; 2161558Srgrimes} 2171558Srgrimes 2181558Srgrimes/* 2191558Srgrimes * Return a count of the number of trackers the thread 'td' already 2201558Srgrimes * has on this CPU for the lock 'rm'. 2211558Srgrimes */ 2221558Srgrimesstatic int 2231558Srgrimesrm_trackers_present(const struct pcpu *pc, const struct rmlock *rm, 22425087Sdfr const struct thread *td) 2259336Sdfr{ 2269336Sdfr struct rm_queue *queue; 22731705Sguido struct rm_priotracker *tracker; 2281558Srgrimes int count; 2291558Srgrimes 2301558Srgrimes count = 0; 2311558Srgrimes for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 2321558Srgrimes queue = queue->rmq_next) { 2331558Srgrimes tracker = (struct rm_priotracker *)queue; 2341558Srgrimes if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td)) 2351558Srgrimes count++; 2361558Srgrimes } 2371558Srgrimes return (count); 2381558Srgrimes} 2391558Srgrimes 2401558Srgrimesstatic void inline 2411558Srgrimesrm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker) 2421558Srgrimes{ 2431558Srgrimes struct rm_queue *next, *prev; 2441558Srgrimes 2451558Srgrimes next = tracker->rmp_cpuQueue.rmq_next; 2461558Srgrimes prev = tracker->rmp_cpuQueue.rmq_prev; 2471558Srgrimes 2481558Srgrimes /* Not used during forward traversal. */ 2491558Srgrimes next->rmq_prev = prev; 2501558Srgrimes 2511558Srgrimes /* Remove from list. */ 2521558Srgrimes prev->rmq_next = next; 2531558Srgrimes} 2541558Srgrimes 2551558Srgrimesstatic void 2561558Srgrimesrm_cleanIPI(void *arg) 2571558Srgrimes{ 2589202Srgrimes struct pcpu *pc; 25932656Sbde struct rmlock *rm = arg; 26023681Speter struct rm_priotracker *tracker; 2611558Srgrimes struct rm_queue *queue; 26223681Speter pc = pcpu_find(curcpu); 26323681Speter 2642999Swollman for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 2652999Swollman queue = queue->rmq_next) { 2662999Swollman tracker = (struct rm_priotracker *)queue; 26723681Speter if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) { 2682999Swollman tracker->rmp_flags = RMPF_ONQUEUE; 26923681Speter mtx_lock_spin(&rm_spinlock); 2702999Swollman LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 2712999Swollman rmp_qentry); 27231665Sguido mtx_unlock_spin(&rm_spinlock); 2731558Srgrimes } 27425087Sdfr } 27525087Sdfr} 27625087Sdfr 2779336Sdfrvoid 2789336Sdfrrm_init_flags(struct rmlock *rm, const char *name, int opts) 2799336Sdfr{ 2809336Sdfr struct lock_class *lc; 2819336Sdfr int liflags; 2829336Sdfr 2838688Sphk liflags = 0; 2848688Sphk if (!(opts & RM_NOWITNESS)) 2858688Sphk liflags |= LO_WITNESS; 28631656Sguido if (opts & RM_RECURSE) 28731656Sguido liflags |= LO_RECURSABLE; 28831656Sguido rm->rm_writecpus = all_cpus; 2891558Srgrimes LIST_INIT(&rm->rm_activeReaders); 29031665Sguido if (opts & RM_SLEEPABLE) { 29131665Sguido liflags |= LO_SLEEPABLE; 2921558Srgrimes lc = &lock_class_rm_sleepable; 2931558Srgrimes sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS); 2941558Srgrimes } else { 2951558Srgrimes lc = &lock_class_rm; 2961558Srgrimes mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS); 2971558Srgrimes } 2981558Srgrimes lock_init(&rm->lock_object, lc, name, NULL, liflags); 2991558Srgrimes} 3001558Srgrimes 3011558Srgrimesvoid 3021558Srgrimesrm_init(struct rmlock *rm, const char *name) 3031558Srgrimes{ 3041558Srgrimes 3051558Srgrimes rm_init_flags(rm, name, 0); 3061558Srgrimes} 3071558Srgrimes 3081558Srgrimesvoid 3091558Srgrimesrm_destroy(struct rmlock *rm) 3101558Srgrimes{ 3111558Srgrimes 3121558Srgrimes rm_assert(rm, RA_UNLOCKED); 3131558Srgrimes LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED; 3141558Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 3151558Srgrimes sx_destroy(&rm->rm_lock_sx); 3161558Srgrimes else 3171558Srgrimes mtx_destroy(&rm->rm_lock_mtx); 3181558Srgrimes lock_destroy(&rm->lock_object); 3191558Srgrimes} 3201558Srgrimes 3211558Srgrimesint 3221558Srgrimesrm_wowned(const struct rmlock *rm) 3231558Srgrimes{ 3241558Srgrimes 3251558Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 32624759Sguido return (sx_xlocked(&rm->rm_lock_sx)); 32724759Sguido else 32832656Sbde return (mtx_owned(&rm->rm_lock_mtx)); 32924759Sguido} 33024759Sguido 33124759Sguidovoid 33224759Sguidorm_sysinit(void *arg) 33324759Sguido{ 33424759Sguido struct rm_args *args = arg; 33524330Sguido 3369202Srgrimes rm_init(args->ra_rm, args->ra_desc); 3379202Srgrimes} 3381558Srgrimes 3391558Srgrimesvoid 3401558Srgrimesrm_sysinit_flags(void *arg) 3419336Sdfr{ 3429336Sdfr struct rm_args_flags *args = arg; 34325087Sdfr 34425087Sdfr rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts); 34525087Sdfr} 34625087Sdfr 34725087Sdfrstatic int 34825087Sdfr_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 3499336Sdfr{ 35025087Sdfr struct pcpu *pc; 3511558Srgrimes 3521558Srgrimes critical_enter(); 3531558Srgrimes pc = pcpu_find(curcpu); 3541558Srgrimes 3551558Srgrimes /* Check if we just need to do a proper critical_exit. */ 3561558Srgrimes if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { 3571558Srgrimes critical_exit(); 3581558Srgrimes return (1); 3591558Srgrimes } 3601558Srgrimes 3611558Srgrimes /* Remove our tracker from the per-cpu list. */ 3621558Srgrimes rm_tracker_remove(pc, tracker); 3631558Srgrimes 3641558Srgrimes /* Check to see if the IPI granted us the lock after all. */ 3651558Srgrimes if (tracker->rmp_flags) { 3661558Srgrimes /* Just add back tracker - we hold the lock. */ 3671558Srgrimes rm_tracker_add(pc, tracker); 3681558Srgrimes critical_exit(); 3699336Sdfr return (1); 3701558Srgrimes } 3711558Srgrimes 3721558Srgrimes /* 37331656Sguido * We allow readers to aquire a lock even if a writer is blocked if 3741558Srgrimes * the lock is recursive and the reader already holds the lock. 3759336Sdfr */ 37623681Speter if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) { 37728911Sguido /* 3789336Sdfr * Just grant the lock if this thread already has a tracker 3791558Srgrimes * for this lock on the per-cpu queue. 3809336Sdfr */ 3819336Sdfr if (rm_trackers_present(pc, rm, curthread) != 0) { 3821558Srgrimes mtx_lock_spin(&rm_spinlock); 38331656Sguido LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 3849336Sdfr rmp_qentry); 3851558Srgrimes tracker->rmp_flags = RMPF_ONQUEUE; 3861558Srgrimes mtx_unlock_spin(&rm_spinlock); 3871558Srgrimes rm_tracker_add(pc, tracker); 3881558Srgrimes critical_exit(); 3891558Srgrimes return (1); 3901558Srgrimes } 3911558Srgrimes } 3929336Sdfr 39331656Sguido sched_unpin(); 39431656Sguido critical_exit(); 39531656Sguido 3961558Srgrimes if (trylock) { 3971558Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) { 3981558Srgrimes if (!sx_try_xlock(&rm->rm_lock_sx)) 3991558Srgrimes return (0); 40031656Sguido } else { 40131656Sguido if (!mtx_trylock(&rm->rm_lock_mtx)) 4021558Srgrimes return (0); 4031558Srgrimes } 4041558Srgrimes } else { 4051558Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 4061558Srgrimes sx_xlock(&rm->rm_lock_sx); 4071558Srgrimes else 4089336Sdfr mtx_lock(&rm->rm_lock_mtx); 4099336Sdfr } 4101558Srgrimes 4111558Srgrimes critical_enter(); 4121558Srgrimes pc = pcpu_find(curcpu); 4139336Sdfr CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); 4149336Sdfr rm_tracker_add(pc, tracker); 4151558Srgrimes sched_pin(); 4161558Srgrimes critical_exit(); 41731656Sguido 41831656Sguido if (rm->lock_object.lo_flags & LO_SLEEPABLE) 41931656Sguido sx_xunlock(&rm->rm_lock_sx); 4201558Srgrimes else 4211558Srgrimes mtx_unlock(&rm->rm_lock_mtx); 42228911Sguido 4231558Srgrimes return (1); 4241558Srgrimes} 4251558Srgrimes 4269336Sdfrint 4271558Srgrimes_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 4289336Sdfr{ 4299336Sdfr struct thread *td = curthread; 4301558Srgrimes struct pcpu *pc; 4319336Sdfr 4321558Srgrimes if (SCHEDULER_STOPPED()) 4331558Srgrimes return (1); 43428911Sguido 43528911Sguido tracker->rmp_flags = 0; 43628911Sguido tracker->rmp_thread = td; 43728911Sguido tracker->rmp_rmlock = rm; 43828911Sguido 43928911Sguido if (rm->lock_object.lo_flags & LO_SLEEPABLE) 44028911Sguido THREAD_NO_SLEEPING(); 4419336Sdfr 4429336Sdfr td->td_critnest++; /* critical_enter(); */ 4439336Sdfr 4449336Sdfr __compiler_membar(); 4459336Sdfr 4461558Srgrimes pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 44723681Speter 4489336Sdfr rm_tracker_add(pc, tracker); 4491558Srgrimes 4501558Srgrimes sched_pin(); 4511558Srgrimes 4521558Srgrimes __compiler_membar(); 4531558Srgrimes 4549336Sdfr td->td_critnest--; 4551558Srgrimes 4561558Srgrimes /* 4579336Sdfr * Fast path to combine two common conditions into a single 4581558Srgrimes * conditional jump. 4591558Srgrimes */ 4601558Srgrimes if (0 == (td->td_owepreempt | 4611558Srgrimes CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus))) 4621558Srgrimes return (1); 4631558Srgrimes 4641558Srgrimes /* We do not have a read token and need to acquire one. */ 46531656Sguido return _rm_rlock_hard(rm, tracker, trylock); 4661558Srgrimes} 4671558Srgrimes 4681558Srgrimesstatic void 46931656Sguido_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker) 47031656Sguido{ 47131656Sguido 47231656Sguido if (td->td_owepreempt) { 47331656Sguido td->td_critnest++; 4741558Srgrimes critical_exit(); 47531656Sguido } 47631656Sguido 47731656Sguido if (!tracker->rmp_flags) 47831656Sguido return; 47928911Sguido 48028911Sguido mtx_lock_spin(&rm_spinlock); 48128911Sguido LIST_REMOVE(tracker, rmp_qentry); 4829336Sdfr 4831558Srgrimes if (tracker->rmp_flags & RMPF_SIGNAL) { 4841558Srgrimes struct rmlock *rm; 4851558Srgrimes struct turnstile *ts; 4861558Srgrimes 48731656Sguido rm = tracker->rmp_rmlock; 48831656Sguido 48931656Sguido turnstile_chain_lock(&rm->lock_object); 49031656Sguido mtx_unlock_spin(&rm_spinlock); 4911558Srgrimes 4921558Srgrimes ts = turnstile_lookup(&rm->lock_object); 4939336Sdfr 49431656Sguido turnstile_signal(ts, TS_EXCLUSIVE_QUEUE); 49531656Sguido turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 49631656Sguido turnstile_chain_unlock(&rm->lock_object); 4971558Srgrimes } else 4981558Srgrimes mtx_unlock_spin(&rm_spinlock); 4991558Srgrimes} 5001558Srgrimes 50131656Sguidovoid 50231656Sguido_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker) 5031558Srgrimes{ 5041558Srgrimes struct pcpu *pc; 5051558Srgrimes struct thread *td = tracker->rmp_thread; 5061558Srgrimes 5071558Srgrimes if (SCHEDULER_STOPPED()) 5081558Srgrimes return; 5091558Srgrimes 5101558Srgrimes td->td_critnest++; /* critical_enter(); */ 51131656Sguido pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 51231656Sguido rm_tracker_remove(pc, tracker); 51331656Sguido td->td_critnest--; 51431656Sguido sched_unpin(); 51531656Sguido 5161558Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 5171558Srgrimes THREAD_SLEEPING_OK(); 5189336Sdfr 51931656Sguido if (0 == (td->td_owepreempt | tracker->rmp_flags)) 52031656Sguido return; 52131656Sguido 5221558Srgrimes _rm_unlock_hard(td, tracker); 5231558Srgrimes} 5241558Srgrimes 5251558Srgrimesvoid 5261558Srgrimes_rm_wlock(struct rmlock *rm) 5271558Srgrimes{ 5281558Srgrimes struct rm_priotracker *prio; 5291558Srgrimes struct turnstile *ts; 53031656Sguido cpuset_t readcpus; 53131656Sguido 53231656Sguido if (SCHEDULER_STOPPED()) 53331656Sguido return; 53431656Sguido 5351558Srgrimes if (rm->lock_object.lo_flags & LO_SLEEPABLE) 5361558Srgrimes sx_xlock(&rm->rm_lock_sx); 5371558Srgrimes else 5381558Srgrimes mtx_lock(&rm->rm_lock_mtx); 53931656Sguido 54031656Sguido if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) { 54131656Sguido /* Get all read tokens back */ 54231656Sguido readcpus = all_cpus; 5431558Srgrimes CPU_NAND(&readcpus, &rm->rm_writecpus); 5441558Srgrimes rm->rm_writecpus = all_cpus; 5451558Srgrimes 5461558Srgrimes /* 5471558Srgrimes * Assumes rm->rm_writecpus update is visible on other CPUs 5481558Srgrimes * before rm_cleanIPI is called. 5491558Srgrimes */ 5501558Srgrimes#ifdef SMP 5511558Srgrimes smp_rendezvous_cpus(readcpus, 5521558Srgrimes smp_no_rendevous_barrier, 5531558Srgrimes rm_cleanIPI, 5541558Srgrimes smp_no_rendevous_barrier, 5551558Srgrimes rm); 5561558Srgrimes 5571558Srgrimes#else 5581558Srgrimes rm_cleanIPI(rm); 5591558Srgrimes#endif 5601558Srgrimes 5611558Srgrimes mtx_lock_spin(&rm_spinlock); 5629336Sdfr while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { 5631558Srgrimes ts = turnstile_trywait(&rm->lock_object); 5641558Srgrimes prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL; 5659336Sdfr mtx_unlock_spin(&rm_spinlock); 5661558Srgrimes turnstile_wait(ts, prio->rmp_thread, 5679336Sdfr TS_EXCLUSIVE_QUEUE); 5681558Srgrimes mtx_lock_spin(&rm_spinlock); 5699336Sdfr } 5709336Sdfr mtx_unlock_spin(&rm_spinlock); 5711558Srgrimes } 5721558Srgrimes} 5731558Srgrimes 5749336Sdfrvoid 5759336Sdfr_rm_wunlock(struct rmlock *rm) 5769336Sdfr{ 5779336Sdfr 5789336Sdfr if (rm->lock_object.lo_flags & LO_SLEEPABLE) 5799336Sdfr sx_xunlock(&rm->rm_lock_sx); 5809336Sdfr else 5819336Sdfr mtx_unlock(&rm->rm_lock_mtx); 5829336Sdfr} 5839336Sdfr 5849336Sdfr#ifdef LOCK_DEBUG 5859336Sdfr 5869336Sdfrvoid 5879336Sdfr_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 5889336Sdfr{ 5899336Sdfr 5909336Sdfr if (SCHEDULER_STOPPED()) 5919336Sdfr return; 5929336Sdfr 5931558Srgrimes KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 5941558Srgrimes ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d", 5951558Srgrimes curthread, rm->lock_object.lo_name, file, line)); 5961558Srgrimes KASSERT(!rm_destroyed(rm), 5971558Srgrimes ("rm_wlock() of destroyed rmlock @ %s:%d", file, line)); 5981558Srgrimes _rm_assert(rm, RA_UNLOCKED, file, line); 5991558Srgrimes 6001558Srgrimes WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, 6011558Srgrimes file, line, NULL); 6021558Srgrimes 6031558Srgrimes _rm_wlock(rm); 6041558Srgrimes 6051558Srgrimes LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line); 6061558Srgrimes 6071558Srgrimes WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 6081558Srgrimes 6091558Srgrimes curthread->td_locks++; 6101558Srgrimes 6111558Srgrimes} 6121558Srgrimes 6131558Srgrimesvoid 6141558Srgrimes_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 6151558Srgrimes{ 6161558Srgrimes 6171558Srgrimes if (SCHEDULER_STOPPED()) 6181558Srgrimes return; 6191558Srgrimes 6201558Srgrimes KASSERT(!rm_destroyed(rm), 6211558Srgrimes ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line)); 6221558Srgrimes _rm_assert(rm, RA_WLOCKED, file, line); 6231558Srgrimes WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 6241558Srgrimes LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); 6251558Srgrimes _rm_wunlock(rm); 6261558Srgrimes curthread->td_locks--; 6271558Srgrimes} 6281558Srgrimes 6291558Srgrimesint 6301558Srgrimes_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 6311558Srgrimes int trylock, const char *file, int line) 6329336Sdfr{ 6339336Sdfr 6341558Srgrimes if (SCHEDULER_STOPPED()) 6359336Sdfr return (1); 6369336Sdfr 6379336Sdfr#ifdef INVARIANTS 6381558Srgrimes if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) { 6391558Srgrimes critical_enter(); 6401558Srgrimes KASSERT(rm_trackers_present(pcpu_find(curcpu), rm, 6411558Srgrimes curthread) == 0, 6421558Srgrimes ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n", 6431558Srgrimes rm->lock_object.lo_name, file, line)); 6441558Srgrimes critical_exit(); 6451558Srgrimes } 6461558Srgrimes#endif 6471558Srgrimes KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 6481558Srgrimes ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d", 6499336Sdfr curthread, rm->lock_object.lo_name, file, line)); 6501558Srgrimes KASSERT(!rm_destroyed(rm), 6511558Srgrimes ("rm_rlock() of destroyed rmlock @ %s:%d", file, line)); 6521558Srgrimes if (!trylock) { 6531558Srgrimes KASSERT(!rm_wowned(rm), 6549336Sdfr ("rm_rlock: wlock already held for %s @ %s:%d", 6551558Srgrimes rm->lock_object.lo_name, file, line)); 6561558Srgrimes WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, 6571558Srgrimes NULL); 6581558Srgrimes } 6591558Srgrimes 6601558Srgrimes if (_rm_rlock(rm, tracker, trylock)) { 6611558Srgrimes if (trylock) 6621558Srgrimes LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file, 6631558Srgrimes line); 6641558Srgrimes else 6651558Srgrimes LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, 6661558Srgrimes line); 6671558Srgrimes WITNESS_LOCK(&rm->lock_object, 0, file, line); 6681558Srgrimes 6691558Srgrimes curthread->td_locks++; 6701558Srgrimes 6711558Srgrimes return (1); 6721558Srgrimes } else if (trylock) 6731558Srgrimes LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line); 6741558Srgrimes 6751558Srgrimes return (0); 6761558Srgrimes} 6771558Srgrimes 6781558Srgrimesvoid 6791558Srgrimes_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 6801558Srgrimes const char *file, int line) 6811558Srgrimes{ 6821558Srgrimes 6831558Srgrimes if (SCHEDULER_STOPPED()) 6841558Srgrimes return; 6851558Srgrimes 6861558Srgrimes KASSERT(!rm_destroyed(rm), 6871558Srgrimes ("rm_runlock() of destroyed rmlock @ %s:%d", file, line)); 6881558Srgrimes _rm_assert(rm, RA_RLOCKED, file, line); 6891558Srgrimes WITNESS_UNLOCK(&rm->lock_object, 0, file, line); 6901558Srgrimes LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); 6911558Srgrimes _rm_runlock(rm, tracker); 6921558Srgrimes curthread->td_locks--; 6931558Srgrimes} 6941558Srgrimes 6951558Srgrimes#else 6961558Srgrimes 6978871Srgrimes/* 6981558Srgrimes * Just strip out file and line arguments if no lock debugging is enabled in 6991558Srgrimes * the kernel - we are called from a kernel module. 7001558Srgrimes */ 7011558Srgrimesvoid 7021558Srgrimes_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 7031558Srgrimes{ 7048871Srgrimes 7051558Srgrimes _rm_wlock(rm); 7061558Srgrimes} 7071558Srgrimes 7081558Srgrimesvoid 7091558Srgrimes_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 7101558Srgrimes{ 7111558Srgrimes 7121558Srgrimes _rm_wunlock(rm); 7131558Srgrimes} 7141558Srgrimes 7151558Srgrimesint 7161558Srgrimes_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 7171558Srgrimes int trylock, const char *file, int line) 7181558Srgrimes{ 7191558Srgrimes 7201558Srgrimes return _rm_rlock(rm, tracker, trylock); 7211558Srgrimes} 7221558Srgrimes 7231558Srgrimesvoid 7241558Srgrimes_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 7251558Srgrimes const char *file, int line) 7261558Srgrimes{ 7271558Srgrimes 7281558Srgrimes _rm_runlock(rm, tracker); 7291558Srgrimes} 7301558Srgrimes 7311558Srgrimes#endif 7321558Srgrimes 7331558Srgrimes#ifdef INVARIANT_SUPPORT 7341558Srgrimes#ifndef INVARIANTS 7351558Srgrimes#undef _rm_assert 7361558Srgrimes#endif 7371558Srgrimes 7381558Srgrimes/* 7391558Srgrimes * Note that this does not need to use witness_assert() for read lock 7401558Srgrimes * assertions since an exact count of read locks held by this thread 7411558Srgrimes * is computable. 7421558Srgrimes */ 7431558Srgrimesvoid 7441558Srgrimes_rm_assert(const struct rmlock *rm, int what, const char *file, int line) 7451558Srgrimes{ 7461558Srgrimes int count; 7471558Srgrimes 7481558Srgrimes if (panicstr != NULL) 7491558Srgrimes return; 7501558Srgrimes switch (what) { 7511558Srgrimes case RA_LOCKED: 7521558Srgrimes case RA_LOCKED | RA_RECURSED: 7531558Srgrimes case RA_LOCKED | RA_NOTRECURSED: 7541558Srgrimes case RA_RLOCKED: 7551558Srgrimes case RA_RLOCKED | RA_RECURSED: 7561558Srgrimes case RA_RLOCKED | RA_NOTRECURSED: 7571558Srgrimes /* 7581558Srgrimes * Handle the write-locked case. Unlike other 7591558Srgrimes * primitives, writers can never recurse. 7601558Srgrimes */ 7611558Srgrimes if (rm_wowned(rm)) { 7621558Srgrimes if (what & RA_RLOCKED) 7631558Srgrimes panic("Lock %s exclusively locked @ %s:%d\n", 7641558Srgrimes rm->lock_object.lo_name, file, line); 7651558Srgrimes if (what & RA_RECURSED) 76623681Speter panic("Lock %s not recursed @ %s:%d\n", 7671558Srgrimes rm->lock_object.lo_name, file, line); 7681558Srgrimes break; 7691558Srgrimes } 7701558Srgrimes 7711558Srgrimes critical_enter(); 7721558Srgrimes count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 7731558Srgrimes critical_exit(); 7749336Sdfr 7751558Srgrimes if (count == 0) 7761558Srgrimes panic("Lock %s not %slocked @ %s:%d\n", 77723681Speter rm->lock_object.lo_name, (what & RA_RLOCKED) ? 77823681Speter "read " : "", file, line); 77923681Speter if (count > 1) { 78023681Speter if (what & RA_NOTRECURSED) 7819336Sdfr panic("Lock %s recursed @ %s:%d\n", 7829336Sdfr rm->lock_object.lo_name, file, line); 7839336Sdfr } else if (what & RA_RECURSED) 7841558Srgrimes panic("Lock %s not recursed @ %s:%d\n", 7851558Srgrimes rm->lock_object.lo_name, file, line); 7861558Srgrimes break; 7871558Srgrimes case RA_WLOCKED: 7881558Srgrimes if (!rm_wowned(rm)) 7891558Srgrimes panic("Lock %s not exclusively locked @ %s:%d\n", 7901558Srgrimes rm->lock_object.lo_name, file, line); 7911558Srgrimes break; 7921558Srgrimes case RA_UNLOCKED: 7931558Srgrimes if (rm_wowned(rm)) 7941558Srgrimes panic("Lock %s exclusively locked @ %s:%d\n", 7951558Srgrimes rm->lock_object.lo_name, file, line); 7961558Srgrimes 7971558Srgrimes critical_enter(); 7981558Srgrimes count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 7991558Srgrimes critical_exit(); 8001558Srgrimes 8011558Srgrimes if (count != 0) 8021558Srgrimes panic("Lock %s read locked @ %s:%d\n", 8031558Srgrimes rm->lock_object.lo_name, file, line); 8041558Srgrimes break; 8051558Srgrimes default: 8061558Srgrimes panic("Unknown rm lock assertion: %d @ %s:%d", what, file, 8071558Srgrimes line); 8081558Srgrimes } 8091558Srgrimes} 8101558Srgrimes#endif /* INVARIANT_SUPPORT */ 8111558Srgrimes 8121558Srgrimes#ifdef DDB 8131558Srgrimesstatic void 8141558Srgrimesprint_tracker(struct rm_priotracker *tr) 8151558Srgrimes{ 8161558Srgrimes struct thread *td; 8171558Srgrimes 8181558Srgrimes td = tr->rmp_thread; 8191558Srgrimes db_printf(" thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid, 8201558Srgrimes td->td_proc->p_pid, td->td_name); 8211558Srgrimes if (tr->rmp_flags & RMPF_ONQUEUE) { 8221558Srgrimes db_printf("ONQUEUE"); 8231558Srgrimes if (tr->rmp_flags & RMPF_SIGNAL) 8241558Srgrimes db_printf(",SIGNAL"); 8251558Srgrimes } else 8261558Srgrimes db_printf("0"); 8271558Srgrimes db_printf("}\n"); 8281558Srgrimes} 8291558Srgrimes 8301558Srgrimesstatic void 8311558Srgrimesdb_show_rm(const struct lock_object *lock) 8321558Srgrimes{ 8331558Srgrimes struct rm_priotracker *tr; 8341558Srgrimes struct rm_queue *queue; 8351558Srgrimes const struct rmlock *rm; 8361558Srgrimes struct lock_class *lc; 8371558Srgrimes struct pcpu *pc; 8381558Srgrimes 8391558Srgrimes rm = (const struct rmlock *)lock; 8401558Srgrimes db_printf(" writecpus: "); 8411558Srgrimes ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus)); 8421558Srgrimes db_printf("\n"); 8431558Srgrimes db_printf(" per-CPU readers:\n"); 8441558Srgrimes STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 8451558Srgrimes for (queue = pc->pc_rm_queue.rmq_next; 8461558Srgrimes queue != &pc->pc_rm_queue; queue = queue->rmq_next) { 8471558Srgrimes tr = (struct rm_priotracker *)queue; 8481558Srgrimes if (tr->rmp_rmlock == rm) 8491558Srgrimes print_tracker(tr); 8501558Srgrimes } 8511558Srgrimes db_printf(" active readers:\n"); 8521558Srgrimes LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry) 8531558Srgrimes print_tracker(tr); 8541558Srgrimes lc = LOCK_CLASS(&rm->rm_wlock_object); 8551558Srgrimes db_printf("Backing write-lock (%s):\n", lc->lc_name); 8561558Srgrimes lc->lc_ddb_show(&rm->rm_wlock_object); 8571558Srgrimes} 8581558Srgrimes#endif 8591558Srgrimes