1173444Sups/*- 2173444Sups * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org> 3173444Sups * All rights reserved. 4173444Sups * 5173444Sups * Redistribution and use in source and binary forms, with or without 6173444Sups * modification, are permitted provided that the following conditions 7173444Sups * are met: 8173444Sups * 1. Redistributions of source code must retain the above copyright 9173444Sups * notice, this list of conditions and the following disclaimer. 10173444Sups * 2. Redistributions in binary form must reproduce the above copyright 11173444Sups * notice, this list of conditions and the following disclaimer in the 12173444Sups * documentation and/or other materials provided with the distribution. 13173444Sups * 3. Neither the name of the author nor the names of any co-contributors 14173444Sups * may be used to endorse or promote products derived from this software 15173444Sups * without specific prior written permission. 16173444Sups * 17173444Sups * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18173444Sups * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19173444Sups * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20173444Sups * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21173444Sups * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22173444Sups * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23173444Sups * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24173444Sups * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25173444Sups * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26173444Sups * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27173444Sups * SUCH DAMAGE. 28173444Sups */ 29173444Sups 30173444Sups/* 31173444Sups * Machine independent bits of reader/writer lock implementation. 32173444Sups */ 33173444Sups 34173444Sups#include <sys/cdefs.h> 35173444Sups__FBSDID("$FreeBSD$"); 36173444Sups 37173444Sups#include "opt_ddb.h" 38192853Ssson#include "opt_kdtrace.h" 39173444Sups 40173444Sups#include <sys/param.h> 41173444Sups#include <sys/systm.h> 42173444Sups 43173444Sups#include <sys/kernel.h> 44244582Sattilio#include <sys/kdb.h> 45173444Sups#include <sys/ktr.h> 46173444Sups#include <sys/lock.h> 47173444Sups#include <sys/mutex.h> 48173444Sups#include <sys/proc.h> 49173444Sups#include <sys/rmlock.h> 50173444Sups#include <sys/sched.h> 51173444Sups#include <sys/smp.h> 52173444Sups#include <sys/turnstile.h> 53173444Sups#include <sys/lock_profile.h> 54173444Sups#include <machine/cpu.h> 55173444Sups 56173444Sups#ifdef DDB 57173444Sups#include <ddb/ddb.h> 58173444Sups#endif 59191539Srwatson 60252209Sjhb/* 61252209Sjhb * A cookie to mark destroyed rmlocks. This is stored in the head of 62252209Sjhb * rm_activeReaders. 63252209Sjhb */ 64252209Sjhb#define RM_DESTROYED ((void *)0xdead) 65252209Sjhb 66252209Sjhb#define rm_destroyed(rm) \ 67252209Sjhb (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED) 68252209Sjhb 69173444Sups#define RMPF_ONQUEUE 1 70173444Sups#define RMPF_SIGNAL 2 71173444Sups 72252209Sjhb#ifndef INVARIANTS 73252209Sjhb#define _rm_assert(c, what, file, line) 74252209Sjhb#endif 75173444Sups 76227588Spjdstatic void assert_rm(const struct lock_object *lock, int what); 77252209Sjhb#ifdef DDB 78252209Sjhbstatic void db_show_rm(const struct lock_object *lock); 79252209Sjhb#endif 80255745Sdavidestatic void lock_rm(struct lock_object *lock, uintptr_t how); 81192853Ssson#ifdef KDTRACE_HOOKS 82227588Spjdstatic int owner_rm(const struct lock_object *lock, struct thread **owner); 83192853Ssson#endif 84255745Sdavidestatic uintptr_t unlock_rm(struct lock_object *lock); 85173444Sups 86173444Supsstruct lock_class lock_class_rm = { 87173444Sups .lc_name = "rm", 88173444Sups .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 89173733Sattilio .lc_assert = assert_rm, 90173444Sups#ifdef DDB 91252209Sjhb .lc_ddb_show = db_show_rm, 92173444Sups#endif 93252209Sjhb .lc_lock = lock_rm, 94252209Sjhb .lc_unlock = unlock_rm, 95252209Sjhb#ifdef KDTRACE_HOOKS 96252209Sjhb .lc_owner = owner_rm, 97173444Sups#endif 98252209Sjhb}; 99252209Sjhb 100252209Sjhbstruct lock_class lock_class_rm_sleepable = { 101252209Sjhb .lc_name = "sleepable rm", 102252209Sjhb .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE, 103252209Sjhb .lc_assert = assert_rm, 104252209Sjhb#ifdef DDB 105252209Sjhb .lc_ddb_show = db_show_rm, 106252209Sjhb#endif 107173444Sups .lc_lock = lock_rm, 108173444Sups .lc_unlock = unlock_rm, 109192853Ssson#ifdef KDTRACE_HOOKS 110192853Ssson .lc_owner = owner_rm, 111192853Ssson#endif 112173444Sups}; 113173444Sups 114173444Supsstatic void 115227588Spjdassert_rm(const struct lock_object *lock, int what) 116173733Sattilio{ 117173733Sattilio 118252209Sjhb rm_assert((const struct rmlock *)lock, what); 119173733Sattilio} 120173733Sattilio 121173733Sattiliostatic void 122255745Sdavidelock_rm(struct lock_object *lock, uintptr_t how) 123191539Srwatson{ 124252209Sjhb struct rmlock *rm; 125255745Sdavide struct rm_priotracker *tracker; 126191539Srwatson 127252209Sjhb rm = (struct rmlock *)lock; 128255745Sdavide if (how == 0) 129252209Sjhb rm_wlock(rm); 130255745Sdavide else { 131255745Sdavide tracker = (struct rm_priotracker *)how; 132255745Sdavide rm_rlock(rm, tracker); 133255745Sdavide } 134173444Sups} 135173444Sups 136255745Sdavidestatic uintptr_t 137191539Srwatsonunlock_rm(struct lock_object *lock) 138191539Srwatson{ 139255745Sdavide struct thread *td; 140255745Sdavide struct pcpu *pc; 141252209Sjhb struct rmlock *rm; 142255745Sdavide struct rm_queue *queue; 143255745Sdavide struct rm_priotracker *tracker; 144255745Sdavide uintptr_t how; 145191539Srwatson 146252209Sjhb rm = (struct rmlock *)lock; 147255745Sdavide tracker = NULL; 148255745Sdavide how = 0; 149255745Sdavide rm_assert(rm, RA_LOCKED | RA_NOTRECURSED); 150255745Sdavide if (rm_wowned(rm)) 151255745Sdavide rm_wunlock(rm); 152255745Sdavide else { 153255745Sdavide /* 154255745Sdavide * Find the right rm_priotracker structure for curthread. 155255745Sdavide * The guarantee about its uniqueness is given by the fact 156255745Sdavide * we already asserted the lock wasn't recursively acquired. 157255745Sdavide */ 158255745Sdavide critical_enter(); 159255745Sdavide td = curthread; 160255745Sdavide pc = pcpu_find(curcpu); 161255745Sdavide for (queue = pc->pc_rm_queue.rmq_next; 162255745Sdavide queue != &pc->pc_rm_queue; queue = queue->rmq_next) { 163255745Sdavide tracker = (struct rm_priotracker *)queue; 164255745Sdavide if ((tracker->rmp_rmlock == rm) && 165255745Sdavide (tracker->rmp_thread == td)) { 166255745Sdavide how = (uintptr_t)tracker; 167255745Sdavide break; 168255745Sdavide } 169255745Sdavide } 170255745Sdavide KASSERT(tracker != NULL, 171255745Sdavide ("rm_priotracker is non-NULL when lock held in read mode")); 172255745Sdavide critical_exit(); 173255745Sdavide rm_runlock(rm, tracker); 174255745Sdavide } 175255745Sdavide return (how); 176173444Sups} 177173444Sups 178192853Ssson#ifdef KDTRACE_HOOKS 179192853Sssonstatic int 180227588Spjdowner_rm(const struct lock_object *lock, struct thread **owner) 181192853Ssson{ 182252209Sjhb const struct rmlock *rm; 183252209Sjhb struct lock_class *lc; 184192853Ssson 185252209Sjhb rm = (const struct rmlock *)lock; 186252209Sjhb lc = LOCK_CLASS(&rm->rm_wlock_object); 187252209Sjhb return (lc->lc_owner(&rm->rm_wlock_object, owner)); 188192853Ssson} 189192853Ssson#endif 190192853Ssson 191173444Supsstatic struct mtx rm_spinlock; 192173444Sups 193173444SupsMTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN); 194173444Sups 195173444Sups/* 196200976Srwatson * Add or remove tracker from per-cpu list. 197191539Srwatson * 198200976Srwatson * The per-cpu list can be traversed at any time in forward direction from an 199191539Srwatson * interrupt on the *local* cpu. 200173444Sups */ 201191539Srwatsonstatic void inline 202191539Srwatsonrm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker) 203191539Srwatson{ 204191539Srwatson struct rm_queue *next; 205191539Srwatson 206173444Sups /* Initialize all tracker pointers */ 207173444Sups tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue; 208173444Sups next = pc->pc_rm_queue.rmq_next; 209173444Sups tracker->rmp_cpuQueue.rmq_next = next; 210191539Srwatson 211191539Srwatson /* rmq_prev is not used during froward traversal. */ 212173444Sups next->rmq_prev = &tracker->rmp_cpuQueue; 213191539Srwatson 214191539Srwatson /* Update pointer to first element. */ 215201000Sbz pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue; 216173444Sups} 217173444Sups 218252209Sjhb/* 219252209Sjhb * Return a count of the number of trackers the thread 'td' already 220252209Sjhb * has on this CPU for the lock 'rm'. 221252209Sjhb */ 222252209Sjhbstatic int 223252209Sjhbrm_trackers_present(const struct pcpu *pc, const struct rmlock *rm, 224252209Sjhb const struct thread *td) 225252209Sjhb{ 226252209Sjhb struct rm_queue *queue; 227252209Sjhb struct rm_priotracker *tracker; 228252209Sjhb int count; 229252209Sjhb 230252209Sjhb count = 0; 231252209Sjhb for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 232252209Sjhb queue = queue->rmq_next) { 233252209Sjhb tracker = (struct rm_priotracker *)queue; 234252209Sjhb if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td)) 235252209Sjhb count++; 236252209Sjhb } 237252209Sjhb return (count); 238252209Sjhb} 239252209Sjhb 240191539Srwatsonstatic void inline 241191539Srwatsonrm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker) 242191539Srwatson{ 243191539Srwatson struct rm_queue *next, *prev; 244173444Sups 245191539Srwatson next = tracker->rmp_cpuQueue.rmq_next; 246191539Srwatson prev = tracker->rmp_cpuQueue.rmq_prev; 247191539Srwatson 248191539Srwatson /* Not used during forward traversal. */ 249173444Sups next->rmq_prev = prev; 250191539Srwatson 251191539Srwatson /* Remove from list. */ 252173444Sups prev->rmq_next = next; 253173444Sups} 254173444Sups 255191539Srwatsonstatic void 256191539Srwatsonrm_cleanIPI(void *arg) 257191539Srwatson{ 258173444Sups struct pcpu *pc; 259191539Srwatson struct rmlock *rm = arg; 260173444Sups struct rm_priotracker *tracker; 261191539Srwatson struct rm_queue *queue; 262173444Sups pc = pcpu_find(curcpu); 263173444Sups 264191539Srwatson for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 265173444Sups queue = queue->rmq_next) { 266191539Srwatson tracker = (struct rm_priotracker *)queue; 267191539Srwatson if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) { 268173444Sups tracker->rmp_flags = RMPF_ONQUEUE; 269173444Sups mtx_lock_spin(&rm_spinlock); 270191539Srwatson LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 271173444Sups rmp_qentry); 272173444Sups mtx_unlock_spin(&rm_spinlock); 273173444Sups } 274173444Sups } 275173444Sups} 276173444Sups 277173444Supsvoid 278193030Srwatsonrm_init_flags(struct rmlock *rm, const char *name, int opts) 279173444Sups{ 280252209Sjhb struct lock_class *lc; 281193030Srwatson int liflags; 282191539Srwatson 283193030Srwatson liflags = 0; 284193030Srwatson if (!(opts & RM_NOWITNESS)) 285193030Srwatson liflags |= LO_WITNESS; 286193030Srwatson if (opts & RM_RECURSE) 287193030Srwatson liflags |= LO_RECURSABLE; 288212112Smlaier rm->rm_writecpus = all_cpus; 289173444Sups LIST_INIT(&rm->rm_activeReaders); 290212112Smlaier if (opts & RM_SLEEPABLE) { 291252209Sjhb liflags |= LO_SLEEPABLE; 292252209Sjhb lc = &lock_class_rm_sleepable; 293252209Sjhb sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS); 294252209Sjhb } else { 295252209Sjhb lc = &lock_class_rm; 296212112Smlaier mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS); 297252209Sjhb } 298252209Sjhb lock_init(&rm->lock_object, lc, name, NULL, liflags); 299173444Sups} 300173444Sups 301173444Supsvoid 302193030Srwatsonrm_init(struct rmlock *rm, const char *name) 303193030Srwatson{ 304193030Srwatson 305193030Srwatson rm_init_flags(rm, name, 0); 306193030Srwatson} 307193030Srwatson 308193030Srwatsonvoid 309173444Supsrm_destroy(struct rmlock *rm) 310173444Sups{ 311191539Srwatson 312252209Sjhb rm_assert(rm, RA_UNLOCKED); 313252209Sjhb LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED; 314252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 315212112Smlaier sx_destroy(&rm->rm_lock_sx); 316212112Smlaier else 317212112Smlaier mtx_destroy(&rm->rm_lock_mtx); 318173444Sups lock_destroy(&rm->lock_object); 319173444Sups} 320173444Sups 321173520Srwatsonint 322227588Spjdrm_wowned(const struct rmlock *rm) 323173520Srwatson{ 324173520Srwatson 325252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 326212112Smlaier return (sx_xlocked(&rm->rm_lock_sx)); 327212112Smlaier else 328212112Smlaier return (mtx_owned(&rm->rm_lock_mtx)); 329173520Srwatson} 330173520Srwatson 331173444Supsvoid 332173444Supsrm_sysinit(void *arg) 333173444Sups{ 334193030Srwatson struct rm_args *args = arg; 335191539Srwatson 336193030Srwatson rm_init(args->ra_rm, args->ra_desc); 337173444Sups} 338173444Sups 339193030Srwatsonvoid 340193030Srwatsonrm_sysinit_flags(void *arg) 341193030Srwatson{ 342193030Srwatson struct rm_args_flags *args = arg; 343193030Srwatson 344193030Srwatson rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts); 345193030Srwatson} 346193030Srwatson 347212112Smlaierstatic int 348212112Smlaier_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 349173444Sups{ 350173444Sups struct pcpu *pc; 351173444Sups 352173444Sups critical_enter(); 353173444Sups pc = pcpu_find(curcpu); 354173444Sups 355191539Srwatson /* Check if we just need to do a proper critical_exit. */ 356223758Sattilio if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { 357173444Sups critical_exit(); 358212112Smlaier return (1); 359173444Sups } 360173444Sups 361200976Srwatson /* Remove our tracker from the per-cpu list. */ 362191539Srwatson rm_tracker_remove(pc, tracker); 363173444Sups 364191539Srwatson /* Check to see if the IPI granted us the lock after all. */ 365191539Srwatson if (tracker->rmp_flags) { 366191539Srwatson /* Just add back tracker - we hold the lock. */ 367191539Srwatson rm_tracker_add(pc, tracker); 368173444Sups critical_exit(); 369212112Smlaier return (1); 370173444Sups } 371173444Sups 372173444Sups /* 373191539Srwatson * We allow readers to aquire a lock even if a writer is blocked if 374191539Srwatson * the lock is recursive and the reader already holds the lock. 375173444Sups */ 376173444Sups if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) { 377173444Sups /* 378200976Srwatson * Just grant the lock if this thread already has a tracker 379200976Srwatson * for this lock on the per-cpu queue. 380173444Sups */ 381252209Sjhb if (rm_trackers_present(pc, rm, curthread) != 0) { 382252209Sjhb mtx_lock_spin(&rm_spinlock); 383252209Sjhb LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 384252209Sjhb rmp_qentry); 385252209Sjhb tracker->rmp_flags = RMPF_ONQUEUE; 386252209Sjhb mtx_unlock_spin(&rm_spinlock); 387252209Sjhb rm_tracker_add(pc, tracker); 388252209Sjhb critical_exit(); 389252209Sjhb return (1); 390173444Sups } 391173444Sups } 392173444Sups 393173444Sups sched_unpin(); 394173444Sups critical_exit(); 395173444Sups 396212112Smlaier if (trylock) { 397252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) { 398212112Smlaier if (!sx_try_xlock(&rm->rm_lock_sx)) 399212112Smlaier return (0); 400212112Smlaier } else { 401212112Smlaier if (!mtx_trylock(&rm->rm_lock_mtx)) 402212112Smlaier return (0); 403212112Smlaier } 404212112Smlaier } else { 405252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 406212112Smlaier sx_xlock(&rm->rm_lock_sx); 407212112Smlaier else 408212112Smlaier mtx_lock(&rm->rm_lock_mtx); 409212112Smlaier } 410212112Smlaier 411173444Sups critical_enter(); 412173444Sups pc = pcpu_find(curcpu); 413223758Sattilio CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); 414191539Srwatson rm_tracker_add(pc, tracker); 415173444Sups sched_pin(); 416173444Sups critical_exit(); 417191539Srwatson 418252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 419212112Smlaier sx_xunlock(&rm->rm_lock_sx); 420212112Smlaier else 421212112Smlaier mtx_unlock(&rm->rm_lock_mtx); 422212112Smlaier 423212112Smlaier return (1); 424173444Sups} 425173444Sups 426212112Smlaierint 427212112Smlaier_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 428173444Sups{ 429173444Sups struct thread *td = curthread; 430173444Sups struct pcpu *pc; 431173444Sups 432228424Savg if (SCHEDULER_STOPPED()) 433228424Savg return (1); 434228424Savg 435173444Sups tracker->rmp_flags = 0; 436173444Sups tracker->rmp_thread = td; 437173444Sups tracker->rmp_rmlock = rm; 438173444Sups 439252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 440252209Sjhb THREAD_NO_SLEEPING(); 441252209Sjhb 442173444Sups td->td_critnest++; /* critical_enter(); */ 443173444Sups 444241374Sattilio __compiler_membar(); 445173444Sups 446173444Sups pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 447173444Sups 448191539Srwatson rm_tracker_add(pc, tracker); 449173444Sups 450193038Srwatson sched_pin(); 451173444Sups 452241374Sattilio __compiler_membar(); 453173444Sups 454173444Sups td->td_critnest--; 455191539Srwatson 456191539Srwatson /* 457191539Srwatson * Fast path to combine two common conditions into a single 458191539Srwatson * conditional jump. 459173444Sups */ 460222813Sattilio if (0 == (td->td_owepreempt | 461223758Sattilio CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus))) 462212112Smlaier return (1); 463173444Sups 464191539Srwatson /* We do not have a read token and need to acquire one. */ 465212112Smlaier return _rm_rlock_hard(rm, tracker, trylock); 466173444Sups} 467173444Sups 468173444Supsstatic void 469191539Srwatson_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker) 470173444Sups{ 471191539Srwatson 472173444Sups if (td->td_owepreempt) { 473173444Sups td->td_critnest++; 474173444Sups critical_exit(); 475173444Sups } 476191539Srwatson 477191539Srwatson if (!tracker->rmp_flags) 478173444Sups return; 479173444Sups 480173444Sups mtx_lock_spin(&rm_spinlock); 481191539Srwatson LIST_REMOVE(tracker, rmp_qentry); 482173444Sups 483173444Sups if (tracker->rmp_flags & RMPF_SIGNAL) { 484173444Sups struct rmlock *rm; 485191539Srwatson struct turnstile *ts; 486173444Sups 487173444Sups rm = tracker->rmp_rmlock; 488191539Srwatson 489173444Sups turnstile_chain_lock(&rm->lock_object); 490173444Sups mtx_unlock_spin(&rm_spinlock); 491173444Sups 492173444Sups ts = turnstile_lookup(&rm->lock_object); 493173444Sups 494173444Sups turnstile_signal(ts, TS_EXCLUSIVE_QUEUE); 495173444Sups turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 496173444Sups turnstile_chain_unlock(&rm->lock_object); 497173444Sups } else 498173444Sups mtx_unlock_spin(&rm_spinlock); 499191539Srwatson} 500173444Sups 501173444Supsvoid 502191539Srwatson_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker) 503173444Sups{ 504173444Sups struct pcpu *pc; 505173444Sups struct thread *td = tracker->rmp_thread; 506173444Sups 507228424Savg if (SCHEDULER_STOPPED()) 508228424Savg return; 509228424Savg 510173444Sups td->td_critnest++; /* critical_enter(); */ 511173444Sups pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 512191539Srwatson rm_tracker_remove(pc, tracker); 513173444Sups td->td_critnest--; 514193038Srwatson sched_unpin(); 515173444Sups 516252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 517252209Sjhb THREAD_SLEEPING_OK(); 518252209Sjhb 519191539Srwatson if (0 == (td->td_owepreempt | tracker->rmp_flags)) 520173444Sups return; 521173444Sups 522191539Srwatson _rm_unlock_hard(td, tracker); 523173444Sups} 524173444Sups 525173444Supsvoid 526173444Sups_rm_wlock(struct rmlock *rm) 527173444Sups{ 528173444Sups struct rm_priotracker *prio; 529173444Sups struct turnstile *ts; 530222813Sattilio cpuset_t readcpus; 531173444Sups 532228424Savg if (SCHEDULER_STOPPED()) 533228424Savg return; 534228424Savg 535252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 536212112Smlaier sx_xlock(&rm->rm_lock_sx); 537212112Smlaier else 538212112Smlaier mtx_lock(&rm->rm_lock_mtx); 539173444Sups 540222813Sattilio if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) { 541173444Sups /* Get all read tokens back */ 542222813Sattilio readcpus = all_cpus; 543222813Sattilio CPU_NAND(&readcpus, &rm->rm_writecpus); 544212112Smlaier rm->rm_writecpus = all_cpus; 545173444Sups 546191539Srwatson /* 547212112Smlaier * Assumes rm->rm_writecpus update is visible on other CPUs 548191539Srwatson * before rm_cleanIPI is called. 549173444Sups */ 550173444Sups#ifdef SMP 551212112Smlaier smp_rendezvous_cpus(readcpus, 552212112Smlaier smp_no_rendevous_barrier, 553173444Sups rm_cleanIPI, 554191539Srwatson smp_no_rendevous_barrier, 555191539Srwatson rm); 556173444Sups 557173444Sups#else 558173444Sups rm_cleanIPI(rm); 559173444Sups#endif 560173444Sups 561173444Sups mtx_lock_spin(&rm_spinlock); 562191539Srwatson while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { 563173444Sups ts = turnstile_trywait(&rm->lock_object); 564173444Sups prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL; 565173444Sups mtx_unlock_spin(&rm_spinlock); 566191539Srwatson turnstile_wait(ts, prio->rmp_thread, 567191539Srwatson TS_EXCLUSIVE_QUEUE); 568173444Sups mtx_lock_spin(&rm_spinlock); 569173444Sups } 570173444Sups mtx_unlock_spin(&rm_spinlock); 571173444Sups } 572173444Sups} 573173444Sups 574173444Supsvoid 575173444Sups_rm_wunlock(struct rmlock *rm) 576173444Sups{ 577191539Srwatson 578252209Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 579212112Smlaier sx_xunlock(&rm->rm_lock_sx); 580212112Smlaier else 581212112Smlaier mtx_unlock(&rm->rm_lock_mtx); 582173444Sups} 583173444Sups 584173444Sups#ifdef LOCK_DEBUG 585173444Sups 586252209Sjhbvoid 587252209Sjhb_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 588173444Sups{ 589173444Sups 590228424Savg if (SCHEDULER_STOPPED()) 591228424Savg return; 592228424Savg 593244582Sattilio KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 594240424Sattilio ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d", 595240424Sattilio curthread, rm->lock_object.lo_name, file, line)); 596252209Sjhb KASSERT(!rm_destroyed(rm), 597252209Sjhb ("rm_wlock() of destroyed rmlock @ %s:%d", file, line)); 598252209Sjhb _rm_assert(rm, RA_UNLOCKED, file, line); 599252209Sjhb 600191539Srwatson WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, 601182914Sjhb file, line, NULL); 602173444Sups 603173444Sups _rm_wlock(rm); 604173444Sups 605173444Sups LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line); 606173444Sups 607252209Sjhb WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 608173444Sups 609173444Sups curthread->td_locks++; 610173444Sups 611173444Sups} 612173444Sups 613191539Srwatsonvoid 614191539Srwatson_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 615173444Sups{ 616191539Srwatson 617228424Savg if (SCHEDULER_STOPPED()) 618228424Savg return; 619228424Savg 620252209Sjhb KASSERT(!rm_destroyed(rm), 621252209Sjhb ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line)); 622252209Sjhb _rm_assert(rm, RA_WLOCKED, file, line); 623252209Sjhb WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 624173444Sups LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); 625173444Sups _rm_wunlock(rm); 626252209Sjhb curthread->td_locks--; 627191539Srwatson} 628173444Sups 629212112Smlaierint 630173444Sups_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 631212112Smlaier int trylock, const char *file, int line) 632173444Sups{ 633228424Savg 634228424Savg if (SCHEDULER_STOPPED()) 635228424Savg return (1); 636228424Savg 637252209Sjhb#ifdef INVARIANTS 638252209Sjhb if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) { 639252209Sjhb critical_enter(); 640252209Sjhb KASSERT(rm_trackers_present(pcpu_find(curcpu), rm, 641252209Sjhb curthread) == 0, 642252209Sjhb ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n", 643252209Sjhb rm->lock_object.lo_name, file, line)); 644252209Sjhb critical_exit(); 645252209Sjhb } 646252209Sjhb#endif 647244582Sattilio KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 648240424Sattilio ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d", 649240424Sattilio curthread, rm->lock_object.lo_name, file, line)); 650252209Sjhb KASSERT(!rm_destroyed(rm), 651252209Sjhb ("rm_rlock() of destroyed rmlock @ %s:%d", file, line)); 652252209Sjhb if (!trylock) { 653252209Sjhb KASSERT(!rm_wowned(rm), 654252209Sjhb ("rm_rlock: wlock already held for %s @ %s:%d", 655252209Sjhb rm->lock_object.lo_name, file, line)); 656252209Sjhb WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, 657252209Sjhb NULL); 658252209Sjhb } 659173444Sups 660212112Smlaier if (_rm_rlock(rm, tracker, trylock)) { 661252209Sjhb if (trylock) 662252209Sjhb LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file, 663252209Sjhb line); 664252209Sjhb else 665252209Sjhb LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, 666252209Sjhb line); 667212112Smlaier WITNESS_LOCK(&rm->lock_object, 0, file, line); 668173444Sups 669212112Smlaier curthread->td_locks++; 670173444Sups 671212112Smlaier return (1); 672252209Sjhb } else if (trylock) 673252209Sjhb LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line); 674212112Smlaier 675212112Smlaier return (0); 676173444Sups} 677173444Sups 678191539Srwatsonvoid 679201000Sbz_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 680191539Srwatson const char *file, int line) 681191539Srwatson{ 682191539Srwatson 683228424Savg if (SCHEDULER_STOPPED()) 684228424Savg return; 685228424Savg 686252209Sjhb KASSERT(!rm_destroyed(rm), 687252209Sjhb ("rm_runlock() of destroyed rmlock @ %s:%d", file, line)); 688252209Sjhb _rm_assert(rm, RA_RLOCKED, file, line); 689191539Srwatson WITNESS_UNLOCK(&rm->lock_object, 0, file, line); 690173444Sups LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); 691173444Sups _rm_runlock(rm, tracker); 692252209Sjhb curthread->td_locks--; 693173444Sups} 694173444Sups 695173444Sups#else 696173444Sups 697191539Srwatson/* 698191539Srwatson * Just strip out file and line arguments if no lock debugging is enabled in 699191539Srwatson * the kernel - we are called from a kernel module. 700191539Srwatson */ 701191539Srwatsonvoid 702191539Srwatson_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 703191539Srwatson{ 704173444Sups 705173444Sups _rm_wlock(rm); 706173444Sups} 707173444Sups 708191539Srwatsonvoid 709191539Srwatson_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 710173444Sups{ 711191539Srwatson 712173444Sups _rm_wunlock(rm); 713191539Srwatson} 714191539Srwatson 715212112Smlaierint 716173444Sups_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 717212112Smlaier int trylock, const char *file, int line) 718173444Sups{ 719191539Srwatson 720212112Smlaier return _rm_rlock(rm, tracker, trylock); 721173444Sups} 722173444Sups 723191539Srwatsonvoid 724201000Sbz_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 725193039Srwatson const char *file, int line) 726193039Srwatson{ 727191539Srwatson 728173444Sups _rm_runlock(rm, tracker); 729173444Sups} 730173444Sups 731173444Sups#endif 732252209Sjhb 733252209Sjhb#ifdef INVARIANT_SUPPORT 734253047Sjhb#ifndef INVARIANTS 735253047Sjhb#undef _rm_assert 736253047Sjhb#endif 737253047Sjhb 738252209Sjhb/* 739252209Sjhb * Note that this does not need to use witness_assert() for read lock 740252209Sjhb * assertions since an exact count of read locks held by this thread 741252209Sjhb * is computable. 742252209Sjhb */ 743252209Sjhbvoid 744252209Sjhb_rm_assert(const struct rmlock *rm, int what, const char *file, int line) 745252209Sjhb{ 746252209Sjhb int count; 747252209Sjhb 748252209Sjhb if (panicstr != NULL) 749252209Sjhb return; 750252209Sjhb switch (what) { 751252209Sjhb case RA_LOCKED: 752252209Sjhb case RA_LOCKED | RA_RECURSED: 753252209Sjhb case RA_LOCKED | RA_NOTRECURSED: 754252209Sjhb case RA_RLOCKED: 755252209Sjhb case RA_RLOCKED | RA_RECURSED: 756252209Sjhb case RA_RLOCKED | RA_NOTRECURSED: 757252209Sjhb /* 758252209Sjhb * Handle the write-locked case. Unlike other 759252209Sjhb * primitives, writers can never recurse. 760252209Sjhb */ 761252209Sjhb if (rm_wowned(rm)) { 762252209Sjhb if (what & RA_RLOCKED) 763252209Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 764252209Sjhb rm->lock_object.lo_name, file, line); 765252209Sjhb if (what & RA_RECURSED) 766252209Sjhb panic("Lock %s not recursed @ %s:%d\n", 767252209Sjhb rm->lock_object.lo_name, file, line); 768252209Sjhb break; 769252209Sjhb } 770252209Sjhb 771252209Sjhb critical_enter(); 772252209Sjhb count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 773252209Sjhb critical_exit(); 774252209Sjhb 775252209Sjhb if (count == 0) 776252209Sjhb panic("Lock %s not %slocked @ %s:%d\n", 777252209Sjhb rm->lock_object.lo_name, (what & RA_RLOCKED) ? 778252209Sjhb "read " : "", file, line); 779252209Sjhb if (count > 1) { 780252209Sjhb if (what & RA_NOTRECURSED) 781252209Sjhb panic("Lock %s recursed @ %s:%d\n", 782252209Sjhb rm->lock_object.lo_name, file, line); 783252209Sjhb } else if (what & RA_RECURSED) 784252209Sjhb panic("Lock %s not recursed @ %s:%d\n", 785252209Sjhb rm->lock_object.lo_name, file, line); 786252209Sjhb break; 787252209Sjhb case RA_WLOCKED: 788252209Sjhb if (!rm_wowned(rm)) 789252209Sjhb panic("Lock %s not exclusively locked @ %s:%d\n", 790252209Sjhb rm->lock_object.lo_name, file, line); 791252209Sjhb break; 792252209Sjhb case RA_UNLOCKED: 793252209Sjhb if (rm_wowned(rm)) 794252209Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 795252209Sjhb rm->lock_object.lo_name, file, line); 796252209Sjhb 797252209Sjhb critical_enter(); 798252209Sjhb count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 799252209Sjhb critical_exit(); 800252209Sjhb 801252209Sjhb if (count != 0) 802252209Sjhb panic("Lock %s read locked @ %s:%d\n", 803252209Sjhb rm->lock_object.lo_name, file, line); 804252209Sjhb break; 805252209Sjhb default: 806252209Sjhb panic("Unknown rm lock assertion: %d @ %s:%d", what, file, 807252209Sjhb line); 808252209Sjhb } 809252209Sjhb} 810252209Sjhb#endif /* INVARIANT_SUPPORT */ 811252209Sjhb 812252209Sjhb#ifdef DDB 813252209Sjhbstatic void 814252209Sjhbprint_tracker(struct rm_priotracker *tr) 815252209Sjhb{ 816252209Sjhb struct thread *td; 817252209Sjhb 818252209Sjhb td = tr->rmp_thread; 819252209Sjhb db_printf(" thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid, 820252209Sjhb td->td_proc->p_pid, td->td_name); 821252209Sjhb if (tr->rmp_flags & RMPF_ONQUEUE) { 822252209Sjhb db_printf("ONQUEUE"); 823252209Sjhb if (tr->rmp_flags & RMPF_SIGNAL) 824252209Sjhb db_printf(",SIGNAL"); 825252209Sjhb } else 826252209Sjhb db_printf("0"); 827252209Sjhb db_printf("}\n"); 828252209Sjhb} 829252209Sjhb 830252209Sjhbstatic void 831252209Sjhbdb_show_rm(const struct lock_object *lock) 832252209Sjhb{ 833252209Sjhb struct rm_priotracker *tr; 834252209Sjhb struct rm_queue *queue; 835252209Sjhb const struct rmlock *rm; 836252209Sjhb struct lock_class *lc; 837252209Sjhb struct pcpu *pc; 838252209Sjhb 839252209Sjhb rm = (const struct rmlock *)lock; 840252209Sjhb db_printf(" writecpus: "); 841252209Sjhb ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus)); 842252209Sjhb db_printf("\n"); 843252209Sjhb db_printf(" per-CPU readers:\n"); 844252209Sjhb STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 845252209Sjhb for (queue = pc->pc_rm_queue.rmq_next; 846252209Sjhb queue != &pc->pc_rm_queue; queue = queue->rmq_next) { 847252209Sjhb tr = (struct rm_priotracker *)queue; 848252209Sjhb if (tr->rmp_rmlock == rm) 849252209Sjhb print_tracker(tr); 850252209Sjhb } 851252209Sjhb db_printf(" active readers:\n"); 852252209Sjhb LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry) 853252209Sjhb print_tracker(tr); 854252209Sjhb lc = LOCK_CLASS(&rm->rm_wlock_object); 855252209Sjhb db_printf("Backing write-lock (%s):\n", lc->lc_name); 856252209Sjhb lc->lc_ddb_show(&rm->rm_wlock_object); 857252209Sjhb} 858252209Sjhb#endif 859