1/*-
2 * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 3. Neither the name of the author nor the names of any co-contributors
14 *    may be used to endorse or promote products derived from this software
15 *    without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30/*
31 * Machine independent bits of reader/writer lock implementation.
32 */
33
34#include <sys/cdefs.h>
35__FBSDID("$FreeBSD$");
36
37#include "opt_ddb.h"
38#include "opt_kdtrace.h"
39
40#include <sys/param.h>
41#include <sys/systm.h>
42
43#include <sys/kernel.h>
44#include <sys/kdb.h>
45#include <sys/ktr.h>
46#include <sys/lock.h>
47#include <sys/mutex.h>
48#include <sys/proc.h>
49#include <sys/rmlock.h>
50#include <sys/sched.h>
51#include <sys/smp.h>
52#include <sys/turnstile.h>
53#include <sys/lock_profile.h>
54#include <machine/cpu.h>
55
56#ifdef DDB
57#include <ddb/ddb.h>
58#endif
59
60/*
61 * A cookie to mark destroyed rmlocks.  This is stored in the head of
62 * rm_activeReaders.
63 */
64#define	RM_DESTROYED	((void *)0xdead)
65
66#define	rm_destroyed(rm)						\
67	(LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
68
69#define RMPF_ONQUEUE	1
70#define RMPF_SIGNAL	2
71
72#ifndef INVARIANTS
73#define	_rm_assert(c, what, file, line)
74#endif
75
76static void	assert_rm(const struct lock_object *lock, int what);
77#ifdef DDB
78static void	db_show_rm(const struct lock_object *lock);
79#endif
80static void	lock_rm(struct lock_object *lock, uintptr_t how);
81#ifdef KDTRACE_HOOKS
82static int	owner_rm(const struct lock_object *lock, struct thread **owner);
83#endif
84static uintptr_t unlock_rm(struct lock_object *lock);
85
86struct lock_class lock_class_rm = {
87	.lc_name = "rm",
88	.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
89	.lc_assert = assert_rm,
90#ifdef DDB
91	.lc_ddb_show = db_show_rm,
92#endif
93	.lc_lock = lock_rm,
94	.lc_unlock = unlock_rm,
95#ifdef KDTRACE_HOOKS
96	.lc_owner = owner_rm,
97#endif
98};
99
100struct lock_class lock_class_rm_sleepable = {
101	.lc_name = "sleepable rm",
102	.lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE,
103	.lc_assert = assert_rm,
104#ifdef DDB
105	.lc_ddb_show = db_show_rm,
106#endif
107	.lc_lock = lock_rm,
108	.lc_unlock = unlock_rm,
109#ifdef KDTRACE_HOOKS
110	.lc_owner = owner_rm,
111#endif
112};
113
114static void
115assert_rm(const struct lock_object *lock, int what)
116{
117
118	rm_assert((const struct rmlock *)lock, what);
119}
120
121static void
122lock_rm(struct lock_object *lock, uintptr_t how)
123{
124	struct rmlock *rm;
125	struct rm_priotracker *tracker;
126
127	rm = (struct rmlock *)lock;
128	if (how == 0)
129		rm_wlock(rm);
130	else {
131		tracker = (struct rm_priotracker *)how;
132		rm_rlock(rm, tracker);
133	}
134}
135
136static uintptr_t
137unlock_rm(struct lock_object *lock)
138{
139	struct thread *td;
140	struct pcpu *pc;
141	struct rmlock *rm;
142	struct rm_queue *queue;
143	struct rm_priotracker *tracker;
144	uintptr_t how;
145
146	rm = (struct rmlock *)lock;
147	tracker = NULL;
148	how = 0;
149	rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
150	if (rm_wowned(rm))
151		rm_wunlock(rm);
152	else {
153		/*
154		 * Find the right rm_priotracker structure for curthread.
155		 * The guarantee about its uniqueness is given by the fact
156		 * we already asserted the lock wasn't recursively acquired.
157		 */
158		critical_enter();
159		td = curthread;
160		pc = pcpu_find(curcpu);
161		for (queue = pc->pc_rm_queue.rmq_next;
162		    queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
163			tracker = (struct rm_priotracker *)queue;
164				if ((tracker->rmp_rmlock == rm) &&
165				    (tracker->rmp_thread == td)) {
166					how = (uintptr_t)tracker;
167					break;
168				}
169		}
170		KASSERT(tracker != NULL,
171		    ("rm_priotracker is non-NULL when lock held in read mode"));
172		critical_exit();
173		rm_runlock(rm, tracker);
174	}
175	return (how);
176}
177
178#ifdef KDTRACE_HOOKS
179static int
180owner_rm(const struct lock_object *lock, struct thread **owner)
181{
182	const struct rmlock *rm;
183	struct lock_class *lc;
184
185	rm = (const struct rmlock *)lock;
186	lc = LOCK_CLASS(&rm->rm_wlock_object);
187	return (lc->lc_owner(&rm->rm_wlock_object, owner));
188}
189#endif
190
191static struct mtx rm_spinlock;
192
193MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN);
194
195/*
196 * Add or remove tracker from per-cpu list.
197 *
198 * The per-cpu list can be traversed at any time in forward direction from an
199 * interrupt on the *local* cpu.
200 */
201static void inline
202rm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker)
203{
204	struct rm_queue *next;
205
206	/* Initialize all tracker pointers */
207	tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue;
208	next = pc->pc_rm_queue.rmq_next;
209	tracker->rmp_cpuQueue.rmq_next = next;
210
211	/* rmq_prev is not used during froward traversal. */
212	next->rmq_prev = &tracker->rmp_cpuQueue;
213
214	/* Update pointer to first element. */
215	pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue;
216}
217
218/*
219 * Return a count of the number of trackers the thread 'td' already
220 * has on this CPU for the lock 'rm'.
221 */
222static int
223rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
224    const struct thread *td)
225{
226	struct rm_queue *queue;
227	struct rm_priotracker *tracker;
228	int count;
229
230	count = 0;
231	for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
232	    queue = queue->rmq_next) {
233		tracker = (struct rm_priotracker *)queue;
234		if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
235			count++;
236	}
237	return (count);
238}
239
240static void inline
241rm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker)
242{
243	struct rm_queue *next, *prev;
244
245	next = tracker->rmp_cpuQueue.rmq_next;
246	prev = tracker->rmp_cpuQueue.rmq_prev;
247
248	/* Not used during forward traversal. */
249	next->rmq_prev = prev;
250
251	/* Remove from list. */
252	prev->rmq_next = next;
253}
254
255static void
256rm_cleanIPI(void *arg)
257{
258	struct pcpu *pc;
259	struct rmlock *rm = arg;
260	struct rm_priotracker *tracker;
261	struct rm_queue *queue;
262	pc = pcpu_find(curcpu);
263
264	for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
265	    queue = queue->rmq_next) {
266		tracker = (struct rm_priotracker *)queue;
267		if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
268			tracker->rmp_flags = RMPF_ONQUEUE;
269			mtx_lock_spin(&rm_spinlock);
270			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
271			    rmp_qentry);
272			mtx_unlock_spin(&rm_spinlock);
273		}
274	}
275}
276
277void
278rm_init_flags(struct rmlock *rm, const char *name, int opts)
279{
280	struct lock_class *lc;
281	int liflags;
282
283	liflags = 0;
284	if (!(opts & RM_NOWITNESS))
285		liflags |= LO_WITNESS;
286	if (opts & RM_RECURSE)
287		liflags |= LO_RECURSABLE;
288	rm->rm_writecpus = all_cpus;
289	LIST_INIT(&rm->rm_activeReaders);
290	if (opts & RM_SLEEPABLE) {
291		liflags |= LO_SLEEPABLE;
292		lc = &lock_class_rm_sleepable;
293		sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS);
294	} else {
295		lc = &lock_class_rm;
296		mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS);
297	}
298	lock_init(&rm->lock_object, lc, name, NULL, liflags);
299}
300
301void
302rm_init(struct rmlock *rm, const char *name)
303{
304
305	rm_init_flags(rm, name, 0);
306}
307
308void
309rm_destroy(struct rmlock *rm)
310{
311
312	rm_assert(rm, RA_UNLOCKED);
313	LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
314	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
315		sx_destroy(&rm->rm_lock_sx);
316	else
317		mtx_destroy(&rm->rm_lock_mtx);
318	lock_destroy(&rm->lock_object);
319}
320
321int
322rm_wowned(const struct rmlock *rm)
323{
324
325	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
326		return (sx_xlocked(&rm->rm_lock_sx));
327	else
328		return (mtx_owned(&rm->rm_lock_mtx));
329}
330
331void
332rm_sysinit(void *arg)
333{
334	struct rm_args *args = arg;
335
336	rm_init(args->ra_rm, args->ra_desc);
337}
338
339void
340rm_sysinit_flags(void *arg)
341{
342	struct rm_args_flags *args = arg;
343
344	rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts);
345}
346
347static int
348_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
349{
350	struct pcpu *pc;
351
352	critical_enter();
353	pc = pcpu_find(curcpu);
354
355	/* Check if we just need to do a proper critical_exit. */
356	if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
357		critical_exit();
358		return (1);
359	}
360
361	/* Remove our tracker from the per-cpu list. */
362	rm_tracker_remove(pc, tracker);
363
364	/* Check to see if the IPI granted us the lock after all. */
365	if (tracker->rmp_flags) {
366		/* Just add back tracker - we hold the lock. */
367		rm_tracker_add(pc, tracker);
368		critical_exit();
369		return (1);
370	}
371
372	/*
373	 * We allow readers to aquire a lock even if a writer is blocked if
374	 * the lock is recursive and the reader already holds the lock.
375	 */
376	if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
377		/*
378		 * Just grant the lock if this thread already has a tracker
379		 * for this lock on the per-cpu queue.
380		 */
381		if (rm_trackers_present(pc, rm, curthread) != 0) {
382			mtx_lock_spin(&rm_spinlock);
383			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
384			    rmp_qentry);
385			tracker->rmp_flags = RMPF_ONQUEUE;
386			mtx_unlock_spin(&rm_spinlock);
387			rm_tracker_add(pc, tracker);
388			critical_exit();
389			return (1);
390		}
391	}
392
393	sched_unpin();
394	critical_exit();
395
396	if (trylock) {
397		if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
398			if (!sx_try_xlock(&rm->rm_lock_sx))
399				return (0);
400		} else {
401			if (!mtx_trylock(&rm->rm_lock_mtx))
402				return (0);
403		}
404	} else {
405		if (rm->lock_object.lo_flags & LO_SLEEPABLE)
406			sx_xlock(&rm->rm_lock_sx);
407		else
408			mtx_lock(&rm->rm_lock_mtx);
409	}
410
411	critical_enter();
412	pc = pcpu_find(curcpu);
413	CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
414	rm_tracker_add(pc, tracker);
415	sched_pin();
416	critical_exit();
417
418	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
419		sx_xunlock(&rm->rm_lock_sx);
420	else
421		mtx_unlock(&rm->rm_lock_mtx);
422
423	return (1);
424}
425
426int
427_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
428{
429	struct thread *td = curthread;
430	struct pcpu *pc;
431
432	if (SCHEDULER_STOPPED())
433		return (1);
434
435	tracker->rmp_flags  = 0;
436	tracker->rmp_thread = td;
437	tracker->rmp_rmlock = rm;
438
439	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
440		THREAD_NO_SLEEPING();
441
442	td->td_critnest++;	/* critical_enter(); */
443
444	__compiler_membar();
445
446	pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
447
448	rm_tracker_add(pc, tracker);
449
450	sched_pin();
451
452	__compiler_membar();
453
454	td->td_critnest--;
455
456	/*
457	 * Fast path to combine two common conditions into a single
458	 * conditional jump.
459	 */
460	if (0 == (td->td_owepreempt |
461	    CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
462		return (1);
463
464	/* We do not have a read token and need to acquire one. */
465	return _rm_rlock_hard(rm, tracker, trylock);
466}
467
468static void
469_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker)
470{
471
472	if (td->td_owepreempt) {
473		td->td_critnest++;
474		critical_exit();
475	}
476
477	if (!tracker->rmp_flags)
478		return;
479
480	mtx_lock_spin(&rm_spinlock);
481	LIST_REMOVE(tracker, rmp_qentry);
482
483	if (tracker->rmp_flags & RMPF_SIGNAL) {
484		struct rmlock *rm;
485		struct turnstile *ts;
486
487		rm = tracker->rmp_rmlock;
488
489		turnstile_chain_lock(&rm->lock_object);
490		mtx_unlock_spin(&rm_spinlock);
491
492		ts = turnstile_lookup(&rm->lock_object);
493
494		turnstile_signal(ts, TS_EXCLUSIVE_QUEUE);
495		turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
496		turnstile_chain_unlock(&rm->lock_object);
497	} else
498		mtx_unlock_spin(&rm_spinlock);
499}
500
501void
502_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
503{
504	struct pcpu *pc;
505	struct thread *td = tracker->rmp_thread;
506
507	if (SCHEDULER_STOPPED())
508		return;
509
510	td->td_critnest++;	/* critical_enter(); */
511	pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
512	rm_tracker_remove(pc, tracker);
513	td->td_critnest--;
514	sched_unpin();
515
516	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
517		THREAD_SLEEPING_OK();
518
519	if (0 == (td->td_owepreempt | tracker->rmp_flags))
520		return;
521
522	_rm_unlock_hard(td, tracker);
523}
524
525void
526_rm_wlock(struct rmlock *rm)
527{
528	struct rm_priotracker *prio;
529	struct turnstile *ts;
530	cpuset_t readcpus;
531
532	if (SCHEDULER_STOPPED())
533		return;
534
535	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
536		sx_xlock(&rm->rm_lock_sx);
537	else
538		mtx_lock(&rm->rm_lock_mtx);
539
540	if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
541		/* Get all read tokens back */
542		readcpus = all_cpus;
543		CPU_NAND(&readcpus, &rm->rm_writecpus);
544		rm->rm_writecpus = all_cpus;
545
546		/*
547		 * Assumes rm->rm_writecpus update is visible on other CPUs
548		 * before rm_cleanIPI is called.
549		 */
550#ifdef SMP
551		smp_rendezvous_cpus(readcpus,
552		    smp_no_rendevous_barrier,
553		    rm_cleanIPI,
554		    smp_no_rendevous_barrier,
555		    rm);
556
557#else
558		rm_cleanIPI(rm);
559#endif
560
561		mtx_lock_spin(&rm_spinlock);
562		while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
563			ts = turnstile_trywait(&rm->lock_object);
564			prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL;
565			mtx_unlock_spin(&rm_spinlock);
566			turnstile_wait(ts, prio->rmp_thread,
567			    TS_EXCLUSIVE_QUEUE);
568			mtx_lock_spin(&rm_spinlock);
569		}
570		mtx_unlock_spin(&rm_spinlock);
571	}
572}
573
574void
575_rm_wunlock(struct rmlock *rm)
576{
577
578	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
579		sx_xunlock(&rm->rm_lock_sx);
580	else
581		mtx_unlock(&rm->rm_lock_mtx);
582}
583
584#ifdef LOCK_DEBUG
585
586void
587_rm_wlock_debug(struct rmlock *rm, const char *file, int line)
588{
589
590	if (SCHEDULER_STOPPED())
591		return;
592
593	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
594	    ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d",
595	    curthread, rm->lock_object.lo_name, file, line));
596	KASSERT(!rm_destroyed(rm),
597	    ("rm_wlock() of destroyed rmlock @ %s:%d", file, line));
598	_rm_assert(rm, RA_UNLOCKED, file, line);
599
600	WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
601	    file, line, NULL);
602
603	_rm_wlock(rm);
604
605	LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
606
607	WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
608
609	curthread->td_locks++;
610
611}
612
613void
614_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
615{
616
617	if (SCHEDULER_STOPPED())
618		return;
619
620	KASSERT(!rm_destroyed(rm),
621	    ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line));
622	_rm_assert(rm, RA_WLOCKED, file, line);
623	WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
624	LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
625	_rm_wunlock(rm);
626	curthread->td_locks--;
627}
628
629int
630_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
631    int trylock, const char *file, int line)
632{
633
634	if (SCHEDULER_STOPPED())
635		return (1);
636
637#ifdef INVARIANTS
638	if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
639		critical_enter();
640		KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
641		    curthread) == 0,
642		    ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
643		    rm->lock_object.lo_name, file, line));
644		critical_exit();
645	}
646#endif
647	KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
648	    ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d",
649	    curthread, rm->lock_object.lo_name, file, line));
650	KASSERT(!rm_destroyed(rm),
651	    ("rm_rlock() of destroyed rmlock @ %s:%d", file, line));
652	if (!trylock) {
653		KASSERT(!rm_wowned(rm),
654		    ("rm_rlock: wlock already held for %s @ %s:%d",
655		    rm->lock_object.lo_name, file, line));
656		WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
657		    NULL);
658	}
659
660	if (_rm_rlock(rm, tracker, trylock)) {
661		if (trylock)
662			LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
663			    line);
664		else
665			LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
666			    line);
667		WITNESS_LOCK(&rm->lock_object, 0, file, line);
668
669		curthread->td_locks++;
670
671		return (1);
672	} else if (trylock)
673		LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
674
675	return (0);
676}
677
678void
679_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
680    const char *file, int line)
681{
682
683	if (SCHEDULER_STOPPED())
684		return;
685
686	KASSERT(!rm_destroyed(rm),
687	    ("rm_runlock() of destroyed rmlock @ %s:%d", file, line));
688	_rm_assert(rm, RA_RLOCKED, file, line);
689	WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
690	LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
691	_rm_runlock(rm, tracker);
692	curthread->td_locks--;
693}
694
695#else
696
697/*
698 * Just strip out file and line arguments if no lock debugging is enabled in
699 * the kernel - we are called from a kernel module.
700 */
701void
702_rm_wlock_debug(struct rmlock *rm, const char *file, int line)
703{
704
705	_rm_wlock(rm);
706}
707
708void
709_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
710{
711
712	_rm_wunlock(rm);
713}
714
715int
716_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
717    int trylock, const char *file, int line)
718{
719
720	return _rm_rlock(rm, tracker, trylock);
721}
722
723void
724_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
725    const char *file, int line)
726{
727
728	_rm_runlock(rm, tracker);
729}
730
731#endif
732
733#ifdef INVARIANT_SUPPORT
734#ifndef INVARIANTS
735#undef _rm_assert
736#endif
737
738/*
739 * Note that this does not need to use witness_assert() for read lock
740 * assertions since an exact count of read locks held by this thread
741 * is computable.
742 */
743void
744_rm_assert(const struct rmlock *rm, int what, const char *file, int line)
745{
746	int count;
747
748	if (panicstr != NULL)
749		return;
750	switch (what) {
751	case RA_LOCKED:
752	case RA_LOCKED | RA_RECURSED:
753	case RA_LOCKED | RA_NOTRECURSED:
754	case RA_RLOCKED:
755	case RA_RLOCKED | RA_RECURSED:
756	case RA_RLOCKED | RA_NOTRECURSED:
757		/*
758		 * Handle the write-locked case.  Unlike other
759		 * primitives, writers can never recurse.
760		 */
761		if (rm_wowned(rm)) {
762			if (what & RA_RLOCKED)
763				panic("Lock %s exclusively locked @ %s:%d\n",
764				    rm->lock_object.lo_name, file, line);
765			if (what & RA_RECURSED)
766				panic("Lock %s not recursed @ %s:%d\n",
767				    rm->lock_object.lo_name, file, line);
768			break;
769		}
770
771		critical_enter();
772		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
773		critical_exit();
774
775		if (count == 0)
776			panic("Lock %s not %slocked @ %s:%d\n",
777			    rm->lock_object.lo_name, (what & RA_RLOCKED) ?
778			    "read " : "", file, line);
779		if (count > 1) {
780			if (what & RA_NOTRECURSED)
781				panic("Lock %s recursed @ %s:%d\n",
782				    rm->lock_object.lo_name, file, line);
783		} else if (what & RA_RECURSED)
784			panic("Lock %s not recursed @ %s:%d\n",
785			    rm->lock_object.lo_name, file, line);
786		break;
787	case RA_WLOCKED:
788		if (!rm_wowned(rm))
789			panic("Lock %s not exclusively locked @ %s:%d\n",
790			    rm->lock_object.lo_name, file, line);
791		break;
792	case RA_UNLOCKED:
793		if (rm_wowned(rm))
794			panic("Lock %s exclusively locked @ %s:%d\n",
795			    rm->lock_object.lo_name, file, line);
796
797		critical_enter();
798		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
799		critical_exit();
800
801		if (count != 0)
802			panic("Lock %s read locked @ %s:%d\n",
803			    rm->lock_object.lo_name, file, line);
804		break;
805	default:
806		panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
807		    line);
808	}
809}
810#endif /* INVARIANT_SUPPORT */
811
812#ifdef DDB
813static void
814print_tracker(struct rm_priotracker *tr)
815{
816	struct thread *td;
817
818	td = tr->rmp_thread;
819	db_printf("   thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid,
820	    td->td_proc->p_pid, td->td_name);
821	if (tr->rmp_flags & RMPF_ONQUEUE) {
822		db_printf("ONQUEUE");
823		if (tr->rmp_flags & RMPF_SIGNAL)
824			db_printf(",SIGNAL");
825	} else
826		db_printf("0");
827	db_printf("}\n");
828}
829
830static void
831db_show_rm(const struct lock_object *lock)
832{
833	struct rm_priotracker *tr;
834	struct rm_queue *queue;
835	const struct rmlock *rm;
836	struct lock_class *lc;
837	struct pcpu *pc;
838
839	rm = (const struct rmlock *)lock;
840	db_printf(" writecpus: ");
841	ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
842	db_printf("\n");
843	db_printf(" per-CPU readers:\n");
844	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
845		for (queue = pc->pc_rm_queue.rmq_next;
846		    queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
847			tr = (struct rm_priotracker *)queue;
848			if (tr->rmp_rmlock == rm)
849				print_tracker(tr);
850		}
851	db_printf(" active readers:\n");
852	LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
853		print_tracker(tr);
854	lc = LOCK_CLASS(&rm->rm_wlock_object);
855	db_printf("Backing write-lock (%s):\n", lc->lc_name);
856	lc->lc_ddb_show(&rm->rm_wlock_object);
857}
858#endif
859