sys/kern/kern_rmlock.c

66 #define	rm_destroyed(rm)						\
67 	(LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
87 	.lc_name = "rm",
101 	.lc_name = "sleepable rm",
124 	struct rmlock *rm;
127 	rm = (struct rmlock *)lock;
129 		rm_wlock(rm);
132 		rm_rlock(rm, tracker);
141 	struct rmlock *rm;
146 	rm = (struct rmlock *)lock;
149 	rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
150 	if (rm_wowned(rm))
151 		rm_wunlock(rm);
164 				if ((tracker->rmp_rmlock == rm) &&
173 		rm_runlock(rm, tracker);
182 	const struct rmlock *rm;
185 	rm = (const struct rmlock *)lock;
186 	lc = LOCK_CLASS(&rm->rm_wlock_object);
187 	return (lc->lc_owner(&rm->rm_wlock_object, owner));
220  * has on this CPU for the lock 'rm'.
223 rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
234 		if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
259 	struct rmlock *rm = arg;
267 		if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
270 			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
278 rm_init_flags(struct rmlock *rm, const char *name, int opts)
288 	rm->rm_writecpus = all_cpus;
289 	LIST_INIT(&rm->rm_activeReaders);
293 		sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS);
296 		mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS);
298 	lock_init(&rm->lock_object, lc, name, NULL, liflags);
302 rm_init(struct rmlock *rm, const char *name)
305 	rm_init_flags(rm, name, 0);
309 rm_destroy(struct rmlock *rm)
312 	rm_assert(rm, RA_UNLOCKED);
313 	LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
314 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
315 		sx_destroy(&rm->rm_lock_sx);
317 		mtx_destroy(&rm->rm_lock_mtx);
318 	lock_destroy(&rm->lock_object);
322 rm_wowned(const struct rmlock *rm)
325 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
326 		return (sx_xlocked(&rm->rm_lock_sx));
328 		return (mtx_owned(&rm->rm_lock_mtx));
348 _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
356 	if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
376 	if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
381 		if (rm_trackers_present(pc, rm, curthread) != 0) {
383 			LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
397 		if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
398 			if (!sx_try_xlock(&rm->rm_lock_sx))
401 			if (!mtx_trylock(&rm->rm_lock_mtx))
405 		if (rm->lock_object.lo_flags & LO_SLEEPABLE)
406 			sx_xlock(&rm->rm_lock_sx);
408 			mtx_lock(&rm->rm_lock_mtx);
413 	CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
418 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
419 		sx_xunlock(&rm->rm_lock_sx);
421 		mtx_unlock(&rm->rm_lock_mtx);
427 _rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
437 	tracker->rmp_rmlock = rm;
439 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
461 	    CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
465 	return _rm_rlock_hard(rm, tracker, trylock);
484 		struct rmlock *rm;
487 		rm = tracker->rmp_rmlock;
489 		turnstile_chain_lock(&rm->lock_object);
492 		ts = turnstile_lookup(&rm->lock_object);
496 		turnstile_chain_unlock(&rm->lock_object);
502 _rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
516 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
526 _rm_wlock(struct rmlock *rm)
535 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
536 		sx_xlock(&rm->rm_lock_sx);
538 		mtx_lock(&rm->rm_lock_mtx);
540 	if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
543 		CPU_NAND(&readcpus, &rm->rm_writecpus);
544 		rm->rm_writecpus = all_cpus;
547 		 * Assumes rm->rm_writecpus update is visible on other CPUs
555 		    rm);
558 		rm_cleanIPI(rm);
562 		while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
563 			ts = turnstile_trywait(&rm->lock_object);
575 _rm_wunlock(struct rmlock *rm)
578 	if (rm->lock_object.lo_flags & LO_SLEEPABLE)
579 		sx_xunlock(&rm->rm_lock_sx);
581 		mtx_unlock(&rm->rm_lock_mtx);
587 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
595 	    curthread, rm->lock_object.lo_name, file, line));
596 	KASSERT(!rm_destroyed(rm),
598 	_rm_assert(rm, RA_UNLOCKED, file, line);
600 	WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
603 	_rm_wlock(rm);
605 	LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
607 	WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
614 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
620 	KASSERT(!rm_destroyed(rm),
622 	_rm_assert(rm, RA_WLOCKED, file, line);
623 	WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
624 	LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
625 	_rm_wunlock(rm);
630 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
638 	if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
640 		KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
643 		    rm->lock_object.lo_name, file, line));
649 	    curthread, rm->lock_object.lo_name, file, line));
650 	KASSERT(!rm_destroyed(rm),
653 		KASSERT(!rm_wowned(rm),
655 		    rm->lock_object.lo_name, file, line));
656 		WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
660 	if (_rm_rlock(rm, tracker, trylock)) {
662 			LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
665 			LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
667 		WITNESS_LOCK(&rm->lock_object, 0, file, line);
673 		LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
679 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
686 	KASSERT(!rm_destroyed(rm),
688 	_rm_assert(rm, RA_RLOCKED, file, line);
689 	WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
690 	LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
691 	_rm_runlock(rm, tracker);
702 _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
705 	_rm_wlock(rm);
709 _rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
712 	_rm_wunlock(rm);
716 _rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
720 	return _rm_rlock(rm, tracker, trylock);
724 _rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
728 	_rm_runlock(rm, tracker);
744 _rm_assert(const struct rmlock *rm, int what, const char *file, int line)
761 		if (rm_wowned(rm)) {
764 				    rm->lock_object.lo_name, file, line);
767 				    rm->lock_object.lo_name, file, line);
772 		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
777 			    rm->lock_object.lo_name, (what & RA_RLOCKED) ?
782 				    rm->lock_object.lo_name, file, line);
785 			    rm->lock_object.lo_name, file, line);
788 		if (!rm_wowned(rm))
790 			    rm->lock_object.lo_name, file, line);
793 		if (rm_wowned(rm))
795 			    rm->lock_object.lo_name, file, line);
798 		count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
803 			    rm->lock_object.lo_name, file, line);
806 		panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
835 	const struct rmlock *rm;
839 	rm = (const struct rmlock *)lock;
841 	ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
848 			if (tr->rmp_rmlock == rm)
852 	LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
854 	lc = LOCK_CLASS(&rm->rm_wlock_object);
856 	lc->lc_ddb_show(&rm->rm_wlock_object);