167352Sjhb/*- 267352Sjhb * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 367352Sjhb * 467352Sjhb * Redistribution and use in source and binary forms, with or without 567352Sjhb * modification, are permitted provided that the following conditions 667352Sjhb * are met: 767352Sjhb * 1. Redistributions of source code must retain the above copyright 867352Sjhb * notice, this list of conditions and the following disclaimer. 967352Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1067352Sjhb * notice, this list of conditions and the following disclaimer in the 1167352Sjhb * documentation and/or other materials provided with the distribution. 1267352Sjhb * 3. Berkeley Software Design Inc's name may not be used to endorse or 1367352Sjhb * promote products derived from this software without specific prior 1467352Sjhb * written permission. 1567352Sjhb * 1667352Sjhb * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1767352Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1867352Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1967352Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2067352Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2167352Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2267352Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2367352Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2467352Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2567352Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2667352Sjhb * SUCH DAMAGE. 2767352Sjhb * 2867352Sjhb * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $ 2967352Sjhb * $FreeBSD: stable/10/sys/sys/mutex.h 323870 2017-09-21 19:24:11Z marius $ 3067352Sjhb */ 3167352Sjhb 3267352Sjhb#ifndef _SYS_MUTEX_H_ 3367352Sjhb#define _SYS_MUTEX_H_ 3467352Sjhb 3567352Sjhb#include <sys/queue.h> 3676166Smarkm#include <sys/_lock.h> 3776166Smarkm#include <sys/_mutex.h> 3867352Sjhb 3967352Sjhb#ifdef _KERNEL 4087702Sjhb#include <sys/pcpu.h> 41165265Skmacy#include <sys/lock_profile.h> 42192853Ssson#include <sys/lockstat.h> 4367352Sjhb#include <machine/atomic.h> 4467352Sjhb#include <machine/cpufunc.h> 4567352Sjhb 4667352Sjhb/* 47145421Sjeff * Mutex types and options passed to mtx_init(). MTX_QUIET and MTX_DUPOK 48145421Sjeff * can also be passed in. 4972200Sbmilekic */ 5072200Sbmilekic#define MTX_DEF 0x00000000 /* DEFAULT (sleep) lock */ 5172200Sbmilekic#define MTX_SPIN 0x00000001 /* Spin lock (disables interrupts) */ 5274912Sjhb#define MTX_RECURSE 0x00000004 /* Option: lock allowed to recurse */ 5374912Sjhb#define MTX_NOWITNESS 0x00000008 /* Don't do any witness checking. */ 54165453Spjd#define MTX_NOPROFILE 0x00000020 /* Don't profile this lock */ 55323870Smarius#define MTX_NEW 0x00000040 /* Don't check for double-init */ 5672200Sbmilekic 5772200Sbmilekic/* 5872200Sbmilekic * Option flags passed to certain lock/unlock routines, through the use 5972200Sbmilekic * of corresponding mtx_{lock,unlock}_flags() interface macros. 6067352Sjhb */ 6174912Sjhb#define MTX_QUIET LOP_QUIET /* Don't log a mutex event */ 62145421Sjeff#define MTX_DUPOK LOP_DUPOK /* Don't log a duplicate acquire */ 6367352Sjhb 6472200Sbmilekic/* 6572200Sbmilekic * State bits kept in mutex->mtx_lock, for the DEFAULT lock type. None of this, 6672200Sbmilekic * with the exception of MTX_UNOWNED, applies to spin locks. 6772200Sbmilekic */ 6872200Sbmilekic#define MTX_RECURSED 0x00000001 /* lock recursed (for MTX_DEF only) */ 6972200Sbmilekic#define MTX_CONTESTED 0x00000002 /* lock contested (for MTX_DEF only) */ 7072200Sbmilekic#define MTX_UNOWNED 0x00000004 /* Cookie for free mutex */ 71159208Sjhb#define MTX_FLAGMASK (MTX_RECURSED | MTX_CONTESTED | MTX_UNOWNED) 7267352Sjhb 73160766Sjhb/* 74160766Sjhb * Value stored in mutex->mtx_lock to denote a destroyed mutex. 75160766Sjhb */ 76160766Sjhb#define MTX_DESTROYED (MTX_CONTESTED | MTX_UNOWNED) 77160766Sjhb 7867352Sjhb/* 7972200Sbmilekic * Prototypes 8072200Sbmilekic * 8172200Sbmilekic * NOTE: Functions prepended with `_' (underscore) are exported to other parts 8283593Sjhb * of the kernel via macros, thus allowing us to use the cpp LOCK_FILE 83242395Sattilio * and LOCK_LINE or for hiding the lock cookie crunching to the 84242395Sattilio * consumers. These functions should not be called directly by any 8574912Sjhb * code using the API. Their macros cover their functionality. 86227758Sattilio * Functions with a `_' suffix are the entrypoint for the common 87227758Sattilio * KPI covering both compat shims and fast path case. These can be 88227758Sattilio * used by consumers willing to pass options, file and line 89227758Sattilio * informations, in an option-independent way. 9072200Sbmilekic * 9172200Sbmilekic * [See below for descriptions] 9272200Sbmilekic * 9372200Sbmilekic */ 94242395Sattiliovoid _mtx_init(volatile uintptr_t *c, const char *name, const char *type, 95242395Sattilio int opts); 96242395Sattiliovoid _mtx_destroy(volatile uintptr_t *c); 9793702Sjhbvoid mtx_sysinit(void *arg); 98242395Sattilioint _mtx_trylock_flags_(volatile uintptr_t *c, int opts, const char *file, 99227758Sattilio int line); 10093702Sjhbvoid mutex_init(void); 101242395Sattiliovoid __mtx_lock_sleep(volatile uintptr_t *c, uintptr_t tid, int opts, 102133137Sjhb const char *file, int line); 103242395Sattiliovoid __mtx_unlock_sleep(volatile uintptr_t *c, int opts, const char *file, 104242395Sattilio int line); 105139733Sjhb#ifdef SMP 106242395Sattiliovoid _mtx_lock_spin_cookie(volatile uintptr_t *c, uintptr_t tid, int opts, 107133137Sjhb const char *file, int line); 108139733Sjhb#endif 109242395Sattiliovoid __mtx_lock_flags(volatile uintptr_t *c, int opts, const char *file, 110242395Sattilio int line); 111242395Sattiliovoid __mtx_unlock_flags(volatile uintptr_t *c, int opts, const char *file, 112242395Sattilio int line); 113242395Sattiliovoid __mtx_lock_spin_flags(volatile uintptr_t *c, int opts, const char *file, 114133137Sjhb int line); 115303552Skibint __mtx_trylock_spin_flags(volatile uintptr_t *c, int opts, 116303552Skib const char *file, int line); 117242395Sattiliovoid __mtx_unlock_spin_flags(volatile uintptr_t *c, int opts, 118242395Sattilio const char *file, int line); 119142821Sjhb#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 120242395Sattiliovoid __mtx_assert(const volatile uintptr_t *c, int what, const char *file, 121242395Sattilio int line); 12273905Sjhb#endif 123227758Sattiliovoid thread_lock_flags_(struct thread *, int, const char *, int); 12467352Sjhb 125170293Sjeff#define thread_lock(tdp) \ 126227758Sattilio thread_lock_flags_((tdp), 0, __FILE__, __LINE__) 127170293Sjeff#define thread_lock_flags(tdp, opt) \ 128227758Sattilio thread_lock_flags_((tdp), (opt), __FILE__, __LINE__) 129170293Sjeff#define thread_unlock(tdp) \ 130170358Sjeff mtx_unlock_spin((tdp)->td_lock) 131170293Sjeff 132242395Sattilio/* 133242395Sattilio * Top-level macros to provide lock cookie once the actual mtx is passed. 134242395Sattilio * They will also prevent passing a malformed object to the mtx KPI by 135242483Sattilio * failing compilation as the mtx_lock reserved member will not be found. 136242395Sattilio */ 137242395Sattilio#define mtx_init(m, n, t, o) \ 138242395Sattilio _mtx_init(&(m)->mtx_lock, n, t, o) 139242395Sattilio#define mtx_destroy(m) \ 140242395Sattilio _mtx_destroy(&(m)->mtx_lock) 141242395Sattilio#define mtx_trylock_flags_(m, o, f, l) \ 142242395Sattilio _mtx_trylock_flags_(&(m)->mtx_lock, o, f, l) 143242395Sattilio#define _mtx_lock_sleep(m, t, o, f, l) \ 144242395Sattilio __mtx_lock_sleep(&(m)->mtx_lock, t, o, f, l) 145242395Sattilio#define _mtx_unlock_sleep(m, o, f, l) \ 146242395Sattilio __mtx_unlock_sleep(&(m)->mtx_lock, o, f, l) 147242395Sattilio#ifdef SMP 148242395Sattilio#define _mtx_lock_spin(m, t, o, f, l) \ 149242395Sattilio _mtx_lock_spin_cookie(&(m)->mtx_lock, t, o, f, l) 150242395Sattilio#endif 151242395Sattilio#define _mtx_lock_flags(m, o, f, l) \ 152242395Sattilio __mtx_lock_flags(&(m)->mtx_lock, o, f, l) 153242395Sattilio#define _mtx_unlock_flags(m, o, f, l) \ 154242395Sattilio __mtx_unlock_flags(&(m)->mtx_lock, o, f, l) 155242395Sattilio#define _mtx_lock_spin_flags(m, o, f, l) \ 156242395Sattilio __mtx_lock_spin_flags(&(m)->mtx_lock, o, f, l) 157303552Skib#define _mtx_trylock_spin_flags(m, o, f, l) \ 158303552Skib __mtx_trylock_spin_flags(&(m)->mtx_lock, o, f, l) 159242395Sattilio#define _mtx_unlock_spin_flags(m, o, f, l) \ 160242395Sattilio __mtx_unlock_spin_flags(&(m)->mtx_lock, o, f, l) 161242395Sattilio#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 162242395Sattilio#define _mtx_assert(m, w, f, l) \ 163242395Sattilio __mtx_assert(&(m)->mtx_lock, w, f, l) 164242395Sattilio#endif 165242395Sattilio 166179025Sattilio#define mtx_recurse lock_object.lo_data 167179025Sattilio 168215054Sjhb/* Very simple operations on mtx_lock. */ 16967352Sjhb 170147556Sjhb/* Try to obtain mtx_lock once. */ 171215054Sjhb#define _mtx_obtain_lock(mp, tid) \ 172148067Sjhb atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) 17367352Sjhb 174147556Sjhb/* Try to release mtx_lock if it is unrecursed and uncontested. */ 175215054Sjhb#define _mtx_release_lock(mp, tid) \ 176148067Sjhb atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), MTX_UNOWNED) 17767352Sjhb 178147556Sjhb/* Release mtx_lock quickly, assuming we own it. */ 179215054Sjhb#define _mtx_release_lock_quick(mp) \ 180148067Sjhb atomic_store_rel_ptr(&(mp)->mtx_lock, MTX_UNOWNED) 18167352Sjhb 18272200Sbmilekic/* 183215054Sjhb * Full lock operations that are suitable to be inlined in non-debug 184215054Sjhb * kernels. If the lock cannot be acquired or released trivially then 185215054Sjhb * the work is deferred to another function. 18672200Sbmilekic */ 187215054Sjhb 188215054Sjhb/* Lock a normal mutex. */ 189215054Sjhb#define __mtx_lock(mp, tid, opts, file, line) do { \ 190148067Sjhb uintptr_t _tid = (uintptr_t)(tid); \ 191215054Sjhb \ 192310979Smjg if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid)))\ 193133137Sjhb _mtx_lock_sleep((mp), _tid, (opts), (file), (line)); \ 194192853Ssson else \ 195192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_LOCK_ACQUIRE, \ 196192853Ssson mp, 0, 0, (file), (line)); \ 19772200Sbmilekic} while (0) 19867352Sjhb 19967352Sjhb/* 200215054Sjhb * Lock a spin mutex. For spinlocks, we handle recursion inline (it 201215054Sjhb * turns out that function calls can be significantly expensive on 202215054Sjhb * some architectures). Since spin locks are not _too_ common, 203215054Sjhb * inlining this code is not too big a deal. 20467352Sjhb */ 205139733Sjhb#ifdef SMP 206215054Sjhb#define __mtx_lock_spin(mp, tid, opts, file, line) do { \ 207148067Sjhb uintptr_t _tid = (uintptr_t)(tid); \ 208215054Sjhb \ 209144637Sjhb spinlock_enter(); \ 210310979Smjg if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid))) {\ 211148067Sjhb if ((mp)->mtx_lock == _tid) \ 21272200Sbmilekic (mp)->mtx_recurse++; \ 213215054Sjhb else \ 214133137Sjhb _mtx_lock_spin((mp), _tid, (opts), (file), (line)); \ 215168329Skmacy } else \ 216192853Ssson LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, \ 217192853Ssson mp, 0, 0, (file), (line)); \ 21872200Sbmilekic} while (0) 219303552Skib#define __mtx_trylock_spin(mp, tid, opts, file, line) __extension__ ({ \ 220303552Skib uintptr_t _tid = (uintptr_t)(tid); \ 221303552Skib int _ret; \ 222303552Skib \ 223303552Skib spinlock_enter(); \ 224303552Skib if (((mp)->mtx_lock != MTX_UNOWNED || !_mtx_obtain_lock((mp), _tid))) {\ 225303552Skib spinlock_exit(); \ 226303552Skib _ret = 0; \ 227303552Skib } else { \ 228303552Skib LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(LS_MTX_SPIN_LOCK_ACQUIRE, \ 229303552Skib mp, 0, 0, file, line); \ 230303552Skib _ret = 1; \ 231303552Skib } \ 232303552Skib _ret; \ 233303552Skib}) 234139733Sjhb#else /* SMP */ 235215054Sjhb#define __mtx_lock_spin(mp, tid, opts, file, line) do { \ 236148067Sjhb uintptr_t _tid = (uintptr_t)(tid); \ 237139733Sjhb \ 238144637Sjhb spinlock_enter(); \ 239148067Sjhb if ((mp)->mtx_lock == _tid) \ 240139733Sjhb (mp)->mtx_recurse++; \ 241139733Sjhb else { \ 242139733Sjhb KASSERT((mp)->mtx_lock == MTX_UNOWNED, ("corrupt spinlock")); \ 243215054Sjhb (mp)->mtx_lock = _tid; \ 244139733Sjhb } \ 245139733Sjhb} while (0) 246303552Skib#define __mtx_trylock_spin(mp, tid, opts, file, line) __extension__ ({ \ 247303552Skib uintptr_t _tid = (uintptr_t)(tid); \ 248303552Skib int _ret; \ 249303552Skib \ 250303552Skib spinlock_enter(); \ 251303552Skib if ((mp)->mtx_lock != MTX_UNOWNED) { \ 252303552Skib spinlock_exit(); \ 253303552Skib _ret = 0; \ 254303552Skib } else { \ 255303552Skib (mp)->mtx_lock = _tid; \ 256303552Skib _ret = 1; \ 257303552Skib } \ 258303552Skib _ret; \ 259303552Skib}) 260139733Sjhb#endif /* SMP */ 26167352Sjhb 262215054Sjhb/* Unlock a normal mutex. */ 263215054Sjhb#define __mtx_unlock(mp, tid, opts, file, line) do { \ 264148067Sjhb uintptr_t _tid = (uintptr_t)(tid); \ 265148067Sjhb \ 266310979Smjg if ((mp)->mtx_lock != _tid || !_mtx_release_lock((mp), _tid)) \ 26774900Sjhb _mtx_unlock_sleep((mp), (opts), (file), (line)); \ 26867352Sjhb} while (0) 26967352Sjhb 27072200Sbmilekic/* 271215054Sjhb * Unlock a spin mutex. For spinlocks, we can handle everything 272215054Sjhb * inline, as it's pretty simple and a function call would be too 273215054Sjhb * expensive (at least on some architectures). Since spin locks are 274215054Sjhb * not _too_ common, inlining this code is not too big a deal. 27588088Sjhb * 276144637Sjhb * Since we always perform a spinlock_enter() when attempting to acquire a 277144637Sjhb * spin lock, we need to always perform a matching spinlock_exit() when 27888088Sjhb * releasing a spin lock. This includes the recursion cases. 27972200Sbmilekic */ 280139733Sjhb#ifdef SMP 281215054Sjhb#define __mtx_unlock_spin(mp) do { \ 28272200Sbmilekic if (mtx_recursed((mp))) \ 28372200Sbmilekic (mp)->mtx_recurse--; \ 284168329Skmacy else { \ 285192853Ssson LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_SPIN_UNLOCK_RELEASE, \ 286192853Ssson mp); \ 287215054Sjhb _mtx_release_lock_quick((mp)); \ 288168329Skmacy } \ 289168329Skmacy spinlock_exit(); \ 29067352Sjhb} while (0) 291139733Sjhb#else /* SMP */ 292215054Sjhb#define __mtx_unlock_spin(mp) do { \ 293139733Sjhb if (mtx_recursed((mp))) \ 294139733Sjhb (mp)->mtx_recurse--; \ 295213271Sjhb else { \ 296213271Sjhb LOCKSTAT_PROFILE_RELEASE_LOCK(LS_MTX_SPIN_UNLOCK_RELEASE, \ 297213271Sjhb mp); \ 298139733Sjhb (mp)->mtx_lock = MTX_UNOWNED; \ 299213271Sjhb } \ 300144637Sjhb spinlock_exit(); \ 301139733Sjhb} while (0) 302139733Sjhb#endif /* SMP */ 30367352Sjhb 30472200Sbmilekic/* 30572200Sbmilekic * Exported lock manipulation interface. 30672200Sbmilekic * 30772200Sbmilekic * mtx_lock(m) locks MTX_DEF mutex `m' 30872200Sbmilekic * 30972200Sbmilekic * mtx_lock_spin(m) locks MTX_SPIN mutex `m' 31072200Sbmilekic * 31172200Sbmilekic * mtx_unlock(m) unlocks MTX_DEF mutex `m' 31272200Sbmilekic * 31372200Sbmilekic * mtx_unlock_spin(m) unlocks MTX_SPIN mutex `m' 31472200Sbmilekic * 31572200Sbmilekic * mtx_lock_spin_flags(m, opts) and mtx_lock_flags(m, opts) locks mutex `m' 31672200Sbmilekic * and passes option flags `opts' to the "hard" function, if required. 31772200Sbmilekic * With these routines, it is possible to pass flags such as MTX_QUIET 31888900Sjhb * to the appropriate lock manipulation routines. 31972200Sbmilekic * 32072200Sbmilekic * mtx_trylock(m) attempts to acquire MTX_DEF mutex `m' but doesn't sleep if 32172200Sbmilekic * it cannot. Rather, it returns 0 on failure and non-zero on success. 32272200Sbmilekic * It does NOT handle recursion as we assume that if a caller is properly 32372200Sbmilekic * using this part of the interface, he will know that the lock in question 32472200Sbmilekic * is _not_ recursed. 32572200Sbmilekic * 32672200Sbmilekic * mtx_trylock_flags(m, opts) is used the same way as mtx_trylock() but accepts 32772200Sbmilekic * relevant option flags `opts.' 32872200Sbmilekic * 329303552Skib * mtx_trylock_spin(m) attempts to acquire MTX_SPIN mutex `m' but doesn't 330303552Skib * spin if it cannot. Rather, it returns 0 on failure and non-zero on 331303552Skib * success. It always returns failure for recursed lock attempts. 332303552Skib * 33374912Sjhb * mtx_initialized(m) returns non-zero if the lock `m' has been initialized. 33474912Sjhb * 33572200Sbmilekic * mtx_owned(m) returns non-zero if the current thread owns the lock `m' 33672200Sbmilekic * 33772200Sbmilekic * mtx_recursed(m) returns non-zero if the lock `m' is presently recursed. 33872200Sbmilekic */ 33973905Sjhb#define mtx_lock(m) mtx_lock_flags((m), 0) 34073905Sjhb#define mtx_lock_spin(m) mtx_lock_spin_flags((m), 0) 34173905Sjhb#define mtx_trylock(m) mtx_trylock_flags((m), 0) 342303552Skib#define mtx_trylock_spin(m) mtx_trylock_spin_flags((m), 0) 34373905Sjhb#define mtx_unlock(m) mtx_unlock_flags((m), 0) 34473905Sjhb#define mtx_unlock_spin(m) mtx_unlock_spin_flags((m), 0) 34572200Sbmilekic 346117494Struckmanstruct mtx_pool; 34786333Sdillon 348117494Struckmanstruct mtx_pool *mtx_pool_create(const char *mtx_name, int pool_size, int opts); 349117494Struckmanvoid mtx_pool_destroy(struct mtx_pool **poolp); 350117494Struckmanstruct mtx *mtx_pool_find(struct mtx_pool *pool, void *ptr); 351117494Struckmanstruct mtx *mtx_pool_alloc(struct mtx_pool *pool); 352117494Struckman#define mtx_pool_lock(pool, ptr) \ 353117494Struckman mtx_lock(mtx_pool_find((pool), (ptr))) 354117494Struckman#define mtx_pool_lock_spin(pool, ptr) \ 355117494Struckman mtx_lock_spin(mtx_pool_find((pool), (ptr))) 356117494Struckman#define mtx_pool_unlock(pool, ptr) \ 357117494Struckman mtx_unlock(mtx_pool_find((pool), (ptr))) 358117494Struckman#define mtx_pool_unlock_spin(pool, ptr) \ 359117494Struckman mtx_unlock_spin(mtx_pool_find((pool), (ptr))) 36086333Sdillon 361117494Struckman/* 362117494Struckman * mtxpool_lockbuilder is a pool of sleep locks that is not witness 363117494Struckman * checked and should only be used for building higher level locks. 364117494Struckman * 365117494Struckman * mtxpool_sleep is a general purpose pool of sleep mutexes. 366117494Struckman */ 367117494Struckmanextern struct mtx_pool *mtxpool_lockbuilder; 368117494Struckmanextern struct mtx_pool *mtxpool_sleep; 369117494Struckman 37085186Sjhb#ifndef LOCK_DEBUG 37185186Sjhb#error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/mutex.h> 37285186Sjhb#endif 373120247Ssam#if LOCK_DEBUG > 0 || defined(MUTEX_NOINLINE) 374227758Sattilio#define mtx_lock_flags_(m, opts, file, line) \ 375227758Sattilio _mtx_lock_flags((m), (opts), (file), (line)) 376227758Sattilio#define mtx_unlock_flags_(m, opts, file, line) \ 377227758Sattilio _mtx_unlock_flags((m), (opts), (file), (line)) 378227758Sattilio#define mtx_lock_spin_flags_(m, opts, file, line) \ 379227758Sattilio _mtx_lock_spin_flags((m), (opts), (file), (line)) 380303552Skib#define mtx_trylock_spin_flags_(m, opts, file, line) \ 381303552Skib _mtx_trylock_spin_flags((m), (opts), (file), (line)) 382227758Sattilio#define mtx_unlock_spin_flags_(m, opts, file, line) \ 383227758Sattilio _mtx_unlock_spin_flags((m), (opts), (file), (line)) 384115125Sscottl#else /* LOCK_DEBUG == 0 && !MUTEX_NOINLINE */ 385227758Sattilio#define mtx_lock_flags_(m, opts, file, line) \ 386227758Sattilio __mtx_lock((m), curthread, (opts), (file), (line)) 387227758Sattilio#define mtx_unlock_flags_(m, opts, file, line) \ 388227758Sattilio __mtx_unlock((m), curthread, (opts), (file), (line)) 389227758Sattilio#define mtx_lock_spin_flags_(m, opts, file, line) \ 390227758Sattilio __mtx_lock_spin((m), curthread, (opts), (file), (line)) 391303552Skib#define mtx_trylock_spin_flags_(m, opts, file, line) \ 392303552Skib __mtx_trylock_spin((m), curthread, (opts), (file), (line)) 393227758Sattilio#define mtx_unlock_spin_flags_(m, opts, file, line) \ 394227758Sattilio __mtx_unlock_spin((m)) 395227758Sattilio#endif /* LOCK_DEBUG > 0 || MUTEX_NOINLINE */ 396227758Sattilio 397227758Sattilio#ifdef INVARIANTS 398227758Sattilio#define mtx_assert_(m, what, file, line) \ 399227758Sattilio _mtx_assert((m), (what), (file), (line)) 400227758Sattilio 401227758Sattilio#define GIANT_REQUIRED mtx_assert_(&Giant, MA_OWNED, __FILE__, __LINE__) 402227758Sattilio 403227758Sattilio#else /* INVARIANTS */ 404227758Sattilio#define mtx_assert_(m, what, file, line) (void)0 405227758Sattilio#define GIANT_REQUIRED 406227758Sattilio#endif /* INVARIANTS */ 407227758Sattilio 40874900Sjhb#define mtx_lock_flags(m, opts) \ 409227758Sattilio mtx_lock_flags_((m), (opts), LOCK_FILE, LOCK_LINE) 41074900Sjhb#define mtx_unlock_flags(m, opts) \ 411227758Sattilio mtx_unlock_flags_((m), (opts), LOCK_FILE, LOCK_LINE) 41274900Sjhb#define mtx_lock_spin_flags(m, opts) \ 413227758Sattilio mtx_lock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE) 41474900Sjhb#define mtx_unlock_spin_flags(m, opts) \ 415227758Sattilio mtx_unlock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE) 41672200Sbmilekic#define mtx_trylock_flags(m, opts) \ 417227758Sattilio mtx_trylock_flags_((m), (opts), LOCK_FILE, LOCK_LINE) 418303552Skib#define mtx_trylock_spin_flags(m, opts) \ 419303552Skib mtx_trylock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE) 420227758Sattilio#define mtx_assert(m, what) \ 421227758Sattilio mtx_assert_((m), (what), __FILE__, __LINE__) 42272200Sbmilekic 423167387Sjhb#define mtx_sleep(chan, mtx, pri, wmesg, timo) \ 424247787Sdavide _sleep((chan), &(mtx)->lock_object, (pri), (wmesg), \ 425247787Sdavide tick_sbt * (timo), 0, C_HARDCLOCK) 426167387Sjhb 427167787Sjhb#define mtx_initialized(m) lock_initalized(&(m)->lock_object) 42874912Sjhb 429159208Sjhb#define mtx_owned(m) (((m)->mtx_lock & ~MTX_FLAGMASK) == (uintptr_t)curthread) 43072200Sbmilekic 43172200Sbmilekic#define mtx_recursed(m) ((m)->mtx_recurse != 0) 43272200Sbmilekic 433167787Sjhb#define mtx_name(m) ((m)->lock_object.lo_name) 43493696Sdes 43572200Sbmilekic/* 43672200Sbmilekic * Global locks. 43772200Sbmilekic */ 43892373Sjhbextern struct mtx Giant; 439170293Sjeffextern struct mtx blocked_lock; 44072200Sbmilekic 44172200Sbmilekic/* 44272200Sbmilekic * Giant lock manipulation and clean exit macros. 44372200Sbmilekic * Used to replace return with an exit Giant and return. 44472200Sbmilekic * 44572200Sbmilekic * Note that DROP_GIANT*() needs to be paired with PICKUP_GIANT() 446108368Sphk * The #ifndef is to allow lint-like tools to redefine DROP_GIANT. 44772200Sbmilekic */ 448108368Sphk#ifndef DROP_GIANT 44967352Sjhb#define DROP_GIANT() \ 45067352Sjhbdo { \ 451168619Sjhb int _giantcnt = 0; \ 45267352Sjhb WITNESS_SAVE_DECL(Giant); \ 45367352Sjhb \ 454168619Sjhb if (mtx_owned(&Giant)) { \ 455167787Sjhb WITNESS_SAVE(&Giant.lock_object, Giant); \ 456228424Savg for (_giantcnt = 0; mtx_owned(&Giant) && \ 457228424Savg !SCHEDULER_STOPPED(); _giantcnt++) \ 458168619Sjhb mtx_unlock(&Giant); \ 459168619Sjhb } 46067352Sjhb 46167352Sjhb#define PICKUP_GIANT() \ 462168074Sjhb PARTIAL_PICKUP_GIANT(); \ 46367352Sjhb} while (0) 46467352Sjhb 46567352Sjhb#define PARTIAL_PICKUP_GIANT() \ 46667352Sjhb mtx_assert(&Giant, MA_NOTOWNED); \ 467168619Sjhb if (_giantcnt > 0) { \ 468168619Sjhb while (_giantcnt--) \ 469168619Sjhb mtx_lock(&Giant); \ 470168619Sjhb WITNESS_RESTORE(&Giant.lock_object, Giant); \ 471168619Sjhb } 472108368Sphk#endif 47367352Sjhb 47493672Sarrstruct mtx_args { 475242395Sattilio void *ma_mtx; 47693672Sarr const char *ma_desc; 47793672Sarr int ma_opts; 47893672Sarr}; 47993672Sarr 48093672Sarr#define MTX_SYSINIT(name, mtx, desc, opts) \ 48193672Sarr static struct mtx_args name##_args = { \ 482108368Sphk (mtx), \ 483108368Sphk (desc), \ 484108368Sphk (opts) \ 48593672Sarr }; \ 48693672Sarr SYSINIT(name##_mtx_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 487148650Sjhb mtx_sysinit, &name##_args); \ 488148650Sjhb SYSUNINIT(name##_mtx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ 489242395Sattilio _mtx_destroy, __DEVOLATILE(void *, &(mtx)->mtx_lock)) 49093672Sarr 49167352Sjhb/* 49272200Sbmilekic * The INVARIANTS-enabled mtx_assert() functionality. 49373372Sbmilekic * 49473372Sbmilekic * The constants need to be defined for INVARIANT_SUPPORT infrastructure 49573372Sbmilekic * support as _mtx_assert() itself uses them and the latter implies that 49673372Sbmilekic * _mtx_assert() must build. 49767352Sjhb */ 498142821Sjhb#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) 499173759Sattilio#define MA_OWNED LA_XLOCKED 500173759Sattilio#define MA_NOTOWNED LA_UNLOCKED 501173759Sattilio#define MA_RECURSED LA_RECURSED 502173759Sattilio#define MA_NOTRECURSED LA_NOTRECURSED 503142821Sjhb#endif 50473372Sbmilekic 50593813Sjhb/* 50693813Sjhb * Common lock type names. 50793813Sjhb */ 50893813Sjhb#define MTX_NETWORK_LOCK "network driver" 50993813Sjhb 51067352Sjhb#endif /* _KERNEL */ 51167352Sjhb#endif /* _SYS_MUTEX_H_ */ 512