kern_umtx.c revision 179970
1139804Simp/*- 2139013Sdavidxu * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 3112904Sjeff * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 4112904Sjeff * All rights reserved. 5112904Sjeff * 6112904Sjeff * Redistribution and use in source and binary forms, with or without 7112904Sjeff * modification, are permitted provided that the following conditions 8112904Sjeff * are met: 9112904Sjeff * 1. Redistributions of source code must retain the above copyright 10112904Sjeff * notice unmodified, this list of conditions, and the following 11112904Sjeff * disclaimer. 12112904Sjeff * 2. Redistributions in binary form must reproduce the above copyright 13112904Sjeff * notice, this list of conditions and the following disclaimer in the 14112904Sjeff * documentation and/or other materials provided with the distribution. 15112904Sjeff * 16112904Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17112904Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18112904Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19112904Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20112904Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21112904Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22112904Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23112904Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24112904Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25112904Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26112904Sjeff */ 27112904Sjeff 28116182Sobrien#include <sys/cdefs.h> 29116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_umtx.c 179970 2008-06-24 07:32:12Z davidxu $"); 30116182Sobrien 31162536Sdavidxu#include "opt_compat.h" 32112904Sjeff#include <sys/param.h> 33112904Sjeff#include <sys/kernel.h> 34131431Smarcel#include <sys/limits.h> 35112904Sjeff#include <sys/lock.h> 36115765Sjeff#include <sys/malloc.h> 37112904Sjeff#include <sys/mutex.h> 38164033Srwatson#include <sys/priv.h> 39112904Sjeff#include <sys/proc.h> 40161678Sdavidxu#include <sys/sched.h> 41165369Sdavidxu#include <sys/smp.h> 42161678Sdavidxu#include <sys/sysctl.h> 43112904Sjeff#include <sys/sysent.h> 44112904Sjeff#include <sys/systm.h> 45112904Sjeff#include <sys/sysproto.h> 46139013Sdavidxu#include <sys/eventhandler.h> 47112904Sjeff#include <sys/umtx.h> 48112904Sjeff 49139013Sdavidxu#include <vm/vm.h> 50139013Sdavidxu#include <vm/vm_param.h> 51139013Sdavidxu#include <vm/pmap.h> 52139013Sdavidxu#include <vm/vm_map.h> 53139013Sdavidxu#include <vm/vm_object.h> 54139013Sdavidxu 55165369Sdavidxu#include <machine/cpu.h> 56165369Sdavidxu 57162536Sdavidxu#ifdef COMPAT_IA32 58162536Sdavidxu#include <compat/freebsd32/freebsd32_proto.h> 59162536Sdavidxu#endif 60162536Sdavidxu 61179421Sdavidxu#define TYPE_SIMPLE_WAIT 0 62179421Sdavidxu#define TYPE_CV 1 63179421Sdavidxu#define TYPE_SIMPLE_LOCK 2 64179421Sdavidxu#define TYPE_NORMAL_UMUTEX 3 65179421Sdavidxu#define TYPE_PI_UMUTEX 4 66179421Sdavidxu#define TYPE_PP_UMUTEX 5 67177848Sdavidxu#define TYPE_RWLOCK 6 68139013Sdavidxu 69179970Sdavidxu#define _UMUTEX_TRY 1 70179970Sdavidxu#define _UMUTEX_WAIT 2 71179970Sdavidxu 72161678Sdavidxu/* Key to represent a unique userland synchronous object */ 73139013Sdavidxustruct umtx_key { 74161678Sdavidxu int hash; 75139013Sdavidxu int type; 76161678Sdavidxu int shared; 77139013Sdavidxu union { 78139013Sdavidxu struct { 79139013Sdavidxu vm_object_t object; 80161678Sdavidxu uintptr_t offset; 81139013Sdavidxu } shared; 82139013Sdavidxu struct { 83161678Sdavidxu struct vmspace *vs; 84161678Sdavidxu uintptr_t addr; 85139013Sdavidxu } private; 86139013Sdavidxu struct { 87161678Sdavidxu void *a; 88161678Sdavidxu uintptr_t b; 89139013Sdavidxu } both; 90139013Sdavidxu } info; 91139013Sdavidxu}; 92139013Sdavidxu 93161678Sdavidxu/* Priority inheritance mutex info. */ 94161678Sdavidxustruct umtx_pi { 95161678Sdavidxu /* Owner thread */ 96161678Sdavidxu struct thread *pi_owner; 97161678Sdavidxu 98161678Sdavidxu /* Reference count */ 99161678Sdavidxu int pi_refcount; 100161678Sdavidxu 101161678Sdavidxu /* List entry to link umtx holding by thread */ 102161678Sdavidxu TAILQ_ENTRY(umtx_pi) pi_link; 103161678Sdavidxu 104161678Sdavidxu /* List entry in hash */ 105161678Sdavidxu TAILQ_ENTRY(umtx_pi) pi_hashlink; 106161678Sdavidxu 107161678Sdavidxu /* List for waiters */ 108161678Sdavidxu TAILQ_HEAD(,umtx_q) pi_blocked; 109161678Sdavidxu 110161678Sdavidxu /* Identify a userland lock object */ 111161678Sdavidxu struct umtx_key pi_key; 112161678Sdavidxu}; 113161678Sdavidxu 114161678Sdavidxu/* A userland synchronous object user. */ 115115765Sjeffstruct umtx_q { 116161678Sdavidxu /* Linked list for the hash. */ 117161678Sdavidxu TAILQ_ENTRY(umtx_q) uq_link; 118161678Sdavidxu 119161678Sdavidxu /* Umtx key. */ 120161678Sdavidxu struct umtx_key uq_key; 121161678Sdavidxu 122161678Sdavidxu /* Umtx flags. */ 123161678Sdavidxu int uq_flags; 124161678Sdavidxu#define UQF_UMTXQ 0x0001 125161678Sdavidxu 126161678Sdavidxu /* The thread waits on. */ 127161678Sdavidxu struct thread *uq_thread; 128161678Sdavidxu 129161678Sdavidxu /* 130161678Sdavidxu * Blocked on PI mutex. read can use chain lock 131170300Sjeff * or umtx_lock, write must have both chain lock and 132170300Sjeff * umtx_lock being hold. 133161678Sdavidxu */ 134161678Sdavidxu struct umtx_pi *uq_pi_blocked; 135161678Sdavidxu 136161678Sdavidxu /* On blocked list */ 137161678Sdavidxu TAILQ_ENTRY(umtx_q) uq_lockq; 138161678Sdavidxu 139161678Sdavidxu /* Thread contending with us */ 140161678Sdavidxu TAILQ_HEAD(,umtx_pi) uq_pi_contested; 141161678Sdavidxu 142161742Sdavidxu /* Inherited priority from PP mutex */ 143161678Sdavidxu u_char uq_inherited_pri; 144115765Sjeff}; 145115765Sjeff 146161678SdavidxuTAILQ_HEAD(umtxq_head, umtx_q); 147161678Sdavidxu 148161678Sdavidxu/* Userland lock object's wait-queue chain */ 149138224Sdavidxustruct umtxq_chain { 150161678Sdavidxu /* Lock for this chain. */ 151161678Sdavidxu struct mtx uc_lock; 152161678Sdavidxu 153161678Sdavidxu /* List of sleep queues. */ 154177848Sdavidxu struct umtxq_head uc_queue[2]; 155177848Sdavidxu#define UMTX_SHARED_QUEUE 0 156177848Sdavidxu#define UMTX_EXCLUSIVE_QUEUE 1 157161678Sdavidxu 158161678Sdavidxu /* Busy flag */ 159161678Sdavidxu char uc_busy; 160161678Sdavidxu 161161678Sdavidxu /* Chain lock waiters */ 162158377Sdavidxu int uc_waiters; 163161678Sdavidxu 164161678Sdavidxu /* All PI in the list */ 165161678Sdavidxu TAILQ_HEAD(,umtx_pi) uc_pi_list; 166138224Sdavidxu}; 167115765Sjeff 168161678Sdavidxu#define UMTXQ_LOCKED_ASSERT(uc) mtx_assert(&(uc)->uc_lock, MA_OWNED) 169161678Sdavidxu 170161678Sdavidxu/* 171161678Sdavidxu * Don't propagate time-sharing priority, there is a security reason, 172161678Sdavidxu * a user can simply introduce PI-mutex, let thread A lock the mutex, 173161678Sdavidxu * and let another thread B block on the mutex, because B is 174161678Sdavidxu * sleeping, its priority will be boosted, this causes A's priority to 175161678Sdavidxu * be boosted via priority propagating too and will never be lowered even 176161678Sdavidxu * if it is using 100%CPU, this is unfair to other processes. 177161678Sdavidxu */ 178161678Sdavidxu 179163709Sjb#define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ 180163709Sjb (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ 181163709Sjb PRI_MAX_TIMESHARE : (td)->td_user_pri) 182161678Sdavidxu 183138224Sdavidxu#define GOLDEN_RATIO_PRIME 2654404609U 184138224Sdavidxu#define UMTX_CHAINS 128 185138224Sdavidxu#define UMTX_SHIFTS (__WORD_BIT - 7) 186115765Sjeff 187161678Sdavidxu#define THREAD_SHARE 0 188161678Sdavidxu#define PROCESS_SHARE 1 189161678Sdavidxu#define AUTO_SHARE 2 190161678Sdavidxu 191161678Sdavidxu#define GET_SHARE(flags) \ 192161678Sdavidxu (((flags) & USYNC_PROCESS_SHARED) == 0 ? THREAD_SHARE : PROCESS_SHARE) 193161678Sdavidxu 194177848Sdavidxu#define BUSY_SPINS 200 195177848Sdavidxu 196161678Sdavidxustatic uma_zone_t umtx_pi_zone; 197179421Sdavidxustatic struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; 198138224Sdavidxustatic MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 199161678Sdavidxustatic int umtx_pi_allocated; 200115310Sjeff 201161678SdavidxuSYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); 202161678SdavidxuSYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, 203161678Sdavidxu &umtx_pi_allocated, 0, "Allocated umtx_pi"); 204161678Sdavidxu 205161678Sdavidxustatic void umtxq_sysinit(void *); 206161678Sdavidxustatic void umtxq_hash(struct umtx_key *key); 207161678Sdavidxustatic struct umtxq_chain *umtxq_getchain(struct umtx_key *key); 208139013Sdavidxustatic void umtxq_lock(struct umtx_key *key); 209139013Sdavidxustatic void umtxq_unlock(struct umtx_key *key); 210139257Sdavidxustatic void umtxq_busy(struct umtx_key *key); 211139257Sdavidxustatic void umtxq_unbusy(struct umtx_key *key); 212177848Sdavidxustatic void umtxq_insert_queue(struct umtx_q *uq, int q); 213177848Sdavidxustatic void umtxq_remove_queue(struct umtx_q *uq, int q); 214161678Sdavidxustatic int umtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo); 215139257Sdavidxustatic int umtxq_count(struct umtx_key *key); 216139013Sdavidxustatic int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2); 217161678Sdavidxustatic int umtx_key_get(void *addr, int type, int share, 218139013Sdavidxu struct umtx_key *key); 219139013Sdavidxustatic void umtx_key_release(struct umtx_key *key); 220163697Sdavidxustatic struct umtx_pi *umtx_pi_alloc(int); 221161678Sdavidxustatic void umtx_pi_free(struct umtx_pi *pi); 222174701Sdavidxustatic void umtx_pi_adjust_locked(struct thread *td, u_char oldpri); 223161678Sdavidxustatic int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); 224161678Sdavidxustatic void umtx_thread_cleanup(struct thread *td); 225161678Sdavidxustatic void umtx_exec_hook(void *arg __unused, struct proc *p __unused, 226161678Sdavidxu struct image_params *imgp __unused); 227161678SdavidxuSYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); 228115310Sjeff 229177848Sdavidxu#define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) 230177848Sdavidxu#define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) 231177848Sdavidxu#define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) 232177848Sdavidxu 233170300Sjeffstatic struct mtx umtx_lock; 234170300Sjeff 235161678Sdavidxustatic void 236161678Sdavidxuumtxq_sysinit(void *arg __unused) 237161678Sdavidxu{ 238179421Sdavidxu int i, j; 239138224Sdavidxu 240161678Sdavidxu umtx_pi_zone = uma_zcreate("umtx pi", sizeof(struct umtx_pi), 241161678Sdavidxu NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 242179421Sdavidxu for (i = 0; i < 2; ++i) { 243179421Sdavidxu for (j = 0; j < UMTX_CHAINS; ++j) { 244179421Sdavidxu mtx_init(&umtxq_chains[i][j].uc_lock, "umtxql", NULL, 245179421Sdavidxu MTX_DEF | MTX_DUPOK); 246179421Sdavidxu TAILQ_INIT(&umtxq_chains[i][j].uc_queue[0]); 247179421Sdavidxu TAILQ_INIT(&umtxq_chains[i][j].uc_queue[1]); 248179421Sdavidxu TAILQ_INIT(&umtxq_chains[i][j].uc_pi_list); 249179421Sdavidxu umtxq_chains[i][j].uc_busy = 0; 250179421Sdavidxu umtxq_chains[i][j].uc_waiters = 0; 251179421Sdavidxu } 252161678Sdavidxu } 253170300Sjeff mtx_init(&umtx_lock, "umtx lock", NULL, MTX_SPIN); 254161678Sdavidxu EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, 255161678Sdavidxu EVENTHANDLER_PRI_ANY); 256161678Sdavidxu} 257161678Sdavidxu 258143149Sdavidxustruct umtx_q * 259143149Sdavidxuumtxq_alloc(void) 260143149Sdavidxu{ 261161678Sdavidxu struct umtx_q *uq; 262161678Sdavidxu 263161678Sdavidxu uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); 264161678Sdavidxu TAILQ_INIT(&uq->uq_pi_contested); 265161678Sdavidxu uq->uq_inherited_pri = PRI_MAX; 266161678Sdavidxu return (uq); 267143149Sdavidxu} 268143149Sdavidxu 269143149Sdavidxuvoid 270143149Sdavidxuumtxq_free(struct umtx_q *uq) 271143149Sdavidxu{ 272143149Sdavidxu free(uq, M_UMTX); 273143149Sdavidxu} 274143149Sdavidxu 275161678Sdavidxustatic inline void 276139013Sdavidxuumtxq_hash(struct umtx_key *key) 277138224Sdavidxu{ 278161678Sdavidxu unsigned n = (uintptr_t)key->info.both.a + key->info.both.b; 279161678Sdavidxu key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; 280138224Sdavidxu} 281138224Sdavidxu 282139013Sdavidxustatic inline int 283139013Sdavidxuumtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 284139013Sdavidxu{ 285139013Sdavidxu return (k1->type == k2->type && 286161678Sdavidxu k1->info.both.a == k2->info.both.a && 287161678Sdavidxu k1->info.both.b == k2->info.both.b); 288139013Sdavidxu} 289139013Sdavidxu 290161678Sdavidxustatic inline struct umtxq_chain * 291161678Sdavidxuumtxq_getchain(struct umtx_key *key) 292139013Sdavidxu{ 293179421Sdavidxu if (key->type <= TYPE_CV) 294179421Sdavidxu return (&umtxq_chains[1][key->hash]); 295179421Sdavidxu return (&umtxq_chains[0][key->hash]); 296139013Sdavidxu} 297139013Sdavidxu 298161678Sdavidxu/* 299177848Sdavidxu * Lock a chain. 300161678Sdavidxu */ 301138224Sdavidxustatic inline void 302177848Sdavidxuumtxq_lock(struct umtx_key *key) 303139257Sdavidxu{ 304161678Sdavidxu struct umtxq_chain *uc; 305139257Sdavidxu 306161678Sdavidxu uc = umtxq_getchain(key); 307177848Sdavidxu mtx_lock(&uc->uc_lock); 308139257Sdavidxu} 309139257Sdavidxu 310161678Sdavidxu/* 311177848Sdavidxu * Unlock a chain. 312161678Sdavidxu */ 313139257Sdavidxustatic inline void 314177848Sdavidxuumtxq_unlock(struct umtx_key *key) 315139257Sdavidxu{ 316161678Sdavidxu struct umtxq_chain *uc; 317139257Sdavidxu 318161678Sdavidxu uc = umtxq_getchain(key); 319177848Sdavidxu mtx_unlock(&uc->uc_lock); 320139257Sdavidxu} 321139257Sdavidxu 322161678Sdavidxu/* 323177848Sdavidxu * Set chain to busy state when following operation 324177848Sdavidxu * may be blocked (kernel mutex can not be used). 325161678Sdavidxu */ 326139257Sdavidxustatic inline void 327177848Sdavidxuumtxq_busy(struct umtx_key *key) 328138224Sdavidxu{ 329161678Sdavidxu struct umtxq_chain *uc; 330161678Sdavidxu 331161678Sdavidxu uc = umtxq_getchain(key); 332177848Sdavidxu mtx_assert(&uc->uc_lock, MA_OWNED); 333177848Sdavidxu if (uc->uc_busy) { 334177880Sdavidxu#ifdef SMP 335177880Sdavidxu if (smp_cpus > 1) { 336177880Sdavidxu int count = BUSY_SPINS; 337177880Sdavidxu if (count > 0) { 338177880Sdavidxu umtxq_unlock(key); 339177880Sdavidxu while (uc->uc_busy && --count > 0) 340177880Sdavidxu cpu_spinwait(); 341177880Sdavidxu umtxq_lock(key); 342177880Sdavidxu } 343177848Sdavidxu } 344177880Sdavidxu#endif 345177880Sdavidxu while (uc->uc_busy) { 346177848Sdavidxu uc->uc_waiters++; 347177848Sdavidxu msleep(uc, &uc->uc_lock, 0, "umtxqb", 0); 348177848Sdavidxu uc->uc_waiters--; 349177848Sdavidxu } 350177848Sdavidxu } 351177848Sdavidxu uc->uc_busy = 1; 352138224Sdavidxu} 353138224Sdavidxu 354161678Sdavidxu/* 355177848Sdavidxu * Unbusy a chain. 356161678Sdavidxu */ 357138225Sdavidxustatic inline void 358177848Sdavidxuumtxq_unbusy(struct umtx_key *key) 359138224Sdavidxu{ 360161678Sdavidxu struct umtxq_chain *uc; 361161678Sdavidxu 362161678Sdavidxu uc = umtxq_getchain(key); 363177848Sdavidxu mtx_assert(&uc->uc_lock, MA_OWNED); 364177848Sdavidxu KASSERT(uc->uc_busy != 0, ("not busy")); 365177848Sdavidxu uc->uc_busy = 0; 366177848Sdavidxu if (uc->uc_waiters) 367177848Sdavidxu wakeup_one(uc); 368138224Sdavidxu} 369138224Sdavidxu 370139013Sdavidxustatic inline void 371177848Sdavidxuumtxq_insert_queue(struct umtx_q *uq, int q) 372115765Sjeff{ 373161678Sdavidxu struct umtxq_chain *uc; 374139013Sdavidxu 375161678Sdavidxu uc = umtxq_getchain(&uq->uq_key); 376161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 377177848Sdavidxu TAILQ_INSERT_TAIL(&uc->uc_queue[q], uq, uq_link); 378158718Sdavidxu uq->uq_flags |= UQF_UMTXQ; 379139013Sdavidxu} 380139013Sdavidxu 381139013Sdavidxustatic inline void 382177848Sdavidxuumtxq_remove_queue(struct umtx_q *uq, int q) 383139013Sdavidxu{ 384161678Sdavidxu struct umtxq_chain *uc; 385161678Sdavidxu 386161678Sdavidxu uc = umtxq_getchain(&uq->uq_key); 387161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 388158718Sdavidxu if (uq->uq_flags & UQF_UMTXQ) { 389177848Sdavidxu TAILQ_REMOVE(&uc->uc_queue[q], uq, uq_link); 390158718Sdavidxu uq->uq_flags &= ~UQF_UMTXQ; 391139013Sdavidxu } 392139013Sdavidxu} 393139013Sdavidxu 394161678Sdavidxu/* 395161678Sdavidxu * Check if there are multiple waiters 396161678Sdavidxu */ 397139013Sdavidxustatic int 398139013Sdavidxuumtxq_count(struct umtx_key *key) 399139013Sdavidxu{ 400161678Sdavidxu struct umtxq_chain *uc; 401115765Sjeff struct umtx_q *uq; 402161678Sdavidxu int count = 0; 403115765Sjeff 404161678Sdavidxu uc = umtxq_getchain(key); 405161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 406177848Sdavidxu TAILQ_FOREACH(uq, &uc->uc_queue[UMTX_SHARED_QUEUE], uq_link) { 407139013Sdavidxu if (umtx_key_match(&uq->uq_key, key)) { 408139013Sdavidxu if (++count > 1) 409139013Sdavidxu break; 410139013Sdavidxu } 411115765Sjeff } 412139013Sdavidxu return (count); 413115765Sjeff} 414115765Sjeff 415161678Sdavidxu/* 416161678Sdavidxu * Check if there are multiple PI waiters and returns first 417161678Sdavidxu * waiter. 418161678Sdavidxu */ 419139257Sdavidxustatic int 420161678Sdavidxuumtxq_count_pi(struct umtx_key *key, struct umtx_q **first) 421161678Sdavidxu{ 422161678Sdavidxu struct umtxq_chain *uc; 423161678Sdavidxu struct umtx_q *uq; 424161678Sdavidxu int count = 0; 425161678Sdavidxu 426161678Sdavidxu *first = NULL; 427161678Sdavidxu uc = umtxq_getchain(key); 428161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 429177848Sdavidxu TAILQ_FOREACH(uq, &uc->uc_queue[UMTX_SHARED_QUEUE], uq_link) { 430161678Sdavidxu if (umtx_key_match(&uq->uq_key, key)) { 431161678Sdavidxu if (++count > 1) 432161678Sdavidxu break; 433161678Sdavidxu *first = uq; 434161678Sdavidxu } 435161678Sdavidxu } 436161678Sdavidxu return (count); 437161678Sdavidxu} 438161678Sdavidxu 439161678Sdavidxu/* 440161678Sdavidxu * Wake up threads waiting on an userland object. 441161678Sdavidxu */ 442177848Sdavidxu 443161678Sdavidxustatic int 444177848Sdavidxuumtxq_signal_queue(struct umtx_key *key, int n_wake, int q) 445115765Sjeff{ 446161678Sdavidxu struct umtxq_chain *uc; 447139257Sdavidxu struct umtx_q *uq, *next; 448161678Sdavidxu int ret; 449115765Sjeff 450139257Sdavidxu ret = 0; 451161678Sdavidxu uc = umtxq_getchain(key); 452161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 453177848Sdavidxu TAILQ_FOREACH_SAFE(uq, &uc->uc_queue[q], uq_link, next) { 454139013Sdavidxu if (umtx_key_match(&uq->uq_key, key)) { 455177848Sdavidxu umtxq_remove_queue(uq, q); 456161678Sdavidxu wakeup(uq); 457139257Sdavidxu if (++ret >= n_wake) 458139257Sdavidxu break; 459139013Sdavidxu } 460139013Sdavidxu } 461139257Sdavidxu return (ret); 462138224Sdavidxu} 463138224Sdavidxu 464177848Sdavidxu 465161678Sdavidxu/* 466161678Sdavidxu * Wake up specified thread. 467161678Sdavidxu */ 468161678Sdavidxustatic inline void 469161678Sdavidxuumtxq_signal_thread(struct umtx_q *uq) 470161678Sdavidxu{ 471161678Sdavidxu struct umtxq_chain *uc; 472161678Sdavidxu 473161678Sdavidxu uc = umtxq_getchain(&uq->uq_key); 474161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 475161678Sdavidxu umtxq_remove(uq); 476161678Sdavidxu wakeup(uq); 477161678Sdavidxu} 478161678Sdavidxu 479161678Sdavidxu/* 480161678Sdavidxu * Put thread into sleep state, before sleeping, check if 481161678Sdavidxu * thread was removed from umtx queue. 482161678Sdavidxu */ 483138224Sdavidxustatic inline int 484161678Sdavidxuumtxq_sleep(struct umtx_q *uq, const char *wmesg, int timo) 485138224Sdavidxu{ 486161678Sdavidxu struct umtxq_chain *uc; 487161678Sdavidxu int error; 488161678Sdavidxu 489161678Sdavidxu uc = umtxq_getchain(&uq->uq_key); 490161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 491161678Sdavidxu if (!(uq->uq_flags & UQF_UMTXQ)) 492161678Sdavidxu return (0); 493161678Sdavidxu error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); 494139751Sdavidxu if (error == EWOULDBLOCK) 495139751Sdavidxu error = ETIMEDOUT; 496139751Sdavidxu return (error); 497138224Sdavidxu} 498138224Sdavidxu 499161678Sdavidxu/* 500161678Sdavidxu * Convert userspace address into unique logical address. 501161678Sdavidxu */ 502139013Sdavidxustatic int 503161678Sdavidxuumtx_key_get(void *addr, int type, int share, struct umtx_key *key) 504139013Sdavidxu{ 505161678Sdavidxu struct thread *td = curthread; 506139013Sdavidxu vm_map_t map; 507139013Sdavidxu vm_map_entry_t entry; 508139013Sdavidxu vm_pindex_t pindex; 509139013Sdavidxu vm_prot_t prot; 510139013Sdavidxu boolean_t wired; 511139013Sdavidxu 512161678Sdavidxu key->type = type; 513161678Sdavidxu if (share == THREAD_SHARE) { 514161678Sdavidxu key->shared = 0; 515161678Sdavidxu key->info.private.vs = td->td_proc->p_vmspace; 516161678Sdavidxu key->info.private.addr = (uintptr_t)addr; 517163677Sdavidxu } else { 518163677Sdavidxu MPASS(share == PROCESS_SHARE || share == AUTO_SHARE); 519161678Sdavidxu map = &td->td_proc->p_vmspace->vm_map; 520161678Sdavidxu if (vm_map_lookup(&map, (vm_offset_t)addr, VM_PROT_WRITE, 521161678Sdavidxu &entry, &key->info.shared.object, &pindex, &prot, 522161678Sdavidxu &wired) != KERN_SUCCESS) { 523161678Sdavidxu return EFAULT; 524161678Sdavidxu } 525161678Sdavidxu 526161678Sdavidxu if ((share == PROCESS_SHARE) || 527161678Sdavidxu (share == AUTO_SHARE && 528161678Sdavidxu VM_INHERIT_SHARE == entry->inheritance)) { 529161678Sdavidxu key->shared = 1; 530161678Sdavidxu key->info.shared.offset = entry->offset + entry->start - 531161678Sdavidxu (vm_offset_t)addr; 532161678Sdavidxu vm_object_reference(key->info.shared.object); 533161678Sdavidxu } else { 534161678Sdavidxu key->shared = 0; 535161678Sdavidxu key->info.private.vs = td->td_proc->p_vmspace; 536161678Sdavidxu key->info.private.addr = (uintptr_t)addr; 537161678Sdavidxu } 538161678Sdavidxu vm_map_lookup_done(map, entry); 539139013Sdavidxu } 540139013Sdavidxu 541161678Sdavidxu umtxq_hash(key); 542139013Sdavidxu return (0); 543139013Sdavidxu} 544139013Sdavidxu 545161678Sdavidxu/* 546161678Sdavidxu * Release key. 547161678Sdavidxu */ 548139013Sdavidxustatic inline void 549139013Sdavidxuumtx_key_release(struct umtx_key *key) 550139013Sdavidxu{ 551161678Sdavidxu if (key->shared) 552139013Sdavidxu vm_object_deallocate(key->info.shared.object); 553139013Sdavidxu} 554139013Sdavidxu 555161678Sdavidxu/* 556161678Sdavidxu * Lock a umtx object. 557161678Sdavidxu */ 558139013Sdavidxustatic int 559163449Sdavidxu_do_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, int timo) 560112904Sjeff{ 561143149Sdavidxu struct umtx_q *uq; 562163449Sdavidxu u_long owner; 563163449Sdavidxu u_long old; 564138224Sdavidxu int error = 0; 565112904Sjeff 566143149Sdavidxu uq = td->td_umtxq; 567161678Sdavidxu 568112904Sjeff /* 569161678Sdavidxu * Care must be exercised when dealing with umtx structure. It 570112904Sjeff * can fault on any access. 571112904Sjeff */ 572112904Sjeff for (;;) { 573112904Sjeff /* 574112904Sjeff * Try the uncontested case. This should be done in userland. 575112904Sjeff */ 576163449Sdavidxu owner = casuword(&umtx->u_owner, UMTX_UNOWNED, id); 577112904Sjeff 578138224Sdavidxu /* The acquire succeeded. */ 579138224Sdavidxu if (owner == UMTX_UNOWNED) 580138224Sdavidxu return (0); 581138224Sdavidxu 582115765Sjeff /* The address was invalid. */ 583115765Sjeff if (owner == -1) 584115765Sjeff return (EFAULT); 585115765Sjeff 586115765Sjeff /* If no one owns it but it is contested try to acquire it. */ 587115765Sjeff if (owner == UMTX_CONTESTED) { 588163449Sdavidxu owner = casuword(&umtx->u_owner, 589139013Sdavidxu UMTX_CONTESTED, id | UMTX_CONTESTED); 590115765Sjeff 591138224Sdavidxu if (owner == UMTX_CONTESTED) 592138224Sdavidxu return (0); 593138224Sdavidxu 594115765Sjeff /* The address was invalid. */ 595115765Sjeff if (owner == -1) 596115765Sjeff return (EFAULT); 597115765Sjeff 598115765Sjeff /* If this failed the lock has changed, restart. */ 599115765Sjeff continue; 600112904Sjeff } 601112904Sjeff 602138224Sdavidxu /* 603138224Sdavidxu * If we caught a signal, we have retried and now 604138224Sdavidxu * exit immediately. 605138224Sdavidxu */ 606161678Sdavidxu if (error != 0) 607138224Sdavidxu return (error); 608112904Sjeff 609161678Sdavidxu if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, 610161678Sdavidxu AUTO_SHARE, &uq->uq_key)) != 0) 611161678Sdavidxu return (error); 612161678Sdavidxu 613161678Sdavidxu umtxq_lock(&uq->uq_key); 614161678Sdavidxu umtxq_busy(&uq->uq_key); 615161678Sdavidxu umtxq_insert(uq); 616161678Sdavidxu umtxq_unbusy(&uq->uq_key); 617161678Sdavidxu umtxq_unlock(&uq->uq_key); 618161678Sdavidxu 619112904Sjeff /* 620112904Sjeff * Set the contested bit so that a release in user space 621112904Sjeff * knows to use the system call for unlock. If this fails 622112904Sjeff * either some one else has acquired the lock or it has been 623112904Sjeff * released. 624112904Sjeff */ 625163449Sdavidxu old = casuword(&umtx->u_owner, owner, owner | UMTX_CONTESTED); 626112904Sjeff 627112904Sjeff /* The address was invalid. */ 628112967Sjake if (old == -1) { 629143149Sdavidxu umtxq_lock(&uq->uq_key); 630143149Sdavidxu umtxq_remove(uq); 631143149Sdavidxu umtxq_unlock(&uq->uq_key); 632143149Sdavidxu umtx_key_release(&uq->uq_key); 633115765Sjeff return (EFAULT); 634112904Sjeff } 635112904Sjeff 636112904Sjeff /* 637115765Sjeff * We set the contested bit, sleep. Otherwise the lock changed 638117685Smtm * and we need to retry or we lost a race to the thread 639117685Smtm * unlocking the umtx. 640112904Sjeff */ 641143149Sdavidxu umtxq_lock(&uq->uq_key); 642161678Sdavidxu if (old == owner) 643161678Sdavidxu error = umtxq_sleep(uq, "umtx", timo); 644143149Sdavidxu umtxq_remove(uq); 645143149Sdavidxu umtxq_unlock(&uq->uq_key); 646143149Sdavidxu umtx_key_release(&uq->uq_key); 647112904Sjeff } 648117743Smtm 649117743Smtm return (0); 650112904Sjeff} 651112904Sjeff 652161678Sdavidxu/* 653161678Sdavidxu * Lock a umtx object. 654161678Sdavidxu */ 655139013Sdavidxustatic int 656163449Sdavidxudo_lock_umtx(struct thread *td, struct umtx *umtx, u_long id, 657140245Sdavidxu struct timespec *timeout) 658112904Sjeff{ 659140245Sdavidxu struct timespec ts, ts2, ts3; 660139013Sdavidxu struct timeval tv; 661140245Sdavidxu int error; 662139013Sdavidxu 663140245Sdavidxu if (timeout == NULL) { 664162536Sdavidxu error = _do_lock_umtx(td, umtx, id, 0); 665162030Sdavidxu /* Mutex locking is restarted if it is interrupted. */ 666162030Sdavidxu if (error == EINTR) 667162030Sdavidxu error = ERESTART; 668139013Sdavidxu } else { 669140245Sdavidxu getnanouptime(&ts); 670140245Sdavidxu timespecadd(&ts, timeout); 671140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 672139013Sdavidxu for (;;) { 673162536Sdavidxu error = _do_lock_umtx(td, umtx, id, tvtohz(&tv)); 674140245Sdavidxu if (error != ETIMEDOUT) 675140245Sdavidxu break; 676140245Sdavidxu getnanouptime(&ts2); 677140245Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 678139751Sdavidxu error = ETIMEDOUT; 679139013Sdavidxu break; 680139013Sdavidxu } 681140245Sdavidxu ts3 = ts; 682140245Sdavidxu timespecsub(&ts3, &ts2); 683140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 684139013Sdavidxu } 685162030Sdavidxu /* Timed-locking is not restarted. */ 686162030Sdavidxu if (error == ERESTART) 687162030Sdavidxu error = EINTR; 688139013Sdavidxu } 689139013Sdavidxu return (error); 690139013Sdavidxu} 691139013Sdavidxu 692161678Sdavidxu/* 693161678Sdavidxu * Unlock a umtx object. 694161678Sdavidxu */ 695139013Sdavidxustatic int 696163449Sdavidxudo_unlock_umtx(struct thread *td, struct umtx *umtx, u_long id) 697139013Sdavidxu{ 698139013Sdavidxu struct umtx_key key; 699163449Sdavidxu u_long owner; 700163449Sdavidxu u_long old; 701139257Sdavidxu int error; 702139257Sdavidxu int count; 703112904Sjeff 704112904Sjeff /* 705112904Sjeff * Make sure we own this mtx. 706112904Sjeff */ 707163449Sdavidxu owner = fuword(__DEVOLATILE(u_long *, &umtx->u_owner)); 708161678Sdavidxu if (owner == -1) 709115765Sjeff return (EFAULT); 710115765Sjeff 711139013Sdavidxu if ((owner & ~UMTX_CONTESTED) != id) 712115765Sjeff return (EPERM); 713112904Sjeff 714161678Sdavidxu /* This should be done in userland */ 715161678Sdavidxu if ((owner & UMTX_CONTESTED) == 0) { 716163449Sdavidxu old = casuword(&umtx->u_owner, owner, UMTX_UNOWNED); 717161678Sdavidxu if (old == -1) 718161678Sdavidxu return (EFAULT); 719161678Sdavidxu if (old == owner) 720161678Sdavidxu return (0); 721161855Sdavidxu owner = old; 722161678Sdavidxu } 723161678Sdavidxu 724117685Smtm /* We should only ever be in here for contested locks */ 725161678Sdavidxu if ((error = umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, 726161678Sdavidxu &key)) != 0) 727139257Sdavidxu return (error); 728139257Sdavidxu 729139257Sdavidxu umtxq_lock(&key); 730139257Sdavidxu umtxq_busy(&key); 731139257Sdavidxu count = umtxq_count(&key); 732139257Sdavidxu umtxq_unlock(&key); 733139257Sdavidxu 734117743Smtm /* 735117743Smtm * When unlocking the umtx, it must be marked as unowned if 736117743Smtm * there is zero or one thread only waiting for it. 737117743Smtm * Otherwise, it must be marked as contested. 738117743Smtm */ 739163449Sdavidxu old = casuword(&umtx->u_owner, owner, 740163449Sdavidxu count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); 741139257Sdavidxu umtxq_lock(&key); 742161678Sdavidxu umtxq_signal(&key,1); 743139257Sdavidxu umtxq_unbusy(&key); 744139257Sdavidxu umtxq_unlock(&key); 745139257Sdavidxu umtx_key_release(&key); 746115765Sjeff if (old == -1) 747115765Sjeff return (EFAULT); 748138224Sdavidxu if (old != owner) 749138224Sdavidxu return (EINVAL); 750115765Sjeff return (0); 751112904Sjeff} 752139013Sdavidxu 753162536Sdavidxu#ifdef COMPAT_IA32 754162536Sdavidxu 755161678Sdavidxu/* 756162536Sdavidxu * Lock a umtx object. 757162536Sdavidxu */ 758162536Sdavidxustatic int 759162536Sdavidxu_do_lock_umtx32(struct thread *td, uint32_t *m, uint32_t id, int timo) 760162536Sdavidxu{ 761162536Sdavidxu struct umtx_q *uq; 762162536Sdavidxu uint32_t owner; 763162536Sdavidxu uint32_t old; 764162536Sdavidxu int error = 0; 765162536Sdavidxu 766162536Sdavidxu uq = td->td_umtxq; 767162536Sdavidxu 768162536Sdavidxu /* 769162536Sdavidxu * Care must be exercised when dealing with umtx structure. It 770162536Sdavidxu * can fault on any access. 771162536Sdavidxu */ 772162536Sdavidxu for (;;) { 773162536Sdavidxu /* 774162536Sdavidxu * Try the uncontested case. This should be done in userland. 775162536Sdavidxu */ 776162536Sdavidxu owner = casuword32(m, UMUTEX_UNOWNED, id); 777162536Sdavidxu 778162536Sdavidxu /* The acquire succeeded. */ 779162536Sdavidxu if (owner == UMUTEX_UNOWNED) 780162536Sdavidxu return (0); 781162536Sdavidxu 782162536Sdavidxu /* The address was invalid. */ 783162536Sdavidxu if (owner == -1) 784162536Sdavidxu return (EFAULT); 785162536Sdavidxu 786162536Sdavidxu /* If no one owns it but it is contested try to acquire it. */ 787162536Sdavidxu if (owner == UMUTEX_CONTESTED) { 788162536Sdavidxu owner = casuword32(m, 789162536Sdavidxu UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 790162536Sdavidxu if (owner == UMUTEX_CONTESTED) 791162536Sdavidxu return (0); 792162536Sdavidxu 793162536Sdavidxu /* The address was invalid. */ 794162536Sdavidxu if (owner == -1) 795162536Sdavidxu return (EFAULT); 796162536Sdavidxu 797162536Sdavidxu /* If this failed the lock has changed, restart. */ 798162536Sdavidxu continue; 799162536Sdavidxu } 800162536Sdavidxu 801162536Sdavidxu /* 802162536Sdavidxu * If we caught a signal, we have retried and now 803162536Sdavidxu * exit immediately. 804162536Sdavidxu */ 805162536Sdavidxu if (error != 0) 806162536Sdavidxu return (error); 807162536Sdavidxu 808162536Sdavidxu if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, 809162536Sdavidxu AUTO_SHARE, &uq->uq_key)) != 0) 810162536Sdavidxu return (error); 811162536Sdavidxu 812162536Sdavidxu umtxq_lock(&uq->uq_key); 813162536Sdavidxu umtxq_busy(&uq->uq_key); 814162536Sdavidxu umtxq_insert(uq); 815162536Sdavidxu umtxq_unbusy(&uq->uq_key); 816162536Sdavidxu umtxq_unlock(&uq->uq_key); 817162536Sdavidxu 818162536Sdavidxu /* 819162536Sdavidxu * Set the contested bit so that a release in user space 820162536Sdavidxu * knows to use the system call for unlock. If this fails 821162536Sdavidxu * either some one else has acquired the lock or it has been 822162536Sdavidxu * released. 823162536Sdavidxu */ 824162536Sdavidxu old = casuword32(m, owner, owner | UMUTEX_CONTESTED); 825162536Sdavidxu 826162536Sdavidxu /* The address was invalid. */ 827162536Sdavidxu if (old == -1) { 828162536Sdavidxu umtxq_lock(&uq->uq_key); 829162536Sdavidxu umtxq_remove(uq); 830162536Sdavidxu umtxq_unlock(&uq->uq_key); 831162536Sdavidxu umtx_key_release(&uq->uq_key); 832162536Sdavidxu return (EFAULT); 833162536Sdavidxu } 834162536Sdavidxu 835162536Sdavidxu /* 836162536Sdavidxu * We set the contested bit, sleep. Otherwise the lock changed 837162536Sdavidxu * and we need to retry or we lost a race to the thread 838162536Sdavidxu * unlocking the umtx. 839162536Sdavidxu */ 840162536Sdavidxu umtxq_lock(&uq->uq_key); 841162536Sdavidxu if (old == owner) 842162536Sdavidxu error = umtxq_sleep(uq, "umtx", timo); 843162536Sdavidxu umtxq_remove(uq); 844162536Sdavidxu umtxq_unlock(&uq->uq_key); 845162536Sdavidxu umtx_key_release(&uq->uq_key); 846162536Sdavidxu } 847162536Sdavidxu 848162536Sdavidxu return (0); 849162536Sdavidxu} 850162536Sdavidxu 851162536Sdavidxu/* 852162536Sdavidxu * Lock a umtx object. 853162536Sdavidxu */ 854162536Sdavidxustatic int 855162536Sdavidxudo_lock_umtx32(struct thread *td, void *m, uint32_t id, 856162536Sdavidxu struct timespec *timeout) 857162536Sdavidxu{ 858162536Sdavidxu struct timespec ts, ts2, ts3; 859162536Sdavidxu struct timeval tv; 860162536Sdavidxu int error; 861162536Sdavidxu 862162536Sdavidxu if (timeout == NULL) { 863162536Sdavidxu error = _do_lock_umtx32(td, m, id, 0); 864162536Sdavidxu /* Mutex locking is restarted if it is interrupted. */ 865162536Sdavidxu if (error == EINTR) 866162536Sdavidxu error = ERESTART; 867162536Sdavidxu } else { 868162536Sdavidxu getnanouptime(&ts); 869162536Sdavidxu timespecadd(&ts, timeout); 870162536Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 871162536Sdavidxu for (;;) { 872162536Sdavidxu error = _do_lock_umtx32(td, m, id, tvtohz(&tv)); 873162536Sdavidxu if (error != ETIMEDOUT) 874162536Sdavidxu break; 875162536Sdavidxu getnanouptime(&ts2); 876162536Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 877162536Sdavidxu error = ETIMEDOUT; 878162536Sdavidxu break; 879162536Sdavidxu } 880162536Sdavidxu ts3 = ts; 881162536Sdavidxu timespecsub(&ts3, &ts2); 882162536Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 883162536Sdavidxu } 884162536Sdavidxu /* Timed-locking is not restarted. */ 885162536Sdavidxu if (error == ERESTART) 886162536Sdavidxu error = EINTR; 887162536Sdavidxu } 888162536Sdavidxu return (error); 889162536Sdavidxu} 890162536Sdavidxu 891162536Sdavidxu/* 892162536Sdavidxu * Unlock a umtx object. 893162536Sdavidxu */ 894162536Sdavidxustatic int 895162536Sdavidxudo_unlock_umtx32(struct thread *td, uint32_t *m, uint32_t id) 896162536Sdavidxu{ 897162536Sdavidxu struct umtx_key key; 898162536Sdavidxu uint32_t owner; 899162536Sdavidxu uint32_t old; 900162536Sdavidxu int error; 901162536Sdavidxu int count; 902162536Sdavidxu 903162536Sdavidxu /* 904162536Sdavidxu * Make sure we own this mtx. 905162536Sdavidxu */ 906162536Sdavidxu owner = fuword32(m); 907162536Sdavidxu if (owner == -1) 908162536Sdavidxu return (EFAULT); 909162536Sdavidxu 910162536Sdavidxu if ((owner & ~UMUTEX_CONTESTED) != id) 911162536Sdavidxu return (EPERM); 912162536Sdavidxu 913162536Sdavidxu /* This should be done in userland */ 914162536Sdavidxu if ((owner & UMUTEX_CONTESTED) == 0) { 915162536Sdavidxu old = casuword32(m, owner, UMUTEX_UNOWNED); 916162536Sdavidxu if (old == -1) 917162536Sdavidxu return (EFAULT); 918162536Sdavidxu if (old == owner) 919162536Sdavidxu return (0); 920162536Sdavidxu owner = old; 921162536Sdavidxu } 922162536Sdavidxu 923162536Sdavidxu /* We should only ever be in here for contested locks */ 924162536Sdavidxu if ((error = umtx_key_get(m, TYPE_SIMPLE_LOCK, AUTO_SHARE, 925162536Sdavidxu &key)) != 0) 926162536Sdavidxu return (error); 927162536Sdavidxu 928162536Sdavidxu umtxq_lock(&key); 929162536Sdavidxu umtxq_busy(&key); 930162536Sdavidxu count = umtxq_count(&key); 931162536Sdavidxu umtxq_unlock(&key); 932162536Sdavidxu 933162536Sdavidxu /* 934162536Sdavidxu * When unlocking the umtx, it must be marked as unowned if 935162536Sdavidxu * there is zero or one thread only waiting for it. 936162536Sdavidxu * Otherwise, it must be marked as contested. 937162536Sdavidxu */ 938162536Sdavidxu old = casuword32(m, owner, 939162536Sdavidxu count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 940162536Sdavidxu umtxq_lock(&key); 941162536Sdavidxu umtxq_signal(&key,1); 942162536Sdavidxu umtxq_unbusy(&key); 943162536Sdavidxu umtxq_unlock(&key); 944162536Sdavidxu umtx_key_release(&key); 945162536Sdavidxu if (old == -1) 946162536Sdavidxu return (EFAULT); 947162536Sdavidxu if (old != owner) 948162536Sdavidxu return (EINVAL); 949162536Sdavidxu return (0); 950162536Sdavidxu} 951162536Sdavidxu#endif 952162536Sdavidxu 953162536Sdavidxu/* 954161678Sdavidxu * Fetch and compare value, sleep on the address if value is not changed. 955161678Sdavidxu */ 956139013Sdavidxustatic int 957163449Sdavidxudo_wait(struct thread *td, void *addr, u_long id, 958178646Sdavidxu struct timespec *timeout, int compat32, int is_private) 959139013Sdavidxu{ 960143149Sdavidxu struct umtx_q *uq; 961140245Sdavidxu struct timespec ts, ts2, ts3; 962139013Sdavidxu struct timeval tv; 963163449Sdavidxu u_long tmp; 964140245Sdavidxu int error = 0; 965139013Sdavidxu 966143149Sdavidxu uq = td->td_umtxq; 967178646Sdavidxu if ((error = umtx_key_get(addr, TYPE_SIMPLE_WAIT, 968178646Sdavidxu is_private ? THREAD_SHARE : AUTO_SHARE, &uq->uq_key)) != 0) 969139013Sdavidxu return (error); 970161678Sdavidxu 971161678Sdavidxu umtxq_lock(&uq->uq_key); 972161678Sdavidxu umtxq_insert(uq); 973161678Sdavidxu umtxq_unlock(&uq->uq_key); 974162536Sdavidxu if (compat32 == 0) 975162536Sdavidxu tmp = fuword(addr); 976162536Sdavidxu else 977162536Sdavidxu tmp = fuword32(addr); 978139427Sdavidxu if (tmp != id) { 979143149Sdavidxu umtxq_lock(&uq->uq_key); 980143149Sdavidxu umtxq_remove(uq); 981143149Sdavidxu umtxq_unlock(&uq->uq_key); 982140245Sdavidxu } else if (timeout == NULL) { 983143149Sdavidxu umtxq_lock(&uq->uq_key); 984164839Sdavidxu error = umtxq_sleep(uq, "uwait", 0); 985161678Sdavidxu umtxq_remove(uq); 986143149Sdavidxu umtxq_unlock(&uq->uq_key); 987139013Sdavidxu } else { 988140245Sdavidxu getnanouptime(&ts); 989140245Sdavidxu timespecadd(&ts, timeout); 990140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 991161678Sdavidxu umtxq_lock(&uq->uq_key); 992139013Sdavidxu for (;;) { 993164839Sdavidxu error = umtxq_sleep(uq, "uwait", tvtohz(&tv)); 994161678Sdavidxu if (!(uq->uq_flags & UQF_UMTXQ)) 995161678Sdavidxu break; 996140245Sdavidxu if (error != ETIMEDOUT) 997140245Sdavidxu break; 998161678Sdavidxu umtxq_unlock(&uq->uq_key); 999140245Sdavidxu getnanouptime(&ts2); 1000140245Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 1001139751Sdavidxu error = ETIMEDOUT; 1002161678Sdavidxu umtxq_lock(&uq->uq_key); 1003139013Sdavidxu break; 1004139013Sdavidxu } 1005140245Sdavidxu ts3 = ts; 1006140245Sdavidxu timespecsub(&ts3, &ts2); 1007140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 1008161678Sdavidxu umtxq_lock(&uq->uq_key); 1009139013Sdavidxu } 1010143149Sdavidxu umtxq_remove(uq); 1011143149Sdavidxu umtxq_unlock(&uq->uq_key); 1012139013Sdavidxu } 1013143149Sdavidxu umtx_key_release(&uq->uq_key); 1014139257Sdavidxu if (error == ERESTART) 1015139257Sdavidxu error = EINTR; 1016139013Sdavidxu return (error); 1017139013Sdavidxu} 1018139013Sdavidxu 1019161678Sdavidxu/* 1020161678Sdavidxu * Wake up threads sleeping on the specified address. 1021161678Sdavidxu */ 1022151692Sdavidxuint 1023178646Sdavidxukern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) 1024139013Sdavidxu{ 1025139013Sdavidxu struct umtx_key key; 1026139257Sdavidxu int ret; 1027139013Sdavidxu 1028178646Sdavidxu if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, 1029178646Sdavidxu is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) 1030139257Sdavidxu return (ret); 1031139258Sdavidxu umtxq_lock(&key); 1032139257Sdavidxu ret = umtxq_signal(&key, n_wake); 1033139258Sdavidxu umtxq_unlock(&key); 1034139257Sdavidxu umtx_key_release(&key); 1035139013Sdavidxu return (0); 1036139013Sdavidxu} 1037139013Sdavidxu 1038161678Sdavidxu/* 1039161678Sdavidxu * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1040161678Sdavidxu */ 1041161678Sdavidxustatic int 1042161678Sdavidxu_do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1043179970Sdavidxu int mode) 1044161678Sdavidxu{ 1045161678Sdavidxu struct umtx_q *uq; 1046161678Sdavidxu uint32_t owner, old, id; 1047161678Sdavidxu int error = 0; 1048161678Sdavidxu 1049161678Sdavidxu id = td->td_tid; 1050161678Sdavidxu uq = td->td_umtxq; 1051161678Sdavidxu 1052161678Sdavidxu /* 1053161678Sdavidxu * Care must be exercised when dealing with umtx structure. It 1054161678Sdavidxu * can fault on any access. 1055161678Sdavidxu */ 1056161678Sdavidxu for (;;) { 1057179970Sdavidxu owner = fuword32(__DEVOLATILE(void *, &m->m_owner)); 1058179970Sdavidxu if (mode == _UMUTEX_WAIT) { 1059179970Sdavidxu if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) 1060179970Sdavidxu return (0); 1061179970Sdavidxu } else { 1062179970Sdavidxu /* 1063179970Sdavidxu * Try the uncontested case. This should be done in userland. 1064179970Sdavidxu */ 1065179970Sdavidxu owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); 1066161678Sdavidxu 1067179970Sdavidxu /* The acquire succeeded. */ 1068179970Sdavidxu if (owner == UMUTEX_UNOWNED) 1069161678Sdavidxu return (0); 1070161678Sdavidxu 1071161678Sdavidxu /* The address was invalid. */ 1072161678Sdavidxu if (owner == -1) 1073161678Sdavidxu return (EFAULT); 1074161678Sdavidxu 1075179970Sdavidxu /* If no one owns it but it is contested try to acquire it. */ 1076179970Sdavidxu if (owner == UMUTEX_CONTESTED) { 1077179970Sdavidxu owner = casuword32(&m->m_owner, 1078179970Sdavidxu UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1079179970Sdavidxu 1080179970Sdavidxu if (owner == UMUTEX_CONTESTED) 1081179970Sdavidxu return (0); 1082179970Sdavidxu 1083179970Sdavidxu /* The address was invalid. */ 1084179970Sdavidxu if (owner == -1) 1085179970Sdavidxu return (EFAULT); 1086179970Sdavidxu 1087179970Sdavidxu /* If this failed the lock has changed, restart. */ 1088179970Sdavidxu continue; 1089179970Sdavidxu } 1090161678Sdavidxu } 1091161678Sdavidxu 1092161678Sdavidxu if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1093161678Sdavidxu (owner & ~UMUTEX_CONTESTED) == id) 1094161678Sdavidxu return (EDEADLK); 1095161678Sdavidxu 1096179970Sdavidxu if (mode == _UMUTEX_TRY) 1097161678Sdavidxu return (EBUSY); 1098161678Sdavidxu 1099161678Sdavidxu /* 1100161678Sdavidxu * If we caught a signal, we have retried and now 1101161678Sdavidxu * exit immediately. 1102161678Sdavidxu */ 1103161678Sdavidxu if (error != 0) 1104161678Sdavidxu return (error); 1105161678Sdavidxu 1106161678Sdavidxu if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, 1107161678Sdavidxu GET_SHARE(flags), &uq->uq_key)) != 0) 1108161678Sdavidxu return (error); 1109161678Sdavidxu 1110161678Sdavidxu umtxq_lock(&uq->uq_key); 1111161678Sdavidxu umtxq_busy(&uq->uq_key); 1112161678Sdavidxu umtxq_insert(uq); 1113161678Sdavidxu umtxq_unlock(&uq->uq_key); 1114161678Sdavidxu 1115161678Sdavidxu /* 1116161678Sdavidxu * Set the contested bit so that a release in user space 1117161678Sdavidxu * knows to use the system call for unlock. If this fails 1118161678Sdavidxu * either some one else has acquired the lock or it has been 1119161678Sdavidxu * released. 1120161678Sdavidxu */ 1121161678Sdavidxu old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); 1122161678Sdavidxu 1123161678Sdavidxu /* The address was invalid. */ 1124161678Sdavidxu if (old == -1) { 1125161678Sdavidxu umtxq_lock(&uq->uq_key); 1126161678Sdavidxu umtxq_remove(uq); 1127179970Sdavidxu umtxq_unbusy(&uq->uq_key); 1128161678Sdavidxu umtxq_unlock(&uq->uq_key); 1129161678Sdavidxu umtx_key_release(&uq->uq_key); 1130161678Sdavidxu return (EFAULT); 1131161678Sdavidxu } 1132161678Sdavidxu 1133161678Sdavidxu /* 1134161678Sdavidxu * We set the contested bit, sleep. Otherwise the lock changed 1135161678Sdavidxu * and we need to retry or we lost a race to the thread 1136161678Sdavidxu * unlocking the umtx. 1137161678Sdavidxu */ 1138161678Sdavidxu umtxq_lock(&uq->uq_key); 1139179970Sdavidxu umtxq_unbusy(&uq->uq_key); 1140161678Sdavidxu if (old == owner) 1141161678Sdavidxu error = umtxq_sleep(uq, "umtxn", timo); 1142161678Sdavidxu umtxq_remove(uq); 1143161678Sdavidxu umtxq_unlock(&uq->uq_key); 1144161678Sdavidxu umtx_key_release(&uq->uq_key); 1145161678Sdavidxu } 1146161678Sdavidxu 1147161678Sdavidxu return (0); 1148161678Sdavidxu} 1149161678Sdavidxu 1150161678Sdavidxu/* 1151161678Sdavidxu * Lock PTHREAD_PRIO_NONE protocol POSIX mutex. 1152161678Sdavidxu */ 1153161678Sdavidxu/* 1154161678Sdavidxu * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. 1155161678Sdavidxu */ 1156161678Sdavidxustatic int 1157161678Sdavidxudo_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) 1158161678Sdavidxu{ 1159161678Sdavidxu struct umtx_key key; 1160161678Sdavidxu uint32_t owner, old, id; 1161161678Sdavidxu int error; 1162161678Sdavidxu int count; 1163161678Sdavidxu 1164161678Sdavidxu id = td->td_tid; 1165161678Sdavidxu /* 1166161678Sdavidxu * Make sure we own this mtx. 1167161678Sdavidxu */ 1168163449Sdavidxu owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1169161678Sdavidxu if (owner == -1) 1170161678Sdavidxu return (EFAULT); 1171161678Sdavidxu 1172161678Sdavidxu if ((owner & ~UMUTEX_CONTESTED) != id) 1173161678Sdavidxu return (EPERM); 1174161678Sdavidxu 1175161678Sdavidxu if ((owner & UMUTEX_CONTESTED) == 0) { 1176161678Sdavidxu old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); 1177161678Sdavidxu if (old == -1) 1178161678Sdavidxu return (EFAULT); 1179161678Sdavidxu if (old == owner) 1180161678Sdavidxu return (0); 1181161855Sdavidxu owner = old; 1182161678Sdavidxu } 1183161678Sdavidxu 1184161678Sdavidxu /* We should only ever be in here for contested locks */ 1185161678Sdavidxu if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1186161678Sdavidxu &key)) != 0) 1187161678Sdavidxu return (error); 1188161678Sdavidxu 1189161678Sdavidxu umtxq_lock(&key); 1190161678Sdavidxu umtxq_busy(&key); 1191161678Sdavidxu count = umtxq_count(&key); 1192161678Sdavidxu umtxq_unlock(&key); 1193161678Sdavidxu 1194161678Sdavidxu /* 1195161678Sdavidxu * When unlocking the umtx, it must be marked as unowned if 1196161678Sdavidxu * there is zero or one thread only waiting for it. 1197161678Sdavidxu * Otherwise, it must be marked as contested. 1198161678Sdavidxu */ 1199161678Sdavidxu old = casuword32(&m->m_owner, owner, 1200161678Sdavidxu count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1201161678Sdavidxu umtxq_lock(&key); 1202161678Sdavidxu umtxq_signal(&key,1); 1203161678Sdavidxu umtxq_unbusy(&key); 1204161678Sdavidxu umtxq_unlock(&key); 1205161678Sdavidxu umtx_key_release(&key); 1206161678Sdavidxu if (old == -1) 1207161678Sdavidxu return (EFAULT); 1208161678Sdavidxu if (old != owner) 1209161678Sdavidxu return (EINVAL); 1210161678Sdavidxu return (0); 1211161678Sdavidxu} 1212161678Sdavidxu 1213179970Sdavidxu/* 1214179970Sdavidxu * Check if the mutex is available and wake up a waiter, 1215179970Sdavidxu * only for simple mutex. 1216179970Sdavidxu */ 1217179970Sdavidxustatic int 1218179970Sdavidxudo_wake_umutex(struct thread *td, struct umutex *m) 1219179970Sdavidxu{ 1220179970Sdavidxu struct umtx_key key; 1221179970Sdavidxu uint32_t owner; 1222179970Sdavidxu uint32_t flags; 1223179970Sdavidxu int error; 1224179970Sdavidxu int count; 1225179970Sdavidxu 1226179970Sdavidxu owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1227179970Sdavidxu if (owner == -1) 1228179970Sdavidxu return (EFAULT); 1229179970Sdavidxu 1230179970Sdavidxu if ((owner & ~UMUTEX_CONTESTED) != 0) 1231179970Sdavidxu return (0); 1232179970Sdavidxu 1233179970Sdavidxu flags = fuword32(&m->m_flags); 1234179970Sdavidxu 1235179970Sdavidxu /* We should only ever be in here for contested locks */ 1236179970Sdavidxu if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), 1237179970Sdavidxu &key)) != 0) 1238179970Sdavidxu return (error); 1239179970Sdavidxu 1240179970Sdavidxu umtxq_lock(&key); 1241179970Sdavidxu umtxq_busy(&key); 1242179970Sdavidxu count = umtxq_count(&key); 1243179970Sdavidxu umtxq_unlock(&key); 1244179970Sdavidxu 1245179970Sdavidxu if (count <= 1) 1246179970Sdavidxu owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED); 1247179970Sdavidxu 1248179970Sdavidxu umtxq_lock(&key); 1249179970Sdavidxu if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) 1250179970Sdavidxu umtxq_signal(&key, 1); 1251179970Sdavidxu umtxq_unbusy(&key); 1252179970Sdavidxu umtxq_unlock(&key); 1253179970Sdavidxu umtx_key_release(&key); 1254179970Sdavidxu return (0); 1255179970Sdavidxu} 1256179970Sdavidxu 1257161678Sdavidxustatic inline struct umtx_pi * 1258163697Sdavidxuumtx_pi_alloc(int flags) 1259161678Sdavidxu{ 1260161678Sdavidxu struct umtx_pi *pi; 1261161678Sdavidxu 1262163697Sdavidxu pi = uma_zalloc(umtx_pi_zone, M_ZERO | flags); 1263161678Sdavidxu TAILQ_INIT(&pi->pi_blocked); 1264161678Sdavidxu atomic_add_int(&umtx_pi_allocated, 1); 1265161678Sdavidxu return (pi); 1266161678Sdavidxu} 1267161678Sdavidxu 1268161678Sdavidxustatic inline void 1269161678Sdavidxuumtx_pi_free(struct umtx_pi *pi) 1270161678Sdavidxu{ 1271161678Sdavidxu uma_zfree(umtx_pi_zone, pi); 1272161678Sdavidxu atomic_add_int(&umtx_pi_allocated, -1); 1273161678Sdavidxu} 1274161678Sdavidxu 1275161678Sdavidxu/* 1276161678Sdavidxu * Adjust the thread's position on a pi_state after its priority has been 1277161678Sdavidxu * changed. 1278161678Sdavidxu */ 1279161678Sdavidxustatic int 1280161678Sdavidxuumtx_pi_adjust_thread(struct umtx_pi *pi, struct thread *td) 1281161678Sdavidxu{ 1282161678Sdavidxu struct umtx_q *uq, *uq1, *uq2; 1283161678Sdavidxu struct thread *td1; 1284161678Sdavidxu 1285170300Sjeff mtx_assert(&umtx_lock, MA_OWNED); 1286161678Sdavidxu if (pi == NULL) 1287161678Sdavidxu return (0); 1288161678Sdavidxu 1289161678Sdavidxu uq = td->td_umtxq; 1290161678Sdavidxu 1291161678Sdavidxu /* 1292161678Sdavidxu * Check if the thread needs to be moved on the blocked chain. 1293161678Sdavidxu * It needs to be moved if either its priority is lower than 1294161678Sdavidxu * the previous thread or higher than the next thread. 1295161678Sdavidxu */ 1296161678Sdavidxu uq1 = TAILQ_PREV(uq, umtxq_head, uq_lockq); 1297161678Sdavidxu uq2 = TAILQ_NEXT(uq, uq_lockq); 1298161678Sdavidxu if ((uq1 != NULL && UPRI(td) < UPRI(uq1->uq_thread)) || 1299161678Sdavidxu (uq2 != NULL && UPRI(td) > UPRI(uq2->uq_thread))) { 1300161678Sdavidxu /* 1301161678Sdavidxu * Remove thread from blocked chain and determine where 1302161678Sdavidxu * it should be moved to. 1303161678Sdavidxu */ 1304161678Sdavidxu TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1305161678Sdavidxu TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1306161678Sdavidxu td1 = uq1->uq_thread; 1307161678Sdavidxu MPASS(td1->td_proc->p_magic == P_MAGIC); 1308161678Sdavidxu if (UPRI(td1) > UPRI(td)) 1309161678Sdavidxu break; 1310161678Sdavidxu } 1311161678Sdavidxu 1312161678Sdavidxu if (uq1 == NULL) 1313161678Sdavidxu TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1314161678Sdavidxu else 1315161678Sdavidxu TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1316161678Sdavidxu } 1317161678Sdavidxu return (1); 1318161678Sdavidxu} 1319161678Sdavidxu 1320161678Sdavidxu/* 1321161678Sdavidxu * Propagate priority when a thread is blocked on POSIX 1322161678Sdavidxu * PI mutex. 1323161678Sdavidxu */ 1324161678Sdavidxustatic void 1325161678Sdavidxuumtx_propagate_priority(struct thread *td) 1326161678Sdavidxu{ 1327161678Sdavidxu struct umtx_q *uq; 1328161678Sdavidxu struct umtx_pi *pi; 1329161678Sdavidxu int pri; 1330161678Sdavidxu 1331170300Sjeff mtx_assert(&umtx_lock, MA_OWNED); 1332161678Sdavidxu pri = UPRI(td); 1333161678Sdavidxu uq = td->td_umtxq; 1334161678Sdavidxu pi = uq->uq_pi_blocked; 1335161678Sdavidxu if (pi == NULL) 1336161678Sdavidxu return; 1337161678Sdavidxu 1338161678Sdavidxu for (;;) { 1339161678Sdavidxu td = pi->pi_owner; 1340161678Sdavidxu if (td == NULL) 1341161678Sdavidxu return; 1342161678Sdavidxu 1343161678Sdavidxu MPASS(td->td_proc != NULL); 1344161678Sdavidxu MPASS(td->td_proc->p_magic == P_MAGIC); 1345161678Sdavidxu 1346161678Sdavidxu if (UPRI(td) <= pri) 1347161678Sdavidxu return; 1348161678Sdavidxu 1349170300Sjeff thread_lock(td); 1350161678Sdavidxu sched_lend_user_prio(td, pri); 1351170300Sjeff thread_unlock(td); 1352161678Sdavidxu 1353161678Sdavidxu /* 1354161678Sdavidxu * Pick up the lock that td is blocked on. 1355161678Sdavidxu */ 1356161678Sdavidxu uq = td->td_umtxq; 1357161678Sdavidxu pi = uq->uq_pi_blocked; 1358161678Sdavidxu /* Resort td on the list if needed. */ 1359161678Sdavidxu if (!umtx_pi_adjust_thread(pi, td)) 1360161678Sdavidxu break; 1361161678Sdavidxu } 1362161678Sdavidxu} 1363161678Sdavidxu 1364161678Sdavidxu/* 1365161678Sdavidxu * Unpropagate priority for a PI mutex when a thread blocked on 1366161678Sdavidxu * it is interrupted by signal or resumed by others. 1367161678Sdavidxu */ 1368161678Sdavidxustatic void 1369161678Sdavidxuumtx_unpropagate_priority(struct umtx_pi *pi) 1370161678Sdavidxu{ 1371161678Sdavidxu struct umtx_q *uq, *uq_owner; 1372161678Sdavidxu struct umtx_pi *pi2; 1373174701Sdavidxu int pri, oldpri; 1374161678Sdavidxu 1375170300Sjeff mtx_assert(&umtx_lock, MA_OWNED); 1376161678Sdavidxu 1377161678Sdavidxu while (pi != NULL && pi->pi_owner != NULL) { 1378161678Sdavidxu pri = PRI_MAX; 1379161678Sdavidxu uq_owner = pi->pi_owner->td_umtxq; 1380161678Sdavidxu 1381161678Sdavidxu TAILQ_FOREACH(pi2, &uq_owner->uq_pi_contested, pi_link) { 1382161678Sdavidxu uq = TAILQ_FIRST(&pi2->pi_blocked); 1383161678Sdavidxu if (uq != NULL) { 1384161678Sdavidxu if (pri > UPRI(uq->uq_thread)) 1385161678Sdavidxu pri = UPRI(uq->uq_thread); 1386161678Sdavidxu } 1387161678Sdavidxu } 1388161678Sdavidxu 1389161678Sdavidxu if (pri > uq_owner->uq_inherited_pri) 1390161678Sdavidxu pri = uq_owner->uq_inherited_pri; 1391170300Sjeff thread_lock(pi->pi_owner); 1392174701Sdavidxu oldpri = pi->pi_owner->td_user_pri; 1393161678Sdavidxu sched_unlend_user_prio(pi->pi_owner, pri); 1394170300Sjeff thread_unlock(pi->pi_owner); 1395174701Sdavidxu umtx_pi_adjust_locked(pi->pi_owner, oldpri); 1396161678Sdavidxu pi = uq_owner->uq_pi_blocked; 1397161678Sdavidxu } 1398161678Sdavidxu} 1399161678Sdavidxu 1400161678Sdavidxu/* 1401161678Sdavidxu * Insert a PI mutex into owned list. 1402161678Sdavidxu */ 1403161678Sdavidxustatic void 1404161678Sdavidxuumtx_pi_setowner(struct umtx_pi *pi, struct thread *owner) 1405161678Sdavidxu{ 1406161678Sdavidxu struct umtx_q *uq_owner; 1407161678Sdavidxu 1408161678Sdavidxu uq_owner = owner->td_umtxq; 1409170300Sjeff mtx_assert(&umtx_lock, MA_OWNED); 1410161678Sdavidxu if (pi->pi_owner != NULL) 1411161678Sdavidxu panic("pi_ower != NULL"); 1412161678Sdavidxu pi->pi_owner = owner; 1413161678Sdavidxu TAILQ_INSERT_TAIL(&uq_owner->uq_pi_contested, pi, pi_link); 1414161678Sdavidxu} 1415161678Sdavidxu 1416161678Sdavidxu/* 1417161678Sdavidxu * Claim ownership of a PI mutex. 1418161678Sdavidxu */ 1419161678Sdavidxustatic int 1420161678Sdavidxuumtx_pi_claim(struct umtx_pi *pi, struct thread *owner) 1421161678Sdavidxu{ 1422161678Sdavidxu struct umtx_q *uq, *uq_owner; 1423161678Sdavidxu 1424161678Sdavidxu uq_owner = owner->td_umtxq; 1425170300Sjeff mtx_lock_spin(&umtx_lock); 1426161678Sdavidxu if (pi->pi_owner == owner) { 1427170300Sjeff mtx_unlock_spin(&umtx_lock); 1428161678Sdavidxu return (0); 1429161678Sdavidxu } 1430161678Sdavidxu 1431161678Sdavidxu if (pi->pi_owner != NULL) { 1432161678Sdavidxu /* 1433161678Sdavidxu * userland may have already messed the mutex, sigh. 1434161678Sdavidxu */ 1435170300Sjeff mtx_unlock_spin(&umtx_lock); 1436161678Sdavidxu return (EPERM); 1437161678Sdavidxu } 1438161678Sdavidxu umtx_pi_setowner(pi, owner); 1439161678Sdavidxu uq = TAILQ_FIRST(&pi->pi_blocked); 1440161678Sdavidxu if (uq != NULL) { 1441161678Sdavidxu int pri; 1442161678Sdavidxu 1443161678Sdavidxu pri = UPRI(uq->uq_thread); 1444170300Sjeff thread_lock(owner); 1445161678Sdavidxu if (pri < UPRI(owner)) 1446161678Sdavidxu sched_lend_user_prio(owner, pri); 1447170300Sjeff thread_unlock(owner); 1448161678Sdavidxu } 1449170300Sjeff mtx_unlock_spin(&umtx_lock); 1450161678Sdavidxu return (0); 1451161678Sdavidxu} 1452161678Sdavidxu 1453174701Sdavidxustatic void 1454174701Sdavidxuumtx_pi_adjust_locked(struct thread *td, u_char oldpri) 1455161599Sdavidxu{ 1456161678Sdavidxu struct umtx_q *uq; 1457161678Sdavidxu struct umtx_pi *pi; 1458161678Sdavidxu 1459161678Sdavidxu uq = td->td_umtxq; 1460161678Sdavidxu /* 1461161678Sdavidxu * Pick up the lock that td is blocked on. 1462161678Sdavidxu */ 1463161678Sdavidxu pi = uq->uq_pi_blocked; 1464161678Sdavidxu MPASS(pi != NULL); 1465161678Sdavidxu 1466161678Sdavidxu /* Resort the turnstile on the list. */ 1467161678Sdavidxu if (!umtx_pi_adjust_thread(pi, td)) 1468161678Sdavidxu return; 1469161678Sdavidxu 1470161678Sdavidxu /* 1471161678Sdavidxu * If our priority was lowered and we are at the head of the 1472161678Sdavidxu * turnstile, then propagate our new priority up the chain. 1473161678Sdavidxu */ 1474161678Sdavidxu if (uq == TAILQ_FIRST(&pi->pi_blocked) && UPRI(td) < oldpri) 1475161678Sdavidxu umtx_propagate_priority(td); 1476161599Sdavidxu} 1477161599Sdavidxu 1478161678Sdavidxu/* 1479174701Sdavidxu * Adjust a thread's order position in its blocked PI mutex, 1480174701Sdavidxu * this may result new priority propagating process. 1481174701Sdavidxu */ 1482174701Sdavidxuvoid 1483174701Sdavidxuumtx_pi_adjust(struct thread *td, u_char oldpri) 1484174701Sdavidxu{ 1485174707Sdavidxu struct umtx_q *uq; 1486174707Sdavidxu struct umtx_pi *pi; 1487174707Sdavidxu 1488174707Sdavidxu uq = td->td_umtxq; 1489174701Sdavidxu mtx_lock_spin(&umtx_lock); 1490174707Sdavidxu /* 1491174707Sdavidxu * Pick up the lock that td is blocked on. 1492174707Sdavidxu */ 1493174707Sdavidxu pi = uq->uq_pi_blocked; 1494174707Sdavidxu if (pi != NULL) 1495174707Sdavidxu umtx_pi_adjust_locked(td, oldpri); 1496174701Sdavidxu mtx_unlock_spin(&umtx_lock); 1497174701Sdavidxu} 1498174701Sdavidxu 1499174701Sdavidxu/* 1500161678Sdavidxu * Sleep on a PI mutex. 1501161678Sdavidxu */ 1502161678Sdavidxustatic int 1503161678Sdavidxuumtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, 1504161678Sdavidxu uint32_t owner, const char *wmesg, int timo) 1505161678Sdavidxu{ 1506161678Sdavidxu struct umtxq_chain *uc; 1507161678Sdavidxu struct thread *td, *td1; 1508161678Sdavidxu struct umtx_q *uq1; 1509161678Sdavidxu int pri; 1510161678Sdavidxu int error = 0; 1511161678Sdavidxu 1512161678Sdavidxu td = uq->uq_thread; 1513161678Sdavidxu KASSERT(td == curthread, ("inconsistent uq_thread")); 1514161678Sdavidxu uc = umtxq_getchain(&uq->uq_key); 1515161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 1516161678Sdavidxu umtxq_insert(uq); 1517161678Sdavidxu if (pi->pi_owner == NULL) { 1518161678Sdavidxu /* XXX 1519161678Sdavidxu * Current, We only support process private PI-mutex, 1520161678Sdavidxu * non-contended PI-mutexes are locked in userland. 1521161678Sdavidxu * Process shared PI-mutex should always be initialized 1522161678Sdavidxu * by kernel and be registered in kernel, locking should 1523161678Sdavidxu * always be done by kernel to avoid security problems. 1524161678Sdavidxu * For process private PI-mutex, we can find owner 1525161678Sdavidxu * thread and boost its priority safely. 1526161678Sdavidxu */ 1527161678Sdavidxu PROC_LOCK(curproc); 1528161678Sdavidxu td1 = thread_find(curproc, owner); 1529170300Sjeff mtx_lock_spin(&umtx_lock); 1530161678Sdavidxu if (td1 != NULL && pi->pi_owner == NULL) { 1531161678Sdavidxu uq1 = td1->td_umtxq; 1532161678Sdavidxu umtx_pi_setowner(pi, td1); 1533161678Sdavidxu } 1534161678Sdavidxu PROC_UNLOCK(curproc); 1535161678Sdavidxu } else { 1536170300Sjeff mtx_lock_spin(&umtx_lock); 1537161678Sdavidxu } 1538161678Sdavidxu 1539161678Sdavidxu TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) { 1540161678Sdavidxu pri = UPRI(uq1->uq_thread); 1541161678Sdavidxu if (pri > UPRI(td)) 1542161678Sdavidxu break; 1543161678Sdavidxu } 1544161678Sdavidxu 1545161678Sdavidxu if (uq1 != NULL) 1546161678Sdavidxu TAILQ_INSERT_BEFORE(uq1, uq, uq_lockq); 1547161678Sdavidxu else 1548161678Sdavidxu TAILQ_INSERT_TAIL(&pi->pi_blocked, uq, uq_lockq); 1549161678Sdavidxu 1550161678Sdavidxu uq->uq_pi_blocked = pi; 1551174701Sdavidxu thread_lock(td); 1552161678Sdavidxu td->td_flags |= TDF_UPIBLOCKED; 1553174701Sdavidxu thread_unlock(td); 1554170300Sjeff mtx_unlock_spin(&umtx_lock); 1555161678Sdavidxu umtxq_unlock(&uq->uq_key); 1556161678Sdavidxu 1557170300Sjeff mtx_lock_spin(&umtx_lock); 1558161678Sdavidxu umtx_propagate_priority(td); 1559170300Sjeff mtx_unlock_spin(&umtx_lock); 1560161678Sdavidxu 1561161678Sdavidxu umtxq_lock(&uq->uq_key); 1562161678Sdavidxu if (uq->uq_flags & UQF_UMTXQ) { 1563161678Sdavidxu error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo); 1564161678Sdavidxu if (error == EWOULDBLOCK) 1565161678Sdavidxu error = ETIMEDOUT; 1566161678Sdavidxu if (uq->uq_flags & UQF_UMTXQ) { 1567161678Sdavidxu umtxq_busy(&uq->uq_key); 1568161678Sdavidxu umtxq_remove(uq); 1569161678Sdavidxu umtxq_unbusy(&uq->uq_key); 1570161678Sdavidxu } 1571161678Sdavidxu } 1572161678Sdavidxu umtxq_unlock(&uq->uq_key); 1573161678Sdavidxu 1574170300Sjeff mtx_lock_spin(&umtx_lock); 1575161678Sdavidxu uq->uq_pi_blocked = NULL; 1576174701Sdavidxu thread_lock(td); 1577161678Sdavidxu td->td_flags &= ~TDF_UPIBLOCKED; 1578174701Sdavidxu thread_unlock(td); 1579161678Sdavidxu TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq); 1580161678Sdavidxu umtx_unpropagate_priority(pi); 1581170300Sjeff mtx_unlock_spin(&umtx_lock); 1582161678Sdavidxu 1583161678Sdavidxu umtxq_lock(&uq->uq_key); 1584161678Sdavidxu 1585161678Sdavidxu return (error); 1586161678Sdavidxu} 1587161678Sdavidxu 1588161678Sdavidxu/* 1589161678Sdavidxu * Add reference count for a PI mutex. 1590161678Sdavidxu */ 1591161678Sdavidxustatic void 1592161678Sdavidxuumtx_pi_ref(struct umtx_pi *pi) 1593161678Sdavidxu{ 1594161678Sdavidxu struct umtxq_chain *uc; 1595161678Sdavidxu 1596161678Sdavidxu uc = umtxq_getchain(&pi->pi_key); 1597161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 1598161678Sdavidxu pi->pi_refcount++; 1599161678Sdavidxu} 1600161678Sdavidxu 1601161678Sdavidxu/* 1602161678Sdavidxu * Decrease reference count for a PI mutex, if the counter 1603161678Sdavidxu * is decreased to zero, its memory space is freed. 1604161678Sdavidxu */ 1605161678Sdavidxustatic void 1606161678Sdavidxuumtx_pi_unref(struct umtx_pi *pi) 1607161678Sdavidxu{ 1608161678Sdavidxu struct umtxq_chain *uc; 1609161678Sdavidxu int free = 0; 1610161678Sdavidxu 1611161678Sdavidxu uc = umtxq_getchain(&pi->pi_key); 1612161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 1613161678Sdavidxu KASSERT(pi->pi_refcount > 0, ("invalid reference count")); 1614161678Sdavidxu if (--pi->pi_refcount == 0) { 1615170300Sjeff mtx_lock_spin(&umtx_lock); 1616161678Sdavidxu if (pi->pi_owner != NULL) { 1617161678Sdavidxu TAILQ_REMOVE(&pi->pi_owner->td_umtxq->uq_pi_contested, 1618161678Sdavidxu pi, pi_link); 1619161678Sdavidxu pi->pi_owner = NULL; 1620161678Sdavidxu } 1621161678Sdavidxu KASSERT(TAILQ_EMPTY(&pi->pi_blocked), 1622161678Sdavidxu ("blocked queue not empty")); 1623170300Sjeff mtx_unlock_spin(&umtx_lock); 1624161678Sdavidxu TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink); 1625161678Sdavidxu free = 1; 1626161678Sdavidxu } 1627161678Sdavidxu if (free) 1628161678Sdavidxu umtx_pi_free(pi); 1629161678Sdavidxu} 1630161678Sdavidxu 1631161678Sdavidxu/* 1632161678Sdavidxu * Find a PI mutex in hash table. 1633161678Sdavidxu */ 1634161678Sdavidxustatic struct umtx_pi * 1635161678Sdavidxuumtx_pi_lookup(struct umtx_key *key) 1636161678Sdavidxu{ 1637161678Sdavidxu struct umtxq_chain *uc; 1638161678Sdavidxu struct umtx_pi *pi; 1639161678Sdavidxu 1640161678Sdavidxu uc = umtxq_getchain(key); 1641161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 1642161678Sdavidxu 1643161678Sdavidxu TAILQ_FOREACH(pi, &uc->uc_pi_list, pi_hashlink) { 1644161678Sdavidxu if (umtx_key_match(&pi->pi_key, key)) { 1645161678Sdavidxu return (pi); 1646161678Sdavidxu } 1647161678Sdavidxu } 1648161678Sdavidxu return (NULL); 1649161678Sdavidxu} 1650161678Sdavidxu 1651161678Sdavidxu/* 1652161678Sdavidxu * Insert a PI mutex into hash table. 1653161678Sdavidxu */ 1654161678Sdavidxustatic inline void 1655161678Sdavidxuumtx_pi_insert(struct umtx_pi *pi) 1656161678Sdavidxu{ 1657161678Sdavidxu struct umtxq_chain *uc; 1658161678Sdavidxu 1659161678Sdavidxu uc = umtxq_getchain(&pi->pi_key); 1660161678Sdavidxu UMTXQ_LOCKED_ASSERT(uc); 1661161678Sdavidxu TAILQ_INSERT_TAIL(&uc->uc_pi_list, pi, pi_hashlink); 1662161678Sdavidxu} 1663161678Sdavidxu 1664161678Sdavidxu/* 1665161678Sdavidxu * Lock a PI mutex. 1666161678Sdavidxu */ 1667161678Sdavidxustatic int 1668161678Sdavidxu_do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1669161678Sdavidxu int try) 1670161678Sdavidxu{ 1671161678Sdavidxu struct umtx_q *uq; 1672161678Sdavidxu struct umtx_pi *pi, *new_pi; 1673161678Sdavidxu uint32_t id, owner, old; 1674161678Sdavidxu int error; 1675161678Sdavidxu 1676161678Sdavidxu id = td->td_tid; 1677161678Sdavidxu uq = td->td_umtxq; 1678161678Sdavidxu 1679161678Sdavidxu if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), 1680161678Sdavidxu &uq->uq_key)) != 0) 1681161678Sdavidxu return (error); 1682163697Sdavidxu umtxq_lock(&uq->uq_key); 1683163697Sdavidxu pi = umtx_pi_lookup(&uq->uq_key); 1684163697Sdavidxu if (pi == NULL) { 1685163697Sdavidxu new_pi = umtx_pi_alloc(M_NOWAIT); 1686163697Sdavidxu if (new_pi == NULL) { 1687161678Sdavidxu umtxq_unlock(&uq->uq_key); 1688163697Sdavidxu new_pi = umtx_pi_alloc(M_WAITOK); 1689161678Sdavidxu new_pi->pi_key = uq->uq_key; 1690161678Sdavidxu umtxq_lock(&uq->uq_key); 1691161678Sdavidxu pi = umtx_pi_lookup(&uq->uq_key); 1692163697Sdavidxu if (pi != NULL) { 1693161678Sdavidxu umtx_pi_free(new_pi); 1694163697Sdavidxu new_pi = NULL; 1695161678Sdavidxu } 1696161678Sdavidxu } 1697163697Sdavidxu if (new_pi != NULL) { 1698163697Sdavidxu new_pi->pi_key = uq->uq_key; 1699163697Sdavidxu umtx_pi_insert(new_pi); 1700163697Sdavidxu pi = new_pi; 1701163697Sdavidxu } 1702163697Sdavidxu } 1703163697Sdavidxu umtx_pi_ref(pi); 1704163697Sdavidxu umtxq_unlock(&uq->uq_key); 1705161678Sdavidxu 1706163697Sdavidxu /* 1707163697Sdavidxu * Care must be exercised when dealing with umtx structure. It 1708163697Sdavidxu * can fault on any access. 1709163697Sdavidxu */ 1710163697Sdavidxu for (;;) { 1711161678Sdavidxu /* 1712161678Sdavidxu * Try the uncontested case. This should be done in userland. 1713161678Sdavidxu */ 1714161678Sdavidxu owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id); 1715161678Sdavidxu 1716161678Sdavidxu /* The acquire succeeded. */ 1717161678Sdavidxu if (owner == UMUTEX_UNOWNED) { 1718161678Sdavidxu error = 0; 1719161678Sdavidxu break; 1720161678Sdavidxu } 1721161678Sdavidxu 1722161678Sdavidxu /* The address was invalid. */ 1723161678Sdavidxu if (owner == -1) { 1724161678Sdavidxu error = EFAULT; 1725161678Sdavidxu break; 1726161678Sdavidxu } 1727161678Sdavidxu 1728161678Sdavidxu /* If no one owns it but it is contested try to acquire it. */ 1729161678Sdavidxu if (owner == UMUTEX_CONTESTED) { 1730161678Sdavidxu owner = casuword32(&m->m_owner, 1731161678Sdavidxu UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1732161678Sdavidxu 1733161678Sdavidxu if (owner == UMUTEX_CONTESTED) { 1734161678Sdavidxu umtxq_lock(&uq->uq_key); 1735161678Sdavidxu error = umtx_pi_claim(pi, td); 1736161678Sdavidxu umtxq_unlock(&uq->uq_key); 1737161678Sdavidxu break; 1738161678Sdavidxu } 1739161678Sdavidxu 1740161678Sdavidxu /* The address was invalid. */ 1741161678Sdavidxu if (owner == -1) { 1742161678Sdavidxu error = EFAULT; 1743161678Sdavidxu break; 1744161678Sdavidxu } 1745161678Sdavidxu 1746161678Sdavidxu /* If this failed the lock has changed, restart. */ 1747161678Sdavidxu continue; 1748161678Sdavidxu } 1749161678Sdavidxu 1750161678Sdavidxu if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1751161678Sdavidxu (owner & ~UMUTEX_CONTESTED) == id) { 1752161678Sdavidxu error = EDEADLK; 1753161678Sdavidxu break; 1754161678Sdavidxu } 1755161678Sdavidxu 1756161678Sdavidxu if (try != 0) { 1757161678Sdavidxu error = EBUSY; 1758161678Sdavidxu break; 1759161678Sdavidxu } 1760161678Sdavidxu 1761161678Sdavidxu /* 1762161678Sdavidxu * If we caught a signal, we have retried and now 1763161678Sdavidxu * exit immediately. 1764161678Sdavidxu */ 1765161678Sdavidxu if (error != 0) 1766161678Sdavidxu break; 1767161678Sdavidxu 1768161678Sdavidxu umtxq_lock(&uq->uq_key); 1769161678Sdavidxu umtxq_busy(&uq->uq_key); 1770161678Sdavidxu umtxq_unlock(&uq->uq_key); 1771161678Sdavidxu 1772161678Sdavidxu /* 1773161678Sdavidxu * Set the contested bit so that a release in user space 1774161678Sdavidxu * knows to use the system call for unlock. If this fails 1775161678Sdavidxu * either some one else has acquired the lock or it has been 1776161678Sdavidxu * released. 1777161678Sdavidxu */ 1778161678Sdavidxu old = casuword32(&m->m_owner, owner, owner | UMUTEX_CONTESTED); 1779161678Sdavidxu 1780161678Sdavidxu /* The address was invalid. */ 1781161678Sdavidxu if (old == -1) { 1782161678Sdavidxu umtxq_lock(&uq->uq_key); 1783161678Sdavidxu umtxq_unbusy(&uq->uq_key); 1784161678Sdavidxu umtxq_unlock(&uq->uq_key); 1785161678Sdavidxu error = EFAULT; 1786161678Sdavidxu break; 1787161678Sdavidxu } 1788161678Sdavidxu 1789161678Sdavidxu umtxq_lock(&uq->uq_key); 1790161678Sdavidxu umtxq_unbusy(&uq->uq_key); 1791161678Sdavidxu /* 1792161678Sdavidxu * We set the contested bit, sleep. Otherwise the lock changed 1793161678Sdavidxu * and we need to retry or we lost a race to the thread 1794161678Sdavidxu * unlocking the umtx. 1795161678Sdavidxu */ 1796161678Sdavidxu if (old == owner) 1797161678Sdavidxu error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, 1798161678Sdavidxu "umtxpi", timo); 1799161678Sdavidxu umtxq_unlock(&uq->uq_key); 1800161678Sdavidxu } 1801161678Sdavidxu 1802163697Sdavidxu umtxq_lock(&uq->uq_key); 1803163697Sdavidxu umtx_pi_unref(pi); 1804163697Sdavidxu umtxq_unlock(&uq->uq_key); 1805161678Sdavidxu 1806161678Sdavidxu umtx_key_release(&uq->uq_key); 1807161678Sdavidxu return (error); 1808161678Sdavidxu} 1809161678Sdavidxu 1810161678Sdavidxu/* 1811161678Sdavidxu * Unlock a PI mutex. 1812161678Sdavidxu */ 1813161678Sdavidxustatic int 1814161678Sdavidxudo_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) 1815161678Sdavidxu{ 1816161678Sdavidxu struct umtx_key key; 1817161678Sdavidxu struct umtx_q *uq_first, *uq_first2, *uq_me; 1818161678Sdavidxu struct umtx_pi *pi, *pi2; 1819161678Sdavidxu uint32_t owner, old, id; 1820161678Sdavidxu int error; 1821161678Sdavidxu int count; 1822161678Sdavidxu int pri; 1823161678Sdavidxu 1824161678Sdavidxu id = td->td_tid; 1825161678Sdavidxu /* 1826161678Sdavidxu * Make sure we own this mtx. 1827161678Sdavidxu */ 1828163449Sdavidxu owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 1829161678Sdavidxu if (owner == -1) 1830161678Sdavidxu return (EFAULT); 1831161678Sdavidxu 1832161678Sdavidxu if ((owner & ~UMUTEX_CONTESTED) != id) 1833161678Sdavidxu return (EPERM); 1834161678Sdavidxu 1835161678Sdavidxu /* This should be done in userland */ 1836161678Sdavidxu if ((owner & UMUTEX_CONTESTED) == 0) { 1837161678Sdavidxu old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED); 1838161678Sdavidxu if (old == -1) 1839161678Sdavidxu return (EFAULT); 1840161678Sdavidxu if (old == owner) 1841161678Sdavidxu return (0); 1842161855Sdavidxu owner = old; 1843161678Sdavidxu } 1844161678Sdavidxu 1845161678Sdavidxu /* We should only ever be in here for contested locks */ 1846161678Sdavidxu if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), 1847161678Sdavidxu &key)) != 0) 1848161678Sdavidxu return (error); 1849161678Sdavidxu 1850161678Sdavidxu umtxq_lock(&key); 1851161678Sdavidxu umtxq_busy(&key); 1852161678Sdavidxu count = umtxq_count_pi(&key, &uq_first); 1853161678Sdavidxu if (uq_first != NULL) { 1854161678Sdavidxu pi = uq_first->uq_pi_blocked; 1855161678Sdavidxu if (pi->pi_owner != curthread) { 1856161678Sdavidxu umtxq_unbusy(&key); 1857161678Sdavidxu umtxq_unlock(&key); 1858161678Sdavidxu /* userland messed the mutex */ 1859161678Sdavidxu return (EPERM); 1860161678Sdavidxu } 1861161678Sdavidxu uq_me = curthread->td_umtxq; 1862170300Sjeff mtx_lock_spin(&umtx_lock); 1863161678Sdavidxu pi->pi_owner = NULL; 1864161678Sdavidxu TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link); 1865161678Sdavidxu uq_first = TAILQ_FIRST(&pi->pi_blocked); 1866161678Sdavidxu pri = PRI_MAX; 1867161678Sdavidxu TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) { 1868161678Sdavidxu uq_first2 = TAILQ_FIRST(&pi2->pi_blocked); 1869161678Sdavidxu if (uq_first2 != NULL) { 1870161678Sdavidxu if (pri > UPRI(uq_first2->uq_thread)) 1871161678Sdavidxu pri = UPRI(uq_first2->uq_thread); 1872161678Sdavidxu } 1873161678Sdavidxu } 1874170300Sjeff thread_lock(curthread); 1875161678Sdavidxu sched_unlend_user_prio(curthread, pri); 1876170300Sjeff thread_unlock(curthread); 1877170300Sjeff mtx_unlock_spin(&umtx_lock); 1878161678Sdavidxu } 1879161678Sdavidxu umtxq_unlock(&key); 1880161678Sdavidxu 1881161678Sdavidxu /* 1882161678Sdavidxu * When unlocking the umtx, it must be marked as unowned if 1883161678Sdavidxu * there is zero or one thread only waiting for it. 1884161678Sdavidxu * Otherwise, it must be marked as contested. 1885161678Sdavidxu */ 1886161678Sdavidxu old = casuword32(&m->m_owner, owner, 1887161678Sdavidxu count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); 1888161678Sdavidxu 1889161678Sdavidxu umtxq_lock(&key); 1890161678Sdavidxu if (uq_first != NULL) 1891161678Sdavidxu umtxq_signal_thread(uq_first); 1892161678Sdavidxu umtxq_unbusy(&key); 1893161678Sdavidxu umtxq_unlock(&key); 1894161678Sdavidxu umtx_key_release(&key); 1895161678Sdavidxu if (old == -1) 1896161678Sdavidxu return (EFAULT); 1897161678Sdavidxu if (old != owner) 1898161678Sdavidxu return (EINVAL); 1899161678Sdavidxu return (0); 1900161678Sdavidxu} 1901161678Sdavidxu 1902161678Sdavidxu/* 1903161678Sdavidxu * Lock a PP mutex. 1904161678Sdavidxu */ 1905161678Sdavidxustatic int 1906161678Sdavidxu_do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, int timo, 1907161678Sdavidxu int try) 1908161678Sdavidxu{ 1909161678Sdavidxu struct umtx_q *uq, *uq2; 1910161678Sdavidxu struct umtx_pi *pi; 1911161678Sdavidxu uint32_t ceiling; 1912161678Sdavidxu uint32_t owner, id; 1913161678Sdavidxu int error, pri, old_inherited_pri, su; 1914161678Sdavidxu 1915161678Sdavidxu id = td->td_tid; 1916161678Sdavidxu uq = td->td_umtxq; 1917161678Sdavidxu if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 1918161678Sdavidxu &uq->uq_key)) != 0) 1919161678Sdavidxu return (error); 1920164033Srwatson su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 1921161678Sdavidxu for (;;) { 1922161678Sdavidxu old_inherited_pri = uq->uq_inherited_pri; 1923161678Sdavidxu umtxq_lock(&uq->uq_key); 1924161678Sdavidxu umtxq_busy(&uq->uq_key); 1925161678Sdavidxu umtxq_unlock(&uq->uq_key); 1926161678Sdavidxu 1927161678Sdavidxu ceiling = RTP_PRIO_MAX - fuword32(&m->m_ceilings[0]); 1928161678Sdavidxu if (ceiling > RTP_PRIO_MAX) { 1929161678Sdavidxu error = EINVAL; 1930161678Sdavidxu goto out; 1931161678Sdavidxu } 1932161678Sdavidxu 1933170300Sjeff mtx_lock_spin(&umtx_lock); 1934161678Sdavidxu if (UPRI(td) < PRI_MIN_REALTIME + ceiling) { 1935170300Sjeff mtx_unlock_spin(&umtx_lock); 1936161678Sdavidxu error = EINVAL; 1937161678Sdavidxu goto out; 1938161678Sdavidxu } 1939161678Sdavidxu if (su && PRI_MIN_REALTIME + ceiling < uq->uq_inherited_pri) { 1940161678Sdavidxu uq->uq_inherited_pri = PRI_MIN_REALTIME + ceiling; 1941170300Sjeff thread_lock(td); 1942161678Sdavidxu if (uq->uq_inherited_pri < UPRI(td)) 1943161678Sdavidxu sched_lend_user_prio(td, uq->uq_inherited_pri); 1944170300Sjeff thread_unlock(td); 1945161678Sdavidxu } 1946170300Sjeff mtx_unlock_spin(&umtx_lock); 1947161678Sdavidxu 1948161678Sdavidxu owner = casuword32(&m->m_owner, 1949161678Sdavidxu UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 1950161678Sdavidxu 1951161678Sdavidxu if (owner == UMUTEX_CONTESTED) { 1952161678Sdavidxu error = 0; 1953161678Sdavidxu break; 1954161678Sdavidxu } 1955161678Sdavidxu 1956161678Sdavidxu /* The address was invalid. */ 1957161678Sdavidxu if (owner == -1) { 1958161678Sdavidxu error = EFAULT; 1959161678Sdavidxu break; 1960161678Sdavidxu } 1961161678Sdavidxu 1962161678Sdavidxu if ((flags & UMUTEX_ERROR_CHECK) != 0 && 1963161678Sdavidxu (owner & ~UMUTEX_CONTESTED) == id) { 1964161678Sdavidxu error = EDEADLK; 1965161678Sdavidxu break; 1966161678Sdavidxu } 1967161678Sdavidxu 1968161678Sdavidxu if (try != 0) { 1969161678Sdavidxu error = EBUSY; 1970161678Sdavidxu break; 1971161678Sdavidxu } 1972161678Sdavidxu 1973161678Sdavidxu /* 1974161678Sdavidxu * If we caught a signal, we have retried and now 1975161678Sdavidxu * exit immediately. 1976161678Sdavidxu */ 1977161678Sdavidxu if (error != 0) 1978161678Sdavidxu break; 1979161678Sdavidxu 1980161678Sdavidxu umtxq_lock(&uq->uq_key); 1981161678Sdavidxu umtxq_insert(uq); 1982161678Sdavidxu umtxq_unbusy(&uq->uq_key); 1983161678Sdavidxu error = umtxq_sleep(uq, "umtxpp", timo); 1984161678Sdavidxu umtxq_remove(uq); 1985161678Sdavidxu umtxq_unlock(&uq->uq_key); 1986161678Sdavidxu 1987170300Sjeff mtx_lock_spin(&umtx_lock); 1988161678Sdavidxu uq->uq_inherited_pri = old_inherited_pri; 1989161678Sdavidxu pri = PRI_MAX; 1990161678Sdavidxu TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 1991161678Sdavidxu uq2 = TAILQ_FIRST(&pi->pi_blocked); 1992161678Sdavidxu if (uq2 != NULL) { 1993161678Sdavidxu if (pri > UPRI(uq2->uq_thread)) 1994161678Sdavidxu pri = UPRI(uq2->uq_thread); 1995161678Sdavidxu } 1996161678Sdavidxu } 1997161678Sdavidxu if (pri > uq->uq_inherited_pri) 1998161678Sdavidxu pri = uq->uq_inherited_pri; 1999170300Sjeff thread_lock(td); 2000161678Sdavidxu sched_unlend_user_prio(td, pri); 2001170300Sjeff thread_unlock(td); 2002170300Sjeff mtx_unlock_spin(&umtx_lock); 2003161678Sdavidxu } 2004161678Sdavidxu 2005161678Sdavidxu if (error != 0) { 2006170300Sjeff mtx_lock_spin(&umtx_lock); 2007161678Sdavidxu uq->uq_inherited_pri = old_inherited_pri; 2008161678Sdavidxu pri = PRI_MAX; 2009161678Sdavidxu TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2010161678Sdavidxu uq2 = TAILQ_FIRST(&pi->pi_blocked); 2011161678Sdavidxu if (uq2 != NULL) { 2012161678Sdavidxu if (pri > UPRI(uq2->uq_thread)) 2013161678Sdavidxu pri = UPRI(uq2->uq_thread); 2014161678Sdavidxu } 2015161678Sdavidxu } 2016161678Sdavidxu if (pri > uq->uq_inherited_pri) 2017161678Sdavidxu pri = uq->uq_inherited_pri; 2018170300Sjeff thread_lock(td); 2019161678Sdavidxu sched_unlend_user_prio(td, pri); 2020170300Sjeff thread_unlock(td); 2021170300Sjeff mtx_unlock_spin(&umtx_lock); 2022161678Sdavidxu } 2023161678Sdavidxu 2024161678Sdavidxuout: 2025161678Sdavidxu umtxq_lock(&uq->uq_key); 2026161678Sdavidxu umtxq_unbusy(&uq->uq_key); 2027161678Sdavidxu umtxq_unlock(&uq->uq_key); 2028161678Sdavidxu umtx_key_release(&uq->uq_key); 2029161678Sdavidxu return (error); 2030161678Sdavidxu} 2031161678Sdavidxu 2032161678Sdavidxu/* 2033161678Sdavidxu * Unlock a PP mutex. 2034161678Sdavidxu */ 2035161678Sdavidxustatic int 2036161678Sdavidxudo_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) 2037161678Sdavidxu{ 2038161678Sdavidxu struct umtx_key key; 2039161678Sdavidxu struct umtx_q *uq, *uq2; 2040161678Sdavidxu struct umtx_pi *pi; 2041161678Sdavidxu uint32_t owner, id; 2042161678Sdavidxu uint32_t rceiling; 2043161926Sdavidxu int error, pri, new_inherited_pri, su; 2044161678Sdavidxu 2045161678Sdavidxu id = td->td_tid; 2046161678Sdavidxu uq = td->td_umtxq; 2047164033Srwatson su = (priv_check(td, PRIV_SCHED_RTPRIO) == 0); 2048161678Sdavidxu 2049161678Sdavidxu /* 2050161678Sdavidxu * Make sure we own this mtx. 2051161678Sdavidxu */ 2052163449Sdavidxu owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner)); 2053161678Sdavidxu if (owner == -1) 2054161678Sdavidxu return (EFAULT); 2055161678Sdavidxu 2056161678Sdavidxu if ((owner & ~UMUTEX_CONTESTED) != id) 2057161678Sdavidxu return (EPERM); 2058161678Sdavidxu 2059161678Sdavidxu error = copyin(&m->m_ceilings[1], &rceiling, sizeof(uint32_t)); 2060161678Sdavidxu if (error != 0) 2061161678Sdavidxu return (error); 2062161678Sdavidxu 2063161678Sdavidxu if (rceiling == -1) 2064161678Sdavidxu new_inherited_pri = PRI_MAX; 2065161678Sdavidxu else { 2066161678Sdavidxu rceiling = RTP_PRIO_MAX - rceiling; 2067161678Sdavidxu if (rceiling > RTP_PRIO_MAX) 2068161678Sdavidxu return (EINVAL); 2069161678Sdavidxu new_inherited_pri = PRI_MIN_REALTIME + rceiling; 2070161678Sdavidxu } 2071161678Sdavidxu 2072161678Sdavidxu if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 2073161678Sdavidxu &key)) != 0) 2074161678Sdavidxu return (error); 2075161678Sdavidxu umtxq_lock(&key); 2076161678Sdavidxu umtxq_busy(&key); 2077161678Sdavidxu umtxq_unlock(&key); 2078161678Sdavidxu /* 2079161678Sdavidxu * For priority protected mutex, always set unlocked state 2080161678Sdavidxu * to UMUTEX_CONTESTED, so that userland always enters kernel 2081161678Sdavidxu * to lock the mutex, it is necessary because thread priority 2082161678Sdavidxu * has to be adjusted for such mutex. 2083161678Sdavidxu */ 2084163449Sdavidxu error = suword32(__DEVOLATILE(uint32_t *, &m->m_owner), 2085163449Sdavidxu UMUTEX_CONTESTED); 2086161678Sdavidxu 2087161678Sdavidxu umtxq_lock(&key); 2088161678Sdavidxu if (error == 0) 2089161678Sdavidxu umtxq_signal(&key, 1); 2090161678Sdavidxu umtxq_unbusy(&key); 2091161678Sdavidxu umtxq_unlock(&key); 2092161678Sdavidxu 2093161678Sdavidxu if (error == -1) 2094161678Sdavidxu error = EFAULT; 2095161678Sdavidxu else { 2096170300Sjeff mtx_lock_spin(&umtx_lock); 2097161926Sdavidxu if (su != 0) 2098161926Sdavidxu uq->uq_inherited_pri = new_inherited_pri; 2099161678Sdavidxu pri = PRI_MAX; 2100161678Sdavidxu TAILQ_FOREACH(pi, &uq->uq_pi_contested, pi_link) { 2101161678Sdavidxu uq2 = TAILQ_FIRST(&pi->pi_blocked); 2102161678Sdavidxu if (uq2 != NULL) { 2103161678Sdavidxu if (pri > UPRI(uq2->uq_thread)) 2104161678Sdavidxu pri = UPRI(uq2->uq_thread); 2105161678Sdavidxu } 2106161678Sdavidxu } 2107161678Sdavidxu if (pri > uq->uq_inherited_pri) 2108161678Sdavidxu pri = uq->uq_inherited_pri; 2109170300Sjeff thread_lock(td); 2110161678Sdavidxu sched_unlend_user_prio(td, pri); 2111170300Sjeff thread_unlock(td); 2112170300Sjeff mtx_unlock_spin(&umtx_lock); 2113161678Sdavidxu } 2114161678Sdavidxu umtx_key_release(&key); 2115161678Sdavidxu return (error); 2116161678Sdavidxu} 2117161678Sdavidxu 2118161678Sdavidxustatic int 2119161678Sdavidxudo_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, 2120161678Sdavidxu uint32_t *old_ceiling) 2121161678Sdavidxu{ 2122161678Sdavidxu struct umtx_q *uq; 2123161678Sdavidxu uint32_t save_ceiling; 2124161678Sdavidxu uint32_t owner, id; 2125161678Sdavidxu uint32_t flags; 2126161678Sdavidxu int error; 2127161678Sdavidxu 2128161678Sdavidxu flags = fuword32(&m->m_flags); 2129161678Sdavidxu if ((flags & UMUTEX_PRIO_PROTECT) == 0) 2130161678Sdavidxu return (EINVAL); 2131161678Sdavidxu if (ceiling > RTP_PRIO_MAX) 2132161678Sdavidxu return (EINVAL); 2133161678Sdavidxu id = td->td_tid; 2134161678Sdavidxu uq = td->td_umtxq; 2135161678Sdavidxu if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), 2136161678Sdavidxu &uq->uq_key)) != 0) 2137161678Sdavidxu return (error); 2138161678Sdavidxu for (;;) { 2139161678Sdavidxu umtxq_lock(&uq->uq_key); 2140161678Sdavidxu umtxq_busy(&uq->uq_key); 2141161678Sdavidxu umtxq_unlock(&uq->uq_key); 2142161678Sdavidxu 2143161678Sdavidxu save_ceiling = fuword32(&m->m_ceilings[0]); 2144161678Sdavidxu 2145161678Sdavidxu owner = casuword32(&m->m_owner, 2146161678Sdavidxu UMUTEX_CONTESTED, id | UMUTEX_CONTESTED); 2147161678Sdavidxu 2148161678Sdavidxu if (owner == UMUTEX_CONTESTED) { 2149161678Sdavidxu suword32(&m->m_ceilings[0], ceiling); 2150163449Sdavidxu suword32(__DEVOLATILE(uint32_t *, &m->m_owner), 2151163449Sdavidxu UMUTEX_CONTESTED); 2152161678Sdavidxu error = 0; 2153161678Sdavidxu break; 2154161678Sdavidxu } 2155161678Sdavidxu 2156161678Sdavidxu /* The address was invalid. */ 2157161678Sdavidxu if (owner == -1) { 2158161678Sdavidxu error = EFAULT; 2159161678Sdavidxu break; 2160161678Sdavidxu } 2161161678Sdavidxu 2162161678Sdavidxu if ((owner & ~UMUTEX_CONTESTED) == id) { 2163161678Sdavidxu suword32(&m->m_ceilings[0], ceiling); 2164161678Sdavidxu error = 0; 2165161678Sdavidxu break; 2166161678Sdavidxu } 2167161678Sdavidxu 2168161678Sdavidxu /* 2169161678Sdavidxu * If we caught a signal, we have retried and now 2170161678Sdavidxu * exit immediately. 2171161678Sdavidxu */ 2172161678Sdavidxu if (error != 0) 2173161678Sdavidxu break; 2174161678Sdavidxu 2175161678Sdavidxu /* 2176161678Sdavidxu * We set the contested bit, sleep. Otherwise the lock changed 2177161678Sdavidxu * and we need to retry or we lost a race to the thread 2178161678Sdavidxu * unlocking the umtx. 2179161678Sdavidxu */ 2180161678Sdavidxu umtxq_lock(&uq->uq_key); 2181161678Sdavidxu umtxq_insert(uq); 2182161678Sdavidxu umtxq_unbusy(&uq->uq_key); 2183161678Sdavidxu error = umtxq_sleep(uq, "umtxpp", 0); 2184161678Sdavidxu umtxq_remove(uq); 2185161678Sdavidxu umtxq_unlock(&uq->uq_key); 2186161678Sdavidxu } 2187161678Sdavidxu umtxq_lock(&uq->uq_key); 2188161678Sdavidxu if (error == 0) 2189161678Sdavidxu umtxq_signal(&uq->uq_key, INT_MAX); 2190161678Sdavidxu umtxq_unbusy(&uq->uq_key); 2191161678Sdavidxu umtxq_unlock(&uq->uq_key); 2192161678Sdavidxu umtx_key_release(&uq->uq_key); 2193161678Sdavidxu if (error == 0 && old_ceiling != NULL) 2194161678Sdavidxu suword32(old_ceiling, save_ceiling); 2195161678Sdavidxu return (error); 2196161678Sdavidxu} 2197161678Sdavidxu 2198162030Sdavidxustatic int 2199162030Sdavidxu_do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo, 2200179970Sdavidxu int mode) 2201162030Sdavidxu{ 2202162030Sdavidxu switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2203162030Sdavidxu case 0: 2204179970Sdavidxu return (_do_lock_normal(td, m, flags, timo, mode)); 2205162030Sdavidxu case UMUTEX_PRIO_INHERIT: 2206179970Sdavidxu return (_do_lock_pi(td, m, flags, timo, mode)); 2207162030Sdavidxu case UMUTEX_PRIO_PROTECT: 2208179970Sdavidxu return (_do_lock_pp(td, m, flags, timo, mode)); 2209162030Sdavidxu } 2210162030Sdavidxu return (EINVAL); 2211162030Sdavidxu} 2212162030Sdavidxu 2213161678Sdavidxu/* 2214161678Sdavidxu * Lock a userland POSIX mutex. 2215161678Sdavidxu */ 2216161678Sdavidxustatic int 2217162030Sdavidxudo_lock_umutex(struct thread *td, struct umutex *m, 2218179970Sdavidxu struct timespec *timeout, int mode) 2219161678Sdavidxu{ 2220162030Sdavidxu struct timespec ts, ts2, ts3; 2221162030Sdavidxu struct timeval tv; 2222161678Sdavidxu uint32_t flags; 2223162030Sdavidxu int error; 2224161678Sdavidxu 2225161678Sdavidxu flags = fuword32(&m->m_flags); 2226161678Sdavidxu if (flags == -1) 2227161678Sdavidxu return (EFAULT); 2228161678Sdavidxu 2229162030Sdavidxu if (timeout == NULL) { 2230179970Sdavidxu error = _do_lock_umutex(td, m, flags, 0, mode); 2231162030Sdavidxu /* Mutex locking is restarted if it is interrupted. */ 2232179970Sdavidxu if (error == EINTR && mode != _UMUTEX_WAIT) 2233162030Sdavidxu error = ERESTART; 2234162030Sdavidxu } else { 2235162030Sdavidxu getnanouptime(&ts); 2236162030Sdavidxu timespecadd(&ts, timeout); 2237162030Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 2238162030Sdavidxu for (;;) { 2239179970Sdavidxu error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode); 2240162030Sdavidxu if (error != ETIMEDOUT) 2241162030Sdavidxu break; 2242162030Sdavidxu getnanouptime(&ts2); 2243162030Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 2244162030Sdavidxu error = ETIMEDOUT; 2245162030Sdavidxu break; 2246162030Sdavidxu } 2247162030Sdavidxu ts3 = ts; 2248162030Sdavidxu timespecsub(&ts3, &ts2); 2249162030Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2250162030Sdavidxu } 2251162030Sdavidxu /* Timed-locking is not restarted. */ 2252162030Sdavidxu if (error == ERESTART) 2253162030Sdavidxu error = EINTR; 2254161742Sdavidxu } 2255162030Sdavidxu return (error); 2256161678Sdavidxu} 2257161678Sdavidxu 2258161678Sdavidxu/* 2259161678Sdavidxu * Unlock a userland POSIX mutex. 2260161678Sdavidxu */ 2261161678Sdavidxustatic int 2262161678Sdavidxudo_unlock_umutex(struct thread *td, struct umutex *m) 2263161678Sdavidxu{ 2264161678Sdavidxu uint32_t flags; 2265161678Sdavidxu 2266161678Sdavidxu flags = fuword32(&m->m_flags); 2267161678Sdavidxu if (flags == -1) 2268161678Sdavidxu return (EFAULT); 2269161678Sdavidxu 2270161855Sdavidxu switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { 2271161855Sdavidxu case 0: 2272161855Sdavidxu return (do_unlock_normal(td, m, flags)); 2273161855Sdavidxu case UMUTEX_PRIO_INHERIT: 2274161855Sdavidxu return (do_unlock_pi(td, m, flags)); 2275161855Sdavidxu case UMUTEX_PRIO_PROTECT: 2276161855Sdavidxu return (do_unlock_pp(td, m, flags)); 2277161855Sdavidxu } 2278161678Sdavidxu 2279161855Sdavidxu return (EINVAL); 2280161678Sdavidxu} 2281161678Sdavidxu 2282164839Sdavidxustatic int 2283164839Sdavidxudo_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, 2284164876Sdavidxu struct timespec *timeout, u_long wflags) 2285164839Sdavidxu{ 2286164839Sdavidxu struct umtx_q *uq; 2287164839Sdavidxu struct timeval tv; 2288164839Sdavidxu struct timespec cts, ets, tts; 2289164839Sdavidxu uint32_t flags; 2290164839Sdavidxu int error; 2291164839Sdavidxu 2292164839Sdavidxu uq = td->td_umtxq; 2293164839Sdavidxu flags = fuword32(&cv->c_flags); 2294164839Sdavidxu error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &uq->uq_key); 2295164839Sdavidxu if (error != 0) 2296164839Sdavidxu return (error); 2297164839Sdavidxu umtxq_lock(&uq->uq_key); 2298164839Sdavidxu umtxq_busy(&uq->uq_key); 2299164839Sdavidxu umtxq_insert(uq); 2300164839Sdavidxu umtxq_unlock(&uq->uq_key); 2301164839Sdavidxu 2302164839Sdavidxu /* 2303164839Sdavidxu * The magic thing is we should set c_has_waiters to 1 before 2304164839Sdavidxu * releasing user mutex. 2305164839Sdavidxu */ 2306164839Sdavidxu suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 1); 2307164839Sdavidxu 2308164839Sdavidxu umtxq_lock(&uq->uq_key); 2309164839Sdavidxu umtxq_unbusy(&uq->uq_key); 2310164839Sdavidxu umtxq_unlock(&uq->uq_key); 2311164839Sdavidxu 2312164839Sdavidxu error = do_unlock_umutex(td, m); 2313164839Sdavidxu 2314164839Sdavidxu umtxq_lock(&uq->uq_key); 2315164839Sdavidxu if (error == 0) { 2316164876Sdavidxu if ((wflags & UMTX_CHECK_UNPARKING) && 2317164876Sdavidxu (td->td_pflags & TDP_WAKEUP)) { 2318164876Sdavidxu td->td_pflags &= ~TDP_WAKEUP; 2319164876Sdavidxu error = EINTR; 2320164876Sdavidxu } else if (timeout == NULL) { 2321164839Sdavidxu error = umtxq_sleep(uq, "ucond", 0); 2322164839Sdavidxu } else { 2323164839Sdavidxu getnanouptime(&ets); 2324164839Sdavidxu timespecadd(&ets, timeout); 2325164839Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 2326164839Sdavidxu for (;;) { 2327164839Sdavidxu error = umtxq_sleep(uq, "ucond", tvtohz(&tv)); 2328164839Sdavidxu if (error != ETIMEDOUT) 2329164839Sdavidxu break; 2330164839Sdavidxu getnanouptime(&cts); 2331164839Sdavidxu if (timespeccmp(&cts, &ets, >=)) { 2332164839Sdavidxu error = ETIMEDOUT; 2333164839Sdavidxu break; 2334164839Sdavidxu } 2335164839Sdavidxu tts = ets; 2336164839Sdavidxu timespecsub(&tts, &cts); 2337164839Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &tts); 2338164839Sdavidxu } 2339164839Sdavidxu } 2340164839Sdavidxu } 2341164839Sdavidxu 2342164839Sdavidxu if (error != 0) { 2343164839Sdavidxu if ((uq->uq_flags & UQF_UMTXQ) == 0) { 2344164839Sdavidxu /* 2345164839Sdavidxu * If we concurrently got do_cv_signal()d 2346164839Sdavidxu * and we got an error or UNIX signals or a timeout, 2347164839Sdavidxu * then, perform another umtxq_signal to avoid 2348164839Sdavidxu * consuming the wakeup. This may cause supurious 2349164839Sdavidxu * wakeup for another thread which was just queued, 2350164839Sdavidxu * but SUSV3 explicitly allows supurious wakeup to 2351164839Sdavidxu * occur, and indeed a kernel based implementation 2352164839Sdavidxu * can not avoid it. 2353164839Sdavidxu */ 2354164876Sdavidxu if (!umtxq_signal(&uq->uq_key, 1)) 2355164876Sdavidxu error = 0; 2356164839Sdavidxu } 2357164839Sdavidxu if (error == ERESTART) 2358164839Sdavidxu error = EINTR; 2359164839Sdavidxu } 2360164839Sdavidxu umtxq_remove(uq); 2361164839Sdavidxu umtxq_unlock(&uq->uq_key); 2362164839Sdavidxu umtx_key_release(&uq->uq_key); 2363164839Sdavidxu return (error); 2364164839Sdavidxu} 2365164839Sdavidxu 2366164839Sdavidxu/* 2367164839Sdavidxu * Signal a userland condition variable. 2368164839Sdavidxu */ 2369164839Sdavidxustatic int 2370164839Sdavidxudo_cv_signal(struct thread *td, struct ucond *cv) 2371164839Sdavidxu{ 2372164839Sdavidxu struct umtx_key key; 2373164839Sdavidxu int error, cnt, nwake; 2374164839Sdavidxu uint32_t flags; 2375164839Sdavidxu 2376164839Sdavidxu flags = fuword32(&cv->c_flags); 2377164839Sdavidxu if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2378164839Sdavidxu return (error); 2379164839Sdavidxu umtxq_lock(&key); 2380164839Sdavidxu umtxq_busy(&key); 2381164839Sdavidxu cnt = umtxq_count(&key); 2382164839Sdavidxu nwake = umtxq_signal(&key, 1); 2383164839Sdavidxu if (cnt <= nwake) { 2384164839Sdavidxu umtxq_unlock(&key); 2385164839Sdavidxu error = suword32( 2386164839Sdavidxu __DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); 2387164839Sdavidxu umtxq_lock(&key); 2388164839Sdavidxu } 2389164839Sdavidxu umtxq_unbusy(&key); 2390164839Sdavidxu umtxq_unlock(&key); 2391164839Sdavidxu umtx_key_release(&key); 2392164839Sdavidxu return (error); 2393164839Sdavidxu} 2394164839Sdavidxu 2395164839Sdavidxustatic int 2396164839Sdavidxudo_cv_broadcast(struct thread *td, struct ucond *cv) 2397164839Sdavidxu{ 2398164839Sdavidxu struct umtx_key key; 2399164839Sdavidxu int error; 2400164839Sdavidxu uint32_t flags; 2401164839Sdavidxu 2402164839Sdavidxu flags = fuword32(&cv->c_flags); 2403164839Sdavidxu if ((error = umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), &key)) != 0) 2404164839Sdavidxu return (error); 2405164839Sdavidxu 2406164839Sdavidxu umtxq_lock(&key); 2407164839Sdavidxu umtxq_busy(&key); 2408164839Sdavidxu umtxq_signal(&key, INT_MAX); 2409164839Sdavidxu umtxq_unlock(&key); 2410164839Sdavidxu 2411164839Sdavidxu error = suword32(__DEVOLATILE(uint32_t *, &cv->c_has_waiters), 0); 2412164839Sdavidxu 2413164839Sdavidxu umtxq_lock(&key); 2414164839Sdavidxu umtxq_unbusy(&key); 2415164839Sdavidxu umtxq_unlock(&key); 2416164839Sdavidxu 2417164839Sdavidxu umtx_key_release(&key); 2418164839Sdavidxu return (error); 2419164839Sdavidxu} 2420164839Sdavidxu 2421177848Sdavidxustatic int 2422177848Sdavidxudo_rw_rdlock(struct thread *td, struct urwlock *rwlock, long fflag, int timo) 2423177848Sdavidxu{ 2424177848Sdavidxu struct umtx_q *uq; 2425177848Sdavidxu uint32_t flags, wrflags; 2426177848Sdavidxu int32_t state, oldstate; 2427177848Sdavidxu int32_t blocked_readers; 2428177848Sdavidxu int error; 2429177848Sdavidxu 2430177848Sdavidxu uq = td->td_umtxq; 2431177848Sdavidxu flags = fuword32(&rwlock->rw_flags); 2432177848Sdavidxu error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2433177848Sdavidxu if (error != 0) 2434177848Sdavidxu return (error); 2435177848Sdavidxu 2436177848Sdavidxu wrflags = URWLOCK_WRITE_OWNER; 2437177848Sdavidxu if (!(fflag & URWLOCK_PREFER_READER) && !(flags & URWLOCK_PREFER_READER)) 2438177848Sdavidxu wrflags |= URWLOCK_WRITE_WAITERS; 2439177848Sdavidxu 2440177848Sdavidxu for (;;) { 2441177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2442177848Sdavidxu /* try to lock it */ 2443177848Sdavidxu while (!(state & wrflags)) { 2444177848Sdavidxu if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) { 2445177848Sdavidxu umtx_key_release(&uq->uq_key); 2446177848Sdavidxu return (EAGAIN); 2447177848Sdavidxu } 2448177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, state + 1); 2449177848Sdavidxu if (oldstate == state) { 2450177848Sdavidxu umtx_key_release(&uq->uq_key); 2451177848Sdavidxu return (0); 2452177848Sdavidxu } 2453177848Sdavidxu state = oldstate; 2454177848Sdavidxu } 2455177848Sdavidxu 2456177848Sdavidxu if (error) 2457177848Sdavidxu break; 2458177848Sdavidxu 2459177848Sdavidxu /* grab monitor lock */ 2460177848Sdavidxu umtxq_lock(&uq->uq_key); 2461177848Sdavidxu umtxq_busy(&uq->uq_key); 2462177848Sdavidxu umtxq_unlock(&uq->uq_key); 2463177848Sdavidxu 2464177848Sdavidxu /* set read contention bit */ 2465177848Sdavidxu while ((state & wrflags) && !(state & URWLOCK_READ_WAITERS)) { 2466177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_READ_WAITERS); 2467177848Sdavidxu if (oldstate == state) 2468177848Sdavidxu goto sleep; 2469177848Sdavidxu state = oldstate; 2470177848Sdavidxu } 2471177848Sdavidxu 2472177848Sdavidxu /* state is changed while setting flags, restart */ 2473177848Sdavidxu if (!(state & wrflags)) { 2474177848Sdavidxu umtxq_lock(&uq->uq_key); 2475177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2476177848Sdavidxu umtxq_unlock(&uq->uq_key); 2477177848Sdavidxu continue; 2478177848Sdavidxu } 2479177848Sdavidxu 2480177848Sdavidxusleep: 2481177848Sdavidxu /* contention bit is set, before sleeping, increase read waiter count */ 2482177848Sdavidxu blocked_readers = fuword32(&rwlock->rw_blocked_readers); 2483177848Sdavidxu suword32(&rwlock->rw_blocked_readers, blocked_readers+1); 2484177848Sdavidxu 2485177848Sdavidxu while (state & wrflags) { 2486177848Sdavidxu umtxq_lock(&uq->uq_key); 2487177848Sdavidxu umtxq_insert(uq); 2488177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2489177848Sdavidxu 2490177848Sdavidxu error = umtxq_sleep(uq, "urdlck", timo); 2491177848Sdavidxu 2492177848Sdavidxu umtxq_busy(&uq->uq_key); 2493177848Sdavidxu umtxq_remove(uq); 2494177848Sdavidxu umtxq_unlock(&uq->uq_key); 2495177848Sdavidxu if (error) 2496177848Sdavidxu break; 2497177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2498177848Sdavidxu } 2499177848Sdavidxu 2500177848Sdavidxu /* decrease read waiter count, and may clear read contention bit */ 2501177848Sdavidxu blocked_readers = fuword32(&rwlock->rw_blocked_readers); 2502177848Sdavidxu suword32(&rwlock->rw_blocked_readers, blocked_readers-1); 2503177848Sdavidxu if (blocked_readers == 1) { 2504177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2505177848Sdavidxu for (;;) { 2506177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, 2507177848Sdavidxu state & ~URWLOCK_READ_WAITERS); 2508177848Sdavidxu if (oldstate == state) 2509177848Sdavidxu break; 2510177848Sdavidxu state = oldstate; 2511177848Sdavidxu } 2512177848Sdavidxu } 2513177848Sdavidxu 2514177848Sdavidxu umtxq_lock(&uq->uq_key); 2515177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2516177848Sdavidxu umtxq_unlock(&uq->uq_key); 2517177848Sdavidxu } 2518177848Sdavidxu umtx_key_release(&uq->uq_key); 2519177848Sdavidxu return (error); 2520177848Sdavidxu} 2521177848Sdavidxu 2522177848Sdavidxustatic int 2523177848Sdavidxudo_rw_rdlock2(struct thread *td, void *obj, long val, struct timespec *timeout) 2524177848Sdavidxu{ 2525177848Sdavidxu struct timespec ts, ts2, ts3; 2526177848Sdavidxu struct timeval tv; 2527177848Sdavidxu int error; 2528177848Sdavidxu 2529177848Sdavidxu getnanouptime(&ts); 2530177848Sdavidxu timespecadd(&ts, timeout); 2531177848Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 2532177848Sdavidxu for (;;) { 2533177848Sdavidxu error = do_rw_rdlock(td, obj, val, tvtohz(&tv)); 2534177848Sdavidxu if (error != ETIMEDOUT) 2535177848Sdavidxu break; 2536177848Sdavidxu getnanouptime(&ts2); 2537177848Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 2538177848Sdavidxu error = ETIMEDOUT; 2539177848Sdavidxu break; 2540177848Sdavidxu } 2541177848Sdavidxu ts3 = ts; 2542177848Sdavidxu timespecsub(&ts3, &ts2); 2543177848Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2544177848Sdavidxu } 2545177849Sdavidxu if (error == ERESTART) 2546177849Sdavidxu error = EINTR; 2547177848Sdavidxu return (error); 2548177848Sdavidxu} 2549177848Sdavidxu 2550177848Sdavidxustatic int 2551177848Sdavidxudo_rw_wrlock(struct thread *td, struct urwlock *rwlock, int timo) 2552177848Sdavidxu{ 2553177848Sdavidxu struct umtx_q *uq; 2554177848Sdavidxu uint32_t flags; 2555177848Sdavidxu int32_t state, oldstate; 2556177848Sdavidxu int32_t blocked_writers; 2557177848Sdavidxu int error; 2558177848Sdavidxu 2559177848Sdavidxu uq = td->td_umtxq; 2560177848Sdavidxu flags = fuword32(&rwlock->rw_flags); 2561177848Sdavidxu error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2562177848Sdavidxu if (error != 0) 2563177848Sdavidxu return (error); 2564177848Sdavidxu 2565177848Sdavidxu for (;;) { 2566177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2567177848Sdavidxu while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { 2568177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER); 2569177848Sdavidxu if (oldstate == state) { 2570177848Sdavidxu umtx_key_release(&uq->uq_key); 2571177848Sdavidxu return (0); 2572177848Sdavidxu } 2573177848Sdavidxu state = oldstate; 2574177848Sdavidxu } 2575177848Sdavidxu 2576177848Sdavidxu if (error) 2577177848Sdavidxu break; 2578177848Sdavidxu 2579177848Sdavidxu /* grab monitor lock */ 2580177848Sdavidxu umtxq_lock(&uq->uq_key); 2581177848Sdavidxu umtxq_busy(&uq->uq_key); 2582177848Sdavidxu umtxq_unlock(&uq->uq_key); 2583177848Sdavidxu 2584177848Sdavidxu while (((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) && 2585177848Sdavidxu (state & URWLOCK_WRITE_WAITERS) == 0) { 2586177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, state | URWLOCK_WRITE_WAITERS); 2587177848Sdavidxu if (oldstate == state) 2588177848Sdavidxu goto sleep; 2589177848Sdavidxu state = oldstate; 2590177848Sdavidxu } 2591177848Sdavidxu 2592177848Sdavidxu if (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { 2593177848Sdavidxu umtxq_lock(&uq->uq_key); 2594177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2595177848Sdavidxu umtxq_unlock(&uq->uq_key); 2596177848Sdavidxu continue; 2597177848Sdavidxu } 2598177848Sdavidxusleep: 2599177848Sdavidxu blocked_writers = fuword32(&rwlock->rw_blocked_writers); 2600177848Sdavidxu suword32(&rwlock->rw_blocked_writers, blocked_writers+1); 2601177848Sdavidxu 2602177848Sdavidxu while ((state & URWLOCK_WRITE_OWNER) || URWLOCK_READER_COUNT(state) != 0) { 2603177848Sdavidxu umtxq_lock(&uq->uq_key); 2604177848Sdavidxu umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE); 2605177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2606177848Sdavidxu 2607177848Sdavidxu error = umtxq_sleep(uq, "uwrlck", timo); 2608177848Sdavidxu 2609177848Sdavidxu umtxq_busy(&uq->uq_key); 2610177848Sdavidxu umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE); 2611177848Sdavidxu umtxq_unlock(&uq->uq_key); 2612177848Sdavidxu if (error) 2613177848Sdavidxu break; 2614177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2615177848Sdavidxu } 2616177848Sdavidxu 2617177848Sdavidxu blocked_writers = fuword32(&rwlock->rw_blocked_writers); 2618177848Sdavidxu suword32(&rwlock->rw_blocked_writers, blocked_writers-1); 2619177848Sdavidxu if (blocked_writers == 1) { 2620177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2621177848Sdavidxu for (;;) { 2622177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, 2623177848Sdavidxu state & ~URWLOCK_WRITE_WAITERS); 2624177848Sdavidxu if (oldstate == state) 2625177848Sdavidxu break; 2626177848Sdavidxu state = oldstate; 2627177848Sdavidxu } 2628177848Sdavidxu } 2629177848Sdavidxu 2630177848Sdavidxu umtxq_lock(&uq->uq_key); 2631177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2632177848Sdavidxu umtxq_unlock(&uq->uq_key); 2633177848Sdavidxu } 2634177848Sdavidxu 2635177848Sdavidxu umtx_key_release(&uq->uq_key); 2636177848Sdavidxu return (error); 2637177848Sdavidxu} 2638177848Sdavidxu 2639177848Sdavidxustatic int 2640177848Sdavidxudo_rw_wrlock2(struct thread *td, void *obj, struct timespec *timeout) 2641177848Sdavidxu{ 2642177848Sdavidxu struct timespec ts, ts2, ts3; 2643177848Sdavidxu struct timeval tv; 2644177848Sdavidxu int error; 2645177848Sdavidxu 2646177848Sdavidxu getnanouptime(&ts); 2647177848Sdavidxu timespecadd(&ts, timeout); 2648177848Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 2649177848Sdavidxu for (;;) { 2650177848Sdavidxu error = do_rw_wrlock(td, obj, tvtohz(&tv)); 2651177848Sdavidxu if (error != ETIMEDOUT) 2652177848Sdavidxu break; 2653177848Sdavidxu getnanouptime(&ts2); 2654177848Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 2655177848Sdavidxu error = ETIMEDOUT; 2656177848Sdavidxu break; 2657177848Sdavidxu } 2658177848Sdavidxu ts3 = ts; 2659177848Sdavidxu timespecsub(&ts3, &ts2); 2660177848Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 2661177848Sdavidxu } 2662177849Sdavidxu if (error == ERESTART) 2663177849Sdavidxu error = EINTR; 2664177848Sdavidxu return (error); 2665177848Sdavidxu} 2666177848Sdavidxu 2667177848Sdavidxustatic int 2668177880Sdavidxudo_rw_unlock(struct thread *td, struct urwlock *rwlock) 2669177848Sdavidxu{ 2670177848Sdavidxu struct umtx_q *uq; 2671177848Sdavidxu uint32_t flags; 2672177848Sdavidxu int32_t state, oldstate; 2673177848Sdavidxu int error, q, count; 2674177848Sdavidxu 2675177848Sdavidxu uq = td->td_umtxq; 2676177848Sdavidxu flags = fuword32(&rwlock->rw_flags); 2677177848Sdavidxu error = umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), &uq->uq_key); 2678177848Sdavidxu if (error != 0) 2679177848Sdavidxu return (error); 2680177848Sdavidxu 2681177848Sdavidxu state = fuword32(__DEVOLATILE(int32_t *, &rwlock->rw_state)); 2682177848Sdavidxu if (state & URWLOCK_WRITE_OWNER) { 2683177848Sdavidxu for (;;) { 2684177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, 2685177848Sdavidxu state & ~URWLOCK_WRITE_OWNER); 2686177848Sdavidxu if (oldstate != state) { 2687177848Sdavidxu state = oldstate; 2688177848Sdavidxu if (!(oldstate & URWLOCK_WRITE_OWNER)) { 2689177848Sdavidxu error = EPERM; 2690177848Sdavidxu goto out; 2691177848Sdavidxu } 2692177848Sdavidxu } else 2693177848Sdavidxu break; 2694177848Sdavidxu } 2695177848Sdavidxu } else if (URWLOCK_READER_COUNT(state) != 0) { 2696177848Sdavidxu for (;;) { 2697177848Sdavidxu oldstate = casuword32(&rwlock->rw_state, state, 2698177848Sdavidxu state - 1); 2699177848Sdavidxu if (oldstate != state) { 2700177848Sdavidxu state = oldstate; 2701177848Sdavidxu if (URWLOCK_READER_COUNT(oldstate) == 0) { 2702177848Sdavidxu error = EPERM; 2703177848Sdavidxu goto out; 2704177848Sdavidxu } 2705177848Sdavidxu } 2706177848Sdavidxu else 2707177848Sdavidxu break; 2708177848Sdavidxu } 2709177848Sdavidxu } else { 2710177848Sdavidxu error = EPERM; 2711177848Sdavidxu goto out; 2712177848Sdavidxu } 2713177848Sdavidxu 2714177848Sdavidxu count = 0; 2715177848Sdavidxu 2716177848Sdavidxu if (!(flags & URWLOCK_PREFER_READER)) { 2717177848Sdavidxu if (state & URWLOCK_WRITE_WAITERS) { 2718177848Sdavidxu count = 1; 2719177848Sdavidxu q = UMTX_EXCLUSIVE_QUEUE; 2720177848Sdavidxu } else if (state & URWLOCK_READ_WAITERS) { 2721177848Sdavidxu count = INT_MAX; 2722177848Sdavidxu q = UMTX_SHARED_QUEUE; 2723177848Sdavidxu } 2724177848Sdavidxu } else { 2725177848Sdavidxu if (state & URWLOCK_READ_WAITERS) { 2726177848Sdavidxu count = INT_MAX; 2727177848Sdavidxu q = UMTX_SHARED_QUEUE; 2728177848Sdavidxu } else if (state & URWLOCK_WRITE_WAITERS) { 2729177848Sdavidxu count = 1; 2730177848Sdavidxu q = UMTX_EXCLUSIVE_QUEUE; 2731177848Sdavidxu } 2732177848Sdavidxu } 2733177848Sdavidxu 2734177848Sdavidxu if (count) { 2735177848Sdavidxu umtxq_lock(&uq->uq_key); 2736177848Sdavidxu umtxq_busy(&uq->uq_key); 2737177848Sdavidxu umtxq_signal_queue(&uq->uq_key, count, q); 2738177848Sdavidxu umtxq_unbusy(&uq->uq_key); 2739177848Sdavidxu umtxq_unlock(&uq->uq_key); 2740177848Sdavidxu } 2741177848Sdavidxuout: 2742177848Sdavidxu umtx_key_release(&uq->uq_key); 2743177848Sdavidxu return (error); 2744177848Sdavidxu} 2745177848Sdavidxu 2746139013Sdavidxuint 2747139013Sdavidxu_umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 2748139013Sdavidxu /* struct umtx *umtx */ 2749139013Sdavidxu{ 2750162536Sdavidxu return _do_lock_umtx(td, uap->umtx, td->td_tid, 0); 2751139013Sdavidxu} 2752139013Sdavidxu 2753139013Sdavidxuint 2754139013Sdavidxu_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 2755139013Sdavidxu /* struct umtx *umtx */ 2756139013Sdavidxu{ 2757162536Sdavidxu return do_unlock_umtx(td, uap->umtx, td->td_tid); 2758139013Sdavidxu} 2759139013Sdavidxu 2760162536Sdavidxustatic int 2761162536Sdavidxu__umtx_op_lock_umtx(struct thread *td, struct _umtx_op_args *uap) 2762139013Sdavidxu{ 2763162536Sdavidxu struct timespec *ts, timeout; 2764139013Sdavidxu int error; 2765139013Sdavidxu 2766162536Sdavidxu /* Allow a null timespec (wait forever). */ 2767162536Sdavidxu if (uap->uaddr2 == NULL) 2768162536Sdavidxu ts = NULL; 2769162536Sdavidxu else { 2770162536Sdavidxu error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2771162536Sdavidxu if (error != 0) 2772162536Sdavidxu return (error); 2773162536Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2774162536Sdavidxu timeout.tv_nsec < 0) { 2775162536Sdavidxu return (EINVAL); 2776161678Sdavidxu } 2777162536Sdavidxu ts = &timeout; 2778162536Sdavidxu } 2779162536Sdavidxu return (do_lock_umtx(td, uap->obj, uap->val, ts)); 2780162536Sdavidxu} 2781162536Sdavidxu 2782162536Sdavidxustatic int 2783162536Sdavidxu__umtx_op_unlock_umtx(struct thread *td, struct _umtx_op_args *uap) 2784162536Sdavidxu{ 2785162536Sdavidxu return (do_unlock_umtx(td, uap->obj, uap->val)); 2786162536Sdavidxu} 2787162536Sdavidxu 2788162536Sdavidxustatic int 2789162536Sdavidxu__umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) 2790162536Sdavidxu{ 2791162536Sdavidxu struct timespec *ts, timeout; 2792162536Sdavidxu int error; 2793162536Sdavidxu 2794162536Sdavidxu if (uap->uaddr2 == NULL) 2795162536Sdavidxu ts = NULL; 2796162536Sdavidxu else { 2797162536Sdavidxu error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2798162536Sdavidxu if (error != 0) 2799162536Sdavidxu return (error); 2800162536Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2801162536Sdavidxu timeout.tv_nsec < 0) 2802162536Sdavidxu return (EINVAL); 2803162536Sdavidxu ts = &timeout; 2804162536Sdavidxu } 2805178646Sdavidxu return do_wait(td, uap->obj, uap->val, ts, 0, 0); 2806162536Sdavidxu} 2807162536Sdavidxu 2808162536Sdavidxustatic int 2809173800Sdavidxu__umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) 2810173800Sdavidxu{ 2811173800Sdavidxu struct timespec *ts, timeout; 2812173800Sdavidxu int error; 2813173800Sdavidxu 2814173800Sdavidxu if (uap->uaddr2 == NULL) 2815173800Sdavidxu ts = NULL; 2816173800Sdavidxu else { 2817173800Sdavidxu error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2818173800Sdavidxu if (error != 0) 2819173800Sdavidxu return (error); 2820173800Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2821173800Sdavidxu timeout.tv_nsec < 0) 2822173800Sdavidxu return (EINVAL); 2823173800Sdavidxu ts = &timeout; 2824173800Sdavidxu } 2825178646Sdavidxu return do_wait(td, uap->obj, uap->val, ts, 1, 0); 2826173800Sdavidxu} 2827173800Sdavidxu 2828173800Sdavidxustatic int 2829178646Sdavidxu__umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) 2830178646Sdavidxu{ 2831178646Sdavidxu struct timespec *ts, timeout; 2832178646Sdavidxu int error; 2833178646Sdavidxu 2834178646Sdavidxu if (uap->uaddr2 == NULL) 2835178646Sdavidxu ts = NULL; 2836178646Sdavidxu else { 2837178646Sdavidxu error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 2838178646Sdavidxu if (error != 0) 2839178646Sdavidxu return (error); 2840178646Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2841178646Sdavidxu timeout.tv_nsec < 0) 2842178646Sdavidxu return (EINVAL); 2843178646Sdavidxu ts = &timeout; 2844178646Sdavidxu } 2845178646Sdavidxu return do_wait(td, uap->obj, uap->val, ts, 1, 1); 2846178646Sdavidxu} 2847178646Sdavidxu 2848178646Sdavidxustatic int 2849162536Sdavidxu__umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) 2850162536Sdavidxu{ 2851178646Sdavidxu return (kern_umtx_wake(td, uap->obj, uap->val, 0)); 2852162536Sdavidxu} 2853162536Sdavidxu 2854162536Sdavidxustatic int 2855178646Sdavidxu__umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) 2856178646Sdavidxu{ 2857178646Sdavidxu return (kern_umtx_wake(td, uap->obj, uap->val, 1)); 2858178646Sdavidxu} 2859178646Sdavidxu 2860178646Sdavidxustatic int 2861162536Sdavidxu__umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) 2862162536Sdavidxu{ 2863162536Sdavidxu struct timespec *ts, timeout; 2864162536Sdavidxu int error; 2865162536Sdavidxu 2866162536Sdavidxu /* Allow a null timespec (wait forever). */ 2867162536Sdavidxu if (uap->uaddr2 == NULL) 2868162536Sdavidxu ts = NULL; 2869162536Sdavidxu else { 2870162536Sdavidxu error = copyin(uap->uaddr2, &timeout, 2871162536Sdavidxu sizeof(timeout)); 2872162536Sdavidxu if (error != 0) 2873162536Sdavidxu return (error); 2874162536Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2875162536Sdavidxu timeout.tv_nsec < 0) { 2876162536Sdavidxu return (EINVAL); 2877139013Sdavidxu } 2878162536Sdavidxu ts = &timeout; 2879139013Sdavidxu } 2880162536Sdavidxu return do_lock_umutex(td, uap->obj, ts, 0); 2881162536Sdavidxu} 2882162536Sdavidxu 2883162536Sdavidxustatic int 2884162536Sdavidxu__umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) 2885162536Sdavidxu{ 2886179970Sdavidxu return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY); 2887162536Sdavidxu} 2888162536Sdavidxu 2889162536Sdavidxustatic int 2890179970Sdavidxu__umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) 2891179970Sdavidxu{ 2892179970Sdavidxu struct timespec *ts, timeout; 2893179970Sdavidxu int error; 2894179970Sdavidxu 2895179970Sdavidxu /* Allow a null timespec (wait forever). */ 2896179970Sdavidxu if (uap->uaddr2 == NULL) 2897179970Sdavidxu ts = NULL; 2898179970Sdavidxu else { 2899179970Sdavidxu error = copyin(uap->uaddr2, &timeout, 2900179970Sdavidxu sizeof(timeout)); 2901179970Sdavidxu if (error != 0) 2902179970Sdavidxu return (error); 2903179970Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2904179970Sdavidxu timeout.tv_nsec < 0) { 2905179970Sdavidxu return (EINVAL); 2906179970Sdavidxu } 2907179970Sdavidxu ts = &timeout; 2908179970Sdavidxu } 2909179970Sdavidxu return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT); 2910179970Sdavidxu} 2911179970Sdavidxu 2912179970Sdavidxustatic int 2913179970Sdavidxu__umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) 2914179970Sdavidxu{ 2915179970Sdavidxu return do_wake_umutex(td, uap->obj); 2916179970Sdavidxu} 2917179970Sdavidxu 2918179970Sdavidxustatic int 2919162536Sdavidxu__umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) 2920162536Sdavidxu{ 2921162536Sdavidxu return do_unlock_umutex(td, uap->obj); 2922162536Sdavidxu} 2923162536Sdavidxu 2924162536Sdavidxustatic int 2925162536Sdavidxu__umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) 2926162536Sdavidxu{ 2927162536Sdavidxu return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); 2928162536Sdavidxu} 2929162536Sdavidxu 2930164839Sdavidxustatic int 2931164839Sdavidxu__umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) 2932164839Sdavidxu{ 2933164839Sdavidxu struct timespec *ts, timeout; 2934164839Sdavidxu int error; 2935164839Sdavidxu 2936164839Sdavidxu /* Allow a null timespec (wait forever). */ 2937164839Sdavidxu if (uap->uaddr2 == NULL) 2938164839Sdavidxu ts = NULL; 2939164839Sdavidxu else { 2940164839Sdavidxu error = copyin(uap->uaddr2, &timeout, 2941164839Sdavidxu sizeof(timeout)); 2942164839Sdavidxu if (error != 0) 2943164839Sdavidxu return (error); 2944164839Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2945164839Sdavidxu timeout.tv_nsec < 0) { 2946164839Sdavidxu return (EINVAL); 2947164839Sdavidxu } 2948164839Sdavidxu ts = &timeout; 2949164839Sdavidxu } 2950164876Sdavidxu return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 2951164839Sdavidxu} 2952164839Sdavidxu 2953164839Sdavidxustatic int 2954164839Sdavidxu__umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) 2955164839Sdavidxu{ 2956164839Sdavidxu return do_cv_signal(td, uap->obj); 2957164839Sdavidxu} 2958164839Sdavidxu 2959164839Sdavidxustatic int 2960164839Sdavidxu__umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) 2961164839Sdavidxu{ 2962164839Sdavidxu return do_cv_broadcast(td, uap->obj); 2963164839Sdavidxu} 2964164839Sdavidxu 2965177848Sdavidxustatic int 2966177848Sdavidxu__umtx_op_rw_rdlock(struct thread *td, struct _umtx_op_args *uap) 2967177848Sdavidxu{ 2968177848Sdavidxu struct timespec timeout; 2969177848Sdavidxu int error; 2970177848Sdavidxu 2971177848Sdavidxu /* Allow a null timespec (wait forever). */ 2972177848Sdavidxu if (uap->uaddr2 == NULL) { 2973177848Sdavidxu error = do_rw_rdlock(td, uap->obj, uap->val, 0); 2974177848Sdavidxu } else { 2975177848Sdavidxu error = copyin(uap->uaddr2, &timeout, 2976177848Sdavidxu sizeof(timeout)); 2977177848Sdavidxu if (error != 0) 2978177848Sdavidxu return (error); 2979177848Sdavidxu if (timeout.tv_nsec >= 1000000000 || 2980177848Sdavidxu timeout.tv_nsec < 0) { 2981177848Sdavidxu return (EINVAL); 2982177848Sdavidxu } 2983177848Sdavidxu error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout); 2984177848Sdavidxu } 2985177848Sdavidxu return (error); 2986177848Sdavidxu} 2987177848Sdavidxu 2988177848Sdavidxustatic int 2989177848Sdavidxu__umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) 2990177848Sdavidxu{ 2991177848Sdavidxu struct timespec timeout; 2992177848Sdavidxu int error; 2993177848Sdavidxu 2994177848Sdavidxu /* Allow a null timespec (wait forever). */ 2995177848Sdavidxu if (uap->uaddr2 == NULL) { 2996177848Sdavidxu error = do_rw_wrlock(td, uap->obj, 0); 2997177848Sdavidxu } else { 2998177848Sdavidxu error = copyin(uap->uaddr2, &timeout, 2999177848Sdavidxu sizeof(timeout)); 3000177848Sdavidxu if (error != 0) 3001177848Sdavidxu return (error); 3002177848Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3003177848Sdavidxu timeout.tv_nsec < 0) { 3004177848Sdavidxu return (EINVAL); 3005177848Sdavidxu } 3006177848Sdavidxu 3007177848Sdavidxu error = do_rw_wrlock2(td, uap->obj, &timeout); 3008177848Sdavidxu } 3009177848Sdavidxu return (error); 3010177848Sdavidxu} 3011177848Sdavidxu 3012177848Sdavidxustatic int 3013177848Sdavidxu__umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) 3014177848Sdavidxu{ 3015177880Sdavidxu return do_rw_unlock(td, uap->obj); 3016177848Sdavidxu} 3017177848Sdavidxu 3018162536Sdavidxutypedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); 3019162536Sdavidxu 3020162536Sdavidxustatic _umtx_op_func op_table[] = { 3021162536Sdavidxu __umtx_op_lock_umtx, /* UMTX_OP_LOCK */ 3022162536Sdavidxu __umtx_op_unlock_umtx, /* UMTX_OP_UNLOCK */ 3023162536Sdavidxu __umtx_op_wait, /* UMTX_OP_WAIT */ 3024162536Sdavidxu __umtx_op_wake, /* UMTX_OP_WAKE */ 3025162536Sdavidxu __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_TRYLOCK */ 3026162536Sdavidxu __umtx_op_lock_umutex, /* UMTX_OP_MUTEX_LOCK */ 3027162536Sdavidxu __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ 3028164839Sdavidxu __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ 3029164839Sdavidxu __umtx_op_cv_wait, /* UMTX_OP_CV_WAIT*/ 3030164839Sdavidxu __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ 3031173800Sdavidxu __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */ 3032177848Sdavidxu __umtx_op_wait_uint, /* UMTX_OP_WAIT_UINT */ 3033177848Sdavidxu __umtx_op_rw_rdlock, /* UMTX_OP_RW_RDLOCK */ 3034177848Sdavidxu __umtx_op_rw_wrlock, /* UMTX_OP_RW_WRLOCK */ 3035178646Sdavidxu __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ 3036178646Sdavidxu __umtx_op_wait_uint_private, /* UMTX_OP_WAIT_UINT_PRIVATE */ 3037179970Sdavidxu __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ 3038179970Sdavidxu __umtx_op_wait_umutex, /* UMTX_OP_UMUTEX_WAIT */ 3039179970Sdavidxu __umtx_op_wake_umutex /* UMTX_OP_UMUTEX_WAKE */ 3040162536Sdavidxu}; 3041162536Sdavidxu 3042162536Sdavidxuint 3043162536Sdavidxu_umtx_op(struct thread *td, struct _umtx_op_args *uap) 3044162536Sdavidxu{ 3045163678Sdavidxu if ((unsigned)uap->op < UMTX_OP_MAX) 3046162536Sdavidxu return (*op_table[uap->op])(td, uap); 3047162536Sdavidxu return (EINVAL); 3048162536Sdavidxu} 3049162536Sdavidxu 3050162536Sdavidxu#ifdef COMPAT_IA32 3051163046Sdavidxuint 3052163046Sdavidxufreebsd32_umtx_lock(struct thread *td, struct freebsd32_umtx_lock_args *uap) 3053163046Sdavidxu /* struct umtx *umtx */ 3054163046Sdavidxu{ 3055163046Sdavidxu return (do_lock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid, NULL)); 3056163046Sdavidxu} 3057163046Sdavidxu 3058163046Sdavidxuint 3059163046Sdavidxufreebsd32_umtx_unlock(struct thread *td, struct freebsd32_umtx_unlock_args *uap) 3060163046Sdavidxu /* struct umtx *umtx */ 3061163046Sdavidxu{ 3062163046Sdavidxu return (do_unlock_umtx32(td, (uint32_t *)uap->umtx, td->td_tid)); 3063163046Sdavidxu} 3064163046Sdavidxu 3065162536Sdavidxustruct timespec32 { 3066162536Sdavidxu u_int32_t tv_sec; 3067162536Sdavidxu u_int32_t tv_nsec; 3068162536Sdavidxu}; 3069162536Sdavidxu 3070162536Sdavidxustatic inline int 3071162536Sdavidxucopyin_timeout32(void *addr, struct timespec *tsp) 3072162536Sdavidxu{ 3073162536Sdavidxu struct timespec32 ts32; 3074162536Sdavidxu int error; 3075162536Sdavidxu 3076162536Sdavidxu error = copyin(addr, &ts32, sizeof(struct timespec32)); 3077162536Sdavidxu if (error == 0) { 3078162536Sdavidxu tsp->tv_sec = ts32.tv_sec; 3079162536Sdavidxu tsp->tv_nsec = ts32.tv_nsec; 3080162536Sdavidxu } 3081140421Sdavidxu return (error); 3082139013Sdavidxu} 3083161678Sdavidxu 3084162536Sdavidxustatic int 3085162536Sdavidxu__umtx_op_lock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) 3086162536Sdavidxu{ 3087162536Sdavidxu struct timespec *ts, timeout; 3088162536Sdavidxu int error; 3089162536Sdavidxu 3090162536Sdavidxu /* Allow a null timespec (wait forever). */ 3091162536Sdavidxu if (uap->uaddr2 == NULL) 3092162536Sdavidxu ts = NULL; 3093162536Sdavidxu else { 3094162536Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3095162536Sdavidxu if (error != 0) 3096162536Sdavidxu return (error); 3097162536Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3098162536Sdavidxu timeout.tv_nsec < 0) { 3099162536Sdavidxu return (EINVAL); 3100162536Sdavidxu } 3101162536Sdavidxu ts = &timeout; 3102162536Sdavidxu } 3103162536Sdavidxu return (do_lock_umtx32(td, uap->obj, uap->val, ts)); 3104162536Sdavidxu} 3105162536Sdavidxu 3106162536Sdavidxustatic int 3107162536Sdavidxu__umtx_op_unlock_umtx_compat32(struct thread *td, struct _umtx_op_args *uap) 3108162536Sdavidxu{ 3109162536Sdavidxu return (do_unlock_umtx32(td, uap->obj, (uint32_t)uap->val)); 3110162536Sdavidxu} 3111162536Sdavidxu 3112162536Sdavidxustatic int 3113162536Sdavidxu__umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 3114162536Sdavidxu{ 3115162536Sdavidxu struct timespec *ts, timeout; 3116162536Sdavidxu int error; 3117162536Sdavidxu 3118162536Sdavidxu if (uap->uaddr2 == NULL) 3119162536Sdavidxu ts = NULL; 3120162536Sdavidxu else { 3121162536Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3122162536Sdavidxu if (error != 0) 3123162536Sdavidxu return (error); 3124162536Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3125162536Sdavidxu timeout.tv_nsec < 0) 3126162536Sdavidxu return (EINVAL); 3127162536Sdavidxu ts = &timeout; 3128162536Sdavidxu } 3129178646Sdavidxu return do_wait(td, uap->obj, uap->val, ts, 1, 0); 3130162536Sdavidxu} 3131162536Sdavidxu 3132162536Sdavidxustatic int 3133162536Sdavidxu__umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 3134162536Sdavidxu{ 3135162536Sdavidxu struct timespec *ts, timeout; 3136162536Sdavidxu int error; 3137162536Sdavidxu 3138162536Sdavidxu /* Allow a null timespec (wait forever). */ 3139162536Sdavidxu if (uap->uaddr2 == NULL) 3140162536Sdavidxu ts = NULL; 3141162536Sdavidxu else { 3142162536Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3143162536Sdavidxu if (error != 0) 3144162536Sdavidxu return (error); 3145162536Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3146162536Sdavidxu timeout.tv_nsec < 0) 3147162536Sdavidxu return (EINVAL); 3148162536Sdavidxu ts = &timeout; 3149162536Sdavidxu } 3150162536Sdavidxu return do_lock_umutex(td, uap->obj, ts, 0); 3151162536Sdavidxu} 3152162536Sdavidxu 3153164839Sdavidxustatic int 3154179970Sdavidxu__umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) 3155179970Sdavidxu{ 3156179970Sdavidxu struct timespec *ts, timeout; 3157179970Sdavidxu int error; 3158179970Sdavidxu 3159179970Sdavidxu /* Allow a null timespec (wait forever). */ 3160179970Sdavidxu if (uap->uaddr2 == NULL) 3161179970Sdavidxu ts = NULL; 3162179970Sdavidxu else { 3163179970Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3164179970Sdavidxu if (error != 0) 3165179970Sdavidxu return (error); 3166179970Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3167179970Sdavidxu timeout.tv_nsec < 0) 3168179970Sdavidxu return (EINVAL); 3169179970Sdavidxu ts = &timeout; 3170179970Sdavidxu } 3171179970Sdavidxu return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT); 3172179970Sdavidxu} 3173179970Sdavidxu 3174179970Sdavidxustatic int 3175164839Sdavidxu__umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap) 3176164839Sdavidxu{ 3177164839Sdavidxu struct timespec *ts, timeout; 3178164839Sdavidxu int error; 3179164839Sdavidxu 3180164839Sdavidxu /* Allow a null timespec (wait forever). */ 3181164839Sdavidxu if (uap->uaddr2 == NULL) 3182164839Sdavidxu ts = NULL; 3183164839Sdavidxu else { 3184164839Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3185164839Sdavidxu if (error != 0) 3186164839Sdavidxu return (error); 3187164839Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3188164839Sdavidxu timeout.tv_nsec < 0) 3189164839Sdavidxu return (EINVAL); 3190164839Sdavidxu ts = &timeout; 3191164839Sdavidxu } 3192164876Sdavidxu return (do_cv_wait(td, uap->obj, uap->uaddr1, ts, uap->val)); 3193164839Sdavidxu} 3194164839Sdavidxu 3195177848Sdavidxustatic int 3196177848Sdavidxu__umtx_op_rw_rdlock_compat32(struct thread *td, struct _umtx_op_args *uap) 3197177848Sdavidxu{ 3198177848Sdavidxu struct timespec timeout; 3199177848Sdavidxu int error; 3200177848Sdavidxu 3201177848Sdavidxu /* Allow a null timespec (wait forever). */ 3202177848Sdavidxu if (uap->uaddr2 == NULL) { 3203177848Sdavidxu error = do_rw_rdlock(td, uap->obj, uap->val, 0); 3204177848Sdavidxu } else { 3205177848Sdavidxu error = copyin(uap->uaddr2, &timeout, 3206177848Sdavidxu sizeof(timeout)); 3207177848Sdavidxu if (error != 0) 3208177848Sdavidxu return (error); 3209177848Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3210177848Sdavidxu timeout.tv_nsec < 0) { 3211177848Sdavidxu return (EINVAL); 3212177848Sdavidxu } 3213177848Sdavidxu error = do_rw_rdlock2(td, uap->obj, uap->val, &timeout); 3214177848Sdavidxu } 3215177848Sdavidxu return (error); 3216177848Sdavidxu} 3217177848Sdavidxu 3218177848Sdavidxustatic int 3219177848Sdavidxu__umtx_op_rw_wrlock_compat32(struct thread *td, struct _umtx_op_args *uap) 3220177848Sdavidxu{ 3221177848Sdavidxu struct timespec timeout; 3222177848Sdavidxu int error; 3223177848Sdavidxu 3224177848Sdavidxu /* Allow a null timespec (wait forever). */ 3225177848Sdavidxu if (uap->uaddr2 == NULL) { 3226177852Sdavidxu error = do_rw_wrlock(td, uap->obj, 0); 3227177848Sdavidxu } else { 3228177848Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3229177848Sdavidxu if (error != 0) 3230177848Sdavidxu return (error); 3231177848Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3232177848Sdavidxu timeout.tv_nsec < 0) { 3233177848Sdavidxu return (EINVAL); 3234177848Sdavidxu } 3235177848Sdavidxu 3236177852Sdavidxu error = do_rw_wrlock2(td, uap->obj, &timeout); 3237177848Sdavidxu } 3238177848Sdavidxu return (error); 3239177848Sdavidxu} 3240177848Sdavidxu 3241178646Sdavidxustatic int 3242178646Sdavidxu__umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *uap) 3243178646Sdavidxu{ 3244178646Sdavidxu struct timespec *ts, timeout; 3245178646Sdavidxu int error; 3246178646Sdavidxu 3247178646Sdavidxu if (uap->uaddr2 == NULL) 3248178646Sdavidxu ts = NULL; 3249178646Sdavidxu else { 3250178646Sdavidxu error = copyin_timeout32(uap->uaddr2, &timeout); 3251178646Sdavidxu if (error != 0) 3252178646Sdavidxu return (error); 3253178646Sdavidxu if (timeout.tv_nsec >= 1000000000 || 3254178646Sdavidxu timeout.tv_nsec < 0) 3255178646Sdavidxu return (EINVAL); 3256178646Sdavidxu ts = &timeout; 3257178646Sdavidxu } 3258178646Sdavidxu return do_wait(td, uap->obj, uap->val, ts, 1, 1); 3259178646Sdavidxu} 3260178646Sdavidxu 3261162536Sdavidxustatic _umtx_op_func op_table_compat32[] = { 3262162536Sdavidxu __umtx_op_lock_umtx_compat32, /* UMTX_OP_LOCK */ 3263162536Sdavidxu __umtx_op_unlock_umtx_compat32, /* UMTX_OP_UNLOCK */ 3264162536Sdavidxu __umtx_op_wait_compat32, /* UMTX_OP_WAIT */ 3265162536Sdavidxu __umtx_op_wake, /* UMTX_OP_WAKE */ 3266162550Sdavidxu __umtx_op_trylock_umutex, /* UMTX_OP_MUTEX_LOCK */ 3267162536Sdavidxu __umtx_op_lock_umutex_compat32, /* UMTX_OP_MUTEX_TRYLOCK */ 3268162536Sdavidxu __umtx_op_unlock_umutex, /* UMTX_OP_MUTEX_UNLOCK */ 3269164839Sdavidxu __umtx_op_set_ceiling, /* UMTX_OP_SET_CEILING */ 3270164839Sdavidxu __umtx_op_cv_wait_compat32, /* UMTX_OP_CV_WAIT*/ 3271164839Sdavidxu __umtx_op_cv_signal, /* UMTX_OP_CV_SIGNAL */ 3272173800Sdavidxu __umtx_op_cv_broadcast, /* UMTX_OP_CV_BROADCAST */ 3273177848Sdavidxu __umtx_op_wait_compat32, /* UMTX_OP_WAIT_UINT */ 3274177848Sdavidxu __umtx_op_rw_rdlock_compat32, /* UMTX_OP_RW_RDLOCK */ 3275177848Sdavidxu __umtx_op_rw_wrlock_compat32, /* UMTX_OP_RW_WRLOCK */ 3276178646Sdavidxu __umtx_op_rw_unlock, /* UMTX_OP_RW_UNLOCK */ 3277178646Sdavidxu __umtx_op_wait_uint_private_compat32, /* UMTX_OP_WAIT_UINT_PRIVATE */ 3278179970Sdavidxu __umtx_op_wake_private, /* UMTX_OP_WAKE_PRIVATE */ 3279179970Sdavidxu __umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */ 3280179970Sdavidxu __umtx_op_wake_umutex /* UMTX_OP_UMUTEX_WAKE */ 3281162536Sdavidxu}; 3282162536Sdavidxu 3283162536Sdavidxuint 3284162536Sdavidxufreebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) 3285162536Sdavidxu{ 3286163678Sdavidxu if ((unsigned)uap->op < UMTX_OP_MAX) 3287162536Sdavidxu return (*op_table_compat32[uap->op])(td, 3288162536Sdavidxu (struct _umtx_op_args *)uap); 3289162536Sdavidxu return (EINVAL); 3290162536Sdavidxu} 3291162536Sdavidxu#endif 3292162536Sdavidxu 3293161678Sdavidxuvoid 3294161678Sdavidxuumtx_thread_init(struct thread *td) 3295161678Sdavidxu{ 3296161678Sdavidxu td->td_umtxq = umtxq_alloc(); 3297161678Sdavidxu td->td_umtxq->uq_thread = td; 3298161678Sdavidxu} 3299161678Sdavidxu 3300161678Sdavidxuvoid 3301161678Sdavidxuumtx_thread_fini(struct thread *td) 3302161678Sdavidxu{ 3303161678Sdavidxu umtxq_free(td->td_umtxq); 3304161678Sdavidxu} 3305161678Sdavidxu 3306161678Sdavidxu/* 3307161678Sdavidxu * It will be called when new thread is created, e.g fork(). 3308161678Sdavidxu */ 3309161678Sdavidxuvoid 3310161678Sdavidxuumtx_thread_alloc(struct thread *td) 3311161678Sdavidxu{ 3312161678Sdavidxu struct umtx_q *uq; 3313161678Sdavidxu 3314161678Sdavidxu uq = td->td_umtxq; 3315161678Sdavidxu uq->uq_inherited_pri = PRI_MAX; 3316161678Sdavidxu 3317161678Sdavidxu KASSERT(uq->uq_flags == 0, ("uq_flags != 0")); 3318161678Sdavidxu KASSERT(uq->uq_thread == td, ("uq_thread != td")); 3319161678Sdavidxu KASSERT(uq->uq_pi_blocked == NULL, ("uq_pi_blocked != NULL")); 3320161678Sdavidxu KASSERT(TAILQ_EMPTY(&uq->uq_pi_contested), ("uq_pi_contested is not empty")); 3321161678Sdavidxu} 3322161678Sdavidxu 3323161678Sdavidxu/* 3324161678Sdavidxu * exec() hook. 3325161678Sdavidxu */ 3326161678Sdavidxustatic void 3327161678Sdavidxuumtx_exec_hook(void *arg __unused, struct proc *p __unused, 3328161678Sdavidxu struct image_params *imgp __unused) 3329161678Sdavidxu{ 3330161678Sdavidxu umtx_thread_cleanup(curthread); 3331161678Sdavidxu} 3332161678Sdavidxu 3333161678Sdavidxu/* 3334161678Sdavidxu * thread_exit() hook. 3335161678Sdavidxu */ 3336161678Sdavidxuvoid 3337161678Sdavidxuumtx_thread_exit(struct thread *td) 3338161678Sdavidxu{ 3339161678Sdavidxu umtx_thread_cleanup(td); 3340161678Sdavidxu} 3341161678Sdavidxu 3342161678Sdavidxu/* 3343161678Sdavidxu * clean up umtx data. 3344161678Sdavidxu */ 3345161678Sdavidxustatic void 3346161678Sdavidxuumtx_thread_cleanup(struct thread *td) 3347161678Sdavidxu{ 3348161678Sdavidxu struct umtx_q *uq; 3349161678Sdavidxu struct umtx_pi *pi; 3350161678Sdavidxu 3351161678Sdavidxu if ((uq = td->td_umtxq) == NULL) 3352161678Sdavidxu return; 3353161678Sdavidxu 3354170300Sjeff mtx_lock_spin(&umtx_lock); 3355161678Sdavidxu uq->uq_inherited_pri = PRI_MAX; 3356161678Sdavidxu while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { 3357161678Sdavidxu pi->pi_owner = NULL; 3358161678Sdavidxu TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); 3359161678Sdavidxu } 3360174701Sdavidxu thread_lock(td); 3361161678Sdavidxu td->td_flags &= ~TDF_UBORROWING; 3362174701Sdavidxu thread_unlock(td); 3363170300Sjeff mtx_unlock_spin(&umtx_lock); 3364161678Sdavidxu} 3365