kern_umtx.c revision 158377
1139804Simp/*- 2139013Sdavidxu * Copyright (c) 2004, David Xu <davidxu@freebsd.org> 3112904Sjeff * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 4112904Sjeff * All rights reserved. 5112904Sjeff * 6112904Sjeff * Redistribution and use in source and binary forms, with or without 7112904Sjeff * modification, are permitted provided that the following conditions 8112904Sjeff * are met: 9112904Sjeff * 1. Redistributions of source code must retain the above copyright 10112904Sjeff * notice unmodified, this list of conditions, and the following 11112904Sjeff * disclaimer. 12112904Sjeff * 2. Redistributions in binary form must reproduce the above copyright 13112904Sjeff * notice, this list of conditions and the following disclaimer in the 14112904Sjeff * documentation and/or other materials provided with the distribution. 15112904Sjeff * 16112904Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17112904Sjeff * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18112904Sjeff * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19112904Sjeff * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20112904Sjeff * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21112904Sjeff * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22112904Sjeff * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23112904Sjeff * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24112904Sjeff * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25112904Sjeff * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26112904Sjeff */ 27112904Sjeff 28116182Sobrien#include <sys/cdefs.h> 29116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/kern_umtx.c 158377 2006-05-09 13:00:46Z davidxu $"); 30116182Sobrien 31112904Sjeff#include <sys/param.h> 32112904Sjeff#include <sys/kernel.h> 33131431Smarcel#include <sys/limits.h> 34112904Sjeff#include <sys/lock.h> 35115765Sjeff#include <sys/malloc.h> 36112904Sjeff#include <sys/mutex.h> 37112904Sjeff#include <sys/proc.h> 38112904Sjeff#include <sys/sysent.h> 39112904Sjeff#include <sys/systm.h> 40112904Sjeff#include <sys/sysproto.h> 41139013Sdavidxu#include <sys/eventhandler.h> 42112904Sjeff#include <sys/thr.h> 43112904Sjeff#include <sys/umtx.h> 44112904Sjeff 45139013Sdavidxu#include <vm/vm.h> 46139013Sdavidxu#include <vm/vm_param.h> 47139013Sdavidxu#include <vm/pmap.h> 48139013Sdavidxu#include <vm/vm_map.h> 49139013Sdavidxu#include <vm/vm_object.h> 50139013Sdavidxu 51139013Sdavidxu#define UMTX_PRIVATE 0 52139013Sdavidxu#define UMTX_SHARED 1 53139013Sdavidxu 54139013Sdavidxustruct umtx_key { 55139013Sdavidxu int type; 56139013Sdavidxu union { 57139013Sdavidxu struct { 58139013Sdavidxu vm_object_t object; 59139013Sdavidxu long offset; 60139013Sdavidxu } shared; 61139013Sdavidxu struct { 62139013Sdavidxu struct umtx *umtx; 63139013Sdavidxu long pid; 64139013Sdavidxu } private; 65139013Sdavidxu struct { 66139013Sdavidxu void *ptr; 67139013Sdavidxu long word; 68139013Sdavidxu } both; 69139013Sdavidxu } info; 70139013Sdavidxu}; 71139013Sdavidxu 72115765Sjeffstruct umtx_q { 73115765Sjeff LIST_ENTRY(umtx_q) uq_next; /* Linked list for the hash. */ 74139013Sdavidxu struct umtx_key uq_key; /* Umtx key. */ 75139257Sdavidxu struct thread *uq_thread; /* The thread waits on. */ 76139013Sdavidxu LIST_ENTRY(umtx_q) uq_rqnext; /* Linked list for requeuing. */ 77139013Sdavidxu vm_offset_t uq_addr; /* Umtx's virtual address. */ 78115765Sjeff}; 79115765Sjeff 80115765SjeffLIST_HEAD(umtx_head, umtx_q); 81138224Sdavidxustruct umtxq_chain { 82139013Sdavidxu struct mtx uc_lock; /* Lock for this chain. */ 83139013Sdavidxu struct umtx_head uc_queue; /* List of sleep queues. */ 84139257Sdavidxu#define UCF_BUSY 0x01 85139257Sdavidxu int uc_flags; 86158377Sdavidxu int uc_waiters; 87138224Sdavidxu}; 88115765Sjeff 89138224Sdavidxu#define GOLDEN_RATIO_PRIME 2654404609U 90138224Sdavidxu#define UMTX_CHAINS 128 91138224Sdavidxu#define UMTX_SHIFTS (__WORD_BIT - 7) 92115765Sjeff 93138224Sdavidxustatic struct umtxq_chain umtxq_chains[UMTX_CHAINS]; 94138224Sdavidxustatic MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); 95115310Sjeff 96139013Sdavidxustatic void umtxq_init_chains(void *); 97139013Sdavidxustatic int umtxq_hash(struct umtx_key *key); 98139013Sdavidxustatic struct mtx *umtxq_mtx(int chain); 99139013Sdavidxustatic void umtxq_lock(struct umtx_key *key); 100139013Sdavidxustatic void umtxq_unlock(struct umtx_key *key); 101139257Sdavidxustatic void umtxq_busy(struct umtx_key *key); 102139257Sdavidxustatic void umtxq_unbusy(struct umtx_key *key); 103139013Sdavidxustatic void umtxq_insert(struct umtx_q *uq); 104139013Sdavidxustatic void umtxq_remove(struct umtx_q *uq); 105139013Sdavidxustatic int umtxq_sleep(struct thread *td, struct umtx_key *key, 106139013Sdavidxu int prio, const char *wmesg, int timo); 107139257Sdavidxustatic int umtxq_count(struct umtx_key *key); 108139257Sdavidxustatic int umtxq_signal(struct umtx_key *key, int nr_wakeup); 109139013Sdavidxustatic int umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2); 110139013Sdavidxustatic int umtx_key_get(struct thread *td, struct umtx *umtx, 111139013Sdavidxu struct umtx_key *key); 112139013Sdavidxustatic void umtx_key_release(struct umtx_key *key); 113115310Sjeff 114139013SdavidxuSYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_init_chains, NULL); 115138224Sdavidxu 116143149Sdavidxustruct umtx_q * 117143149Sdavidxuumtxq_alloc(void) 118143149Sdavidxu{ 119143149Sdavidxu return (malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK)); 120143149Sdavidxu} 121143149Sdavidxu 122143149Sdavidxuvoid 123143149Sdavidxuumtxq_free(struct umtx_q *uq) 124143149Sdavidxu{ 125143149Sdavidxu free(uq, M_UMTX); 126143149Sdavidxu} 127143149Sdavidxu 128138224Sdavidxustatic void 129139013Sdavidxuumtxq_init_chains(void *arg __unused) 130138224Sdavidxu{ 131138224Sdavidxu int i; 132138224Sdavidxu 133138224Sdavidxu for (i = 0; i < UMTX_CHAINS; ++i) { 134138224Sdavidxu mtx_init(&umtxq_chains[i].uc_lock, "umtxq_lock", NULL, 135138224Sdavidxu MTX_DEF | MTX_DUPOK); 136139013Sdavidxu LIST_INIT(&umtxq_chains[i].uc_queue); 137139257Sdavidxu umtxq_chains[i].uc_flags = 0; 138158377Sdavidxu umtxq_chains[i].uc_waiters = 0; 139138224Sdavidxu } 140138224Sdavidxu} 141138224Sdavidxu 142138224Sdavidxustatic inline int 143139013Sdavidxuumtxq_hash(struct umtx_key *key) 144138224Sdavidxu{ 145139013Sdavidxu unsigned n = (uintptr_t)key->info.both.ptr + key->info.both.word; 146138224Sdavidxu return (((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS); 147138224Sdavidxu} 148138224Sdavidxu 149139013Sdavidxustatic inline int 150139013Sdavidxuumtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 151139013Sdavidxu{ 152139013Sdavidxu return (k1->type == k2->type && 153139013Sdavidxu k1->info.both.ptr == k2->info.both.ptr && 154139013Sdavidxu k1->info.both.word == k2->info.both.word); 155139013Sdavidxu} 156139013Sdavidxu 157139013Sdavidxustatic inline struct mtx * 158139013Sdavidxuumtxq_mtx(int chain) 159139013Sdavidxu{ 160139013Sdavidxu return (&umtxq_chains[chain].uc_lock); 161139013Sdavidxu} 162139013Sdavidxu 163138224Sdavidxustatic inline void 164139257Sdavidxuumtxq_busy(struct umtx_key *key) 165139257Sdavidxu{ 166139257Sdavidxu int chain = umtxq_hash(key); 167139257Sdavidxu 168139257Sdavidxu mtx_assert(umtxq_mtx(chain), MA_OWNED); 169139257Sdavidxu while (umtxq_chains[chain].uc_flags & UCF_BUSY) { 170158377Sdavidxu umtxq_chains[chain].uc_waiters++; 171139257Sdavidxu msleep(&umtxq_chains[chain], umtxq_mtx(chain), 172157815Sjhb 0, "umtxq_busy", 0); 173158377Sdavidxu umtxq_chains[chain].uc_waiters--; 174139257Sdavidxu } 175139257Sdavidxu umtxq_chains[chain].uc_flags |= UCF_BUSY; 176139257Sdavidxu} 177139257Sdavidxu 178139257Sdavidxustatic inline void 179139257Sdavidxuumtxq_unbusy(struct umtx_key *key) 180139257Sdavidxu{ 181139257Sdavidxu int chain = umtxq_hash(key); 182139257Sdavidxu 183139257Sdavidxu mtx_assert(umtxq_mtx(chain), MA_OWNED); 184139258Sdavidxu KASSERT(umtxq_chains[chain].uc_flags & UCF_BUSY, ("not busy")); 185139257Sdavidxu umtxq_chains[chain].uc_flags &= ~UCF_BUSY; 186158377Sdavidxu if (umtxq_chains[chain].uc_waiters) 187158377Sdavidxu wakeup_one(&umtxq_chains[chain]); 188139257Sdavidxu} 189139257Sdavidxu 190139257Sdavidxustatic inline void 191139013Sdavidxuumtxq_lock(struct umtx_key *key) 192138224Sdavidxu{ 193139013Sdavidxu int chain = umtxq_hash(key); 194139013Sdavidxu mtx_lock(umtxq_mtx(chain)); 195138224Sdavidxu} 196138224Sdavidxu 197138225Sdavidxustatic inline void 198139013Sdavidxuumtxq_unlock(struct umtx_key *key) 199138224Sdavidxu{ 200139013Sdavidxu int chain = umtxq_hash(key); 201139013Sdavidxu mtx_unlock(umtxq_mtx(chain)); 202138224Sdavidxu} 203138224Sdavidxu 204139013Sdavidxu/* 205139013Sdavidxu * Insert a thread onto the umtx queue. 206139013Sdavidxu */ 207139013Sdavidxustatic inline void 208139013Sdavidxuumtxq_insert(struct umtx_q *uq) 209115765Sjeff{ 210115765Sjeff struct umtx_head *head; 211139013Sdavidxu int chain = umtxq_hash(&uq->uq_key); 212139013Sdavidxu 213139257Sdavidxu mtx_assert(umtxq_mtx(chain), MA_OWNED); 214139013Sdavidxu head = &umtxq_chains[chain].uc_queue; 215139013Sdavidxu LIST_INSERT_HEAD(head, uq, uq_next); 216139013Sdavidxu mtx_lock_spin(&sched_lock); 217139013Sdavidxu uq->uq_thread->td_flags |= TDF_UMTXQ; 218139013Sdavidxu mtx_unlock_spin(&sched_lock); 219139013Sdavidxu} 220139013Sdavidxu 221139013Sdavidxu/* 222139013Sdavidxu * Remove thread from the umtx queue. 223139013Sdavidxu */ 224139013Sdavidxustatic inline void 225139013Sdavidxuumtxq_remove(struct umtx_q *uq) 226139013Sdavidxu{ 227139257Sdavidxu mtx_assert(umtxq_mtx(umtxq_hash(&uq->uq_key)), MA_OWNED); 228139013Sdavidxu if (uq->uq_thread->td_flags & TDF_UMTXQ) { 229139013Sdavidxu LIST_REMOVE(uq, uq_next); 230139013Sdavidxu /* turning off TDF_UMTXQ should be the last thing. */ 231139013Sdavidxu mtx_lock_spin(&sched_lock); 232139013Sdavidxu uq->uq_thread->td_flags &= ~TDF_UMTXQ; 233139013Sdavidxu mtx_unlock_spin(&sched_lock); 234139013Sdavidxu } 235139013Sdavidxu} 236139013Sdavidxu 237139013Sdavidxustatic int 238139013Sdavidxuumtxq_count(struct umtx_key *key) 239139013Sdavidxu{ 240115765Sjeff struct umtx_q *uq; 241139013Sdavidxu struct umtx_head *head; 242139013Sdavidxu int chain, count = 0; 243115765Sjeff 244139013Sdavidxu chain = umtxq_hash(key); 245139257Sdavidxu mtx_assert(umtxq_mtx(chain), MA_OWNED); 246139013Sdavidxu head = &umtxq_chains[chain].uc_queue; 247115765Sjeff LIST_FOREACH(uq, head, uq_next) { 248139013Sdavidxu if (umtx_key_match(&uq->uq_key, key)) { 249139013Sdavidxu if (++count > 1) 250139013Sdavidxu break; 251139013Sdavidxu } 252115765Sjeff } 253139013Sdavidxu return (count); 254115765Sjeff} 255115765Sjeff 256139257Sdavidxustatic int 257139257Sdavidxuumtxq_signal(struct umtx_key *key, int n_wake) 258115765Sjeff{ 259139257Sdavidxu struct umtx_q *uq, *next; 260115765Sjeff struct umtx_head *head; 261139013Sdavidxu struct thread *blocked = NULL; 262139257Sdavidxu int chain, ret; 263115765Sjeff 264139257Sdavidxu ret = 0; 265139013Sdavidxu chain = umtxq_hash(key); 266139257Sdavidxu mtx_assert(umtxq_mtx(chain), MA_OWNED); 267139013Sdavidxu head = &umtxq_chains[chain].uc_queue; 268139257Sdavidxu for (uq = LIST_FIRST(head); uq; uq = next) { 269139013Sdavidxu next = LIST_NEXT(uq, uq_next); 270139013Sdavidxu if (umtx_key_match(&uq->uq_key, key)) { 271139013Sdavidxu blocked = uq->uq_thread; 272139013Sdavidxu umtxq_remove(uq); 273139013Sdavidxu wakeup(blocked); 274139257Sdavidxu if (++ret >= n_wake) 275139257Sdavidxu break; 276139013Sdavidxu } 277139013Sdavidxu } 278139257Sdavidxu return (ret); 279138224Sdavidxu} 280138224Sdavidxu 281138224Sdavidxustatic inline int 282139013Sdavidxuumtxq_sleep(struct thread *td, struct umtx_key *key, int priority, 283139013Sdavidxu const char *wmesg, int timo) 284138224Sdavidxu{ 285139013Sdavidxu int chain = umtxq_hash(key); 286139751Sdavidxu int error = msleep(td, umtxq_mtx(chain), priority, wmesg, timo); 287139751Sdavidxu if (error == EWOULDBLOCK) 288139751Sdavidxu error = ETIMEDOUT; 289139751Sdavidxu return (error); 290138224Sdavidxu} 291138224Sdavidxu 292139013Sdavidxustatic int 293139013Sdavidxuumtx_key_get(struct thread *td, struct umtx *umtx, struct umtx_key *key) 294139013Sdavidxu{ 295139013Sdavidxu vm_map_t map; 296139013Sdavidxu vm_map_entry_t entry; 297139013Sdavidxu vm_pindex_t pindex; 298139013Sdavidxu vm_prot_t prot; 299139013Sdavidxu boolean_t wired; 300139013Sdavidxu 301139013Sdavidxu map = &td->td_proc->p_vmspace->vm_map; 302139013Sdavidxu if (vm_map_lookup(&map, (vm_offset_t)umtx, VM_PROT_WRITE, 303139013Sdavidxu &entry, &key->info.shared.object, &pindex, &prot, 304139013Sdavidxu &wired) != KERN_SUCCESS) { 305139013Sdavidxu return EFAULT; 306139013Sdavidxu } 307139013Sdavidxu 308139013Sdavidxu if (VM_INHERIT_SHARE == entry->inheritance) { 309139013Sdavidxu key->type = UMTX_SHARED; 310139013Sdavidxu key->info.shared.offset = entry->offset + entry->start - 311139013Sdavidxu (vm_offset_t)umtx; 312139013Sdavidxu vm_object_reference(key->info.shared.object); 313139013Sdavidxu } else { 314139013Sdavidxu key->type = UMTX_PRIVATE; 315139013Sdavidxu key->info.private.umtx = umtx; 316139013Sdavidxu key->info.private.pid = td->td_proc->p_pid; 317139013Sdavidxu } 318139013Sdavidxu vm_map_lookup_done(map, entry); 319139013Sdavidxu return (0); 320139013Sdavidxu} 321139013Sdavidxu 322139013Sdavidxustatic inline void 323139013Sdavidxuumtx_key_release(struct umtx_key *key) 324139013Sdavidxu{ 325139013Sdavidxu if (key->type == UMTX_SHARED) 326139013Sdavidxu vm_object_deallocate(key->info.shared.object); 327139013Sdavidxu} 328139013Sdavidxu 329138224Sdavidxustatic inline int 330139013Sdavidxuumtxq_queue_me(struct thread *td, struct umtx *umtx, struct umtx_q *uq) 331138224Sdavidxu{ 332139013Sdavidxu int error; 333138224Sdavidxu 334139013Sdavidxu if ((error = umtx_key_get(td, umtx, &uq->uq_key)) != 0) 335139013Sdavidxu return (error); 336139013Sdavidxu 337139013Sdavidxu uq->uq_addr = (vm_offset_t)umtx; 338139013Sdavidxu uq->uq_thread = td; 339139013Sdavidxu umtxq_lock(&uq->uq_key); 340139257Sdavidxu /* hmm, for condition variable, we don't need busy flag. */ 341139257Sdavidxu umtxq_busy(&uq->uq_key); 342139013Sdavidxu umtxq_insert(uq); 343139257Sdavidxu umtxq_unbusy(&uq->uq_key); 344139013Sdavidxu umtxq_unlock(&uq->uq_key); 345139013Sdavidxu return (0); 346138224Sdavidxu} 347138224Sdavidxu 348139013Sdavidxustatic int 349139013Sdavidxu_do_lock(struct thread *td, struct umtx *umtx, long id, int timo) 350112904Sjeff{ 351143149Sdavidxu struct umtx_q *uq; 352112904Sjeff intptr_t owner; 353112967Sjake intptr_t old; 354138224Sdavidxu int error = 0; 355112904Sjeff 356143149Sdavidxu uq = td->td_umtxq; 357112904Sjeff /* 358139013Sdavidxu * Care must be exercised when dealing with umtx structure. It 359112904Sjeff * can fault on any access. 360112904Sjeff */ 361112904Sjeff 362112904Sjeff for (;;) { 363112904Sjeff /* 364112904Sjeff * Try the uncontested case. This should be done in userland. 365112904Sjeff */ 366112904Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, 367139013Sdavidxu UMTX_UNOWNED, id); 368112904Sjeff 369138224Sdavidxu /* The acquire succeeded. */ 370138224Sdavidxu if (owner == UMTX_UNOWNED) 371138224Sdavidxu return (0); 372138224Sdavidxu 373115765Sjeff /* The address was invalid. */ 374115765Sjeff if (owner == -1) 375115765Sjeff return (EFAULT); 376115765Sjeff 377115765Sjeff /* If no one owns it but it is contested try to acquire it. */ 378115765Sjeff if (owner == UMTX_CONTESTED) { 379115765Sjeff owner = casuptr((intptr_t *)&umtx->u_owner, 380139013Sdavidxu UMTX_CONTESTED, id | UMTX_CONTESTED); 381115765Sjeff 382138224Sdavidxu if (owner == UMTX_CONTESTED) 383138224Sdavidxu return (0); 384138224Sdavidxu 385115765Sjeff /* The address was invalid. */ 386115765Sjeff if (owner == -1) 387115765Sjeff return (EFAULT); 388115765Sjeff 389115765Sjeff /* If this failed the lock has changed, restart. */ 390115765Sjeff continue; 391112904Sjeff } 392112904Sjeff 393138224Sdavidxu /* 394138224Sdavidxu * If we caught a signal, we have retried and now 395138224Sdavidxu * exit immediately. 396138224Sdavidxu */ 397143149Sdavidxu if (error || (error = umtxq_queue_me(td, umtx, uq)) != 0) 398138224Sdavidxu return (error); 399112904Sjeff 400112904Sjeff /* 401112904Sjeff * Set the contested bit so that a release in user space 402112904Sjeff * knows to use the system call for unlock. If this fails 403112904Sjeff * either some one else has acquired the lock or it has been 404112904Sjeff * released. 405112904Sjeff */ 406112967Sjake old = casuptr((intptr_t *)&umtx->u_owner, owner, 407112967Sjake owner | UMTX_CONTESTED); 408112904Sjeff 409112904Sjeff /* The address was invalid. */ 410112967Sjake if (old == -1) { 411143149Sdavidxu umtxq_lock(&uq->uq_key); 412143149Sdavidxu umtxq_busy(&uq->uq_key); 413143149Sdavidxu umtxq_remove(uq); 414143149Sdavidxu umtxq_unbusy(&uq->uq_key); 415143149Sdavidxu umtxq_unlock(&uq->uq_key); 416143149Sdavidxu umtx_key_release(&uq->uq_key); 417115765Sjeff return (EFAULT); 418112904Sjeff } 419112904Sjeff 420112904Sjeff /* 421115765Sjeff * We set the contested bit, sleep. Otherwise the lock changed 422117685Smtm * and we need to retry or we lost a race to the thread 423117685Smtm * unlocking the umtx. 424112904Sjeff */ 425143149Sdavidxu umtxq_lock(&uq->uq_key); 426139013Sdavidxu if (old == owner && (td->td_flags & TDF_UMTXQ)) { 427157815Sjhb error = umtxq_sleep(td, &uq->uq_key, PCATCH, 428139013Sdavidxu "umtx", timo); 429138224Sdavidxu } 430143149Sdavidxu umtxq_busy(&uq->uq_key); 431143149Sdavidxu umtxq_remove(uq); 432143149Sdavidxu umtxq_unbusy(&uq->uq_key); 433143149Sdavidxu umtxq_unlock(&uq->uq_key); 434143149Sdavidxu umtx_key_release(&uq->uq_key); 435112904Sjeff } 436117743Smtm 437117743Smtm return (0); 438112904Sjeff} 439112904Sjeff 440139013Sdavidxustatic int 441139013Sdavidxudo_lock(struct thread *td, struct umtx *umtx, long id, 442140245Sdavidxu struct timespec *timeout) 443112904Sjeff{ 444140245Sdavidxu struct timespec ts, ts2, ts3; 445139013Sdavidxu struct timeval tv; 446140245Sdavidxu int error; 447139013Sdavidxu 448140245Sdavidxu if (timeout == NULL) { 449139013Sdavidxu error = _do_lock(td, umtx, id, 0); 450139013Sdavidxu } else { 451140245Sdavidxu getnanouptime(&ts); 452140245Sdavidxu timespecadd(&ts, timeout); 453140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 454139013Sdavidxu for (;;) { 455140245Sdavidxu error = _do_lock(td, umtx, id, tvtohz(&tv)); 456140245Sdavidxu if (error != ETIMEDOUT) 457140245Sdavidxu break; 458140245Sdavidxu getnanouptime(&ts2); 459140245Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 460139751Sdavidxu error = ETIMEDOUT; 461139013Sdavidxu break; 462139013Sdavidxu } 463140245Sdavidxu ts3 = ts; 464140245Sdavidxu timespecsub(&ts3, &ts2); 465140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 466139013Sdavidxu } 467139013Sdavidxu } 468139258Sdavidxu /* 469139258Sdavidxu * This lets userland back off critical region if needed. 470139258Sdavidxu */ 471139258Sdavidxu if (error == ERESTART) 472139258Sdavidxu error = EINTR; 473139013Sdavidxu return (error); 474139013Sdavidxu} 475139013Sdavidxu 476139013Sdavidxustatic int 477139013Sdavidxudo_unlock(struct thread *td, struct umtx *umtx, long id) 478139013Sdavidxu{ 479139013Sdavidxu struct umtx_key key; 480112904Sjeff intptr_t owner; 481112967Sjake intptr_t old; 482139257Sdavidxu int error; 483139257Sdavidxu int count; 484112904Sjeff 485112904Sjeff /* 486112904Sjeff * Make sure we own this mtx. 487112904Sjeff * 488112904Sjeff * XXX Need a {fu,su}ptr this is not correct on arch where 489112904Sjeff * sizeof(intptr_t) != sizeof(long). 490112904Sjeff */ 491115765Sjeff if ((owner = fuword(&umtx->u_owner)) == -1) 492115765Sjeff return (EFAULT); 493115765Sjeff 494139013Sdavidxu if ((owner & ~UMTX_CONTESTED) != id) 495115765Sjeff return (EPERM); 496112904Sjeff 497117685Smtm /* We should only ever be in here for contested locks */ 498119836Stjr if ((owner & UMTX_CONTESTED) == 0) 499119836Stjr return (EINVAL); 500112904Sjeff 501139257Sdavidxu if ((error = umtx_key_get(td, umtx, &key)) != 0) 502139257Sdavidxu return (error); 503139257Sdavidxu 504139257Sdavidxu umtxq_lock(&key); 505139257Sdavidxu umtxq_busy(&key); 506139257Sdavidxu count = umtxq_count(&key); 507139257Sdavidxu umtxq_unlock(&key); 508139257Sdavidxu 509117743Smtm /* 510117743Smtm * When unlocking the umtx, it must be marked as unowned if 511117743Smtm * there is zero or one thread only waiting for it. 512117743Smtm * Otherwise, it must be marked as contested. 513117743Smtm */ 514139257Sdavidxu old = casuptr((intptr_t *)&umtx->u_owner, owner, 515139257Sdavidxu count <= 1 ? UMTX_UNOWNED : UMTX_CONTESTED); 516139257Sdavidxu umtxq_lock(&key); 517139257Sdavidxu umtxq_signal(&key, 0); 518139257Sdavidxu umtxq_unbusy(&key); 519139257Sdavidxu umtxq_unlock(&key); 520139257Sdavidxu umtx_key_release(&key); 521115765Sjeff if (old == -1) 522115765Sjeff return (EFAULT); 523138224Sdavidxu if (old != owner) 524138224Sdavidxu return (EINVAL); 525115765Sjeff return (0); 526112904Sjeff} 527139013Sdavidxu 528139013Sdavidxustatic int 529140245Sdavidxudo_wait(struct thread *td, struct umtx *umtx, long id, struct timespec *timeout) 530139013Sdavidxu{ 531143149Sdavidxu struct umtx_q *uq; 532140245Sdavidxu struct timespec ts, ts2, ts3; 533139013Sdavidxu struct timeval tv; 534139427Sdavidxu long tmp; 535140245Sdavidxu int error = 0; 536139013Sdavidxu 537143149Sdavidxu uq = td->td_umtxq; 538143149Sdavidxu if ((error = umtxq_queue_me(td, umtx, uq)) != 0) 539139013Sdavidxu return (error); 540139427Sdavidxu tmp = fuword(&umtx->u_owner); 541139427Sdavidxu if (tmp != id) { 542143149Sdavidxu umtxq_lock(&uq->uq_key); 543143149Sdavidxu umtxq_remove(uq); 544143149Sdavidxu umtxq_unlock(&uq->uq_key); 545140245Sdavidxu } else if (timeout == NULL) { 546143149Sdavidxu umtxq_lock(&uq->uq_key); 547139013Sdavidxu if (td->td_flags & TDF_UMTXQ) 548143149Sdavidxu error = umtxq_sleep(td, &uq->uq_key, 549157815Sjhb PCATCH, "ucond", 0); 550139257Sdavidxu if (!(td->td_flags & TDF_UMTXQ)) 551139257Sdavidxu error = 0; 552139257Sdavidxu else 553143149Sdavidxu umtxq_remove(uq); 554143149Sdavidxu umtxq_unlock(&uq->uq_key); 555139013Sdavidxu } else { 556140245Sdavidxu getnanouptime(&ts); 557140245Sdavidxu timespecadd(&ts, timeout); 558140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, timeout); 559139013Sdavidxu for (;;) { 560143149Sdavidxu umtxq_lock(&uq->uq_key); 561140245Sdavidxu if (td->td_flags & TDF_UMTXQ) { 562157815Sjhb error = umtxq_sleep(td, &uq->uq_key, PCATCH, 563140245Sdavidxu "ucond", tvtohz(&tv)); 564140245Sdavidxu } 565140245Sdavidxu if (!(td->td_flags & TDF_UMTXQ)) { 566143149Sdavidxu umtxq_unlock(&uq->uq_key); 567140245Sdavidxu goto out; 568140245Sdavidxu } 569143149Sdavidxu umtxq_unlock(&uq->uq_key); 570140245Sdavidxu if (error != ETIMEDOUT) 571140245Sdavidxu break; 572140245Sdavidxu getnanouptime(&ts2); 573140245Sdavidxu if (timespeccmp(&ts2, &ts, >=)) { 574139751Sdavidxu error = ETIMEDOUT; 575139013Sdavidxu break; 576139013Sdavidxu } 577140245Sdavidxu ts3 = ts; 578140245Sdavidxu timespecsub(&ts3, &ts2); 579140245Sdavidxu TIMESPEC_TO_TIMEVAL(&tv, &ts3); 580139013Sdavidxu } 581143149Sdavidxu umtxq_lock(&uq->uq_key); 582143149Sdavidxu umtxq_remove(uq); 583143149Sdavidxu umtxq_unlock(&uq->uq_key); 584139013Sdavidxu } 585140245Sdavidxuout: 586143149Sdavidxu umtx_key_release(&uq->uq_key); 587139257Sdavidxu if (error == ERESTART) 588139257Sdavidxu error = EINTR; 589139013Sdavidxu return (error); 590139013Sdavidxu} 591139013Sdavidxu 592151692Sdavidxuint 593151692Sdavidxukern_umtx_wake(struct thread *td, void *uaddr, int n_wake) 594139013Sdavidxu{ 595139013Sdavidxu struct umtx_key key; 596139257Sdavidxu int ret; 597139013Sdavidxu 598139257Sdavidxu if ((ret = umtx_key_get(td, uaddr, &key)) != 0) 599139257Sdavidxu return (ret); 600139258Sdavidxu umtxq_lock(&key); 601139257Sdavidxu ret = umtxq_signal(&key, n_wake); 602139258Sdavidxu umtxq_unlock(&key); 603139257Sdavidxu umtx_key_release(&key); 604139013Sdavidxu return (0); 605139013Sdavidxu} 606139013Sdavidxu 607139013Sdavidxuint 608139013Sdavidxu_umtx_lock(struct thread *td, struct _umtx_lock_args *uap) 609139013Sdavidxu /* struct umtx *umtx */ 610139013Sdavidxu{ 611139013Sdavidxu return _do_lock(td, uap->umtx, td->td_tid, 0); 612139013Sdavidxu} 613139013Sdavidxu 614139013Sdavidxuint 615139013Sdavidxu_umtx_unlock(struct thread *td, struct _umtx_unlock_args *uap) 616139013Sdavidxu /* struct umtx *umtx */ 617139013Sdavidxu{ 618139013Sdavidxu return do_unlock(td, uap->umtx, td->td_tid); 619139013Sdavidxu} 620139013Sdavidxu 621139013Sdavidxuint 622139013Sdavidxu_umtx_op(struct thread *td, struct _umtx_op_args *uap) 623139013Sdavidxu{ 624140245Sdavidxu struct timespec timeout; 625139013Sdavidxu struct timespec *ts; 626139013Sdavidxu int error; 627139013Sdavidxu 628139013Sdavidxu switch(uap->op) { 629139013Sdavidxu case UMTX_OP_LOCK: 630139013Sdavidxu /* Allow a null timespec (wait forever). */ 631139292Sdavidxu if (uap->uaddr2 == NULL) 632139013Sdavidxu ts = NULL; 633139013Sdavidxu else { 634140245Sdavidxu error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 635139013Sdavidxu if (error != 0) 636140102Sdavidxu break; 637140245Sdavidxu if (timeout.tv_nsec >= 1000000000 || 638140245Sdavidxu timeout.tv_nsec < 0) { 639140102Sdavidxu error = EINVAL; 640140102Sdavidxu break; 641140102Sdavidxu } 642140245Sdavidxu ts = &timeout; 643139013Sdavidxu } 644140102Sdavidxu error = do_lock(td, uap->umtx, uap->id, ts); 645140102Sdavidxu break; 646139013Sdavidxu case UMTX_OP_UNLOCK: 647140102Sdavidxu error = do_unlock(td, uap->umtx, uap->id); 648140102Sdavidxu break; 649139427Sdavidxu case UMTX_OP_WAIT: 650139013Sdavidxu /* Allow a null timespec (wait forever). */ 651139292Sdavidxu if (uap->uaddr2 == NULL) 652139013Sdavidxu ts = NULL; 653139013Sdavidxu else { 654140245Sdavidxu error = copyin(uap->uaddr2, &timeout, sizeof(timeout)); 655139013Sdavidxu if (error != 0) 656140102Sdavidxu break; 657140245Sdavidxu if (timeout.tv_nsec >= 1000000000 || 658140245Sdavidxu timeout.tv_nsec < 0) { 659140102Sdavidxu error = EINVAL; 660140102Sdavidxu break; 661140102Sdavidxu } 662140245Sdavidxu ts = &timeout; 663139013Sdavidxu } 664140102Sdavidxu error = do_wait(td, uap->umtx, uap->id, ts); 665140102Sdavidxu break; 666139013Sdavidxu case UMTX_OP_WAKE: 667151692Sdavidxu error = kern_umtx_wake(td, uap->umtx, uap->id); 668140102Sdavidxu break; 669139013Sdavidxu default: 670140102Sdavidxu error = EINVAL; 671140102Sdavidxu break; 672139013Sdavidxu } 673140421Sdavidxu return (error); 674139013Sdavidxu} 675