1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 */ 29 30#ifndef _SYS_UMTXVAR_H_ 31#define _SYS_UMTXVAR_H_ 32 33#ifdef _KERNEL 34 35#include <sys/_timespec.h> 36 37/* 38 * The umtx_key structure is used by both the Linux futex code and the 39 * umtx implementation to map userland addresses to unique keys. 40 */ 41enum { 42 TYPE_SIMPLE_WAIT, 43 TYPE_CV, 44 TYPE_SEM, 45 TYPE_SIMPLE_LOCK, 46 TYPE_NORMAL_UMUTEX, 47 TYPE_PI_UMUTEX, 48 TYPE_PP_UMUTEX, 49 TYPE_RWLOCK, 50 TYPE_FUTEX, 51 TYPE_SHM, 52 TYPE_PI_ROBUST_UMUTEX, 53 TYPE_PP_ROBUST_UMUTEX, 54 TYPE_PI_FUTEX, 55}; 56 57/* Key to represent a unique userland synchronous object */ 58struct umtx_key { 59 int hash; 60 int type; 61 int shared; 62 union { 63 struct { 64 struct vm_object *object; 65 uintptr_t offset; 66 } shared; 67 struct { 68 struct vmspace *vs; 69 uintptr_t addr; 70 } private; 71 struct { 72 void *a; 73 uintptr_t b; 74 } both; 75 } info; 76}; 77 78#define THREAD_SHARE 0 79#define PROCESS_SHARE 1 80#define AUTO_SHARE 2 81 82struct umtx_abs_timeout { 83 int clockid; 84 bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */ 85 struct timespec cur; 86 struct timespec end; 87}; 88 89struct thread; 90 91/* Priority inheritance mutex info. */ 92struct umtx_pi { 93 /* Owner thread */ 94 struct thread *pi_owner; 95 96 /* Reference count */ 97 int pi_refcount; 98 99 /* List entry to link umtx holding by thread */ 100 TAILQ_ENTRY(umtx_pi) pi_link; 101 102 /* List entry in hash */ 103 TAILQ_ENTRY(umtx_pi) pi_hashlink; 104 105 /* List for waiters */ 106 TAILQ_HEAD(,umtx_q) pi_blocked; 107 108 /* Identify a userland lock object */ 109 struct umtx_key pi_key; 110}; 111 112/* A userland synchronous object user. */ 113struct umtx_q { 114 /* Linked list for the hash. */ 115 TAILQ_ENTRY(umtx_q) uq_link; 116 117 /* Umtx key. */ 118 struct umtx_key uq_key; 119 120 /* Umtx flags. */ 121 int uq_flags; 122#define UQF_UMTXQ 0x0001 123 124 /* Futex bitset mask */ 125 u_int uq_bitset; 126 127 /* The thread waits on. */ 128 struct thread *uq_thread; 129 130 /* 131 * Blocked on PI mutex. read can use chain lock 132 * or umtx_lock, write must have both chain lock and 133 * umtx_lock being hold. 134 */ 135 struct umtx_pi *uq_pi_blocked; 136 137 /* On blocked list */ 138 TAILQ_ENTRY(umtx_q) uq_lockq; 139 140 /* Thread contending with us */ 141 TAILQ_HEAD(,umtx_pi) uq_pi_contested; 142 143 /* Inherited priority from PP mutex */ 144 u_char uq_inherited_pri; 145 146 /* Spare queue ready to be reused */ 147 struct umtxq_queue *uq_spare_queue; 148 149 /* The queue we on */ 150 struct umtxq_queue *uq_cur_queue; 151}; 152 153TAILQ_HEAD(umtxq_head, umtx_q); 154 155/* Per-key wait-queue */ 156struct umtxq_queue { 157 struct umtxq_head head; 158 struct umtx_key key; 159 LIST_ENTRY(umtxq_queue) link; 160 int length; 161}; 162 163LIST_HEAD(umtxq_list, umtxq_queue); 164 165/* Userland lock object's wait-queue chain */ 166struct umtxq_chain { 167 /* Lock for this chain. */ 168 struct mtx uc_lock; 169 170 /* List of sleep queues. */ 171 struct umtxq_list uc_queue[2]; 172#define UMTX_SHARED_QUEUE 0 173#define UMTX_EXCLUSIVE_QUEUE 1 174 175 LIST_HEAD(, umtxq_queue) uc_spare_queue; 176 177 /* Busy flag */ 178 char uc_busy; 179 180 /* Chain lock waiters */ 181 int uc_waiters; 182 183 /* All PI in the list */ 184 TAILQ_HEAD(,umtx_pi) uc_pi_list; 185 186#ifdef UMTX_PROFILING 187 u_int length; 188 u_int max_length; 189#endif 190}; 191 192static inline int 193umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2) 194{ 195 196 return (k1->type == k2->type && 197 k1->info.both.a == k2->info.both.a && 198 k1->info.both.b == k2->info.both.b); 199} 200 201void umtx_abs_timeout_init(struct umtx_abs_timeout *, int, int, 202 const struct timespec *); 203int umtx_copyin_timeout(const void *, struct timespec *); 204void umtx_exec(struct proc *p); 205int umtx_key_get(const void *, int, int, struct umtx_key *); 206void umtx_key_release(struct umtx_key *); 207struct umtx_q *umtxq_alloc(void); 208void umtxq_busy(struct umtx_key *); 209int umtxq_count(struct umtx_key *); 210void umtxq_free(struct umtx_q *); 211struct umtxq_chain *umtxq_getchain(struct umtx_key *); 212void umtxq_insert_queue(struct umtx_q *, int); 213void umtxq_remove_queue(struct umtx_q *, int); 214int umtxq_requeue(struct umtx_key *, int, struct umtx_key *, int); 215int umtxq_signal_mask(struct umtx_key *, int, u_int); 216int umtxq_sleep(struct umtx_q *, const char *, 217 struct umtx_abs_timeout *); 218int umtxq_sleep_pi(struct umtx_q *, struct umtx_pi *, uint32_t, 219 const char *, struct umtx_abs_timeout *, bool); 220void umtxq_unbusy(struct umtx_key *); 221void umtxq_unbusy_unlocked(struct umtx_key *); 222int kern_umtx_wake(struct thread *, void *, int, int); 223void umtx_pi_adjust(struct thread *, u_char); 224struct umtx_pi *umtx_pi_alloc(int); 225int umtx_pi_claim(struct umtx_pi *, struct thread *); 226int umtx_pi_drop(struct thread *, struct umtx_key *, bool, int *); 227void umtx_pi_free(struct umtx_pi *); 228void umtx_pi_insert(struct umtx_pi *); 229struct umtx_pi *umtx_pi_lookup(struct umtx_key *); 230void umtx_pi_ref(struct umtx_pi *); 231void umtx_pi_unref(struct umtx_pi *); 232void umtx_thread_init(struct thread *); 233void umtx_thread_fini(struct thread *); 234void umtx_thread_alloc(struct thread *); 235void umtx_thread_exit(struct thread *); 236 237#define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE) 238#define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE) 239 240/* 241 * Lock a chain. 242 * 243 * The code is a macro so that file/line information is taken from the caller. 244 */ 245#define umtxq_lock(key) do { \ 246 struct umtx_key *_key = (key); \ 247 struct umtxq_chain *_uc; \ 248 \ 249 _uc = umtxq_getchain(_key); \ 250 mtx_lock(&_uc->uc_lock); \ 251} while (0) 252 253/* 254 * Unlock a chain. 255 */ 256static inline void 257umtxq_unlock(struct umtx_key *key) 258{ 259 struct umtxq_chain *uc; 260 261 uc = umtxq_getchain(key); 262 mtx_unlock(&uc->uc_lock); 263} 264 265#endif /* _KERNEL */ 266#endif /* !_SYS_UMTXVAR_H_ */ 267