1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright 1999, 2000 John D. Polstra. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * from: FreeBSD: src/libexec/rtld-elf/sparc64/lockdflt.c,v 1.3 2002/10/09 28 */ 29 30/* 31 * Thread locking implementation for the dynamic linker. 32 * 33 * We use the "simple, non-scalable reader-preference lock" from: 34 * 35 * J. M. Mellor-Crummey and M. L. Scott. "Scalable Reader-Writer 36 * Synchronization for Shared-Memory Multiprocessors." 3rd ACM Symp. on 37 * Principles and Practice of Parallel Programming, April 1991. 38 * 39 * In this algorithm the lock is a single word. Its low-order bit is 40 * set when a writer holds the lock. The remaining high-order bits 41 * contain a count of readers desiring the lock. The algorithm requires 42 * atomic "compare_and_store" and "add" operations, which we take 43 * from machine/atomic.h. 44 */ 45 46#include <sys/param.h> 47#include <sys/signalvar.h> 48#include <signal.h> 49#include <stdlib.h> 50#include <time.h> 51 52#include "debug.h" 53#include "rtld.h" 54#include "rtld_machdep.h" 55#include "rtld_libc.h" 56 57void _rtld_thread_init(struct RtldLockInfo *) __exported; 58void _rtld_atfork_pre(int *) __exported; 59void _rtld_atfork_post(int *) __exported; 60 61static char def_dlerror_msg[512]; 62static int def_dlerror_seen_val = 1; 63 64static char * 65def_dlerror_loc(void) 66{ 67 return (def_dlerror_msg); 68} 69 70static int * 71def_dlerror_seen(void) 72{ 73 return (&def_dlerror_seen_val); 74} 75 76#define WAFLAG 0x1 /* A writer holds the lock */ 77#define RC_INCR 0x2 /* Adjusts count of readers desiring lock */ 78 79typedef struct Struct_Lock { 80 volatile u_int lock; 81 void *base; 82} Lock; 83 84static sigset_t fullsigmask, oldsigmask; 85static int thread_flag, wnested; 86static uint32_t fsigblock; 87 88static void * 89def_lock_create(void) 90{ 91 void *base; 92 char *p; 93 uintptr_t r; 94 Lock *l; 95 96 /* 97 * Arrange for the lock to occupy its own cache line. First, we 98 * optimistically allocate just a cache line, hoping that malloc 99 * will give us a well-aligned block of memory. If that doesn't 100 * work, we allocate a larger block and take a well-aligned cache 101 * line from it. 102 */ 103 base = xmalloc(CACHE_LINE_SIZE); 104 p = base; 105 if ((uintptr_t)p % CACHE_LINE_SIZE != 0) { 106 free(base); 107 base = xmalloc(2 * CACHE_LINE_SIZE); 108 p = base; 109 if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0) 110 p += CACHE_LINE_SIZE - r; 111 } 112 l = (Lock *)p; 113 l->base = base; 114 l->lock = 0; 115 return (l); 116} 117 118static void 119def_lock_destroy(void *lock) 120{ 121 Lock *l = lock; 122 123 free(l->base); 124} 125 126static void 127sig_fastunblock(void) 128{ 129 uint32_t oldval; 130 131 assert((fsigblock & ~SIGFASTBLOCK_FLAGS) >= SIGFASTBLOCK_INC); 132 oldval = atomic_fetchadd_32(&fsigblock, -SIGFASTBLOCK_INC); 133 if (oldval == (SIGFASTBLOCK_PEND | SIGFASTBLOCK_INC)) 134 __sys_sigfastblock(SIGFASTBLOCK_UNBLOCK, NULL); 135} 136 137static bool 138def_lock_acquire_set(Lock *l, bool wlock) 139{ 140 if (wlock) { 141 if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) 142 return (true); 143 } else { 144 atomic_add_acq_int(&l->lock, RC_INCR); 145 if ((l->lock & WAFLAG) == 0) 146 return (true); 147 atomic_add_int(&l->lock, -RC_INCR); 148 } 149 return (false); 150} 151 152static void 153def_lock_acquire(Lock *l, bool wlock) 154{ 155 sigset_t tmp_oldsigmask; 156 157 if (ld_fast_sigblock) { 158 for (;;) { 159 atomic_add_32(&fsigblock, SIGFASTBLOCK_INC); 160 if (def_lock_acquire_set(l, wlock)) 161 break; 162 sig_fastunblock(); 163 } 164 } else { 165 for (;;) { 166 sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask); 167 if (def_lock_acquire_set(l, wlock)) 168 break; 169 sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL); 170 } 171 if (atomic_fetchadd_int(&wnested, 1) == 0) 172 oldsigmask = tmp_oldsigmask; 173 } 174} 175 176static void 177def_rlock_acquire(void *lock) 178{ 179 def_lock_acquire(lock, false); 180} 181 182static void 183def_wlock_acquire(void *lock) 184{ 185 def_lock_acquire(lock, true); 186} 187 188static void 189def_lock_release(void *lock) 190{ 191 Lock *l = lock; 192 193 atomic_add_rel_int(&l->lock, -((l->lock & WAFLAG) == 0 ? 194 RC_INCR : WAFLAG)); 195 if (ld_fast_sigblock) 196 sig_fastunblock(); 197 else if (atomic_fetchadd_int(&wnested, -1) == 1) 198 sigprocmask(SIG_SETMASK, &oldsigmask, NULL); 199} 200 201static int 202def_thread_set_flag(int mask) 203{ 204 int old_val = thread_flag; 205 206 thread_flag |= mask; 207 return (old_val); 208} 209 210static int 211def_thread_clr_flag(int mask) 212{ 213 int old_val = thread_flag; 214 215 thread_flag &= ~mask; 216 return (old_val); 217} 218 219/* 220 * Public interface exposed to the rest of the dynamic linker. 221 */ 222struct RtldLockInfo lockinfo; 223static struct RtldLockInfo deflockinfo; 224 225static __inline int 226thread_mask_set(int mask) 227{ 228 return (lockinfo.thread_set_flag(mask)); 229} 230 231static __inline void 232thread_mask_clear(int mask) 233{ 234 lockinfo.thread_clr_flag(mask); 235} 236 237#define RTLD_LOCK_CNT 3 238static struct rtld_lock { 239 void *handle; 240 int mask; 241} rtld_locks[RTLD_LOCK_CNT]; 242 243rtld_lock_t rtld_bind_lock = &rtld_locks[0]; 244rtld_lock_t rtld_libc_lock = &rtld_locks[1]; 245rtld_lock_t rtld_phdr_lock = &rtld_locks[2]; 246 247void 248rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) 249{ 250 251 if (lockstate == NULL) 252 return; 253 254 if (thread_mask_set(lock->mask) & lock->mask) { 255 dbg("rlock_acquire: recursed"); 256 lockstate->lockstate = RTLD_LOCK_UNLOCKED; 257 return; 258 } 259 lockinfo.rlock_acquire(lock->handle); 260 lockstate->lockstate = RTLD_LOCK_RLOCKED; 261} 262 263void 264wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) 265{ 266 267 if (lockstate == NULL) 268 return; 269 270 if (thread_mask_set(lock->mask) & lock->mask) { 271 dbg("wlock_acquire: recursed"); 272 lockstate->lockstate = RTLD_LOCK_UNLOCKED; 273 return; 274 } 275 lockinfo.wlock_acquire(lock->handle); 276 lockstate->lockstate = RTLD_LOCK_WLOCKED; 277} 278 279void 280lock_release(rtld_lock_t lock, RtldLockState *lockstate) 281{ 282 283 if (lockstate == NULL) 284 return; 285 286 switch (lockstate->lockstate) { 287 case RTLD_LOCK_UNLOCKED: 288 break; 289 case RTLD_LOCK_RLOCKED: 290 case RTLD_LOCK_WLOCKED: 291 thread_mask_clear(lock->mask); 292 lockinfo.lock_release(lock->handle); 293 break; 294 default: 295 assert(0); 296 } 297} 298 299void 300lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate) 301{ 302 303 if (lockstate == NULL) 304 return; 305 306 lock_release(lock, lockstate); 307 wlock_acquire(lock, lockstate); 308} 309 310void 311lock_restart_for_upgrade(RtldLockState *lockstate) 312{ 313 314 if (lockstate == NULL) 315 return; 316 317 switch (lockstate->lockstate) { 318 case RTLD_LOCK_UNLOCKED: 319 case RTLD_LOCK_WLOCKED: 320 break; 321 case RTLD_LOCK_RLOCKED: 322 siglongjmp(lockstate->env, 1); 323 break; 324 default: 325 assert(0); 326 } 327} 328 329void 330dlerror_dflt_init(void) 331{ 332 lockinfo.dlerror_loc = def_dlerror_loc; 333 lockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg); 334 lockinfo.dlerror_seen = def_dlerror_seen; 335} 336 337void 338lockdflt_init(void) 339{ 340 int i; 341 342 deflockinfo.rtli_version = RTLI_VERSION; 343 deflockinfo.lock_create = def_lock_create; 344 deflockinfo.lock_destroy = def_lock_destroy; 345 deflockinfo.rlock_acquire = def_rlock_acquire; 346 deflockinfo.wlock_acquire = def_wlock_acquire; 347 deflockinfo.lock_release = def_lock_release; 348 deflockinfo.thread_set_flag = def_thread_set_flag; 349 deflockinfo.thread_clr_flag = def_thread_clr_flag; 350 deflockinfo.at_fork = NULL; 351 deflockinfo.dlerror_loc = def_dlerror_loc; 352 deflockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg); 353 deflockinfo.dlerror_seen = def_dlerror_seen; 354 355 for (i = 0; i < RTLD_LOCK_CNT; i++) { 356 rtld_locks[i].mask = (1 << i); 357 rtld_locks[i].handle = NULL; 358 } 359 360 memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo)); 361 _rtld_thread_init(NULL); 362 if (ld_fast_sigblock) { 363 __sys_sigfastblock(SIGFASTBLOCK_SETPTR, &fsigblock); 364 } else { 365 /* 366 * Construct a mask to block all signals. Note that 367 * blocked traps mean that the process is terminated 368 * if trap occurs while we are in locked section, with 369 * the default settings for kern.forcesigexit. 370 */ 371 sigfillset(&fullsigmask); 372 } 373} 374 375/* 376 * Callback function to allow threads implementation to 377 * register their own locking primitives if the default 378 * one is not suitable. 379 * The current context should be the only context 380 * executing at the invocation time. 381 */ 382void 383_rtld_thread_init(struct RtldLockInfo *pli) 384{ 385 const Obj_Entry *obj; 386 SymLook req; 387 void *locks[RTLD_LOCK_CNT]; 388 int flags, i, res; 389 390 if (pli == NULL) { 391 lockinfo.rtli_version = RTLI_VERSION; 392 } else { 393 lockinfo.rtli_version = RTLI_VERSION_ONE; 394 obj = obj_from_addr(pli->lock_create); 395 if (obj != NULL) { 396 symlook_init(&req, "_pli_rtli_version"); 397 res = symlook_obj(&req, obj); 398 if (res == 0) 399 lockinfo.rtli_version = pli->rtli_version; 400 } 401 } 402 403 /* disable all locking while this function is running */ 404 flags = thread_mask_set(~0); 405 406 if (pli == NULL) 407 pli = &deflockinfo; 408 else if (ld_fast_sigblock) { 409 fsigblock = 0; 410 __sys_sigfastblock(SIGFASTBLOCK_UNSETPTR, NULL); 411 } 412 413 for (i = 0; i < RTLD_LOCK_CNT; i++) 414 if ((locks[i] = pli->lock_create()) == NULL) 415 break; 416 417 if (i < RTLD_LOCK_CNT) { 418 while (--i >= 0) 419 pli->lock_destroy(locks[i]); 420 abort(); 421 } 422 423 for (i = 0; i < RTLD_LOCK_CNT; i++) { 424 if (rtld_locks[i].handle == NULL) 425 continue; 426 if (flags & rtld_locks[i].mask) 427 lockinfo.lock_release(rtld_locks[i].handle); 428 lockinfo.lock_destroy(rtld_locks[i].handle); 429 } 430 431 for (i = 0; i < RTLD_LOCK_CNT; i++) { 432 rtld_locks[i].handle = locks[i]; 433 if (flags & rtld_locks[i].mask) 434 pli->wlock_acquire(rtld_locks[i].handle); 435 } 436 437 lockinfo.lock_create = pli->lock_create; 438 lockinfo.lock_destroy = pli->lock_destroy; 439 lockinfo.rlock_acquire = pli->rlock_acquire; 440 lockinfo.wlock_acquire = pli->wlock_acquire; 441 lockinfo.lock_release = pli->lock_release; 442 lockinfo.thread_set_flag = pli->thread_set_flag; 443 lockinfo.thread_clr_flag = pli->thread_clr_flag; 444 lockinfo.at_fork = pli->at_fork; 445 if (lockinfo.rtli_version > RTLI_VERSION_ONE && pli != NULL) { 446 strlcpy(pli->dlerror_loc(), lockinfo.dlerror_loc(), 447 lockinfo.dlerror_loc_sz); 448 lockinfo.dlerror_loc = pli->dlerror_loc; 449 lockinfo.dlerror_loc_sz = pli->dlerror_loc_sz; 450 lockinfo.dlerror_seen = pli->dlerror_seen; 451 } 452 453 /* restore thread locking state, this time with new locks */ 454 thread_mask_clear(~0); 455 thread_mask_set(flags); 456 dbg("_rtld_thread_init: done"); 457} 458 459void 460_rtld_atfork_pre(int *locks) 461{ 462 RtldLockState ls[2]; 463 464 if (locks == NULL) 465 return; 466 467 /* 468 * Warning: this did not worked well with the rtld compat 469 * locks above, when the thread signal mask was corrupted (set 470 * to all signals blocked) if two locks were taken 471 * simultaneously in the write mode. The caller of the 472 * _rtld_atfork_pre() must provide the working implementation 473 * of the locks anyway, and libthr locks are fine. 474 */ 475 wlock_acquire(rtld_phdr_lock, &ls[0]); 476 wlock_acquire(rtld_bind_lock, &ls[1]); 477 478 /* XXXKIB: I am really sorry for this. */ 479 locks[0] = ls[1].lockstate; 480 locks[2] = ls[0].lockstate; 481} 482 483void 484_rtld_atfork_post(int *locks) 485{ 486 RtldLockState ls[2]; 487 488 if (locks == NULL) 489 return; 490 491 bzero(ls, sizeof(ls)); 492 ls[0].lockstate = locks[2]; 493 ls[1].lockstate = locks[0]; 494 lock_release(rtld_bind_lock, &ls[1]); 495 lock_release(rtld_phdr_lock, &ls[0]); 496} 497