1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright 1999, 2000 John D. Polstra.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 *	from: FreeBSD: src/libexec/rtld-elf/sparc64/lockdflt.c,v 1.3 2002/10/09
28 */
29
30/*
31 * Thread locking implementation for the dynamic linker.
32 *
33 * We use the "simple, non-scalable reader-preference lock" from:
34 *
35 *   J. M. Mellor-Crummey and M. L. Scott. "Scalable Reader-Writer
36 *   Synchronization for Shared-Memory Multiprocessors." 3rd ACM Symp. on
37 *   Principles and Practice of Parallel Programming, April 1991.
38 *
39 * In this algorithm the lock is a single word.  Its low-order bit is
40 * set when a writer holds the lock.  The remaining high-order bits
41 * contain a count of readers desiring the lock.  The algorithm requires
42 * atomic "compare_and_store" and "add" operations, which we take
43 * from machine/atomic.h.
44 */
45
46#include <sys/param.h>
47#include <sys/signalvar.h>
48#include <signal.h>
49#include <stdlib.h>
50#include <time.h>
51
52#include "debug.h"
53#include "rtld.h"
54#include "rtld_machdep.h"
55#include "rtld_libc.h"
56
57void _rtld_thread_init(struct RtldLockInfo *) __exported;
58void _rtld_atfork_pre(int *) __exported;
59void _rtld_atfork_post(int *) __exported;
60
61static char def_dlerror_msg[512];
62static int def_dlerror_seen_val = 1;
63
64static char *
65def_dlerror_loc(void)
66{
67	return (def_dlerror_msg);
68}
69
70static int *
71def_dlerror_seen(void)
72{
73	return (&def_dlerror_seen_val);
74}
75
76#define WAFLAG		0x1	/* A writer holds the lock */
77#define RC_INCR		0x2	/* Adjusts count of readers desiring lock */
78
79typedef struct Struct_Lock {
80	volatile u_int lock;
81	void *base;
82} Lock;
83
84static sigset_t fullsigmask, oldsigmask;
85static int thread_flag, wnested;
86static uint32_t fsigblock;
87
88static void *
89def_lock_create(void)
90{
91	void *base;
92	char *p;
93	uintptr_t r;
94	Lock *l;
95
96	/*
97	 * Arrange for the lock to occupy its own cache line.  First, we
98	 * optimistically allocate just a cache line, hoping that malloc
99	 * will give us a well-aligned block of memory.  If that doesn't
100	 * work, we allocate a larger block and take a well-aligned cache
101	 * line from it.
102	 */
103	base = xmalloc(CACHE_LINE_SIZE);
104	p = base;
105	if ((uintptr_t)p % CACHE_LINE_SIZE != 0) {
106		free(base);
107		base = xmalloc(2 * CACHE_LINE_SIZE);
108		p = base;
109		if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0)
110			p += CACHE_LINE_SIZE - r;
111	}
112	l = (Lock *)p;
113	l->base = base;
114	l->lock = 0;
115	return (l);
116}
117
118static void
119def_lock_destroy(void *lock)
120{
121	Lock *l = lock;
122
123	free(l->base);
124}
125
126static void
127sig_fastunblock(void)
128{
129	uint32_t oldval;
130
131	assert((fsigblock & ~SIGFASTBLOCK_FLAGS) >= SIGFASTBLOCK_INC);
132	oldval = atomic_fetchadd_32(&fsigblock, -SIGFASTBLOCK_INC);
133	if (oldval == (SIGFASTBLOCK_PEND | SIGFASTBLOCK_INC))
134		__sys_sigfastblock(SIGFASTBLOCK_UNBLOCK, NULL);
135}
136
137static bool
138def_lock_acquire_set(Lock *l, bool wlock)
139{
140	if (wlock) {
141		if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG))
142			return (true);
143	} else {
144		atomic_add_acq_int(&l->lock, RC_INCR);
145		if ((l->lock & WAFLAG) == 0)
146			return (true);
147		atomic_add_int(&l->lock, -RC_INCR);
148	}
149	return (false);
150}
151
152static void
153def_lock_acquire(Lock *l, bool wlock)
154{
155	sigset_t tmp_oldsigmask;
156
157	if (ld_fast_sigblock) {
158		for (;;) {
159			atomic_add_32(&fsigblock, SIGFASTBLOCK_INC);
160			if (def_lock_acquire_set(l, wlock))
161				break;
162			sig_fastunblock();
163		}
164	} else {
165		for (;;) {
166			sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask);
167			if (def_lock_acquire_set(l, wlock))
168				break;
169			sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL);
170		}
171		if (atomic_fetchadd_int(&wnested, 1) == 0)
172			oldsigmask = tmp_oldsigmask;
173	}
174}
175
176static void
177def_rlock_acquire(void *lock)
178{
179	def_lock_acquire(lock, false);
180}
181
182static void
183def_wlock_acquire(void *lock)
184{
185	def_lock_acquire(lock, true);
186}
187
188static void
189def_lock_release(void *lock)
190{
191	Lock *l = lock;
192
193	atomic_add_rel_int(&l->lock, -((l->lock & WAFLAG) == 0 ?
194	    RC_INCR : WAFLAG));
195	if (ld_fast_sigblock)
196		sig_fastunblock();
197	else if (atomic_fetchadd_int(&wnested, -1) == 1)
198		sigprocmask(SIG_SETMASK, &oldsigmask, NULL);
199}
200
201static int
202def_thread_set_flag(int mask)
203{
204	int old_val = thread_flag;
205
206	thread_flag |= mask;
207	return (old_val);
208}
209
210static int
211def_thread_clr_flag(int mask)
212{
213	int old_val = thread_flag;
214
215	thread_flag &= ~mask;
216	return (old_val);
217}
218
219/*
220 * Public interface exposed to the rest of the dynamic linker.
221 */
222struct RtldLockInfo lockinfo;
223static struct RtldLockInfo deflockinfo;
224
225static __inline int
226thread_mask_set(int mask)
227{
228	return (lockinfo.thread_set_flag(mask));
229}
230
231static __inline void
232thread_mask_clear(int mask)
233{
234	lockinfo.thread_clr_flag(mask);
235}
236
237#define	RTLD_LOCK_CNT	3
238static struct rtld_lock {
239	void	*handle;
240	int	 mask;
241} rtld_locks[RTLD_LOCK_CNT];
242
243rtld_lock_t	rtld_bind_lock = &rtld_locks[0];
244rtld_lock_t	rtld_libc_lock = &rtld_locks[1];
245rtld_lock_t	rtld_phdr_lock = &rtld_locks[2];
246
247void
248rlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
249{
250
251	if (lockstate == NULL)
252		return;
253
254	if (thread_mask_set(lock->mask) & lock->mask) {
255		dbg("rlock_acquire: recursed");
256		lockstate->lockstate = RTLD_LOCK_UNLOCKED;
257		return;
258	}
259	lockinfo.rlock_acquire(lock->handle);
260	lockstate->lockstate = RTLD_LOCK_RLOCKED;
261}
262
263void
264wlock_acquire(rtld_lock_t lock, RtldLockState *lockstate)
265{
266
267	if (lockstate == NULL)
268		return;
269
270	if (thread_mask_set(lock->mask) & lock->mask) {
271		dbg("wlock_acquire: recursed");
272		lockstate->lockstate = RTLD_LOCK_UNLOCKED;
273		return;
274	}
275	lockinfo.wlock_acquire(lock->handle);
276	lockstate->lockstate = RTLD_LOCK_WLOCKED;
277}
278
279void
280lock_release(rtld_lock_t lock, RtldLockState *lockstate)
281{
282
283	if (lockstate == NULL)
284		return;
285
286	switch (lockstate->lockstate) {
287	case RTLD_LOCK_UNLOCKED:
288		break;
289	case RTLD_LOCK_RLOCKED:
290	case RTLD_LOCK_WLOCKED:
291		thread_mask_clear(lock->mask);
292		lockinfo.lock_release(lock->handle);
293		break;
294	default:
295		assert(0);
296	}
297}
298
299void
300lock_upgrade(rtld_lock_t lock, RtldLockState *lockstate)
301{
302
303	if (lockstate == NULL)
304		return;
305
306	lock_release(lock, lockstate);
307	wlock_acquire(lock, lockstate);
308}
309
310void
311lock_restart_for_upgrade(RtldLockState *lockstate)
312{
313
314	if (lockstate == NULL)
315		return;
316
317	switch (lockstate->lockstate) {
318	case RTLD_LOCK_UNLOCKED:
319	case RTLD_LOCK_WLOCKED:
320		break;
321	case RTLD_LOCK_RLOCKED:
322		siglongjmp(lockstate->env, 1);
323		break;
324	default:
325		assert(0);
326	}
327}
328
329void
330dlerror_dflt_init(void)
331{
332	lockinfo.dlerror_loc = def_dlerror_loc;
333	lockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg);
334	lockinfo.dlerror_seen = def_dlerror_seen;
335}
336
337void
338lockdflt_init(void)
339{
340	int i;
341
342	deflockinfo.rtli_version = RTLI_VERSION;
343	deflockinfo.lock_create = def_lock_create;
344	deflockinfo.lock_destroy = def_lock_destroy;
345	deflockinfo.rlock_acquire = def_rlock_acquire;
346	deflockinfo.wlock_acquire = def_wlock_acquire;
347	deflockinfo.lock_release = def_lock_release;
348	deflockinfo.thread_set_flag = def_thread_set_flag;
349	deflockinfo.thread_clr_flag = def_thread_clr_flag;
350	deflockinfo.at_fork = NULL;
351	deflockinfo.dlerror_loc = def_dlerror_loc;
352	deflockinfo.dlerror_loc_sz = sizeof(def_dlerror_msg);
353	deflockinfo.dlerror_seen = def_dlerror_seen;
354
355	for (i = 0; i < RTLD_LOCK_CNT; i++) {
356		rtld_locks[i].mask   = (1 << i);
357		rtld_locks[i].handle = NULL;
358	}
359
360	memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo));
361	_rtld_thread_init(NULL);
362	if (ld_fast_sigblock) {
363		__sys_sigfastblock(SIGFASTBLOCK_SETPTR, &fsigblock);
364	} else {
365		/*
366		 * Construct a mask to block all signals.  Note that
367		 * blocked traps mean that the process is terminated
368		 * if trap occurs while we are in locked section, with
369		 * the default settings for kern.forcesigexit.
370		 */
371		sigfillset(&fullsigmask);
372	}
373}
374
375/*
376 * Callback function to allow threads implementation to
377 * register their own locking primitives if the default
378 * one is not suitable.
379 * The current context should be the only context
380 * executing at the invocation time.
381 */
382void
383_rtld_thread_init(struct RtldLockInfo *pli)
384{
385	const Obj_Entry *obj;
386	SymLook req;
387	void *locks[RTLD_LOCK_CNT];
388	int flags, i, res;
389
390	if (pli == NULL) {
391		lockinfo.rtli_version = RTLI_VERSION;
392	} else {
393		lockinfo.rtli_version = RTLI_VERSION_ONE;
394		obj = obj_from_addr(pli->lock_create);
395		if (obj != NULL) {
396			symlook_init(&req, "_pli_rtli_version");
397			res = symlook_obj(&req, obj);
398			if (res == 0)
399				lockinfo.rtli_version = pli->rtli_version;
400		}
401	}
402
403	/* disable all locking while this function is running */
404	flags =	thread_mask_set(~0);
405
406	if (pli == NULL)
407		pli = &deflockinfo;
408	else if (ld_fast_sigblock) {
409		fsigblock = 0;
410		__sys_sigfastblock(SIGFASTBLOCK_UNSETPTR, NULL);
411	}
412
413	for (i = 0; i < RTLD_LOCK_CNT; i++)
414		if ((locks[i] = pli->lock_create()) == NULL)
415			break;
416
417	if (i < RTLD_LOCK_CNT) {
418		while (--i >= 0)
419			pli->lock_destroy(locks[i]);
420		abort();
421	}
422
423	for (i = 0; i < RTLD_LOCK_CNT; i++) {
424		if (rtld_locks[i].handle == NULL)
425			continue;
426		if (flags & rtld_locks[i].mask)
427			lockinfo.lock_release(rtld_locks[i].handle);
428		lockinfo.lock_destroy(rtld_locks[i].handle);
429	}
430
431	for (i = 0; i < RTLD_LOCK_CNT; i++) {
432		rtld_locks[i].handle = locks[i];
433		if (flags & rtld_locks[i].mask)
434			pli->wlock_acquire(rtld_locks[i].handle);
435	}
436
437	lockinfo.lock_create = pli->lock_create;
438	lockinfo.lock_destroy = pli->lock_destroy;
439	lockinfo.rlock_acquire = pli->rlock_acquire;
440	lockinfo.wlock_acquire = pli->wlock_acquire;
441	lockinfo.lock_release  = pli->lock_release;
442	lockinfo.thread_set_flag = pli->thread_set_flag;
443	lockinfo.thread_clr_flag = pli->thread_clr_flag;
444	lockinfo.at_fork = pli->at_fork;
445	if (lockinfo.rtli_version > RTLI_VERSION_ONE && pli != NULL) {
446		strlcpy(pli->dlerror_loc(), lockinfo.dlerror_loc(),
447		    lockinfo.dlerror_loc_sz);
448		lockinfo.dlerror_loc = pli->dlerror_loc;
449		lockinfo.dlerror_loc_sz = pli->dlerror_loc_sz;
450		lockinfo.dlerror_seen = pli->dlerror_seen;
451	}
452
453	/* restore thread locking state, this time with new locks */
454	thread_mask_clear(~0);
455	thread_mask_set(flags);
456	dbg("_rtld_thread_init: done");
457}
458
459void
460_rtld_atfork_pre(int *locks)
461{
462	RtldLockState ls[2];
463
464	if (locks == NULL)
465		return;
466
467	/*
468	 * Warning: this did not worked well with the rtld compat
469	 * locks above, when the thread signal mask was corrupted (set
470	 * to all signals blocked) if two locks were taken
471	 * simultaneously in the write mode.  The caller of the
472	 * _rtld_atfork_pre() must provide the working implementation
473	 * of the locks anyway, and libthr locks are fine.
474	 */
475	wlock_acquire(rtld_phdr_lock, &ls[0]);
476	wlock_acquire(rtld_bind_lock, &ls[1]);
477
478	/* XXXKIB: I am really sorry for this. */
479	locks[0] = ls[1].lockstate;
480	locks[2] = ls[0].lockstate;
481}
482
483void
484_rtld_atfork_post(int *locks)
485{
486	RtldLockState ls[2];
487
488	if (locks == NULL)
489		return;
490
491	bzero(ls, sizeof(ls));
492	ls[0].lockstate = locks[2];
493	ls[1].lockstate = locks[0];
494	lock_release(rtld_bind_lock, &ls[1]);
495	lock_release(rtld_phdr_lock, &ls[0]);
496}
497