1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2003, Jeffrey Roberson <jeff@freebsd.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice unmodified, this list of conditions, and the following
12 *    disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include "opt_posix.h"
30#include "opt_hwpmc_hooks.h"
31
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/limits.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/priv.h>
38#include <sys/proc.h>
39#include <sys/posix4.h>
40#include <sys/ptrace.h>
41#include <sys/racct.h>
42#include <sys/resourcevar.h>
43#include <sys/rtprio.h>
44#include <sys/rwlock.h>
45#include <sys/sched.h>
46#include <sys/sysctl.h>
47#include <sys/smp.h>
48#include <sys/syscallsubr.h>
49#include <sys/sysent.h>
50#include <sys/sysproto.h>
51#include <sys/signalvar.h>
52#include <sys/sysctl.h>
53#include <sys/thr.h>
54#include <sys/ucontext.h>
55#include <sys/umtxvar.h>
56#ifdef	HWPMC_HOOKS
57#include <sys/pmckern.h>
58#endif
59
60#include <machine/frame.h>
61
62#include <security/audit/audit.h>
63
64static SYSCTL_NODE(_kern, OID_AUTO, threads, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
65    "thread allocation");
66
67int max_threads_per_proc = 1500;
68SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_per_proc, CTLFLAG_RW,
69    &max_threads_per_proc, 0, "Limit on threads per proc");
70
71static int max_threads_hits;
72SYSCTL_INT(_kern_threads, OID_AUTO, max_threads_hits, CTLFLAG_RD,
73    &max_threads_hits, 0, "kern.threads.max_threads_per_proc hit count");
74
75#ifdef COMPAT_FREEBSD32
76
77static inline int
78suword_lwpid(void *addr, lwpid_t lwpid)
79{
80	int error;
81
82	if (SV_CURPROC_FLAG(SV_LP64))
83		error = suword(addr, lwpid);
84	else
85		error = suword32(addr, lwpid);
86	return (error);
87}
88
89#else
90#define suword_lwpid	suword
91#endif
92
93/*
94 * System call interface.
95 */
96
97struct thr_create_initthr_args {
98	ucontext_t ctx;
99	long *tid;
100};
101
102static int
103thr_create_initthr(struct thread *td, void *thunk)
104{
105	struct thr_create_initthr_args *args;
106
107	/* Copy out the child tid. */
108	args = thunk;
109	if (args->tid != NULL && suword_lwpid(args->tid, td->td_tid))
110		return (EFAULT);
111
112	return (set_mcontext(td, &args->ctx.uc_mcontext));
113}
114
115int
116sys_thr_create(struct thread *td, struct thr_create_args *uap)
117    /* ucontext_t *ctx, long *id, int flags */
118{
119	struct thr_create_initthr_args args;
120	int error;
121
122	if ((error = copyin(uap->ctx, &args.ctx, sizeof(args.ctx))))
123		return (error);
124	args.tid = uap->id;
125	return (thread_create(td, NULL, thr_create_initthr, &args));
126}
127
128int
129sys_thr_new(struct thread *td, struct thr_new_args *uap)
130    /* struct thr_param * */
131{
132	struct thr_param param;
133	int error;
134
135	if (uap->param_size < 0 || uap->param_size > sizeof(param))
136		return (EINVAL);
137	bzero(&param, sizeof(param));
138	if ((error = copyin(uap->param, &param, uap->param_size)))
139		return (error);
140	return (kern_thr_new(td, &param));
141}
142
143static int
144thr_new_initthr(struct thread *td, void *thunk)
145{
146	stack_t stack;
147	struct thr_param *param;
148	int error;
149
150	/*
151	 * Here we copy out tid to two places, one for child and one
152	 * for parent, because pthread can create a detached thread,
153	 * if parent wants to safely access child tid, it has to provide
154	 * its storage, because child thread may exit quickly and
155	 * memory is freed before parent thread can access it.
156	 */
157	param = thunk;
158	if ((param->child_tid != NULL &&
159	    suword_lwpid(param->child_tid, td->td_tid)) ||
160	    (param->parent_tid != NULL &&
161	    suword_lwpid(param->parent_tid, td->td_tid)))
162		return (EFAULT);
163
164	/* Set up our machine context. */
165	stack.ss_sp = param->stack_base;
166	stack.ss_size = param->stack_size;
167	/* Set upcall address to user thread entry function. */
168	error = cpu_set_upcall(td, param->start_func, param->arg, &stack);
169	if (error != 0)
170		return (error);
171	/* Setup user TLS address and TLS pointer register. */
172	return (cpu_set_user_tls(td, param->tls_base));
173}
174
175int
176kern_thr_new(struct thread *td, struct thr_param *param)
177{
178	struct rtprio rtp, *rtpp;
179	int error;
180
181	rtpp = NULL;
182	if (param->rtp != 0) {
183		error = copyin(param->rtp, &rtp, sizeof(struct rtprio));
184		if (error)
185			return (error);
186		rtpp = &rtp;
187	}
188	return (thread_create(td, rtpp, thr_new_initthr, param));
189}
190
191int
192thread_create(struct thread *td, struct rtprio *rtp,
193    int (*initialize_thread)(struct thread *, void *), void *thunk)
194{
195	struct thread *newtd;
196	struct proc *p;
197	int error;
198
199	p = td->td_proc;
200
201	if (rtp != NULL) {
202		switch(rtp->type) {
203		case RTP_PRIO_REALTIME:
204		case RTP_PRIO_FIFO:
205			/* Only root can set scheduler policy */
206			if (priv_check(td, PRIV_SCHED_SETPOLICY) != 0)
207				return (EPERM);
208			if (rtp->prio > RTP_PRIO_MAX)
209				return (EINVAL);
210			break;
211		case RTP_PRIO_NORMAL:
212			rtp->prio = 0;
213			break;
214		default:
215			return (EINVAL);
216		}
217	}
218
219#ifdef RACCT
220	if (racct_enable) {
221		PROC_LOCK(p);
222		error = racct_add(p, RACCT_NTHR, 1);
223		PROC_UNLOCK(p);
224		if (error != 0)
225			return (EPROCLIM);
226	}
227#endif
228
229	/* Initialize our td */
230	error = kern_thr_alloc(p, 0, &newtd);
231	if (error)
232		goto fail;
233
234	bzero(&newtd->td_startzero,
235	    __rangeof(struct thread, td_startzero, td_endzero));
236	bcopy(&td->td_startcopy, &newtd->td_startcopy,
237	    __rangeof(struct thread, td_startcopy, td_endcopy));
238	newtd->td_proc = td->td_proc;
239	newtd->td_rb_list = newtd->td_rbp_list = newtd->td_rb_inact = 0;
240	thread_cow_get(newtd, td);
241
242	cpu_copy_thread(newtd, td);
243
244	error = initialize_thread(newtd, thunk);
245	if (error != 0) {
246		thread_cow_free(newtd);
247		thread_free(newtd);
248		goto fail;
249	}
250
251	PROC_LOCK(p);
252	p->p_flag |= P_HADTHREADS;
253	thread_link(newtd, p);
254	bcopy(p->p_comm, newtd->td_name, sizeof(newtd->td_name));
255	thread_lock(td);
256	/* let the scheduler know about these things. */
257	sched_fork_thread(td, newtd);
258	thread_unlock(td);
259	if (P_SHOULDSTOP(p))
260		ast_sched(newtd, TDA_SUSPEND);
261	if (p->p_ptevents & PTRACE_LWP)
262		newtd->td_dbgflags |= TDB_BORN;
263
264	PROC_UNLOCK(p);
265#ifdef	HWPMC_HOOKS
266	if (PMC_PROC_IS_USING_PMCS(p))
267		PMC_CALL_HOOK(newtd, PMC_FN_THR_CREATE, NULL);
268	else if (PMC_SYSTEM_SAMPLING_ACTIVE())
269		PMC_CALL_HOOK_UNLOCKED(newtd, PMC_FN_THR_CREATE_LOG, NULL);
270#endif
271
272	tidhash_add(newtd);
273
274	/* ignore timesharing class */
275	if (rtp != NULL && !(td->td_pri_class == PRI_TIMESHARE &&
276	    rtp->type == RTP_PRIO_NORMAL))
277		rtp_to_pri(rtp, newtd);
278
279	thread_lock(newtd);
280	TD_SET_CAN_RUN(newtd);
281	sched_add(newtd, SRQ_BORING);
282
283	return (0);
284
285fail:
286#ifdef RACCT
287	if (racct_enable) {
288		PROC_LOCK(p);
289		racct_sub(p, RACCT_NTHR, 1);
290		PROC_UNLOCK(p);
291	}
292#endif
293	return (error);
294}
295
296int
297sys_thr_self(struct thread *td, struct thr_self_args *uap)
298    /* long *id */
299{
300	int error;
301
302	error = suword_lwpid(uap->id, (unsigned)td->td_tid);
303	if (error == -1)
304		return (EFAULT);
305	return (0);
306}
307
308int
309sys_thr_exit(struct thread *td, struct thr_exit_args *uap)
310    /* long *state */
311{
312
313	umtx_thread_exit(td);
314
315	/* Signal userland that it can free the stack. */
316	if ((void *)uap->state != NULL) {
317		(void)suword_lwpid(uap->state, 1);
318		(void)kern_umtx_wake(td, uap->state, INT_MAX, 0);
319	}
320
321	return (kern_thr_exit(td));
322}
323
324int
325kern_thr_exit(struct thread *td)
326{
327	struct proc *p;
328
329	p = td->td_proc;
330
331	/*
332	 * If all of the threads in a process call this routine to
333	 * exit (e.g. all threads call pthread_exit()), exactly one
334	 * thread should return to the caller to terminate the process
335	 * instead of the thread.
336	 *
337	 * Checking p_numthreads alone is not sufficient since threads
338	 * might be committed to terminating while the PROC_LOCK is
339	 * dropped in either ptracestop() or while removing this thread
340	 * from the tidhash.  Instead, the p_pendingexits field holds
341	 * the count of threads in either of those states and a thread
342	 * is considered the "last" thread if all of the other threads
343	 * in a process are already terminating.
344	 */
345	PROC_LOCK(p);
346	if (p->p_numthreads == p->p_pendingexits + 1) {
347		/*
348		 * Ignore attempts to shut down last thread in the
349		 * proc.  This will actually call _exit(2) in the
350		 * usermode trampoline when it returns.
351		 */
352		PROC_UNLOCK(p);
353		return (0);
354	}
355
356	if (p->p_sysent->sv_ontdexit != NULL)
357		p->p_sysent->sv_ontdexit(td);
358
359	td->td_dbgflags |= TDB_EXIT;
360	if (p->p_ptevents & PTRACE_LWP) {
361		p->p_pendingexits++;
362		ptracestop(td, SIGTRAP, NULL);
363		p->p_pendingexits--;
364	}
365	tidhash_remove(td);
366
367	/*
368	 * The check above should prevent all other threads from this
369	 * process from exiting while the PROC_LOCK is dropped, so
370	 * there must be at least one other thread other than the
371	 * current thread.
372	 */
373	KASSERT(p->p_numthreads > 1, ("too few threads"));
374	racct_sub(p, RACCT_NTHR, 1);
375	tdsigcleanup(td);
376
377#ifdef AUDIT
378	AUDIT_SYSCALL_EXIT(0, td);
379#endif
380
381	PROC_SLOCK(p);
382	thread_stopped(p);
383	thread_exit();
384	/* NOTREACHED */
385}
386
387int
388sys_thr_kill(struct thread *td, struct thr_kill_args *uap)
389    /* long id, int sig */
390{
391	ksiginfo_t ksi;
392	struct thread *ttd;
393	struct proc *p;
394	int error;
395
396	p = td->td_proc;
397	ksiginfo_init(&ksi);
398	ksi.ksi_signo = uap->sig;
399	ksi.ksi_code = SI_LWP;
400	ksi.ksi_pid = p->p_pid;
401	ksi.ksi_uid = td->td_ucred->cr_ruid;
402	if (uap->id == -1) {
403		if (uap->sig != 0 && !_SIG_VALID(uap->sig)) {
404			error = EINVAL;
405		} else {
406			error = ESRCH;
407			PROC_LOCK(p);
408			FOREACH_THREAD_IN_PROC(p, ttd) {
409				if (ttd != td) {
410					error = 0;
411					if (uap->sig == 0)
412						break;
413					tdksignal(ttd, uap->sig, &ksi);
414				}
415			}
416			PROC_UNLOCK(p);
417		}
418	} else {
419		error = 0;
420		ttd = tdfind((lwpid_t)uap->id, p->p_pid);
421		if (ttd == NULL)
422			return (ESRCH);
423		if (uap->sig == 0)
424			;
425		else if (!_SIG_VALID(uap->sig))
426			error = EINVAL;
427		else
428			tdksignal(ttd, uap->sig, &ksi);
429		PROC_UNLOCK(ttd->td_proc);
430	}
431	return (error);
432}
433
434int
435sys_thr_kill2(struct thread *td, struct thr_kill2_args *uap)
436    /* pid_t pid, long id, int sig */
437{
438	ksiginfo_t ksi;
439	struct thread *ttd;
440	struct proc *p;
441	int error;
442
443	AUDIT_ARG_SIGNUM(uap->sig);
444
445	ksiginfo_init(&ksi);
446	ksi.ksi_signo = uap->sig;
447	ksi.ksi_code = SI_LWP;
448	ksi.ksi_pid = td->td_proc->p_pid;
449	ksi.ksi_uid = td->td_ucred->cr_ruid;
450	if (uap->id == -1) {
451		if ((p = pfind(uap->pid)) == NULL)
452			return (ESRCH);
453		AUDIT_ARG_PROCESS(p);
454		error = p_cansignal(td, p, uap->sig);
455		if (error) {
456			PROC_UNLOCK(p);
457			return (error);
458		}
459		if (uap->sig != 0 && !_SIG_VALID(uap->sig)) {
460			error = EINVAL;
461		} else {
462			error = ESRCH;
463			FOREACH_THREAD_IN_PROC(p, ttd) {
464				if (ttd != td) {
465					error = 0;
466					if (uap->sig == 0)
467						break;
468					tdksignal(ttd, uap->sig, &ksi);
469				}
470			}
471		}
472		PROC_UNLOCK(p);
473	} else {
474		ttd = tdfind((lwpid_t)uap->id, uap->pid);
475		if (ttd == NULL)
476			return (ESRCH);
477		p = ttd->td_proc;
478		AUDIT_ARG_PROCESS(p);
479		error = p_cansignal(td, p, uap->sig);
480		if (uap->sig == 0)
481			;
482		else if (!_SIG_VALID(uap->sig))
483			error = EINVAL;
484		else
485			tdksignal(ttd, uap->sig, &ksi);
486		PROC_UNLOCK(p);
487	}
488	return (error);
489}
490
491int
492sys_thr_suspend(struct thread *td, struct thr_suspend_args *uap)
493	/* const struct timespec *timeout */
494{
495	struct timespec ts, *tsp;
496	int error;
497
498	tsp = NULL;
499	if (uap->timeout != NULL) {
500		error = umtx_copyin_timeout(uap->timeout, &ts);
501		if (error != 0)
502			return (error);
503		tsp = &ts;
504	}
505
506	return (kern_thr_suspend(td, tsp));
507}
508
509int
510kern_thr_suspend(struct thread *td, struct timespec *tsp)
511{
512	struct proc *p = td->td_proc;
513	struct timeval tv;
514	int error = 0;
515	int timo = 0;
516
517	if (td->td_pflags & TDP_WAKEUP) {
518		td->td_pflags &= ~TDP_WAKEUP;
519		return (0);
520	}
521
522	if (tsp != NULL) {
523		if (tsp->tv_sec == 0 && tsp->tv_nsec == 0)
524			error = EWOULDBLOCK;
525		else {
526			TIMESPEC_TO_TIMEVAL(&tv, tsp);
527			timo = tvtohz(&tv);
528		}
529	}
530
531	PROC_LOCK(p);
532	if (error == 0 && (td->td_flags & TDF_THRWAKEUP) == 0)
533		error = msleep((void *)td, &p->p_mtx,
534			 PCATCH, "lthr", timo);
535
536	if (td->td_flags & TDF_THRWAKEUP) {
537		thread_lock(td);
538		td->td_flags &= ~TDF_THRWAKEUP;
539		thread_unlock(td);
540		PROC_UNLOCK(p);
541		return (0);
542	}
543	PROC_UNLOCK(p);
544	if (error == EWOULDBLOCK)
545		error = ETIMEDOUT;
546	else if (error == ERESTART) {
547		if (timo != 0)
548			error = EINTR;
549	}
550	return (error);
551}
552
553int
554sys_thr_wake(struct thread *td, struct thr_wake_args *uap)
555	/* long id */
556{
557	struct proc *p;
558	struct thread *ttd;
559
560	if (uap->id == td->td_tid) {
561		td->td_pflags |= TDP_WAKEUP;
562		return (0);
563	}
564
565	p = td->td_proc;
566	ttd = tdfind((lwpid_t)uap->id, p->p_pid);
567	if (ttd == NULL)
568		return (ESRCH);
569	thread_lock(ttd);
570	ttd->td_flags |= TDF_THRWAKEUP;
571	thread_unlock(ttd);
572	wakeup((void *)ttd);
573	PROC_UNLOCK(p);
574	return (0);
575}
576
577int
578sys_thr_set_name(struct thread *td, struct thr_set_name_args *uap)
579{
580	struct proc *p;
581	char name[MAXCOMLEN + 1];
582	struct thread *ttd;
583	int error;
584
585	error = 0;
586	name[0] = '\0';
587	if (uap->name != NULL) {
588		error = copyinstr(uap->name, name, sizeof(name), NULL);
589		if (error == ENAMETOOLONG) {
590			error = copyin(uap->name, name, sizeof(name) - 1);
591			name[sizeof(name) - 1] = '\0';
592		}
593		if (error)
594			return (error);
595	}
596	p = td->td_proc;
597	ttd = tdfind((lwpid_t)uap->id, p->p_pid);
598	if (ttd == NULL)
599		return (ESRCH);
600	strcpy(ttd->td_name, name);
601#ifdef HWPMC_HOOKS
602	if (PMC_PROC_IS_USING_PMCS(p) || PMC_SYSTEM_SAMPLING_ACTIVE())
603		PMC_CALL_HOOK_UNLOCKED(ttd, PMC_FN_THR_CREATE_LOG, NULL);
604#endif
605#ifdef KTR
606	sched_clear_tdname(ttd);
607#endif
608	PROC_UNLOCK(p);
609	return (error);
610}
611
612int
613kern_thr_alloc(struct proc *p, int pages, struct thread **ntd)
614{
615
616	/* Have race condition but it is cheap. */
617	if (p->p_numthreads >= max_threads_per_proc) {
618		++max_threads_hits;
619		return (EPROCLIM);
620	}
621
622	*ntd = thread_alloc(pages);
623	if (*ntd == NULL)
624		return (ENOMEM);
625
626	return (0);
627}
628