1139825Simp/*-
277957Sbenno * Copyright (C) 1995, 1996 Wolfgang Solfrank.
377957Sbenno * Copyright (C) 1995, 1996 TooLs GmbH.
477957Sbenno * All rights reserved.
577957Sbenno *
677957Sbenno * Redistribution and use in source and binary forms, with or without
777957Sbenno * modification, are permitted provided that the following conditions
877957Sbenno * are met:
977957Sbenno * 1. Redistributions of source code must retain the above copyright
1077957Sbenno *    notice, this list of conditions and the following disclaimer.
1177957Sbenno * 2. Redistributions in binary form must reproduce the above copyright
1277957Sbenno *    notice, this list of conditions and the following disclaimer in the
1377957Sbenno *    documentation and/or other materials provided with the distribution.
1477957Sbenno * 3. All advertising materials mentioning features or use of this software
1577957Sbenno *    must display the following acknowledgement:
1677957Sbenno *	This product includes software developed by TooLs GmbH.
1777957Sbenno * 4. The name of TooLs GmbH may not be used to endorse or promote products
1877957Sbenno *    derived from this software without specific prior written permission.
1977957Sbenno *
2077957Sbenno * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
2177957Sbenno * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
2277957Sbenno * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
2377957Sbenno * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2477957Sbenno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
2577957Sbenno * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
2677957Sbenno * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
2777957Sbenno * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
2877957Sbenno * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
2977957Sbenno * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3077957Sbenno *
3196255Sbenno * $NetBSD: trap.c,v 1.58 2002/03/04 04:07:35 dbj Exp $
3277957Sbenno */
3377957Sbenno
34113038Sobrien#include <sys/cdefs.h>
35113038Sobrien__FBSDID("$FreeBSD$");
3677957Sbenno
37228869Sjhibbits#include "opt_hwpmc_hooks.h"
38242723Sjhibbits#include "opt_kdtrace.h"
39228869Sjhibbits
4077957Sbenno#include <sys/param.h>
41132074Sgrehan#include <sys/kdb.h>
4296938Sbenno#include <sys/proc.h>
4396938Sbenno#include <sys/ktr.h>
4486067Smp#include <sys/lock.h>
4586067Smp#include <sys/mutex.h>
4696938Sbenno#include <sys/pioctl.h>
47160764Sjhb#include <sys/ptrace.h>
4877957Sbenno#include <sys/reboot.h>
4977957Sbenno#include <sys/syscall.h>
5096938Sbenno#include <sys/sysent.h>
5196255Sbenno#include <sys/systm.h>
5296906Sbenno#include <sys/uio.h>
53143633Sgrehan#include <sys/signalvar.h>
5496938Sbenno#include <sys/vmmeter.h>
55228869Sjhibbits#ifdef HWPMC_HOOKS
56228869Sjhibbits#include <sys/pmckern.h>
57228869Sjhibbits#endif
5877957Sbenno
59162361Srwatson#include <security/audit/audit.h>
60162361Srwatson
6177957Sbenno#include <vm/vm.h>
6286067Smp#include <vm/pmap.h>
6386067Smp#include <vm/vm_extern.h>
6496255Sbenno#include <vm/vm_param.h>
6577957Sbenno#include <vm/vm_kern.h>
6686067Smp#include <vm/vm_map.h>
6796255Sbenno#include <vm/vm_page.h>
6877957Sbenno
69209975Snwhitehorn#include <machine/_inttypes.h>
70188860Snwhitehorn#include <machine/altivec.h>
7177957Sbenno#include <machine/cpu.h>
7296255Sbenno#include <machine/db_machdep.h>
7396255Sbenno#include <machine/fpu.h>
7477957Sbenno#include <machine/frame.h>
7577957Sbenno#include <machine/pcb.h>
7696255Sbenno#include <machine/pmap.h>
7777957Sbenno#include <machine/psl.h>
7877957Sbenno#include <machine/trap.h>
7996255Sbenno#include <machine/spr.h>
8096255Sbenno#include <machine/sr.h>
8177957Sbenno
8296938Sbennostatic void	trap_fatal(struct trapframe *frame);
8396938Sbennostatic void	printtrap(u_int vector, struct trapframe *frame, int isfatal,
8496938Sbenno		    int user);
8596938Sbennostatic int	trap_pfault(struct trapframe *frame, int user);
8696938Sbennostatic int	fix_unaligned(struct thread *td, struct trapframe *frame);
87204197Snwhitehornstatic int	ppc_instr_emulate(struct trapframe *frame);
8896938Sbennostatic int	handle_onfault(struct trapframe *frame);
8996938Sbennostatic void	syscall(struct trapframe *frame);
9096938Sbenno
91209975Snwhitehorn#ifdef __powerpc64__
92230123Snwhitehorn       void	handle_kernel_slb_spill(int, register_t, register_t);
93230123Snwhitehornstatic int	handle_user_slb_spill(pmap_t pm, vm_offset_t addr);
94230123Snwhitehornextern int	n_slbs;
95209975Snwhitehorn#endif
96209975Snwhitehorn
9796906Sbennoint	setfault(faultbuf);		/* defined in locore.S */
9896906Sbenno
9996255Sbenno/* Why are these not defined in a header? */
10096906Sbennoint	badaddr(void *, size_t);
10196906Sbennoint	badaddr_read(void *, size_t, int *);
10286067Smp
10396938Sbennostruct powerpc_exception {
10496938Sbenno	u_int	vector;
10596938Sbenno	char	*name;
10696938Sbenno};
10796938Sbenno
108242723Sjhibbits#ifdef KDTRACE_HOOKS
109242723Sjhibbits#include <sys/dtrace_bsd.h>
110242723Sjhibbits
111242723Sjhibbits/*
112242723Sjhibbits * This is a hook which is initialised by the dtrace module
113242723Sjhibbits * to handle traps which might occur during DTrace probe
114242723Sjhibbits * execution.
115242723Sjhibbits */
116242723Sjhibbitsdtrace_trap_func_t	dtrace_trap_func;
117242723Sjhibbits
118242723Sjhibbitsdtrace_doubletrap_func_t	dtrace_doubletrap_func;
119242723Sjhibbits
120242723Sjhibbits/*
121242723Sjhibbits * This is a hook which is initialised by the systrace module
122242723Sjhibbits * when it is loaded. This keeps the DTrace syscall provider
123242723Sjhibbits * implementation opaque.
124242723Sjhibbits */
125242723Sjhibbitssystrace_probe_func_t	systrace_probe_func;
126242723Sjhibbits
127242723Sjhibbits/*
128242723Sjhibbits * These hooks are necessary for the pid, usdt and fasttrap providers.
129242723Sjhibbits */
130242723Sjhibbitsdtrace_fasttrap_probe_ptr_t	dtrace_fasttrap_probe_ptr;
131242723Sjhibbitsdtrace_pid_probe_ptr_t		dtrace_pid_probe_ptr;
132242723Sjhibbitsdtrace_return_probe_ptr_t	dtrace_return_probe_ptr;
133248457Sjhibbitsint (*dtrace_invop_jump_addr)(struct trapframe *);
134242723Sjhibbits#endif
135242723Sjhibbits
13696938Sbennostatic struct powerpc_exception powerpc_exceptions[] = {
13796938Sbenno	{ 0x0100, "system reset" },
13896938Sbenno	{ 0x0200, "machine check" },
13996938Sbenno	{ 0x0300, "data storage interrupt" },
140209975Snwhitehorn	{ 0x0380, "data segment exception" },
14196938Sbenno	{ 0x0400, "instruction storage interrupt" },
142209975Snwhitehorn	{ 0x0480, "instruction segment exception" },
14396938Sbenno	{ 0x0500, "external interrupt" },
14496938Sbenno	{ 0x0600, "alignment" },
14596938Sbenno	{ 0x0700, "program" },
14696938Sbenno	{ 0x0800, "floating-point unavailable" },
14796938Sbenno	{ 0x0900, "decrementer" },
14896938Sbenno	{ 0x0c00, "system call" },
14996938Sbenno	{ 0x0d00, "trace" },
15096938Sbenno	{ 0x0e00, "floating-point assist" },
15196938Sbenno	{ 0x0f00, "performance monitoring" },
15296938Sbenno	{ 0x0f20, "altivec unavailable" },
15396938Sbenno	{ 0x1000, "instruction tlb miss" },
15496938Sbenno	{ 0x1100, "data load tlb miss" },
15596938Sbenno	{ 0x1200, "data store tlb miss" },
15696938Sbenno	{ 0x1300, "instruction breakpoint" },
157131698Sgrehan	{ 0x1400, "system management" },
15896938Sbenno	{ 0x1600, "altivec assist" },
15996938Sbenno	{ 0x1700, "thermal management" },
16096938Sbenno	{ 0x2000, "run mode/trace" },
16196938Sbenno	{ 0x3000, NULL }
16296938Sbenno};
16396938Sbenno
16496938Sbennostatic const char *
16596938Sbennotrapname(u_int vector)
16696938Sbenno{
16796938Sbenno	struct	powerpc_exception *pe;
16896938Sbenno
16996938Sbenno	for (pe = powerpc_exceptions; pe->vector != 0x3000; pe++) {
17096938Sbenno		if (pe->vector == vector)
17196938Sbenno			return (pe->name);
17296938Sbenno	}
17396938Sbenno
17496938Sbenno	return ("unknown");
17596938Sbenno}
17696938Sbenno
17796255Sbennovoid
17896906Sbennotrap(struct trapframe *frame)
17986067Smp{
180112429Sgrehan	struct thread	*td;
18196906Sbenno	struct proc	*p;
18296938Sbenno	int		sig, type, user;
183155455Sphk	u_int		ucode;
184151316Sdavidxu	ksiginfo_t	ksi;
18586067Smp
186170291Sattilio	PCPU_INC(cnt.v_trap);
18796938Sbenno
188223485Snwhitehorn	td = curthread;
18996906Sbenno	p = td->td_proc;
19086067Smp
19196938Sbenno	type = ucode = frame->exc;
19296938Sbenno	sig = 0;
19396938Sbenno	user = frame->srr1 & PSL_PR;
19486067Smp
195173601Sjulian	CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name,
19696938Sbenno	    trapname(type), user ? "user" : "kernel");
19786067Smp
198228869Sjhibbits#ifdef HWPMC_HOOKS
199228869Sjhibbits	if (type == EXC_PERF && (pmc_intr != NULL)) {
200255165Sjhibbits		(*pmc_intr)(PCPU_GET(cpuid), frame);
201255165Sjhibbits		if (user)
202255165Sjhibbits			userret(td, frame);
203228869Sjhibbits		return;
204228869Sjhibbits	}
205228869Sjhibbits#endif
206242723Sjhibbits#ifdef KDTRACE_HOOKS
207242723Sjhibbits	/*
208242723Sjhibbits	 * A trap can occur while DTrace executes a probe. Before
209242723Sjhibbits	 * executing the probe, DTrace blocks re-scheduling and sets
210242723Sjhibbits	 * a flag in it's per-cpu flags to indicate that it doesn't
211242723Sjhibbits	 * want to fault. On returning from the probe, the no-fault
212242723Sjhibbits	 * flag is cleared and finally re-scheduling is enabled.
213242723Sjhibbits	 *
214242723Sjhibbits	 * If the DTrace kernel module has registered a trap handler,
215242723Sjhibbits	 * call it and if it returns non-zero, assume that it has
216242723Sjhibbits	 * handled the trap and modified the trap frame so that this
217242723Sjhibbits	 * function can return normally.
218242723Sjhibbits	 */
219242723Sjhibbits	/*
220242723Sjhibbits	 * XXXDTRACE: add fasttrap and pid  probes handlers here (if ever)
221242723Sjhibbits	 */
222248457Sjhibbits	if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type))
223248457Sjhibbits		return;
224242723Sjhibbits#endif
225242723Sjhibbits
22696938Sbenno	if (user) {
227155455Sphk		td->td_pticks = 0;
22896938Sbenno		td->td_frame = frame;
22996938Sbenno		if (td->td_ucred != p->p_ucred)
23096938Sbenno			cred_update_thread(td);
23186067Smp
23296938Sbenno		/* User Mode Traps */
23396938Sbenno		switch (type) {
23496938Sbenno		case EXC_RUNMODETRC:
23596938Sbenno		case EXC_TRC:
23696938Sbenno			frame->srr1 &= ~PSL_SE;
23796938Sbenno			sig = SIGTRAP;
23896938Sbenno			break;
23977957Sbenno
240209975Snwhitehorn#ifdef __powerpc64__
241209975Snwhitehorn		case EXC_ISE:
242209975Snwhitehorn		case EXC_DSE:
243230123Snwhitehorn			if (handle_user_slb_spill(&p->p_vmspace->vm_pmap,
244209975Snwhitehorn			    (type == EXC_ISE) ? frame->srr0 :
245209975Snwhitehorn			    frame->cpu.aim.dar) != 0)
246209975Snwhitehorn				sig = SIGSEGV;
247209975Snwhitehorn			break;
248209975Snwhitehorn#endif
24996938Sbenno		case EXC_DSI:
25096938Sbenno		case EXC_ISI:
25196938Sbenno			sig = trap_pfault(frame, 1);
25296938Sbenno			break;
25396938Sbenno
25496938Sbenno		case EXC_SC:
25596938Sbenno			syscall(frame);
25696938Sbenno			break;
25796938Sbenno
25896938Sbenno		case EXC_FPU:
259112429Sgrehan			KASSERT((td->td_pcb->pcb_flags & PCB_FPU) != PCB_FPU,
260112429Sgrehan			    ("FPU already enabled for thread"));
26196938Sbenno			enable_fpu(td);
26296938Sbenno			break;
26396938Sbenno
26496938Sbenno		case EXC_VEC:
265188860Snwhitehorn			KASSERT((td->td_pcb->pcb_flags & PCB_VEC) != PCB_VEC,
266188860Snwhitehorn			    ("Altivec already enabled for thread"));
26796938Sbenno			enable_vec(td);
26896938Sbenno			break;
269188860Snwhitehorn
270213456Snwhitehorn		case EXC_VECAST_G4:
271213456Snwhitehorn		case EXC_VECAST_G5:
272213456Snwhitehorn			/*
273213456Snwhitehorn			 * We get a VPU assist exception for IEEE mode
274213456Snwhitehorn			 * vector operations on denormalized floats.
275213456Snwhitehorn			 * Emulating this is a giant pain, so for now,
276213456Snwhitehorn			 * just switch off IEEE mode and treat them as
277213456Snwhitehorn			 * zero.
278213456Snwhitehorn			 */
279213456Snwhitehorn
280213456Snwhitehorn			save_vec(td);
281213456Snwhitehorn			td->td_pcb->pcb_vec.vscr |= ALTIVEC_VSCR_NJ;
282213456Snwhitehorn			enable_vec(td);
283148568Sgrehan			break;
28496938Sbenno
28596938Sbenno		case EXC_ALI:
28696938Sbenno			if (fix_unaligned(td, frame) != 0)
28796938Sbenno				sig = SIGBUS;
28877957Sbenno			else
28996938Sbenno				frame->srr0 += 4;
29096938Sbenno			break;
29196938Sbenno
29296938Sbenno		case EXC_PGM:
293191261Snwhitehorn			/* Identify the trap reason */
294191261Snwhitehorn			if (frame->srr1 & EXC_PGM_TRAP)
29596938Sbenno				sig = SIGTRAP;
296248457Sjhibbits			else if (ppc_instr_emulate(frame) == 0)
297204197Snwhitehorn				frame->srr0 += 4;
298204197Snwhitehorn			else
29996938Sbenno				sig = SIGILL;
30096938Sbenno			break;
30196938Sbenno
30296938Sbenno		default:
30396938Sbenno			trap_fatal(frame);
30477957Sbenno		}
30596938Sbenno	} else {
30696938Sbenno		/* Kernel Mode Traps */
30796938Sbenno
30896938Sbenno		KASSERT(cold || td->td_ucred != NULL,
30996938Sbenno		    ("kernel trap doesn't have ucred"));
31096938Sbenno		switch (type) {
311248457Sjhibbits#ifdef KDTRACE_HOOKS
312248457Sjhibbits		case EXC_PGM:
313248457Sjhibbits			if (frame->srr1 & EXC_PGM_TRAP) {
314248457Sjhibbits				if (*(uintptr_t *)frame->srr0 == 0x7c810808) {
315248457Sjhibbits					if (dtrace_invop_jump_addr != NULL) {
316248457Sjhibbits						dtrace_invop_jump_addr(frame);
317255165Sjhibbits						return;
318248457Sjhibbits					}
319248457Sjhibbits				}
320248457Sjhibbits			}
321255165Sjhibbits			break;
322248457Sjhibbits#endif
323209975Snwhitehorn#ifdef __powerpc64__
324214574Snwhitehorn		case EXC_DSE:
325214574Snwhitehorn			if ((frame->cpu.aim.dar & SEGMENT_MASK) == USER_ADDR) {
326214574Snwhitehorn				__asm __volatile ("slbmte %0, %1" ::
327230123Snwhitehorn					"r"(td->td_pcb->pcb_cpu.aim.usr_vsid),
328230123Snwhitehorn					"r"(USER_SLB_SLBE));
329214574Snwhitehorn				return;
330214574Snwhitehorn			}
331230123Snwhitehorn			break;
332209975Snwhitehorn#endif
333230123Snwhitehorn		case EXC_DSI:
334230123Snwhitehorn			if (trap_pfault(frame, 0) == 0)
335230123Snwhitehorn 				return;
336230123Snwhitehorn			break;
33796938Sbenno		case EXC_MCHK:
33896938Sbenno			if (handle_onfault(frame))
33996938Sbenno 				return;
34096938Sbenno			break;
34196938Sbenno		default:
342132994Sgrehan			break;
34396255Sbenno		}
344132994Sgrehan		trap_fatal(frame);
34586067Smp	}
34696938Sbenno
34796938Sbenno	if (sig != 0) {
34896938Sbenno		if (p->p_sysent->sv_transtrap != NULL)
34996938Sbenno			sig = (p->p_sysent->sv_transtrap)(sig, type);
350151316Sdavidxu		ksiginfo_init_trap(&ksi);
351151316Sdavidxu		ksi.ksi_signo = sig;
352151316Sdavidxu		ksi.ksi_code = (int) ucode; /* XXX, not POSIX */
353151316Sdavidxu		/* ksi.ksi_addr = ? */
354151316Sdavidxu		ksi.ksi_trapno = type;
355151316Sdavidxu		trapsignal(td, &ksi);
35696938Sbenno	}
35796938Sbenno
358155455Sphk	userret(td, frame);
35996938Sbenno}
36096938Sbenno
36196938Sbennostatic void
36296938Sbennotrap_fatal(struct trapframe *frame)
36396938Sbenno{
36496938Sbenno
36596938Sbenno	printtrap(frame->exc, frame, 1, (frame->srr1 & PSL_PR));
366132074Sgrehan#ifdef KDB
367132074Sgrehan	if ((debugger_on_panic || kdb_active) &&
368132074Sgrehan	    kdb_trap(frame->exc, 0, frame))
36996938Sbenno		return;
37096938Sbenno#endif
37196938Sbenno	panic("%s trap", trapname(frame->exc));
37296938Sbenno}
37396938Sbenno
37496938Sbennostatic void
37596938Sbennoprinttrap(u_int vector, struct trapframe *frame, int isfatal, int user)
37696938Sbenno{
37796938Sbenno
37896938Sbenno	printf("\n");
37996938Sbenno	printf("%s %s trap:\n", isfatal ? "fatal" : "handled",
38096938Sbenno	    user ? "user" : "kernel");
38196938Sbenno	printf("\n");
382230123Snwhitehorn	printf("   exception       = 0x%x (%s)\n", vector, trapname(vector));
38396938Sbenno	switch (vector) {
384209975Snwhitehorn	case EXC_DSE:
38596938Sbenno	case EXC_DSI:
386209975Snwhitehorn		printf("   virtual address = 0x%" PRIxPTR "\n",
387209975Snwhitehorn		    frame->cpu.aim.dar);
388249129Sjhibbits		printf("   dsisr           = 0x%" PRIxPTR "\n",
389249129Sjhibbits		    frame->cpu.aim.dsisr);
39096255Sbenno		break;
391209975Snwhitehorn	case EXC_ISE:
39296255Sbenno	case EXC_ISI:
393209975Snwhitehorn		printf("   virtual address = 0x%" PRIxPTR "\n", frame->srr0);
39496255Sbenno		break;
39596938Sbenno	}
396209975Snwhitehorn	printf("   srr0            = 0x%" PRIxPTR "\n", frame->srr0);
397209975Snwhitehorn	printf("   srr1            = 0x%" PRIxPTR "\n", frame->srr1);
398209975Snwhitehorn	printf("   lr              = 0x%" PRIxPTR "\n", frame->lr);
39996938Sbenno	printf("   curthread       = %p\n", curthread);
40096938Sbenno	if (curthread != NULL)
40196938Sbenno		printf("          pid = %d, comm = %s\n",
402173600Sjulian		    curthread->td_proc->p_pid, curthread->td_name);
40396938Sbenno	printf("\n");
40496938Sbenno}
40586067Smp
40696938Sbenno/*
40796938Sbenno * Handles a fatal fault when we have onfault state to recover.  Returns
40896938Sbenno * non-zero if there was onfault recovery state available.
40996938Sbenno */
41096938Sbennostatic int
41196938Sbennohandle_onfault(struct trapframe *frame)
41296938Sbenno{
41396938Sbenno	struct		thread *td;
41496938Sbenno	faultbuf	*fb;
41586067Smp
41696938Sbenno	td = curthread;
41796938Sbenno	fb = td->td_pcb->pcb_onfault;
41896938Sbenno	if (fb != NULL) {
41996938Sbenno		frame->srr0 = (*fb)[0];
42096938Sbenno		frame->fixreg[1] = (*fb)[1];
42196938Sbenno		frame->fixreg[2] = (*fb)[2];
422131867Sgrehan		frame->fixreg[3] = 1;
42396938Sbenno		frame->cr = (*fb)[3];
42496938Sbenno		bcopy(&(*fb)[4], &frame->fixreg[13],
42596938Sbenno		    19 * sizeof(register_t));
42696938Sbenno		return (1);
42796938Sbenno	}
42896938Sbenno	return (0);
42996938Sbenno}
43086067Smp
431208453Skibint
432208453Skibcpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
43396938Sbenno{
434208453Skib	struct proc *p;
435208453Skib	struct trapframe *frame;
436208453Skib	caddr_t	params;
437209975Snwhitehorn	size_t argsz;
438209975Snwhitehorn	int error, n, i;
43986067Smp
44096938Sbenno	p = td->td_proc;
441208453Skib	frame = td->td_frame;
44286067Smp
443208453Skib	sa->code = frame->fixreg[0];
44496938Sbenno	params = (caddr_t)(frame->fixreg + FIRSTARG);
44596938Sbenno	n = NARGREG;
446131698Sgrehan
447208453Skib	if (sa->code == SYS_syscall) {
44896938Sbenno		/*
44996938Sbenno		 * code is first argument,
45096938Sbenno		 * followed by actual args.
45196938Sbenno		 */
452209975Snwhitehorn		sa->code = *(register_t *) params;
453103607Sgrehan		params += sizeof(register_t);
45496938Sbenno		n -= 1;
455208453Skib	} else if (sa->code == SYS___syscall) {
45696938Sbenno		/*
45796938Sbenno		 * Like syscall, but code is a quad,
45896938Sbenno		 * so as to maintain quad alignment
45996938Sbenno		 * for the rest of the args.
46096938Sbenno		 */
461217896Sdchagin		if (SV_PROC_FLAG(p, SV_ILP32)) {
462209975Snwhitehorn			params += sizeof(register_t);
463209975Snwhitehorn			sa->code = *(register_t *) params;
464209975Snwhitehorn			params += sizeof(register_t);
465209975Snwhitehorn			n -= 2;
466209975Snwhitehorn		} else {
467209975Snwhitehorn			sa->code = *(register_t *) params;
468209975Snwhitehorn			params += sizeof(register_t);
469209975Snwhitehorn			n -= 1;
470209975Snwhitehorn		}
47196938Sbenno	}
47296452Sbenno
47396938Sbenno 	if (p->p_sysent->sv_mask)
474208453Skib		sa->code &= p->p_sysent->sv_mask;
475208453Skib	if (sa->code >= p->p_sysent->sv_size)
476208453Skib		sa->callp = &p->p_sysent->sv_table[0];
477208453Skib	else
478208453Skib		sa->callp = &p->p_sysent->sv_table[sa->code];
47986067Smp
480208453Skib	sa->narg = sa->callp->sy_narg;
48186067Smp
482217896Sdchagin	if (SV_PROC_FLAG(p, SV_ILP32)) {
483209975Snwhitehorn		argsz = sizeof(uint32_t);
484209975Snwhitehorn
485209975Snwhitehorn		for (i = 0; i < n; i++)
486209975Snwhitehorn			sa->args[i] = ((u_register_t *)(params))[i] &
487209975Snwhitehorn			    0xffffffff;
488209975Snwhitehorn	} else {
489209975Snwhitehorn		argsz = sizeof(uint64_t);
490209975Snwhitehorn
491209975Snwhitehorn		for (i = 0; i < n; i++)
492209975Snwhitehorn			sa->args[i] = ((u_register_t *)(params))[i];
493209975Snwhitehorn	}
494209975Snwhitehorn
495209975Snwhitehorn	if (sa->narg > n)
496208453Skib		error = copyin(MOREARGS(frame->fixreg[1]), sa->args + n,
497209975Snwhitehorn			       (sa->narg - n) * argsz);
498209975Snwhitehorn	else
49998001Sjhb		error = 0;
50098001Sjhb
501209975Snwhitehorn#ifdef __powerpc64__
502217896Sdchagin	if (SV_PROC_FLAG(p, SV_ILP32) && sa->narg > n) {
503209975Snwhitehorn		/* Expand the size of arguments copied from the stack */
504209975Snwhitehorn
505209975Snwhitehorn		for (i = sa->narg; i >= n; i--)
506209975Snwhitehorn			sa->args[i] = ((uint32_t *)(&sa->args[n]))[i-n];
507209975Snwhitehorn	}
508209975Snwhitehorn#endif
509209975Snwhitehorn
51098001Sjhb	if (error == 0) {
51198001Sjhb		td->td_retval[0] = 0;
51298001Sjhb		td->td_retval[1] = frame->fixreg[FIRSTARG + 1];
51398001Sjhb	}
514208453Skib	return (error);
515208453Skib}
51677957Sbenno
517225474Skib#include "../../kern/subr_syscall.c"
518225474Skib
519208453Skibvoid
520208453Skibsyscall(struct trapframe *frame)
521208453Skib{
522208453Skib	struct thread *td;
523208453Skib	struct syscall_args sa;
524208453Skib	int error;
525199135Skib
526223485Snwhitehorn	td = curthread;
527208453Skib	td->td_frame = frame;
528160773Sjhb
529214749Snwhitehorn#ifdef __powerpc64__
530214739Snwhitehorn	/*
531214739Snwhitehorn	 * Speculatively restore last user SLB segment, which we know is
532214739Snwhitehorn	 * invalid already, since we are likely to do copyin()/copyout().
533214739Snwhitehorn	 */
534214739Snwhitehorn	__asm __volatile ("slbmte %0, %1; isync" ::
535214739Snwhitehorn            "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), "r"(USER_SLB_SLBE));
536214749Snwhitehorn#endif
537214739Snwhitehorn
538208453Skib	error = syscallenter(td, &sa);
539208453Skib	syscallret(td, error, &sa);
54077957Sbenno}
54177957Sbenno
542209975Snwhitehorn#ifdef __powerpc64__
543230123Snwhitehorn/* Handle kernel SLB faults -- runs in real mode, all seat belts off */
544230123Snwhitehornvoid
545230123Snwhitehornhandle_kernel_slb_spill(int type, register_t dar, register_t srr0)
546230123Snwhitehorn{
547230123Snwhitehorn	struct slb *slbcache;
548230123Snwhitehorn	uint64_t slbe, slbv;
549230123Snwhitehorn	uint64_t esid, addr;
550230123Snwhitehorn	int i;
551230123Snwhitehorn
552230123Snwhitehorn	addr = (type == EXC_ISE) ? srr0 : dar;
553230123Snwhitehorn	slbcache = PCPU_GET(slb);
554230123Snwhitehorn	esid = (uintptr_t)addr >> ADDR_SR_SHFT;
555230123Snwhitehorn	slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
556230123Snwhitehorn
557230123Snwhitehorn	/* See if the hardware flushed this somehow (can happen in LPARs) */
558230123Snwhitehorn	for (i = 0; i < n_slbs; i++)
559230123Snwhitehorn		if (slbcache[i].slbe == (slbe | (uint64_t)i))
560230123Snwhitehorn			return;
561230123Snwhitehorn
562230123Snwhitehorn	/* Not in the map, needs to actually be added */
563230123Snwhitehorn	slbv = kernel_va_to_slbv(addr);
564230123Snwhitehorn	if (slbcache[USER_SLB_SLOT].slbe == 0) {
565230123Snwhitehorn		for (i = 0; i < n_slbs; i++) {
566230123Snwhitehorn			if (i == USER_SLB_SLOT)
567230123Snwhitehorn				continue;
568230123Snwhitehorn			if (!(slbcache[i].slbe & SLBE_VALID))
569230123Snwhitehorn				goto fillkernslb;
570230123Snwhitehorn		}
571230123Snwhitehorn
572230123Snwhitehorn		if (i == n_slbs)
573230123Snwhitehorn			slbcache[USER_SLB_SLOT].slbe = 1;
574230123Snwhitehorn	}
575230123Snwhitehorn
576230123Snwhitehorn	/* Sacrifice a random SLB entry that is not the user entry */
577230123Snwhitehorn	i = mftb() % n_slbs;
578230123Snwhitehorn	if (i == USER_SLB_SLOT)
579230123Snwhitehorn		i = (i+1) % n_slbs;
580230123Snwhitehorn
581230123Snwhitehornfillkernslb:
582230123Snwhitehorn	/* Write new entry */
583230123Snwhitehorn	slbcache[i].slbv = slbv;
584230123Snwhitehorn	slbcache[i].slbe = slbe | (uint64_t)i;
585230123Snwhitehorn
586230123Snwhitehorn	/* Trap handler will restore from cache on exit */
587230123Snwhitehorn}
588230123Snwhitehorn
589209975Snwhitehornstatic int
590230123Snwhitehornhandle_user_slb_spill(pmap_t pm, vm_offset_t addr)
591209975Snwhitehorn{
592212722Snwhitehorn	struct slb *user_entry;
593212715Snwhitehorn	uint64_t esid;
594212715Snwhitehorn	int i;
595209975Snwhitehorn
596212715Snwhitehorn	esid = (uintptr_t)addr >> ADDR_SR_SHFT;
597212715Snwhitehorn
598209975Snwhitehorn	PMAP_LOCK(pm);
599212715Snwhitehorn	user_entry = user_va_to_slb_entry(pm, addr);
600212715Snwhitehorn
601212715Snwhitehorn	if (user_entry == NULL) {
602212715Snwhitehorn		/* allocate_vsid auto-spills it */
603212722Snwhitehorn		(void)allocate_user_vsid(pm, esid, 0);
604212715Snwhitehorn	} else {
605209975Snwhitehorn		/*
606209975Snwhitehorn		 * Check that another CPU has not already mapped this.
607209975Snwhitehorn		 * XXX: Per-thread SLB caches would be better.
608209975Snwhitehorn		 */
609212722Snwhitehorn		for (i = 0; i < pm->pm_slb_len; i++)
610212722Snwhitehorn			if (pm->pm_slb[i] == user_entry)
611209975Snwhitehorn				break;
612209975Snwhitehorn
613212722Snwhitehorn		if (i == pm->pm_slb_len)
614212722Snwhitehorn			slb_insert_user(pm, user_entry);
615209975Snwhitehorn	}
616209975Snwhitehorn	PMAP_UNLOCK(pm);
617209975Snwhitehorn
618209975Snwhitehorn	return (0);
619209975Snwhitehorn}
620209975Snwhitehorn#endif
621209975Snwhitehorn
62296938Sbennostatic int
62396938Sbennotrap_pfault(struct trapframe *frame, int user)
62477957Sbenno{
62596938Sbenno	vm_offset_t	eva, va;
62696938Sbenno	struct		thread *td;
62796938Sbenno	struct		proc *p;
62896938Sbenno	vm_map_t	map;
62996938Sbenno	vm_prot_t	ftype;
63096938Sbenno	int		rv;
631209975Snwhitehorn	register_t	user_sr;
63277957Sbenno
63396938Sbenno	td = curthread;
63496906Sbenno	p = td->td_proc;
63596938Sbenno	if (frame->exc == EXC_ISI) {
63696938Sbenno		eva = frame->srr0;
637217341Snwhitehorn		ftype = VM_PROT_EXECUTE;
638217341Snwhitehorn		if (frame->srr1 & SRR1_ISI_PFAULT)
639217341Snwhitehorn			ftype |= VM_PROT_READ;
64096938Sbenno	} else {
641176742Sraj		eva = frame->cpu.aim.dar;
642176742Sraj		if (frame->cpu.aim.dsisr & DSISR_STORE)
643103607Sgrehan			ftype = VM_PROT_WRITE;
64496938Sbenno		else
64596938Sbenno			ftype = VM_PROT_READ;
64696938Sbenno	}
64796906Sbenno
64896938Sbenno	if (user) {
64996938Sbenno		map = &p->p_vmspace->vm_map;
65096938Sbenno	} else {
651209975Snwhitehorn		if ((eva >> ADDR_SR_SHFT) == (USER_ADDR >> ADDR_SR_SHFT)) {
65296938Sbenno			if (p->p_vmspace == NULL)
65396938Sbenno				return (SIGSEGV);
654131698Sgrehan
655209975Snwhitehorn			map = &p->p_vmspace->vm_map;
656209975Snwhitehorn
657212715Snwhitehorn			user_sr = td->td_pcb->pcb_cpu.aim.usr_segm;
65896938Sbenno			eva &= ADDR_PIDX | ADDR_POFF;
65996938Sbenno			eva |= user_sr << ADDR_SR_SHFT;
66096938Sbenno		} else {
66196938Sbenno			map = kernel_map;
66296938Sbenno		}
66396938Sbenno	}
66496938Sbenno	va = trunc_page(eva);
66577957Sbenno
66696938Sbenno	if (map != kernel_map) {
66796938Sbenno		/*
66896938Sbenno		 * Keep swapout from messing with us during this
66996938Sbenno		 *	critical time.
67096938Sbenno		 */
67196255Sbenno		PROC_LOCK(p);
67296938Sbenno		++p->p_lock;
67396255Sbenno		PROC_UNLOCK(p);
67496938Sbenno
67596938Sbenno		/* Fault in the user page: */
676199868Salc		rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
67796938Sbenno
67896938Sbenno		PROC_LOCK(p);
67996938Sbenno		--p->p_lock;
68096938Sbenno		PROC_UNLOCK(p);
681242723Sjhibbits		/*
682242723Sjhibbits		 * XXXDTRACE: add dtrace_doubletrap_func here?
683242723Sjhibbits		 */
68496938Sbenno	} else {
68596938Sbenno		/*
68696938Sbenno		 * Don't have to worry about process locking or stacks in the
68796938Sbenno		 * kernel.
68896938Sbenno		 */
68996938Sbenno		rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
69096255Sbenno	}
69196938Sbenno
69296938Sbenno	if (rv == KERN_SUCCESS)
69396938Sbenno		return (0);
69496938Sbenno
69596938Sbenno	if (!user && handle_onfault(frame))
69696938Sbenno		return (0);
69796938Sbenno
69896938Sbenno	return (SIGSEGV);
69977957Sbenno}
70077957Sbenno
70177957Sbennoint
70296906Sbennobadaddr(void *addr, size_t size)
70377957Sbenno{
70496906Sbenno	return (badaddr_read(addr, size, NULL));
70577957Sbenno}
70677957Sbenno
70777957Sbennoint
70896906Sbennobadaddr_read(void *addr, size_t size, int *rptr)
70977957Sbenno{
71096906Sbenno	struct thread	*td;
71196906Sbenno	faultbuf	env;
71296906Sbenno	int		x;
71377957Sbenno
71477957Sbenno	/* Get rid of any stale machine checks that have been waiting.  */
71577957Sbenno	__asm __volatile ("sync; isync");
71677957Sbenno
717223485Snwhitehorn	td = curthread;
71896255Sbenno
71977957Sbenno	if (setfault(env)) {
72096255Sbenno		td->td_pcb->pcb_onfault = 0;
72177957Sbenno		__asm __volatile ("sync");
72277957Sbenno		return 1;
72377957Sbenno	}
72477957Sbenno
72577957Sbenno	__asm __volatile ("sync");
72677957Sbenno
72777957Sbenno	switch (size) {
72877957Sbenno	case 1:
72977957Sbenno		x = *(volatile int8_t *)addr;
73077957Sbenno		break;
73177957Sbenno	case 2:
73277957Sbenno		x = *(volatile int16_t *)addr;
73377957Sbenno		break;
73477957Sbenno	case 4:
73577957Sbenno		x = *(volatile int32_t *)addr;
73677957Sbenno		break;
73777957Sbenno	default:
738209975Snwhitehorn		panic("badaddr: invalid size (%zd)", size);
73977957Sbenno	}
74077957Sbenno
74177957Sbenno	/* Make sure we took the machine check, if we caused one. */
74277957Sbenno	__asm __volatile ("sync; isync");
74377957Sbenno
74496255Sbenno	td->td_pcb->pcb_onfault = 0;
74577957Sbenno	__asm __volatile ("sync");	/* To be sure. */
74677957Sbenno
74777957Sbenno	/* Use the value to avoid reorder. */
74877957Sbenno	if (rptr)
74977957Sbenno		*rptr = x;
75077957Sbenno
75196906Sbenno	return (0);
75277957Sbenno}
75377957Sbenno
75477957Sbenno/*
75577957Sbenno * For now, this only deals with the particular unaligned access case
75677957Sbenno * that gcc tends to generate.  Eventually it should handle all of the
75777957Sbenno * possibilities that can happen on a 32-bit PowerPC in big-endian mode.
75877957Sbenno */
75977957Sbenno
76077957Sbennostatic int
76196906Sbennofix_unaligned(struct thread *td, struct trapframe *frame)
76277957Sbenno{
76396906Sbenno	struct thread	*fputhread;
76496906Sbenno	int		indicator, reg;
76596906Sbenno	double		*fpr;
76677957Sbenno
767176742Sraj	indicator = EXC_ALI_OPCODE_INDICATOR(frame->cpu.aim.dsisr);
76896906Sbenno
76977957Sbenno	switch (indicator) {
77077957Sbenno	case EXC_ALI_LFD:
77177957Sbenno	case EXC_ALI_STFD:
772176742Sraj		reg = EXC_ALI_RST(frame->cpu.aim.dsisr);
77396906Sbenno		fpr = &td->td_pcb->pcb_fpu.fpr[reg];
77496906Sbenno		fputhread = PCPU_GET(fputhread);
77577957Sbenno
77696906Sbenno		/* Juggle the FPU to ensure that we've initialized
77796906Sbenno		 * the FPRs, and that their current state is in
77896906Sbenno		 * the PCB.
77996906Sbenno		 */
78096906Sbenno		if (fputhread != td) {
78196906Sbenno			if (fputhread)
78296906Sbenno				save_fpu(fputhread);
78396906Sbenno			enable_fpu(td);
78496906Sbenno		}
78596906Sbenno		save_fpu(td);
78677957Sbenno
78796906Sbenno		if (indicator == EXC_ALI_LFD) {
788176742Sraj			if (copyin((void *)frame->cpu.aim.dar, fpr,
78996906Sbenno			    sizeof(double)) != 0)
79096906Sbenno				return -1;
79196906Sbenno			enable_fpu(td);
79296906Sbenno		} else {
793176742Sraj			if (copyout(fpr, (void *)frame->cpu.aim.dar,
79496906Sbenno			    sizeof(double)) != 0)
79596906Sbenno				return -1;
79677957Sbenno		}
79796906Sbenno		return 0;
79877957Sbenno		break;
79977957Sbenno	}
80077957Sbenno
80177957Sbenno	return -1;
80277957Sbenno}
803204197Snwhitehorn
804204197Snwhitehornstatic int
805204197Snwhitehornppc_instr_emulate(struct trapframe *frame)
806204197Snwhitehorn{
807204197Snwhitehorn	uint32_t instr;
808204197Snwhitehorn	int reg;
809204197Snwhitehorn
810204197Snwhitehorn	instr = fuword32((void *)frame->srr0);
811204197Snwhitehorn
812204197Snwhitehorn	if ((instr & 0xfc1fffff) == 0x7c1f42a6) {	/* mfpvr */
813204197Snwhitehorn		reg = (instr & ~0xfc1fffff) >> 21;
814204197Snwhitehorn		frame->fixreg[reg] = mfpvr();
815204197Snwhitehorn		return (0);
816204197Snwhitehorn	}
817204197Snwhitehorn
818204197Snwhitehorn	return (-1);
819204197Snwhitehorn}
820204197Snwhitehorn
821