180708Sjake/*- 281896Sjake * Copyright (c) 1982, 1986 The Regents of the University of California. 381896Sjake * Copyright (c) 1989, 1990 William Jolitz 481896Sjake * Copyright (c) 1994 John Dyson 580708Sjake * Copyright (c) 2001 Jake Burkholder. 680708Sjake * All rights reserved. 780708Sjake * 881896Sjake * This code is derived from software contributed to Berkeley by 981896Sjake * the Systems Programming Group of the University of Utah Computer 1081896Sjake * Science Department, and William Jolitz. 1181896Sjake * 1280708Sjake * Redistribution and use in source and binary forms, with or without 1380708Sjake * modification, are permitted provided that the following conditions 1480708Sjake * are met: 1580708Sjake * 1. Redistributions of source code must retain the above copyright 1680708Sjake * notice, this list of conditions and the following disclaimer. 1780708Sjake * 2. Redistributions in binary form must reproduce the above copyright 1880708Sjake * notice, this list of conditions and the following disclaimer in the 1980708Sjake * documentation and/or other materials provided with the distribution. 2081896Sjake * 4. Neither the name of the University nor the names of its contributors 2181896Sjake * may be used to endorse or promote products derived from this software 2281896Sjake * without specific prior written permission. 2380708Sjake * 2481896Sjake * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2580708Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2680708Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2781896Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2880708Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2980708Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3080708Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3180708Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3280708Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3380708Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3480708Sjake * SUCH DAMAGE. 3580708Sjake * 3681896Sjake * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91 3781896Sjake * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$ 38191980Smarius * from: FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.167 2001/07/12 3980708Sjake */ 4080708Sjake 41191980Smarius#include <sys/cdefs.h> 42191980Smarius__FBSDID("$FreeBSD$"); 43191980Smarius 44108700Sjake#include "opt_pmap.h" 45108700Sjake 4680708Sjake#include <sys/param.h> 4780708Sjake#include <sys/systm.h> 4880708Sjake#include <sys/bio.h> 4980708Sjake#include <sys/buf.h> 50119563Salc#include <sys/kernel.h> 51191980Smarius#include <sys/malloc.h> 52119563Salc#include <sys/mbuf.h> 53119563Salc#include <sys/mutex.h> 54191980Smarius#include <sys/proc.h> 55199135Skib#include <sys/sysent.h> 56122780Salc#include <sys/sf_buf.h> 57191980Smarius#include <sys/sched.h> 58108700Sjake#include <sys/sysctl.h> 5980709Sjake#include <sys/unistd.h> 6082014Sjake#include <sys/vmmeter.h> 6180708Sjake 6280709Sjake#include <dev/ofw/openfirm.h> 6380709Sjake 6480708Sjake#include <vm/vm.h> 6580708Sjake#include <vm/vm_extern.h> 6682908Sjake#include <vm/pmap.h> 67119563Salc#include <vm/vm_kern.h> 6886523Sjake#include <vm/vm_map.h> 6986523Sjake#include <vm/vm_page.h> 70109615Sjeff#include <vm/vm_pageout.h> 7191612Sjake#include <vm/vm_param.h> 72108302Sjake#include <vm/uma.h> 73108302Sjake#include <vm/uma_int.h> 7480708Sjake 7586523Sjake#include <machine/cache.h> 7680709Sjake#include <machine/cpu.h> 77112920Sjake#include <machine/fp.h> 78191980Smarius#include <machine/frame.h> 7988643Sjake#include <machine/fsr.h> 8080708Sjake#include <machine/md_var.h> 8186147Stmm#include <machine/ofw_machdep.h> 82112306Sjake#include <machine/ofw_mem.h> 83138129Sdas#include <machine/pcb.h> 84108302Sjake#include <machine/tlb.h> 8582014Sjake#include <machine/tstate.h> 8680708Sjake 87129906Sbmilekic#ifndef NSFBUFS 88129906Sbmilekic#define NSFBUFS (512 + maxusers * 16) 89129906Sbmilekic#endif 90129906Sbmilekic 91255786Sglebiusstatic int nsfbufs; 92255786Sglebiusstatic int nsfbufspeak; 93255786Sglebiusstatic int nsfbufsused; 94255786Sglebius 95255786SglebiusSYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufs, CTLFLAG_RDTUN, &nsfbufs, 0, 96255786Sglebius "Maximum number of sendfile(2) sf_bufs available"); 97255786SglebiusSYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufspeak, CTLFLAG_RD, &nsfbufspeak, 0, 98255786Sglebius "Number of sendfile(2) sf_bufs at peak usage"); 99255786SglebiusSYSCTL_INT(_kern_ipc, OID_AUTO, nsfbufsused, CTLFLAG_RD, &nsfbufsused, 0, 100255786Sglebius "Number of sendfile(2) sf_bufs in use"); 101255786Sglebius 102119563Salcstatic void sf_buf_init(void *arg); 103177253SrwatsonSYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL); 104119563Salc 105119563Salc/* 106191980Smarius * Expanded sf_freelist head. Really an SLIST_HEAD() in disguise, with the 107119563Salc * sf_freelist head with the sf_lock mutex. 108119563Salc */ 109119563Salcstatic struct { 110119563Salc SLIST_HEAD(, sf_buf) sf_head; 111119563Salc struct mtx sf_lock; 112119563Salc} sf_freelist; 113119563Salc 114119563Salcstatic u_int sf_buf_alloc_want; 115119563Salc 116108700SjakePMAP_STATS_VAR(uma_nsmall_alloc); 117108700SjakePMAP_STATS_VAR(uma_nsmall_alloc_oc); 118108700SjakePMAP_STATS_VAR(uma_nsmall_free); 119108700Sjake 12080708Sjakevoid 12183366Sjuliancpu_exit(struct thread *td) 12280708Sjake{ 12391339Sjake struct proc *p; 12488782Sjake 12591339Sjake p = td->td_proc; 12695744Sjake p->p_md.md_sigtramp = NULL; 127140485Sjhb if (p->p_md.md_utrap != NULL) { 128140485Sjhb utrap_free(p->p_md.md_utrap); 12991339Sjake p->p_md.md_utrap = NULL; 13088782Sjake } 13180708Sjake} 13280708Sjake 13398765Sjakevoid 13499072Sjuliancpu_thread_exit(struct thread *td) 13599072Sjulian{ 136191980Smarius 13799072Sjulian} 13899072Sjulian 13999072Sjulianvoid 140107719Sjuliancpu_thread_clean(struct thread *td) 141107180Smux{ 142191980Smarius 143107180Smux} 144107180Smux 145107180Smuxvoid 146173615Smarcelcpu_thread_alloc(struct thread *td) 14799072Sjulian{ 148112968Sjake struct pcb *pcb; 149112968Sjake 150127545Skensmith pcb = (struct pcb *)((td->td_kstack + td->td_kstack_pages * PAGE_SIZE - 151112968Sjake sizeof(struct pcb)) & ~0x3fUL); 152135853Skensmith pcb->pcb_nsaved = 0; 153112968Sjake td->td_frame = (struct trapframe *)pcb - 1; 154112968Sjake td->td_pcb = pcb; 15599072Sjulian} 15699072Sjulian 15799072Sjulianvoid 158173615Smarcelcpu_thread_free(struct thread *td) 159173615Smarcel{ 160191980Smarius 161173615Smarcel} 162191980Smarius 163173615Smarcelvoid 164119004Smarcelcpu_thread_swapin(struct thread *td) 165119004Smarcel{ 166191980Smarius 167119004Smarcel} 168119004Smarcel 169119004Smarcelvoid 170119004Smarcelcpu_thread_swapout(struct thread *td) 171119004Smarcel{ 172191980Smarius 173119004Smarcel} 174119004Smarcel 175119004Smarcelvoid 176199135Skibcpu_set_syscall_retval(struct thread *td, int error) 177199135Skib{ 178199135Skib 179199135Skib switch (error) { 180199135Skib case 0: 181199135Skib td->td_frame->tf_out[0] = td->td_retval[0]; 182199135Skib td->td_frame->tf_out[1] = td->td_retval[1]; 183199135Skib td->td_frame->tf_tstate &= ~TSTATE_XCC_C; 184199135Skib break; 185199135Skib 186199135Skib case ERESTART: 187199135Skib /* 188199135Skib * Undo the tpc advancement we have done on syscall 189199135Skib * enter, we want to reexecute the system call. 190199135Skib */ 191199135Skib td->td_frame->tf_tpc = td->td_pcb->pcb_tpc; 192199135Skib td->td_frame->tf_tnpc -= 4; 193199135Skib break; 194199135Skib 195199135Skib case EJUSTRETURN: 196199135Skib break; 197199135Skib 198199135Skib default: 199199135Skib if (td->td_proc->p_sysent->sv_errsize) { 200199135Skib if (error >= td->td_proc->p_sysent->sv_errsize) 201199135Skib error = -1; /* XXX */ 202199135Skib else 203199135Skib error = td->td_proc->p_sysent->sv_errtbl[error]; 204199135Skib } 205199135Skib td->td_frame->tf_out[0] = error; 206199135Skib td->td_frame->tf_tstate |= TSTATE_XCC_C; 207199135Skib break; 208199135Skib } 209199135Skib} 210199135Skib 211199135Skibvoid 212115858Smarcelcpu_set_upcall(struct thread *td, struct thread *td0) 21399072Sjulian{ 214112921Sjake struct trapframe *tf; 215112921Sjake struct frame *fr; 216112968Sjake struct pcb *pcb; 217112921Sjake 218115858Smarcel bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe)); 219115858Smarcel 220112968Sjake pcb = td->td_pcb; 221112968Sjake tf = td->td_frame; 222112921Sjake fr = (struct frame *)tf - 1; 223112921Sjake fr->fr_local[0] = (u_long)fork_return; 224112921Sjake fr->fr_local[1] = (u_long)td; 225112921Sjake fr->fr_local[2] = (u_long)tf; 226112921Sjake pcb->pcb_pc = (u_long)fork_trampoline - 8; 227112921Sjake pcb->pcb_sp = (u_long)fr - SPOFF; 228144637Sjhb 229170305Sjeff /* Setup to release the spin count in fork_exit(). */ 230144637Sjhb td->td_md.md_spinlock_count = 1; 231144637Sjhb td->td_md.md_saved_pil = 0; 23299072Sjulian} 23399072Sjulian 23499072Sjulianvoid 235145433Sdavidxucpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg, 236211049Smarius stack_t *stack) 23799072Sjulian{ 238119622Sjake struct trapframe *tf; 239119622Sjake uint64_t sp; 240119622Sjake 241145433Sdavidxu if (td == curthread) 242145433Sdavidxu flushw(); 243119622Sjake tf = td->td_frame; 244145433Sdavidxu sp = (uint64_t)stack->ss_sp + stack->ss_size; 245145433Sdavidxu tf->tf_out[0] = (uint64_t)arg; 246119622Sjake tf->tf_out[6] = sp - SPOFF - sizeof(struct frame); 247145433Sdavidxu tf->tf_tpc = (uint64_t)entry; 248119622Sjake tf->tf_tnpc = tf->tf_tpc + 4; 249119622Sjake 250119622Sjake td->td_retval[0] = tf->tf_out[0]; 251119622Sjake td->td_retval[1] = tf->tf_out[1]; 25299072Sjulian} 25399072Sjulian 254147889Sdavidxuint 255145433Sdavidxucpu_set_user_tls(struct thread *td, void *tls_base) 256145433Sdavidxu{ 257145433Sdavidxu 258145433Sdavidxu if (td == curthread) 259145433Sdavidxu flushw(); 260211049Smarius td->td_frame->tf_global[7] = (uint64_t)tls_base; 261147889Sdavidxu return (0); 262145433Sdavidxu} 263145433Sdavidxu 26482908Sjake/* 26582908Sjake * Finish a fork operation, with process p2 nearly set up. 26682908Sjake * Copy and update the pcb, set up the stack so that the child 26782908Sjake * ready to run and return to user mode. 26882908Sjake */ 26980708Sjakevoid 27090361Sjuliancpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) 27180708Sjake{ 27280709Sjake struct trapframe *tf; 27382014Sjake struct frame *fp; 27491612Sjake struct pcb *pcb1; 27591612Sjake struct pcb *pcb2; 27691612Sjake vm_offset_t sp; 27791612Sjake int error; 27891612Sjake int i; 27980709Sjake 28090361Sjulian KASSERT(td1 == curthread || td1 == &thread0, 28182908Sjake ("cpu_fork: p1 not curproc and not proc0")); 28282908Sjake 28380709Sjake if ((flags & RFPROC) == 0) 28480709Sjake return; 28580709Sjake 28695744Sjake p2->p_md.md_sigtramp = td1->td_proc->p_md.md_sigtramp; 287140485Sjhb p2->p_md.md_utrap = utrap_hold(td1->td_proc->p_md.md_utrap); 28888782Sjake 28986147Stmm /* The pcb must be aligned on a 64-byte boundary. */ 29091612Sjake pcb1 = td1->td_pcb; 291127545Skensmith pcb2 = (struct pcb *)((td2->td_kstack + td2->td_kstack_pages * 292127545Skensmith PAGE_SIZE - sizeof(struct pcb)) & ~0x3fUL); 29391612Sjake td2->td_pcb = pcb2; 29483366Sjulian 29582908Sjake /* 29682908Sjake * Ensure that p1's pcb is up to date. 29782908Sjake */ 298112920Sjake critical_enter(); 299112920Sjake if ((td1->td_frame->tf_fprs & FPRS_FEF) != 0) 300112920Sjake savefpctx(pcb1->pcb_ufp); 301112920Sjake critical_exit(); 30282014Sjake /* Make sure the copied windows are spilled. */ 30389044Sjake flushw(); 30482014Sjake /* Copy the pcb (this will copy the windows saved in the pcb, too). */ 30591612Sjake bcopy(pcb1, pcb2, sizeof(*pcb1)); 30680709Sjake 30782908Sjake /* 30891612Sjake * If we're creating a new user process and we're sharing the address 30991612Sjake * space, the parent's top most frame must be saved in the pcb. The 31091612Sjake * child will pop the frame when it returns to user mode, and may 31191612Sjake * overwrite it with its own data causing much suffering for the 31291612Sjake * parent. We check if its already in the pcb, and if not copy it 31391612Sjake * in. Its unlikely that the copyin will fail, but if so there's not 31491612Sjake * much we can do. The parent will likely crash soon anyway in that 31591612Sjake * case. 31691612Sjake */ 31791612Sjake if ((flags & RFMEM) != 0 && td1 != &thread0) { 31891612Sjake sp = td1->td_frame->tf_sp; 31991612Sjake for (i = 0; i < pcb1->pcb_nsaved; i++) { 32091612Sjake if (pcb1->pcb_rwsp[i] == sp) 32191612Sjake break; 32291612Sjake } 32391612Sjake if (i == pcb1->pcb_nsaved) { 32491612Sjake error = copyin((caddr_t)sp + SPOFF, &pcb1->pcb_rw[i], 32591612Sjake sizeof(struct rwindow)); 32691612Sjake if (error == 0) { 32791612Sjake pcb1->pcb_rwsp[i] = sp; 32891612Sjake pcb1->pcb_nsaved++; 32991612Sjake } 33091612Sjake } 33191612Sjake } 33291612Sjake 33391612Sjake /* 33482908Sjake * Create a new fresh stack for the new process. 33582908Sjake * Copy the trap frame for the return to user mode as if from a 33682908Sjake * syscall. This copies most of the user mode register values. 33782908Sjake */ 33891612Sjake tf = (struct trapframe *)pcb2 - 1; 33983366Sjulian bcopy(td1->td_frame, tf, sizeof(*tf)); 34082908Sjake 34182908Sjake tf->tf_out[0] = 0; /* Child returns zero */ 34288643Sjake tf->tf_out[1] = 0; 34388643Sjake tf->tf_tstate &= ~TSTATE_XCC_C; /* success */ 34488643Sjake tf->tf_fprs = 0; 34582908Sjake 34683366Sjulian td2->td_frame = tf; 34782014Sjake fp = (struct frame *)tf - 1; 34894254Sjake fp->fr_local[0] = (u_long)fork_return; 34994254Sjake fp->fr_local[1] = (u_long)td2; 35094254Sjake fp->fr_local[2] = (u_long)tf; 351127343Stmm /* Terminate stack traces at this frame. */ 352127343Stmm fp->fr_pc = fp->fr_fp = 0; 353112917Sjake pcb2->pcb_sp = (u_long)fp - SPOFF; 35491612Sjake pcb2->pcb_pc = (u_long)fork_trampoline - 8; 35582908Sjake 356170305Sjeff /* Setup to release the spin count in fork_exit(). */ 357144637Sjhb td2->td_md.md_spinlock_count = 1; 358144637Sjhb td2->td_md.md_saved_pil = 0; 359144637Sjhb 36082908Sjake /* 36182908Sjake * Now, cpu_switch() can schedule the new process. 36282908Sjake */ 36380708Sjake} 36480708Sjake 36580708Sjakevoid 36680708Sjakecpu_reset(void) 36780708Sjake{ 36886147Stmm static char bspec[64] = ""; 36986147Stmm phandle_t chosen; 37086147Stmm static struct { 37186147Stmm cell_t name; 37286147Stmm cell_t nargs; 37386147Stmm cell_t nreturns; 37486147Stmm cell_t bootspec; 37586147Stmm } args = { 37686147Stmm (cell_t)"boot", 37786147Stmm 1, 37886147Stmm 0, 37986147Stmm (cell_t)bspec 38086147Stmm }; 381191980Smarius 382228201Sjchandra if ((chosen = OF_finddevice("/chosen")) != -1) { 38386147Stmm if (OF_getprop(chosen, "bootpath", bspec, sizeof(bspec)) == -1) 38486147Stmm bspec[0] = '\0'; 38586147Stmm bspec[sizeof(bspec) - 1] = '\0'; 38686147Stmm } 38786523Sjake 388191981Smarius cpu_shutdown(&args); 38980708Sjake} 39080708Sjake 39182908Sjake/* 39282908Sjake * Intercept the return address from a freshly forked process that has NOT 39382908Sjake * been scheduled yet. 39482908Sjake * 39582908Sjake * This is needed to make kernel threads stay in kernel mode. 39682908Sjake */ 39780708Sjakevoid 39883366Sjuliancpu_set_fork_handler(struct thread *td, void (*func)(void *), void *arg) 39980708Sjake{ 40080709Sjake struct frame *fp; 40180709Sjake struct pcb *pcb; 40280709Sjake 40383366Sjulian pcb = td->td_pcb; 404112917Sjake fp = (struct frame *)(pcb->pcb_sp + SPOFF); 40594254Sjake fp->fr_local[0] = (u_long)func; 40694254Sjake fp->fr_local[1] = (u_long)arg; 40780708Sjake} 40880708Sjake 40984847Stmmint 410113238Sjakeis_physical_memory(vm_paddr_t addr) 41184847Stmm{ 412112306Sjake struct ofw_mem_region *mr; 41384847Stmm 414112306Sjake for (mr = sparc64_memreg; mr < sparc64_memreg + sparc64_nmemreg; mr++) 415112306Sjake if (addr >= mr->mr_start && addr < mr->mr_start + mr->mr_size) 416112306Sjake return (1); 417112306Sjake return (0); 41884847Stmm} 41984847Stmm 420119563Salc/* 421119563Salc * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-)) 422119563Salc */ 423119563Salcstatic void 424119563Salcsf_buf_init(void *arg) 425119563Salc{ 426119563Salc struct sf_buf *sf_bufs; 427119563Salc vm_offset_t sf_base; 428119563Salc int i; 429119563Salc 430129906Sbmilekic nsfbufs = NSFBUFS; 431129906Sbmilekic TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs); 432129906Sbmilekic 433119563Salc mtx_init(&sf_freelist.sf_lock, "sf_bufs list lock", NULL, MTX_DEF); 434119563Salc SLIST_INIT(&sf_freelist.sf_head); 435254025Sjeff sf_base = kva_alloc(nsfbufs * PAGE_SIZE); 436119563Salc sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP, 437119563Salc M_NOWAIT | M_ZERO); 438119563Salc for (i = 0; i < nsfbufs; i++) { 439119563Salc sf_bufs[i].kva = sf_base + i * PAGE_SIZE; 440119563Salc SLIST_INSERT_HEAD(&sf_freelist.sf_head, &sf_bufs[i], free_list); 441119563Salc } 442119563Salc sf_buf_alloc_want = 0; 443119563Salc} 444119563Salc 445119563Salc/* 446191980Smarius * Get an sf_buf from the freelist. Will block if none are available. 447119563Salc */ 448119563Salcstruct sf_buf * 449137372Salcsf_buf_alloc(struct vm_page *m, int flags) 450119563Salc{ 451119563Salc struct sf_buf *sf; 452119563Salc int error; 453119563Salc 454119563Salc mtx_lock(&sf_freelist.sf_lock); 455119563Salc while ((sf = SLIST_FIRST(&sf_freelist.sf_head)) == NULL) { 456137372Salc if (flags & SFB_NOWAIT) 457137376Salc break; 458119563Salc sf_buf_alloc_want++; 459253351Sae SFSTAT_INC(sf_allocwait); 460137372Salc error = msleep(&sf_freelist, &sf_freelist.sf_lock, 461137372Salc (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0); 462119563Salc sf_buf_alloc_want--; 463119563Salc 464119563Salc /* 465191980Smarius * If we got a signal, don't risk going back to sleep. 466119563Salc */ 467119563Salc if (error) 468119563Salc break; 469119563Salc } 470119563Salc if (sf != NULL) { 471119563Salc SLIST_REMOVE_HEAD(&sf_freelist.sf_head, free_list); 472119563Salc sf->m = m; 473123884Ssilby nsfbufsused++; 474123920Ssilby nsfbufspeak = imax(nsfbufspeak, nsfbufsused); 475119563Salc pmap_qenter(sf->kva, &sf->m, 1); 476119563Salc } 477119563Salc mtx_unlock(&sf_freelist.sf_lock); 478119563Salc return (sf); 479119563Salc} 480119563Salc 481119563Salc/* 482127086Salc * Release resources back to the system. 483119563Salc */ 48480708Sjakevoid 485127086Salcsf_buf_free(struct sf_buf *sf) 486119563Salc{ 487119563Salc 488127086Salc pmap_qremove(sf->kva, 1); 489119563Salc mtx_lock(&sf_freelist.sf_lock); 490119563Salc SLIST_INSERT_HEAD(&sf_freelist.sf_head, sf, free_list); 491123884Ssilby nsfbufsused--; 492119563Salc if (sf_buf_alloc_want > 0) 493217561Skib wakeup(&sf_freelist); 494119563Salc mtx_unlock(&sf_freelist.sf_lock); 495119563Salc} 496119563Salc 497119563Salcvoid 49880708Sjakeswi_vm(void *v) 49980708Sjake{ 50086523Sjake 501212663Smarius /* Nothing to do here - busdma bounce buffers are not implemented. */ 50280708Sjake} 50380708Sjake 504108302Sjakevoid * 505108302Sjakeuma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 506108302Sjake{ 507113238Sjake vm_paddr_t pa; 508108302Sjake vm_page_t m; 509108302Sjake int pflags; 510108302Sjake void *va; 511108302Sjake 512108700Sjake PMAP_STATS_INC(uma_nsmall_alloc); 513108700Sjake 514108302Sjake *flags = UMA_SLAB_PRIV; 515243040Skib pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED; 516108302Sjake 517109615Sjeff for (;;) { 518228522Salc m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ); 519109615Sjeff if (m == NULL) { 520109615Sjeff if (wait & M_NOWAIT) 521109615Sjeff return (NULL); 522109615Sjeff else 523109615Sjeff VM_WAIT; 524109615Sjeff } else 525109615Sjeff break; 526109615Sjeff } 527108302Sjake 528109615Sjeff pa = VM_PAGE_TO_PHYS(m); 529211049Smarius if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) { 530109615Sjeff KASSERT(m->md.colors[0] == 0 && m->md.colors[1] == 0, 531223801Smarius ("uma_small_alloc: free page %p still has mappings!", m)); 532109615Sjeff PMAP_STATS_INC(uma_nsmall_alloc_oc); 533109615Sjeff m->md.color = DCACHE_COLOR(pa); 534109615Sjeff dcache_page_inval(pa); 535108302Sjake } 536109615Sjeff va = (void *)TLB_PHYS_TO_DIRECT(pa); 537116510Salc if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) 538212663Smarius cpu_block_zero(va, PAGE_SIZE); 539109615Sjeff return (va); 540108302Sjake} 541108302Sjake 542108302Sjakevoid 543108302Sjakeuma_small_free(void *mem, int size, u_int8_t flags) 544108302Sjake{ 545108302Sjake vm_page_t m; 546108302Sjake 547108700Sjake PMAP_STATS_INC(uma_nsmall_free); 548108302Sjake m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS((vm_offset_t)mem)); 549172189Salc m->wire_count--; 550108302Sjake vm_page_free(m); 551172189Salc atomic_subtract_int(&cnt.v_wire_count, 1); 552108302Sjake} 553