1139825Simp/*- 279263Sdillon * Copyright (c) 1991 Regents of the University of California. 379263Sdillon * All rights reserved. 479263Sdillon * 579263Sdillon * This code is derived from software contributed to Berkeley by 679263Sdillon * The Mach Operating System project at Carnegie-Mellon University. 779263Sdillon * 879263Sdillon * Redistribution and use in source and binary forms, with or without 979263Sdillon * modification, are permitted provided that the following conditions 1079263Sdillon * are met: 1179263Sdillon * 1. Redistributions of source code must retain the above copyright 1279263Sdillon * notice, this list of conditions and the following disclaimer. 1379263Sdillon * 2. Redistributions in binary form must reproduce the above copyright 1479263Sdillon * notice, this list of conditions and the following disclaimer in the 1579263Sdillon * documentation and/or other materials provided with the distribution. 1679263Sdillon * 4. Neither the name of the University nor the names of its contributors 1779263Sdillon * may be used to endorse or promote products derived from this software 1879263Sdillon * without specific prior written permission. 1979263Sdillon * 2079263Sdillon * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2179263Sdillon * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2279263Sdillon * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2379263Sdillon * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2479263Sdillon * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2579263Sdillon * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2679263Sdillon * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2779263Sdillon * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2879263Sdillon * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2979263Sdillon * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3079263Sdillon * SUCH DAMAGE. 3179263Sdillon * 3279263Sdillon * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91 3379263Sdillon */ 3479263Sdillon 35139825Simp/*- 3679263Sdillon * Copyright (c) 1987, 1990 Carnegie-Mellon University. 3779263Sdillon * All rights reserved. 3879263Sdillon * 3979263Sdillon * Authors: Avadis Tevanian, Jr., Michael Wayne Young 4079263Sdillon * 4179263Sdillon * Permission to use, copy, modify and distribute this software and 4279263Sdillon * its documentation is hereby granted, provided that both the copyright 4379263Sdillon * notice and this permission notice appear in all copies of the 4479263Sdillon * software, derivative works or modified versions, and any portions 4579263Sdillon * thereof, and that both notices appear in supporting documentation. 4679263Sdillon * 4779263Sdillon * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 4879263Sdillon * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 4979263Sdillon * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 5079263Sdillon * 5179263Sdillon * Carnegie Mellon requests users of this software to return to 5279263Sdillon * 5379263Sdillon * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 5479263Sdillon * School of Computer Science 5579263Sdillon * Carnegie Mellon University 5679263Sdillon * Pittsburgh PA 15213-3890 5779263Sdillon * 5879263Sdillon * any improvements or extensions that they make and grant Carnegie the 5979263Sdillon * rights to redistribute these changes. 6079263Sdillon */ 6179263Sdillon 62116226Sobrien#include <sys/cdefs.h> 63116226Sobrien__FBSDID("$FreeBSD$"); 64116226Sobrien 6579263Sdillon#include <sys/param.h> 6679263Sdillon#include <sys/systm.h> 67240757Salc#include <sys/eventhandler.h> 6879263Sdillon#include <sys/lock.h> 69170529Salc#include <sys/mount.h> 7079263Sdillon#include <sys/mutex.h> 7179263Sdillon#include <sys/proc.h> 72132379Sgreen#include <sys/kernel.h> 73132379Sgreen#include <sys/sysctl.h> 7479263Sdillon#include <sys/vmmeter.h> 7579263Sdillon#include <sys/vnode.h> 7679263Sdillon 7779263Sdillon#include <vm/vm.h> 7879263Sdillon#include <vm/vm_param.h> 7979263Sdillon#include <vm/vm_kern.h> 8084869Sdillon#include <vm/pmap.h> 8184869Sdillon#include <vm/vm_map.h> 8279263Sdillon#include <vm/vm_object.h> 8379263Sdillon#include <vm/vm_page.h> 8479263Sdillon#include <vm/vm_pageout.h> 8579263Sdillon#include <vm/vm_pager.h> 8679263Sdillon#include <vm/vm_extern.h> 8779263Sdillon 88100031Salcstatic int 89240757Salcvm_contig_launder_page(vm_page_t m, vm_page_t *next, int tries) 90100031Salc{ 91100031Salc vm_object_t object; 92132379Sgreen vm_page_t m_tmp; 93121226Salc struct vnode *vp; 94156225Stegge struct mount *mp; 95207519Salc int vfslocked; 96100031Salc 97207519Salc mtx_assert(&vm_page_queue_mtx, MA_OWNED); 98240757Salc if (!vm_pageout_page_lock(m, next) || m->hold_count != 0) { 99240757Salc vm_page_unlock(m); 100240757Salc return (EAGAIN); 101240757Salc } 102136924Salc object = m->object; 103173918Salc if (!VM_OBJECT_TRYLOCK(object) && 104216807Salc (!vm_pageout_fallback_object_lock(m, next) || m->hold_count != 0)) { 105207519Salc vm_page_unlock(m); 106173918Salc VM_OBJECT_UNLOCK(object); 107136924Salc return (EAGAIN); 108173918Salc } 109240757Salc if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { 110240757Salc if (tries == 0) { 111240757Salc vm_page_unlock(m); 112240757Salc VM_OBJECT_UNLOCK(object); 113240757Salc return (EAGAIN); 114240757Salc } 115240757Salc vm_page_sleep(m, "vpctw0"); 116136924Salc VM_OBJECT_UNLOCK(object); 117207519Salc vm_page_lock_queues(); 118132379Sgreen return (EBUSY); 119132379Sgreen } 120132379Sgreen vm_page_test_dirty(m); 121216807Salc if (m->dirty == 0) 122132379Sgreen pmap_remove_all(m); 123207519Salc if (m->dirty != 0) { 124207450Skmacy vm_page_unlock(m); 125240757Salc if (tries == 0 || (object->flags & OBJ_DEAD) != 0) { 126156415Stegge VM_OBJECT_UNLOCK(object); 127156415Stegge return (EAGAIN); 128156415Stegge } 129132379Sgreen if (object->type == OBJT_VNODE) { 130207519Salc vm_page_unlock_queues(); 131132379Sgreen vp = object->handle; 132156224Stegge vm_object_reference_locked(object); 133132379Sgreen VM_OBJECT_UNLOCK(object); 134156225Stegge (void) vn_start_write(vp, &mp, V_WAIT); 135170529Salc vfslocked = VFS_LOCK_GIANT(vp->v_mount); 136175202Sattilio vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 137132379Sgreen VM_OBJECT_LOCK(object); 138132379Sgreen vm_object_page_clean(object, 0, 0, OBJPC_SYNC); 139132379Sgreen VM_OBJECT_UNLOCK(object); 140175294Sattilio VOP_UNLOCK(vp, 0); 141170529Salc VFS_UNLOCK_GIANT(vfslocked); 142156224Stegge vm_object_deallocate(object); 143156225Stegge vn_finished_write(mp); 144207519Salc vm_page_lock_queues(); 145132379Sgreen return (0); 146132379Sgreen } else if (object->type == OBJT_SWAP || 147132379Sgreen object->type == OBJT_DEFAULT) { 148207519Salc vm_page_unlock_queues(); 149132379Sgreen m_tmp = m; 150233728Skib vm_pageout_flush(&m_tmp, 1, VM_PAGER_PUT_SYNC, 0, 151233728Skib NULL, NULL); 152132379Sgreen VM_OBJECT_UNLOCK(object); 153207519Salc vm_page_lock_queues(); 154132379Sgreen return (0); 155132379Sgreen } 156207519Salc } else { 157216807Salc vm_page_cache(m); 158207450Skmacy vm_page_unlock(m); 159207450Skmacy } 160136924Salc VM_OBJECT_UNLOCK(object); 161240757Salc return (EAGAIN); 162132379Sgreen} 163132379Sgreen 164192360Skmacystatic int 165240757Salcvm_contig_launder(int queue, int tries, vm_paddr_t low, vm_paddr_t high) 166132379Sgreen{ 167132379Sgreen vm_page_t m, next; 168208794Sjchandra vm_paddr_t pa; 169132379Sgreen int error; 170132379Sgreen 171173901Salc TAILQ_FOREACH_SAFE(m, &vm_page_queues[queue].pl, pageq, next) { 172240757Salc KASSERT(m->queue == queue, 173240757Salc ("vm_contig_launder: page %p's queue is not %d", m, queue)); 174148997Stegge if ((m->flags & PG_MARKER) != 0) 175148997Stegge continue; 176208794Sjchandra pa = VM_PAGE_TO_PHYS(m); 177208794Sjchandra if (pa < low || pa + PAGE_SIZE > high) 178208794Sjchandra continue; 179240757Salc error = vm_contig_launder_page(m, &next, tries); 180132379Sgreen if (error == 0) 181100031Salc return (TRUE); 182132379Sgreen if (error == EBUSY) 183132379Sgreen return (FALSE); 184100031Salc } 185100031Salc return (FALSE); 186100031Salc} 187100031Salc 188173901Salc/* 189240757Salc * Increase the number of cached pages. The specified value, "tries", 190240757Salc * determines which categories of pages are cached: 191240757Salc * 192240757Salc * 0: All clean, inactive pages within the specified physical address range 193240757Salc * are cached. Will not sleep. 194240757Salc * 1: The vm_lowmem handlers are called. All inactive pages within 195240757Salc * the specified physical address range are cached. May sleep. 196240757Salc * 2: The vm_lowmem handlers are called. All inactive and active pages 197240757Salc * within the specified physical address range are cached. May sleep. 198206409Salc */ 199208794Sjchandravoid 200208794Sjchandravm_contig_grow_cache(int tries, vm_paddr_t low, vm_paddr_t high) 201206409Salc{ 202206409Salc int actl, actmax, inactl, inactmax; 203206409Salc 204240757Salc if (tries > 0) { 205240757Salc /* 206240757Salc * Decrease registered cache sizes. The vm_lowmem handlers 207240757Salc * may acquire locks and/or sleep, so they can only be invoked 208240757Salc * when "tries" is greater than zero. 209240757Salc */ 210240757Salc EVENTHANDLER_INVOKE(vm_lowmem, 0); 211240757Salc 212240757Salc /* 213240757Salc * We do this explicitly after the caches have been drained 214240757Salc * above. 215240757Salc */ 216240757Salc uma_reclaim(); 217240757Salc } 218206409Salc vm_page_lock_queues(); 219206409Salc inactl = 0; 220240757Salc inactmax = cnt.v_inactive_count; 221206409Salc actl = 0; 222206409Salc actmax = tries < 2 ? 0 : cnt.v_active_count; 223206409Salcagain: 224240757Salc if (inactl < inactmax && vm_contig_launder(PQ_INACTIVE, tries, low, 225240757Salc high)) { 226206409Salc inactl++; 227206409Salc goto again; 228206409Salc } 229240757Salc if (actl < actmax && vm_contig_launder(PQ_ACTIVE, tries, low, high)) { 230206409Salc actl++; 231206409Salc goto again; 232206409Salc } 233206409Salc vm_page_unlock_queues(); 234206409Salc} 235206409Salc 236206409Salc/* 237206409Salc * Allocates a region from the kernel address map and pages within the 238206409Salc * specified physical address range to the kernel object, creates a wired 239206409Salc * mapping from the region to these pages, and returns the region's starting 240206409Salc * virtual address. The allocated pages are not necessarily physically 241206409Salc * contiguous. If M_ZERO is specified through the given flags, then the pages 242206409Salc * are zeroed before they are mapped. 243206409Salc */ 244206409Salcvm_offset_t 245206409Salckmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, 246206409Salc vm_paddr_t high, vm_memattr_t memattr) 247206409Salc{ 248206409Salc vm_object_t object = kernel_object; 249262933Sdumbbell vm_offset_t addr; 250262933Sdumbbell vm_ooffset_t end_offset, offset; 251206409Salc vm_page_t m; 252262933Sdumbbell int pflags, tries; 253206409Salc 254206409Salc size = round_page(size); 255206409Salc vm_map_lock(map); 256206409Salc if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 257206409Salc vm_map_unlock(map); 258206409Salc return (0); 259206409Salc } 260206409Salc offset = addr - VM_MIN_KERNEL_ADDRESS; 261206409Salc vm_object_reference(object); 262206409Salc vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL, 263206409Salc VM_PROT_ALL, 0); 264262933Sdumbbell if ((flags & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT) 265262933Sdumbbell pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOBUSY; 266262933Sdumbbell else 267262933Sdumbbell pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOBUSY; 268262933Sdumbbell if (flags & M_ZERO) 269262933Sdumbbell pflags |= VM_ALLOC_ZERO; 270206409Salc VM_OBJECT_LOCK(object); 271262933Sdumbbell end_offset = offset + size; 272262933Sdumbbell for (; offset < end_offset; offset += PAGE_SIZE) { 273206409Salc tries = 0; 274206409Salcretry: 275262933Sdumbbell m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 1, 276262933Sdumbbell low, high, PAGE_SIZE, 0, memattr); 277206409Salc if (m == NULL) { 278262933Sdumbbell VM_OBJECT_UNLOCK(object); 279206409Salc if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { 280206409Salc vm_map_unlock(map); 281208794Sjchandra vm_contig_grow_cache(tries, low, high); 282206409Salc vm_map_lock(map); 283206409Salc VM_OBJECT_LOCK(object); 284224689Salc tries++; 285206409Salc goto retry; 286206409Salc } 287262933Sdumbbell /* 288262933Sdumbbell * Since the pages that were allocated by any previous 289262933Sdumbbell * iterations of this loop are not busy, they can be 290262933Sdumbbell * freed by vm_object_page_remove(), which is called 291262933Sdumbbell * by vm_map_delete(). 292262933Sdumbbell */ 293206409Salc vm_map_delete(map, addr, addr + size); 294206409Salc vm_map_unlock(map); 295206409Salc return (0); 296206409Salc } 297206409Salc if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 298206409Salc pmap_zero_page(m); 299206409Salc m->valid = VM_PAGE_BITS_ALL; 300206409Salc } 301206409Salc VM_OBJECT_UNLOCK(object); 302206409Salc vm_map_unlock(map); 303206409Salc vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM | 304206409Salc VM_MAP_WIRE_NOHOLES); 305206409Salc return (addr); 306206409Salc} 307206409Salc 308206409Salc/* 309173901Salc * Allocates a region from the kernel address map, inserts the 310173901Salc * given physically contiguous pages into the kernel object, 311173901Salc * creates a wired mapping from the region to the pages, and 312173901Salc * returns the region's starting virtual address. If M_ZERO is 313173901Salc * specified through the given flags, then the pages are zeroed 314173901Salc * before they are mapped. 315173901Salc */ 316262933Sdumbbellvm_offset_t 317262933Sdumbbellkmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, 318262933Sdumbbell vm_paddr_t high, u_long alignment, u_long boundary, 319262933Sdumbbell vm_memattr_t memattr) 320132379Sgreen{ 321132379Sgreen vm_object_t object = kernel_object; 322262933Sdumbbell vm_offset_t addr; 323262933Sdumbbell vm_ooffset_t offset; 324262933Sdumbbell vm_page_t end_m, m; 325262933Sdumbbell int pflags, tries; 326132379Sgreen 327262933Sdumbbell size = round_page(size); 328132379Sgreen vm_map_lock(map); 329194337Salc if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 330132379Sgreen vm_map_unlock(map); 331194376Salc return (0); 332132379Sgreen } 333262933Sdumbbell offset = addr - VM_MIN_KERNEL_ADDRESS; 334132379Sgreen vm_object_reference(object); 335262933Sdumbbell vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL, 336262933Sdumbbell VM_PROT_ALL, 0); 337262933Sdumbbell if ((flags & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT) 338262933Sdumbbell pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOBUSY; 339262933Sdumbbell else 340262933Sdumbbell pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOBUSY; 341262933Sdumbbell if (flags & M_ZERO) 342262933Sdumbbell pflags |= VM_ALLOC_ZERO; 343132379Sgreen VM_OBJECT_LOCK(object); 344170816Salc tries = 0; 345170816Salcretry: 346262933Sdumbbell m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 347262933Sdumbbell atop(size), low, high, alignment, boundary, memattr); 348262933Sdumbbell if (m == NULL) { 349262933Sdumbbell VM_OBJECT_UNLOCK(object); 350170816Salc if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { 351262933Sdumbbell vm_map_unlock(map); 352208794Sjchandra vm_contig_grow_cache(tries, low, high); 353262933Sdumbbell vm_map_lock(map); 354262933Sdumbbell VM_OBJECT_LOCK(object); 355170816Salc tries++; 356170816Salc goto retry; 357170816Salc } 358262933Sdumbbell vm_map_delete(map, addr, addr + size); 359262933Sdumbbell vm_map_unlock(map); 360262933Sdumbbell return (0); 361132379Sgreen } 362262933Sdumbbell end_m = m + atop(size); 363262933Sdumbbell for (; m < end_m; m++) { 364262933Sdumbbell if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 365262933Sdumbbell pmap_zero_page(m); 366262933Sdumbbell m->valid = VM_PAGE_BITS_ALL; 367262933Sdumbbell } 368262933Sdumbbell VM_OBJECT_UNLOCK(object); 369262933Sdumbbell vm_map_unlock(map); 370262933Sdumbbell vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM | 371262933Sdumbbell VM_MAP_WIRE_NOHOLES); 372262933Sdumbbell return (addr); 37379263Sdillon} 374