device_pager.c revision 179074
1139825Simp/*- 21541Srgrimes * Copyright (c) 1990 University of Utah. 31541Srgrimes * Copyright (c) 1991, 1993 41541Srgrimes * The Regents of the University of California. All rights reserved. 51541Srgrimes * 61541Srgrimes * This code is derived from software contributed to Berkeley by 71541Srgrimes * the Systems Programming Group of the University of Utah Computer 81541Srgrimes * Science Department. 91541Srgrimes * 101541Srgrimes * Redistribution and use in source and binary forms, with or without 111541Srgrimes * modification, are permitted provided that the following conditions 121541Srgrimes * are met: 131541Srgrimes * 1. Redistributions of source code must retain the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer. 151541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 161541Srgrimes * notice, this list of conditions and the following disclaimer in the 171541Srgrimes * documentation and/or other materials provided with the distribution. 181541Srgrimes * 4. Neither the name of the University nor the names of its contributors 191541Srgrimes * may be used to endorse or promote products derived from this software 201541Srgrimes * without specific prior written permission. 211541Srgrimes * 221541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 231541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 241541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 251541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 261541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 271541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 281541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 291541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 301541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 311541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 321541Srgrimes * SUCH DAMAGE. 331541Srgrimes * 341549Srgrimes * @(#)device_pager.c 8.1 (Berkeley) 6/11/93 351541Srgrimes */ 361541Srgrimes 37116226Sobrien#include <sys/cdefs.h> 38116226Sobrien__FBSDID("$FreeBSD: head/sys/vm/device_pager.c 179074 2008-05-17 16:26:34Z alc $"); 39116226Sobrien 401541Srgrimes#include <sys/param.h> 411541Srgrimes#include <sys/systm.h> 421541Srgrimes#include <sys/conf.h> 4376166Smarkm#include <sys/lock.h> 4479224Sdillon#include <sys/proc.h> 4576166Smarkm#include <sys/mutex.h> 461541Srgrimes#include <sys/mman.h> 4775675Salfred#include <sys/sx.h> 481541Srgrimes 491541Srgrimes#include <vm/vm.h> 5012662Sdg#include <vm/vm_object.h> 511541Srgrimes#include <vm/vm_page.h> 529507Sdg#include <vm/vm_pager.h> 5392748Sjeff#include <vm/uma.h> 541541Srgrimes 5592727Salfredstatic void dev_pager_init(void); 5692727Salfredstatic vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, 5792727Salfred vm_ooffset_t); 5892727Salfredstatic void dev_pager_dealloc(vm_object_t); 5992727Salfredstatic int dev_pager_getpages(vm_object_t, vm_page_t *, int, int); 6092727Salfredstatic void dev_pager_putpages(vm_object_t, vm_page_t *, int, 6192727Salfred boolean_t, int *); 6292727Salfredstatic boolean_t dev_pager_haspage(vm_object_t, vm_pindex_t, int *, 6392727Salfred int *); 641541Srgrimes 6512820Sphk/* list of device pager objects */ 6612820Sphkstatic struct pagerlst dev_pager_object_list; 6775675Salfred/* protect list manipulation */ 6875675Salfredstatic struct mtx dev_pager_mtx; 6912820Sphk 7075675Salfred 7192748Sjeffstatic uma_zone_t fakepg_zone; 7212820Sphk 73112569Sjakestatic vm_page_t dev_pager_getfake(vm_paddr_t); 7492727Salfredstatic void dev_pager_putfake(vm_page_t); 75132884Sdfrstatic void dev_pager_updatefake(vm_page_t, vm_paddr_t); 761541Srgrimes 771541Srgrimesstruct pagerops devicepagerops = { 78118466Sphk .pgo_init = dev_pager_init, 79118466Sphk .pgo_alloc = dev_pager_alloc, 80118466Sphk .pgo_dealloc = dev_pager_dealloc, 81118466Sphk .pgo_getpages = dev_pager_getpages, 82118466Sphk .pgo_putpages = dev_pager_putpages, 83118466Sphk .pgo_haspage = dev_pager_haspage, 841541Srgrimes}; 851541Srgrimes 8612820Sphkstatic void 871541Srgrimesdev_pager_init() 881541Srgrimes{ 899507Sdg TAILQ_INIT(&dev_pager_object_list); 9093818Sjhb mtx_init(&dev_pager_mtx, "dev_pager list", NULL, MTX_DEF); 9192748Sjeff fakepg_zone = uma_zcreate("DP fakepg", sizeof(struct vm_page), 92120739Sjeff NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 93120739Sjeff UMA_ZONE_NOFREE|UMA_ZONE_VM); 941541Srgrimes} 951541Srgrimes 9698630Salc/* 9798630Salc * MPSAFE 9898630Salc */ 9912820Sphkstatic vm_object_t 10040286Sdgdev_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff) 1011541Srgrimes{ 102130585Sphk struct cdev *dev; 103171779Skib vm_object_t object, object1; 104124133Salc vm_pindex_t pindex; 10541004Sdfr unsigned int npages; 106112569Sjake vm_paddr_t paddr; 107112569Sjake vm_offset_t off; 108135707Sphk struct cdevsw *csw; 1091541Srgrimes 1101541Srgrimes /* 11198630Salc * Offset should be page aligned. 11298630Salc */ 11398630Salc if (foff & PAGE_MASK) 11498630Salc return (NULL); 11598630Salc 11698630Salc size = round_page(size); 117124133Salc pindex = OFF_TO_IDX(foff + size); 11898630Salc 11998630Salc /* 1201541Srgrimes * Make sure this device can be mapped. 1211541Srgrimes */ 12247111Sbde dev = handle; 123135707Sphk csw = dev_refthread(dev); 124135707Sphk if (csw == NULL) 125135707Sphk return (NULL); 1261541Srgrimes 1271541Srgrimes /* 1285455Sdg * Check that the specified range of the device allows the desired 1295455Sdg * protection. 1308876Srgrimes * 1311541Srgrimes * XXX assumes VM_PROT_* == PROT_* 1321541Srgrimes */ 13340286Sdg npages = OFF_TO_IDX(size); 1341541Srgrimes for (off = foff; npages--; off += PAGE_SIZE) 135135707Sphk if ((*csw->d_mmap)(dev, off, &paddr, (int)prot) != 0) { 136135707Sphk dev_relthread(dev); 1375455Sdg return (NULL); 13898630Salc } 1391541Srgrimes 140171779Skib mtx_lock(&dev_pager_mtx); 1419507Sdg 1429507Sdg /* 1431541Srgrimes * Look up pager, creating as necessary. 1441541Srgrimes */ 145171779Skib object1 = NULL; 1469507Sdg object = vm_pager_object_lookup(&dev_pager_object_list, handle); 1479507Sdg if (object == NULL) { 1481541Srgrimes /* 149179074Salc * Allocate object and associate it with the pager. Initialize 150179074Salc * the object's pg_color based upon the physical address of the 151179074Salc * device's memory. 1521541Srgrimes */ 153171779Skib mtx_unlock(&dev_pager_mtx); 154171779Skib object1 = vm_object_allocate(OBJT_DEVICE, pindex); 155179074Salc object1->flags |= OBJ_COLORED; 156179074Salc object1->pg_color = atop(paddr) - OFF_TO_IDX(off - PAGE_SIZE); 15775675Salfred mtx_lock(&dev_pager_mtx); 158171779Skib object = vm_pager_object_lookup(&dev_pager_object_list, handle); 159171779Skib if (object != NULL) { 160171779Skib /* 161171779Skib * We raced with other thread while allocating object. 162171779Skib */ 163171779Skib if (pindex > object->size) 164171779Skib object->size = pindex; 165171779Skib } else { 166171779Skib object = object1; 167171779Skib object1 = NULL; 168171779Skib object->handle = handle; 169171779Skib TAILQ_INIT(&object->un_pager.devp.devp_pglist); 170171779Skib TAILQ_INSERT_TAIL(&dev_pager_object_list, object, 171171779Skib pager_object_list); 172171779Skib } 1731541Srgrimes } else { 174124133Salc if (pindex > object->size) 175124133Salc object->size = pindex; 1761541Srgrimes } 177171779Skib mtx_unlock(&dev_pager_mtx); 178135707Sphk dev_relthread(dev); 179171779Skib vm_object_deallocate(object1); 1809507Sdg return (object); 1811541Srgrimes} 1821541Srgrimes 18312820Sphkstatic void 1849507Sdgdev_pager_dealloc(object) 1859507Sdg vm_object_t object; 1861541Srgrimes{ 1871541Srgrimes vm_page_t m; 1881541Srgrimes 189171779Skib VM_OBJECT_UNLOCK(object); 19075675Salfred mtx_lock(&dev_pager_mtx); 1919507Sdg TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list); 19275675Salfred mtx_unlock(&dev_pager_mtx); 193171779Skib VM_OBJECT_LOCK(object); 1941541Srgrimes /* 1951541Srgrimes * Free up our fake pages. 1961541Srgrimes */ 19715809Sdyson while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist)) != 0) { 1989507Sdg TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, pageq); 1991541Srgrimes dev_pager_putfake(m); 2001541Srgrimes } 2011541Srgrimes} 2021541Srgrimes 20312820Sphkstatic int 2049507Sdgdev_pager_getpages(object, m, count, reqpage) 2059507Sdg vm_object_t object; 2069507Sdg vm_page_t *m; 2079507Sdg int count; 2089507Sdg int reqpage; 2091541Srgrimes{ 21098824Siedowse vm_pindex_t offset; 211112569Sjake vm_paddr_t paddr; 2121541Srgrimes vm_page_t page; 213130585Sphk struct cdev *dev; 214111462Smux int i, ret; 21512591Sbde int prot; 216135707Sphk struct cdevsw *csw; 2171541Srgrimes 218116793Salc VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 21947111Sbde dev = object->handle; 22042957Sdillon offset = m[reqpage]->pindex; 221116279Salc VM_OBJECT_UNLOCK(object); 222135707Sphk csw = dev_refthread(dev); 223135707Sphk if (csw == NULL) 224135707Sphk panic("dev_pager_getpage: no cdevsw"); 2251541Srgrimes prot = PROT_READ; /* XXX should pass in? */ 2261549Srgrimes 227135707Sphk ret = (*csw->d_mmap)(dev, (vm_offset_t)offset << PAGE_SHIFT, &paddr, prot); 228111462Smux KASSERT(ret == 0, ("dev_pager_getpage: map function returns error")); 229135707Sphk dev_relthread(dev); 230128570Salc 231132884Sdfr if ((m[reqpage]->flags & PG_FICTITIOUS) != 0) { 232132884Sdfr /* 233132884Sdfr * If the passed in reqpage page is a fake page, update it with 234132884Sdfr * the new physical address. 235132884Sdfr */ 236133113Sdfr VM_OBJECT_LOCK(object); 237132884Sdfr dev_pager_updatefake(m[reqpage], paddr); 238132884Sdfr if (count > 1) { 239132884Sdfr vm_page_lock_queues(); 240132884Sdfr for (i = 0; i < count; i++) { 241132884Sdfr if (i != reqpage) 242132884Sdfr vm_page_free(m[i]); 243132884Sdfr } 244132884Sdfr vm_page_unlock_queues(); 245132884Sdfr } 246132884Sdfr } else { 247132884Sdfr /* 248132884Sdfr * Replace the passed in reqpage page with our own fake page and 249132884Sdfr * free up the all of the original pages. 250132884Sdfr */ 251132884Sdfr page = dev_pager_getfake(paddr); 252132884Sdfr VM_OBJECT_LOCK(object); 253132884Sdfr TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq); 254132884Sdfr vm_page_lock_queues(); 255132884Sdfr for (i = 0; i < count; i++) 256132884Sdfr vm_page_free(m[i]); 257132884Sdfr vm_page_unlock_queues(); 258132884Sdfr vm_page_insert(page, object, offset); 259132884Sdfr m[reqpage] = page; 260132884Sdfr } 2611541Srgrimes 2625455Sdg return (VM_PAGER_OK); 2631541Srgrimes} 2641541Srgrimes 26543129Sdillonstatic void 2669507Sdgdev_pager_putpages(object, m, count, sync, rtvals) 2679507Sdg vm_object_t object; 2689507Sdg vm_page_t *m; 2699507Sdg int count; 2701541Srgrimes boolean_t sync; 2719507Sdg int *rtvals; 2721541Srgrimes{ 2731541Srgrimes panic("dev_pager_putpage called"); 2741541Srgrimes} 2751541Srgrimes 27612820Sphkstatic boolean_t 27712767Sdysondev_pager_haspage(object, pindex, before, after) 2789507Sdg vm_object_t object; 27912767Sdyson vm_pindex_t pindex; 2809507Sdg int *before; 2819507Sdg int *after; 2821541Srgrimes{ 2839507Sdg if (before != NULL) 2849507Sdg *before = 0; 2859507Sdg if (after != NULL) 2869507Sdg *after = 0; 2875455Sdg return (TRUE); 2881541Srgrimes} 2891541Srgrimes 290147262Salc/* 291147262Salc * Instantiate a fictitious page. Unlike physical memory pages, only 292147262Salc * the machine-independent fields must be initialized. 293147262Salc */ 2941541Srgrimesstatic vm_page_t 2951541Srgrimesdev_pager_getfake(paddr) 296112569Sjake vm_paddr_t paddr; 2971541Srgrimes{ 2981541Srgrimes vm_page_t m; 2991541Srgrimes 300111119Simp m = uma_zalloc(fakepg_zone, M_WAITOK); 3011549Srgrimes 302163604Salc m->flags = PG_FICTITIOUS; 303163604Salc m->oflags = VPO_BUSY; 3049507Sdg m->valid = VM_PAGE_BITS_ALL; 3055455Sdg m->dirty = 0; 3065455Sdg m->busy = 0; 30713490Sdyson m->queue = PQ_NONE; 30840557Sdg m->object = NULL; 3091549Srgrimes 3101549Srgrimes m->wire_count = 1; 31114430Sdyson m->hold_count = 0; 3121541Srgrimes m->phys_addr = paddr; 3131549Srgrimes 3145455Sdg return (m); 3151541Srgrimes} 3161541Srgrimes 3171541Srgrimesstatic void 3181541Srgrimesdev_pager_putfake(m) 3191541Srgrimes vm_page_t m; 3201541Srgrimes{ 3211541Srgrimes if (!(m->flags & PG_FICTITIOUS)) 3221541Srgrimes panic("dev_pager_putfake: bad page"); 32392748Sjeff uma_zfree(fakepg_zone, m); 3241541Srgrimes} 325132884Sdfr 326132884Sdfrstatic void 327132884Sdfrdev_pager_updatefake(m, paddr) 328132884Sdfr vm_page_t m; 329132884Sdfr vm_paddr_t paddr; 330132884Sdfr{ 331132884Sdfr if (!(m->flags & PG_FICTITIOUS)) 332132884Sdfr panic("dev_pager_updatefake: bad page"); 333132884Sdfr m->phys_addr = paddr; 334133113Sdfr m->valid = VM_PAGE_BITS_ALL; 335132884Sdfr} 336