1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2000 Peter Wemm 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/param.h> 29#include <sys/systm.h> 30#include <sys/conf.h> 31#include <sys/kernel.h> 32#include <sys/lock.h> 33#include <sys/proc.h> 34#include <sys/mutex.h> 35#include <sys/mman.h> 36#include <sys/rwlock.h> 37#include <sys/sysctl.h> 38#include <sys/user.h> 39 40#include <vm/vm.h> 41#include <vm/vm_param.h> 42#include <vm/vm_object.h> 43#include <vm/vm_page.h> 44#include <vm/vm_pageout.h> 45#include <vm/vm_pager.h> 46 47/* list of phys pager objects */ 48static struct pagerlst phys_pager_object_list; 49/* protect access to phys_pager_object_list */ 50static struct mtx phys_pager_mtx; 51 52static int default_phys_pager_getpages(vm_object_t object, vm_page_t *m, 53 int count, int *rbehind, int *rahead); 54static int default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx, 55 int fault_type, vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last); 56static boolean_t default_phys_pager_haspage(vm_object_t object, 57 vm_pindex_t pindex, int *before, int *after); 58const struct phys_pager_ops default_phys_pg_ops = { 59 .phys_pg_getpages = default_phys_pager_getpages, 60 .phys_pg_populate = default_phys_pager_populate, 61 .phys_pg_haspage = default_phys_pager_haspage, 62 .phys_pg_ctor = NULL, 63 .phys_pg_dtor = NULL, 64}; 65 66static void 67phys_pager_init(void) 68{ 69 70 TAILQ_INIT(&phys_pager_object_list); 71 mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF); 72} 73 74vm_object_t 75phys_pager_allocate(void *handle, const struct phys_pager_ops *ops, void *data, 76 vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) 77{ 78 vm_object_t object, object1; 79 vm_pindex_t pindex; 80 bool init; 81 82 /* 83 * Offset should be page aligned. 84 */ 85 if (foff & PAGE_MASK) 86 return (NULL); 87 88 pindex = OFF_TO_IDX(foff + PAGE_MASK + size); 89 init = true; 90 91 if (handle != NULL) { 92 mtx_lock(&phys_pager_mtx); 93 /* 94 * Look up pager, creating as necessary. 95 */ 96 object1 = NULL; 97 object = vm_pager_object_lookup(&phys_pager_object_list, handle); 98 if (object == NULL) { 99 /* 100 * Allocate object and associate it with the pager. 101 */ 102 mtx_unlock(&phys_pager_mtx); 103 object1 = vm_object_allocate(OBJT_PHYS, pindex); 104 mtx_lock(&phys_pager_mtx); 105 object = vm_pager_object_lookup(&phys_pager_object_list, 106 handle); 107 if (object != NULL) { 108 /* 109 * We raced with other thread while 110 * allocating object. 111 */ 112 if (pindex > object->size) 113 object->size = pindex; 114 init = false; 115 } else { 116 object = object1; 117 object1 = NULL; 118 object->handle = handle; 119 object->un_pager.phys.ops = ops; 120 object->un_pager.phys.data_ptr = data; 121 if (ops->phys_pg_populate != NULL) 122 vm_object_set_flag(object, OBJ_POPULATE); 123 TAILQ_INSERT_TAIL(&phys_pager_object_list, 124 object, pager_object_list); 125 } 126 } else { 127 if (pindex > object->size) 128 object->size = pindex; 129 } 130 mtx_unlock(&phys_pager_mtx); 131 vm_object_deallocate(object1); 132 } else { 133 object = vm_object_allocate(OBJT_PHYS, pindex); 134 object->un_pager.phys.ops = ops; 135 object->un_pager.phys.data_ptr = data; 136 if (ops->phys_pg_populate != NULL) 137 vm_object_set_flag(object, OBJ_POPULATE); 138 } 139 if (init && ops->phys_pg_ctor != NULL) 140 ops->phys_pg_ctor(object, prot, foff, cred); 141 142 return (object); 143} 144 145static vm_object_t 146phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 147 vm_ooffset_t foff, struct ucred *ucred) 148{ 149 return (phys_pager_allocate(handle, &default_phys_pg_ops, NULL, 150 size, prot, foff, ucred)); 151} 152 153static void 154phys_pager_dealloc(vm_object_t object) 155{ 156 157 if (object->handle != NULL) { 158 VM_OBJECT_WUNLOCK(object); 159 mtx_lock(&phys_pager_mtx); 160 TAILQ_REMOVE(&phys_pager_object_list, object, pager_object_list); 161 mtx_unlock(&phys_pager_mtx); 162 VM_OBJECT_WLOCK(object); 163 } 164 object->type = OBJT_DEAD; 165 if (object->un_pager.phys.ops->phys_pg_dtor != NULL) 166 object->un_pager.phys.ops->phys_pg_dtor(object); 167 object->handle = NULL; 168} 169 170/* 171 * Fill as many pages as vm_fault has allocated for us. 172 */ 173static int 174default_phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, 175 int *rbehind, int *rahead) 176{ 177 int i; 178 179 for (i = 0; i < count; i++) { 180 if (vm_page_none_valid(m[i])) { 181 if ((m[i]->flags & PG_ZERO) == 0) 182 pmap_zero_page(m[i]); 183 vm_page_valid(m[i]); 184 } 185 KASSERT(vm_page_all_valid(m[i]), 186 ("phys_pager_getpages: partially valid page %p", m[i])); 187 KASSERT(m[i]->dirty == 0, 188 ("phys_pager_getpages: dirty page %p", m[i])); 189 } 190 if (rbehind) 191 *rbehind = 0; 192 if (rahead) 193 *rahead = 0; 194 return (VM_PAGER_OK); 195} 196 197static int 198phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, 199 int *rahead) 200{ 201 return (object->un_pager.phys.ops->phys_pg_getpages(object, m, 202 count, rbehind, rahead)); 203} 204 205/* 206 * Implement a pretty aggressive clustered getpages strategy. Hint that 207 * everything in an entire 4MB window should be prefaulted at once. 208 * 209 * 4MB (1024 slots per page table page) is convenient for x86, 210 * but may not be for other arches. 211 */ 212#ifndef PHYSCLUSTER 213#define PHYSCLUSTER 1024 214#endif 215static int phys_pager_cluster = PHYSCLUSTER; 216SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN, 217 &phys_pager_cluster, 0, 218 "prefault window size for phys pager"); 219 220/* 221 * Max hint to vm_page_alloc() about the further allocation needs 222 * inside the phys_pager_populate() loop. The number of bits used to 223 * implement VM_ALLOC_COUNT() determines the hard limit on this value. 224 * That limit is currently 65535. 225 */ 226#define PHYSALLOC 16 227 228static int 229default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx, 230 int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first, 231 vm_pindex_t *last) 232{ 233 vm_page_t m; 234 vm_pindex_t base, end, i; 235 int ahead; 236 237 base = rounddown(pidx, phys_pager_cluster); 238 end = base + phys_pager_cluster - 1; 239 if (end >= object->size) 240 end = object->size - 1; 241 if (*first > base) 242 base = *first; 243 if (end > *last) 244 end = *last; 245 *first = base; 246 *last = end; 247 248 for (i = base; i <= end; i++) { 249 ahead = MIN(end - i, PHYSALLOC); 250 m = vm_page_grab(object, i, 251 VM_ALLOC_NORMAL | VM_ALLOC_COUNT(ahead)); 252 if (!vm_page_all_valid(m)) 253 vm_page_zero_invalid(m, TRUE); 254 KASSERT(m->dirty == 0, 255 ("phys_pager_populate: dirty page %p", m)); 256 } 257 return (VM_PAGER_OK); 258} 259 260static int 261phys_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type, 262 vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 263{ 264 return (object->un_pager.phys.ops->phys_pg_populate(object, pidx, 265 fault_type, max_prot, first, last)); 266} 267 268static void 269phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags, 270 int *rtvals) 271{ 272 273 panic("phys_pager_putpage called"); 274} 275 276static boolean_t 277default_phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 278 int *after) 279{ 280 vm_pindex_t base, end; 281 282 base = rounddown(pindex, phys_pager_cluster); 283 end = base + phys_pager_cluster - 1; 284 if (before != NULL) 285 *before = pindex - base; 286 if (after != NULL) 287 *after = end - pindex; 288 return (TRUE); 289} 290 291static boolean_t 292phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 293 int *after) 294{ 295 return (object->un_pager.phys.ops->phys_pg_haspage(object, pindex, 296 before, after)); 297} 298 299const struct pagerops physpagerops = { 300 .pgo_kvme_type = KVME_TYPE_PHYS, 301 .pgo_init = phys_pager_init, 302 .pgo_alloc = phys_pager_alloc, 303 .pgo_dealloc = phys_pager_dealloc, 304 .pgo_getpages = phys_pager_getpages, 305 .pgo_putpages = phys_pager_putpages, 306 .pgo_haspage = phys_pager_haspage, 307 .pgo_populate = phys_pager_populate, 308}; 309