1/* $OpenBSD: uvm_vnode.c,v 1.132 2023/04/10 04:21:20 jsg Exp $ */ 2/* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */ 3 4/* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 41 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 42 */ 43 44/* 45 * uvm_vnode.c: the vnode pager. 46 */ 47 48#include <sys/param.h> 49#include <sys/systm.h> 50#include <sys/proc.h> 51#include <sys/malloc.h> 52#include <sys/vnode.h> 53#include <sys/lock.h> 54#include <sys/disklabel.h> 55#include <sys/fcntl.h> 56#include <sys/conf.h> 57#include <sys/rwlock.h> 58#include <sys/dkio.h> 59#include <sys/specdev.h> 60 61#include <uvm/uvm.h> 62#include <uvm/uvm_vnode.h> 63 64/* 65 * private global data structure 66 * 67 * we keep a list of writeable active vnode-backed VM objects for sync op. 68 * we keep a simpleq of vnodes that are currently being sync'd. 69 */ 70 71LIST_HEAD(, uvm_vnode) uvn_wlist; /* [K] writeable uvns */ 72SIMPLEQ_HEAD(, uvm_vnode) uvn_sync_q; /* [S] sync'ing uvns */ 73struct rwlock uvn_sync_lock; /* locks sync operation */ 74 75extern int rebooting; 76 77/* 78 * functions 79 */ 80void uvn_cluster(struct uvm_object *, voff_t, voff_t *, voff_t *); 81void uvn_detach(struct uvm_object *); 82boolean_t uvn_flush(struct uvm_object *, voff_t, voff_t, int); 83int uvn_get(struct uvm_object *, voff_t, vm_page_t *, int *, int, 84 vm_prot_t, int, int); 85void uvn_init(void); 86int uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int); 87int uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t); 88void uvn_reference(struct uvm_object *); 89 90/* 91 * master pager structure 92 */ 93const struct uvm_pagerops uvm_vnodeops = { 94 .pgo_init = uvn_init, 95 .pgo_reference = uvn_reference, 96 .pgo_detach = uvn_detach, 97 .pgo_flush = uvn_flush, 98 .pgo_get = uvn_get, 99 .pgo_put = uvn_put, 100 .pgo_cluster = uvn_cluster, 101 /* use generic version of this: see uvm_pager.c */ 102 .pgo_mk_pcluster = uvm_mk_pcluster, 103}; 104 105/* 106 * the ops! 107 */ 108/* 109 * uvn_init 110 * 111 * init pager private data structures. 112 */ 113void 114uvn_init(void) 115{ 116 117 LIST_INIT(&uvn_wlist); 118 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 119 rw_init_flags(&uvn_sync_lock, "uvnsync", RWL_IS_VNODE); 120} 121 122/* 123 * uvn_attach 124 * 125 * attach a vnode structure to a VM object. if the vnode is already 126 * attached, then just bump the reference count by one and return the 127 * VM object. if not already attached, attach and return the new VM obj. 128 * the "accessprot" tells the max access the attaching thread wants to 129 * our pages. 130 * 131 * => in fact, nothing should be locked so that we can sleep here. 132 * => note that uvm_object is first thing in vnode structure, so their 133 * pointers are equiv. 134 */ 135struct uvm_object * 136uvn_attach(struct vnode *vp, vm_prot_t accessprot) 137{ 138 struct uvm_vnode *uvn = vp->v_uvm; 139 struct vattr vattr; 140 int oldflags, result; 141 struct partinfo pi; 142 u_quad_t used_vnode_size = 0; 143 144 /* if we're mapping a BLK device, make sure it is a disk. */ 145 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 146 return NULL; 147 } 148 149 /* first get a lock on the uvn. */ 150 rw_enter(uvn->u_obj.vmobjlock, RW_WRITE); 151 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 152 uvn->u_flags |= UVM_VNODE_WANTED; 153 rwsleep_nsec(uvn, uvn->u_obj.vmobjlock, PVM, "uvn_attach", 154 INFSLP); 155 } 156 157 /* 158 * now uvn must not be in a blocked state. 159 * first check to see if it is already active, in which case 160 * we can bump the reference count, check to see if we need to 161 * add it to the writeable list, and then return. 162 */ 163 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 164 165 /* regain vref if we were persisting */ 166 if (uvn->u_obj.uo_refs == 0) { 167 vref(vp); 168 } 169 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 170 171 /* check for new writeable uvn */ 172 if ((accessprot & PROT_WRITE) != 0 && 173 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 174 uvn->u_flags |= UVM_VNODE_WRITEABLE; 175 KERNEL_ASSERT_LOCKED(); 176 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 177 } 178 179 rw_exit(uvn->u_obj.vmobjlock); 180 return (&uvn->u_obj); 181 } 182 183 /* 184 * need to call VOP_GETATTR() to get the attributes, but that could 185 * block (due to I/O), so we want to unlock the object before calling. 186 * however, we want to keep anyone else from playing with the object 187 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 188 * prevents anyone from attaching to the vnode until we are done with 189 * it. 190 */ 191 uvn->u_flags = UVM_VNODE_ALOCK; 192 rw_exit(uvn->u_obj.vmobjlock); 193 194 if (vp->v_type == VBLK) { 195 /* 196 * We could implement this as a specfs getattr call, but: 197 * 198 * (1) VOP_GETATTR() would get the file system 199 * vnode operation, not the specfs operation. 200 * 201 * (2) All we want is the size, anyhow. 202 */ 203 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 204 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 205 if (result == 0) { 206 /* XXX should remember blocksize */ 207 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 208 (u_quad_t)DL_GETPSIZE(pi.part); 209 } 210 } else { 211 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 212 if (result == 0) 213 used_vnode_size = vattr.va_size; 214 } 215 216 if (result != 0) { 217 rw_enter(uvn->u_obj.vmobjlock, RW_WRITE); 218 if (uvn->u_flags & UVM_VNODE_WANTED) 219 wakeup(uvn); 220 uvn->u_flags = 0; 221 rw_exit(uvn->u_obj.vmobjlock); 222 return NULL; 223 } 224 225 /* 226 * make sure that the newsize fits within a vaddr_t 227 * XXX: need to revise addressing data types 228 */ 229#ifdef DEBUG 230 if (vp->v_type == VBLK) 231 printf("used_vnode_size = %llu\n", (long long)used_vnode_size); 232#endif 233 234 /* now set up the uvn. */ 235 KASSERT(uvn->u_obj.uo_refs == 0); 236 uvn->u_obj.uo_refs++; 237 oldflags = uvn->u_flags; 238 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 239 uvn->u_nio = 0; 240 uvn->u_size = used_vnode_size; 241 242 /* 243 * add a reference to the vnode. this reference will stay as long 244 * as there is a valid mapping of the vnode. dropped when the 245 * reference count goes to zero [and we either free or persist]. 246 */ 247 vref(vp); 248 249 /* if write access, we need to add it to the wlist */ 250 if (accessprot & PROT_WRITE) { 251 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 252 KERNEL_ASSERT_LOCKED(); 253 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 254 } 255 256 if (oldflags & UVM_VNODE_WANTED) 257 wakeup(uvn); 258 259 return &uvn->u_obj; 260} 261 262 263/* 264 * uvn_reference 265 * 266 * duplicate a reference to a VM object. Note that the reference 267 * count must already be at least one (the passed in reference) so 268 * there is no chance of the uvn being killed out here. 269 * 270 * => caller must be using the same accessprot as was used at attach time 271 */ 272 273 274void 275uvn_reference(struct uvm_object *uobj) 276{ 277#ifdef DEBUG 278 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 279#endif 280 281 rw_enter(uobj->vmobjlock, RW_WRITE); 282#ifdef DEBUG 283 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 284 printf("uvn_reference: ref=%d, flags=0x%x\n", 285 uobj->uo_refs, uvn->u_flags); 286 panic("uvn_reference: invalid state"); 287 } 288#endif 289 uobj->uo_refs++; 290 rw_exit(uobj->vmobjlock); 291} 292 293/* 294 * uvn_detach 295 * 296 * remove a reference to a VM object. 297 * 298 * => caller must call with map locked. 299 * => this starts the detach process, but doesn't have to finish it 300 * (async i/o could still be pending). 301 */ 302void 303uvn_detach(struct uvm_object *uobj) 304{ 305 struct uvm_vnode *uvn; 306 struct vnode *vp; 307 int oldflags; 308 309 rw_enter(uobj->vmobjlock, RW_WRITE); 310 uobj->uo_refs--; /* drop ref! */ 311 if (uobj->uo_refs) { /* still more refs */ 312 rw_exit(uobj->vmobjlock); 313 return; 314 } 315 316 /* get other pointers ... */ 317 uvn = (struct uvm_vnode *) uobj; 318 vp = uvn->u_vnode; 319 320 /* 321 * clear VTEXT flag now that there are no mappings left (VTEXT is used 322 * to keep an active text file from being overwritten). 323 */ 324 vp->v_flag &= ~VTEXT; 325 326 /* 327 * we just dropped the last reference to the uvn. see if we can 328 * let it "stick around". 329 */ 330 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 331 /* won't block */ 332 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 333 goto out; 334 } 335 336 /* its a goner! */ 337 uvn->u_flags |= UVM_VNODE_DYING; 338 339 /* 340 * even though we may unlock in flush, no one can gain a reference 341 * to us until we clear the "dying" flag [because it blocks 342 * attaches]. we will not do that until after we've disposed of all 343 * the pages with uvn_flush(). note that before the flush the only 344 * pages that could be marked PG_BUSY are ones that are in async 345 * pageout by the daemon. (there can't be any pending "get"'s 346 * because there are no references to the object). 347 */ 348 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 349 350 /* 351 * given the structure of this pager, the above flush request will 352 * create the following state: all the pages that were in the object 353 * have either been free'd or they are marked PG_BUSY and in the 354 * middle of an async io. If we still have pages we set the "relkill" 355 * state, so that in the case the vnode gets terminated we know 356 * to leave it alone. Otherwise we'll kill the vnode when it's empty. 357 */ 358 uvn->u_flags |= UVM_VNODE_RELKILL; 359 /* wait on any outstanding io */ 360 while (uobj->uo_npages && uvn->u_flags & UVM_VNODE_RELKILL) { 361 uvn->u_flags |= UVM_VNODE_IOSYNC; 362 rwsleep_nsec(&uvn->u_nio, uobj->vmobjlock, PVM, "uvn_term", 363 INFSLP); 364 } 365 366 if ((uvn->u_flags & UVM_VNODE_RELKILL) == 0) { 367 rw_exit(uobj->vmobjlock); 368 return; 369 } 370 371 /* 372 * kill object now. note that we can't be on the sync q because 373 * all references are gone. 374 */ 375 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 376 LIST_REMOVE(uvn, u_wlist); 377 } 378 KASSERT(RBT_EMPTY(uvm_objtree, &uobj->memt)); 379 oldflags = uvn->u_flags; 380 uvn->u_flags = 0; 381 382 /* wake up any sleepers */ 383 if (oldflags & UVM_VNODE_WANTED) 384 wakeup(uvn); 385out: 386 rw_exit(uobj->vmobjlock); 387 388 /* drop our reference to the vnode. */ 389 vrele(vp); 390 391 return; 392} 393 394/* 395 * uvm_vnp_terminate: external hook to clear out a vnode's VM 396 * 397 * called in two cases: 398 * [1] when a persisting vnode vm object (i.e. one with a zero reference 399 * count) needs to be freed so that a vnode can be reused. this 400 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 401 * the free list is still attached (i.e. not VBAD) then vgone is 402 * called. as part of the vgone trace this should get called to 403 * free the vm object. this is the common case. 404 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 405 * "umount -f") the vgone() function is called on active vnodes 406 * on the mounted file systems to kill their data (the vnodes become 407 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 408 * call here (even if the uvn is still in use -- i.e. has a non-zero 409 * reference count). this case happens at "umount -f" and during a 410 * "reboot/halt" operation. 411 * 412 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 413 * [protects us from getting a vnode that is already in the DYING 414 * state...] 415 * => in case [2] the uvn is still alive after this call, but all I/O 416 * ops will fail (due to the backing vnode now being "dead"). this 417 * will prob. kill any process using the uvn due to pgo_get failing. 418 */ 419void 420uvm_vnp_terminate(struct vnode *vp) 421{ 422 struct uvm_vnode *uvn = vp->v_uvm; 423 struct uvm_object *uobj = &uvn->u_obj; 424 int oldflags; 425 426 /* check if it is valid */ 427 rw_enter(uobj->vmobjlock, RW_WRITE); 428 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 429 rw_exit(uobj->vmobjlock); 430 return; 431 } 432 433 /* 434 * must be a valid uvn that is not already dying (because XLOCK 435 * protects us from that). the uvn can't in the ALOCK state 436 * because it is valid, and uvn's that are in the ALOCK state haven't 437 * been marked valid yet. 438 */ 439#ifdef DEBUG 440 /* 441 * debug check: are we yanking the vnode out from under our uvn? 442 */ 443 if (uvn->u_obj.uo_refs) { 444 printf("uvm_vnp_terminate(%p): terminating active vnode " 445 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 446 } 447#endif 448 449 /* 450 * it is possible that the uvn was detached and is in the relkill 451 * state [i.e. waiting for async i/o to finish]. 452 * we take over the vnode now and cancel the relkill. 453 * we want to know when the i/o is done so we can recycle right 454 * away. note that a uvn can only be in the RELKILL state if it 455 * has a zero reference count. 456 */ 457 if (uvn->u_flags & UVM_VNODE_RELKILL) 458 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 459 460 /* 461 * block the uvn by setting the dying flag, and then flush the 462 * pages. 463 * 464 * also, note that we tell I/O that we are already VOP_LOCK'd so 465 * that uvn_io doesn't attempt to VOP_LOCK again. 466 * 467 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 468 * due to a forceful unmount might not be a good idea. maybe we 469 * need a way to pass in this info to uvn_flush through a 470 * pager-defined PGO_ constant [currently there are none]. 471 */ 472 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 473 474 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 475 476 /* 477 * as we just did a flush we expect all the pages to be gone or in 478 * the process of going. sleep to wait for the rest to go [via iosync]. 479 */ 480 while (uvn->u_obj.uo_npages) { 481#ifdef DEBUG 482 struct vm_page *pp; 483 RBT_FOREACH(pp, uvm_objtree, &uvn->u_obj.memt) { 484 if ((pp->pg_flags & PG_BUSY) == 0) 485 panic("uvm_vnp_terminate: detected unbusy pg"); 486 } 487 if (uvn->u_nio == 0) 488 panic("uvm_vnp_terminate: no I/O to wait for?"); 489 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 490 /* 491 * XXXCDC: this is unlikely to happen without async i/o so we 492 * put a printf in just to keep an eye on it. 493 */ 494#endif 495 uvn->u_flags |= UVM_VNODE_IOSYNC; 496 rwsleep_nsec(&uvn->u_nio, uobj->vmobjlock, PVM, "uvn_term", 497 INFSLP); 498 } 499 500 /* 501 * done. now we free the uvn if its reference count is zero 502 * (true if we are zapping a persisting uvn). however, if we are 503 * terminating a uvn with active mappings we let it live ... future 504 * calls down to the vnode layer will fail. 505 */ 506 oldflags = uvn->u_flags; 507 if (uvn->u_obj.uo_refs) { 508 /* 509 * uvn must live on it is dead-vnode state until all references 510 * are gone. restore flags. clear CANPERSIST state. 511 */ 512 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 513 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 514 } else { 515 /* 516 * free the uvn now. note that the vref reference is already 517 * gone [it is dropped when we enter the persist state]. 518 */ 519 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 520 panic("uvm_vnp_terminate: io sync wanted bit set"); 521 522 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 523 LIST_REMOVE(uvn, u_wlist); 524 } 525 uvn->u_flags = 0; /* uvn is history, clear all bits */ 526 } 527 528 if (oldflags & UVM_VNODE_WANTED) 529 wakeup(uvn); 530 531 rw_exit(uobj->vmobjlock); 532} 533 534/* 535 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 536 * through the buffer cache and allow I/O in any size. These VOPs use 537 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 538 * go through the buffer cache or allow I/O sizes larger than a 539 * block]. we will eventually want to change this. 540 * 541 * issues to consider: 542 * uvm provides the uvm_aiodesc structure for async i/o management. 543 * there are two tailq's in the uvm. structure... one for pending async 544 * i/o and one for "done" async i/o. to do an async i/o one puts 545 * an aiodesc on the "pending" list (protected by splbio()), starts the 546 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 547 * some sort of "i/o done" function to be called (at splbio(), interrupt 548 * time). this function should remove the aiodesc from the pending list 549 * and place it on the "done" list and wakeup the daemon. the daemon 550 * will run at normal spl() and will remove all items from the "done" 551 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 552 * [in the old vm code, this was done by calling the "put" routine with 553 * null arguments which made the code harder to read and understand because 554 * you had one function ("put") doing two things.] 555 * 556 * so the current pager needs: 557 * int uvn_aiodone(struct uvm_aiodesc *) 558 * 559 * => return 0 (aio finished, free it). otherwise requeue for later collection. 560 * => called with pageq's locked by the daemon. 561 * 562 * general outline: 563 * - drop "u_nio" (this req is done!) 564 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 565 * - get "page" structures (atop?). 566 * - handle "wanted" pages 567 * dont forget to look at "object" wanted flag in all cases. 568 */ 569 570/* 571 * uvn_flush: flush pages out of a uvm object. 572 * 573 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 574 * might want to unlock higher level resources (e.g. vm_map) 575 * before calling flush. 576 * => if PGO_CLEANIT is not set, then we will not block 577 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 578 * for flushing. 579 * => NOTE: we are allowed to lock the page queues, so the caller 580 * must not be holding the lock on them [e.g. pagedaemon had 581 * better not call us with the queues locked] 582 * => we return TRUE unless we encountered some sort of I/O error 583 * 584 * comment on "cleaning" object and PG_BUSY pages: 585 * this routine is holding the lock on the object. the only time 586 * that it can run into a PG_BUSY page that it does not own is if 587 * some other process has started I/O on the page (e.g. either 588 * a pagein, or a pageout). if the PG_BUSY page is being paged 589 * in, then it can not be dirty (!PG_CLEAN) because no one has 590 * had a chance to modify it yet. if the PG_BUSY page is being 591 * paged out then it means that someone else has already started 592 * cleaning the page for us (how nice!). in this case, if we 593 * have syncio specified, then after we make our pass through the 594 * object we need to wait for the other PG_BUSY pages to clear 595 * off (i.e. we need to do an iosync). also note that once a 596 * page is PG_BUSY it must stay in its object until it is un-busyed. 597 */ 598boolean_t 599uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) 600{ 601 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 602 struct vm_page *pp, *ptmp; 603 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 604 struct pglist dead; 605 int npages, result, lcv; 606 boolean_t retval, need_iosync, needs_clean; 607 voff_t curoff; 608 609 KASSERT(rw_write_held(uobj->vmobjlock)); 610 TAILQ_INIT(&dead); 611 612 /* get init vals and determine how we are going to traverse object */ 613 need_iosync = FALSE; 614 retval = TRUE; /* return value */ 615 if (flags & PGO_ALLPAGES) { 616 start = 0; 617 stop = round_page(uvn->u_size); 618 } else { 619 start = trunc_page(start); 620 stop = MIN(round_page(stop), round_page(uvn->u_size)); 621 } 622 623 /* 624 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 625 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 626 * is wrong it will only prevent us from clustering... it won't break 627 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 628 * will set them as it syncs PG_CLEAN. This is only an issue if we 629 * are looking at non-inactive pages (because inactive page's PG_CLEAN 630 * bit is always up to date since there are no mappings). 631 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 632 */ 633 if ((flags & PGO_CLEANIT) != 0) { 634 KASSERT(uobj->pgops->pgo_mk_pcluster != 0); 635 for (curoff = start ; curoff < stop; curoff += PAGE_SIZE) { 636 if ((pp = uvm_pagelookup(uobj, curoff)) != NULL) 637 atomic_clearbits_int(&pp->pg_flags, 638 PG_CLEANCHK); 639 } 640 } 641 642 ppsp = NULL; /* XXX: shut up gcc */ 643 uvm_lock_pageq(); 644 /* locked: both page queues */ 645 for (curoff = start; curoff < stop; curoff += PAGE_SIZE) { 646 if ((pp = uvm_pagelookup(uobj, curoff)) == NULL) 647 continue; 648 /* 649 * handle case where we do not need to clean page (either 650 * because we are not clean or because page is not dirty or 651 * is busy): 652 * 653 * NOTE: we are allowed to deactivate a non-wired active 654 * PG_BUSY page, but once a PG_BUSY page is on the inactive 655 * queue it must stay put until it is !PG_BUSY (so as not to 656 * confuse pagedaemon). 657 */ 658 if ((flags & PGO_CLEANIT) == 0 || (pp->pg_flags & PG_BUSY) != 0) { 659 needs_clean = FALSE; 660 if ((pp->pg_flags & PG_BUSY) != 0 && 661 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 662 (PGO_CLEANIT|PGO_SYNCIO)) 663 need_iosync = TRUE; 664 } else { 665 /* 666 * freeing: nuke all mappings so we can sync 667 * PG_CLEAN bit with no race 668 */ 669 if ((pp->pg_flags & PG_CLEAN) != 0 && 670 (flags & PGO_FREE) != 0 && 671 (pp->pg_flags & PQ_ACTIVE) != 0) 672 pmap_page_protect(pp, PROT_NONE); 673 if ((pp->pg_flags & PG_CLEAN) != 0 && 674 pmap_is_modified(pp)) 675 atomic_clearbits_int(&pp->pg_flags, PG_CLEAN); 676 atomic_setbits_int(&pp->pg_flags, PG_CLEANCHK); 677 678 needs_clean = ((pp->pg_flags & PG_CLEAN) == 0); 679 } 680 681 /* if we don't need a clean, deactivate/free pages then cont. */ 682 if (!needs_clean) { 683 if (flags & PGO_DEACTIVATE) { 684 if (pp->wire_count == 0) { 685 pmap_page_protect(pp, PROT_NONE); 686 uvm_pagedeactivate(pp); 687 } 688 } else if (flags & PGO_FREE) { 689 if (pp->pg_flags & PG_BUSY) { 690 uvm_unlock_pageq(); 691 uvm_pagewait(pp, uobj->vmobjlock, 692 "uvn_flsh"); 693 rw_enter(uobj->vmobjlock, RW_WRITE); 694 uvm_lock_pageq(); 695 curoff -= PAGE_SIZE; 696 continue; 697 } else { 698 pmap_page_protect(pp, PROT_NONE); 699 /* removed page from object */ 700 uvm_pageclean(pp); 701 TAILQ_INSERT_HEAD(&dead, pp, pageq); 702 } 703 } 704 continue; 705 } 706 707 /* 708 * pp points to a page in the object that we are 709 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 710 * for cleaning (PGO_CLEANIT). we clean it now. 711 * 712 * let uvm_pager_put attempted a clustered page out. 713 * note: locked: page queues. 714 */ 715 atomic_setbits_int(&pp->pg_flags, PG_BUSY); 716 UVM_PAGE_OWN(pp, "uvn_flush"); 717 pmap_page_protect(pp, PROT_READ); 718 /* if we're async, free the page in aiodoned */ 719 if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE) 720 atomic_setbits_int(&pp->pg_flags, PG_RELEASED); 721ReTry: 722 ppsp = pps; 723 npages = sizeof(pps) / sizeof(struct vm_page *); 724 725 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 726 flags | PGO_DOACTCLUST, start, stop); 727 728 /* 729 * if we did an async I/O it is remotely possible for the 730 * async i/o to complete and the page "pp" be freed or what 731 * not before we get a chance to relock the object. Therefore, 732 * we only touch it when it won't be freed, RELEASED took care 733 * of the rest. 734 */ 735 uvm_lock_pageq(); 736 737 /* 738 * VM_PAGER_AGAIN: given the structure of this pager, this 739 * can only happen when we are doing async I/O and can't 740 * map the pages into kernel memory (pager_map) due to lack 741 * of vm space. if this happens we drop back to sync I/O. 742 */ 743 if (result == VM_PAGER_AGAIN) { 744 /* 745 * it is unlikely, but page could have been released 746 * we ignore this now and retry the I/O. 747 * we will detect and 748 * handle the released page after the syncio I/O 749 * completes. 750 */ 751#ifdef DIAGNOSTIC 752 if (flags & PGO_SYNCIO) 753 panic("%s: PGO_SYNCIO return 'try again' error (impossible)", __func__); 754#endif 755 flags |= PGO_SYNCIO; 756 if (flags & PGO_FREE) 757 atomic_clearbits_int(&pp->pg_flags, 758 PG_RELEASED); 759 760 goto ReTry; 761 } 762 763 /* 764 * the cleaning operation is now done. finish up. note that 765 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 766 * if success (OK, PEND) then uvm_pager_put returns the cluster 767 * to us in ppsp/npages. 768 */ 769 /* 770 * for pending async i/o if we are not deactivating 771 * we can move on to the next page. aiodoned deals with 772 * the freeing case for us. 773 */ 774 if (result == VM_PAGER_PEND && (flags & PGO_DEACTIVATE) == 0) 775 continue; 776 777 /* 778 * need to look at each page of the I/O operation, and do what 779 * we gotta do. 780 */ 781 for (lcv = 0 ; lcv < npages; lcv++) { 782 ptmp = ppsp[lcv]; 783 /* 784 * verify the page didn't get moved 785 */ 786 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 787 continue; 788 789 /* 790 * unbusy the page if I/O is done. note that for 791 * pending I/O it is possible that the I/O op 792 * finished 793 * (in which case the page is no longer busy). 794 */ 795 if (result != VM_PAGER_PEND) { 796 if (ptmp->pg_flags & PG_WANTED) 797 wakeup(ptmp); 798 799 atomic_clearbits_int(&ptmp->pg_flags, 800 PG_WANTED|PG_BUSY); 801 UVM_PAGE_OWN(ptmp, NULL); 802 atomic_setbits_int(&ptmp->pg_flags, 803 PG_CLEAN|PG_CLEANCHK); 804 if ((flags & PGO_FREE) == 0) 805 pmap_clear_modify(ptmp); 806 } 807 808 /* dispose of page */ 809 if (flags & PGO_DEACTIVATE) { 810 if (ptmp->wire_count == 0) { 811 pmap_page_protect(ptmp, PROT_NONE); 812 uvm_pagedeactivate(ptmp); 813 } 814 } else if (flags & PGO_FREE && 815 result != VM_PAGER_PEND) { 816 if (result != VM_PAGER_OK) { 817 static struct timeval lasttime; 818 static const struct timeval interval = 819 { 5, 0 }; 820 821 if (ratecheck(&lasttime, &interval)) { 822 printf("%s: obj=%p, " 823 "offset=0x%llx. error " 824 "during pageout.\n", 825 __func__, pp->uobject, 826 (long long)pp->offset); 827 printf("%s: WARNING: " 828 "changes to page may be " 829 "lost!\n", __func__); 830 } 831 retval = FALSE; 832 } 833 pmap_page_protect(ptmp, PROT_NONE); 834 uvm_pageclean(ptmp); 835 TAILQ_INSERT_TAIL(&dead, ptmp, pageq); 836 } 837 838 } /* end of "lcv" for loop */ 839 840 } /* end of "pp" for loop */ 841 842 /* done with pagequeues: unlock */ 843 uvm_unlock_pageq(); 844 845 /* now wait for all I/O if required. */ 846 if (need_iosync) { 847 while (uvn->u_nio != 0) { 848 uvn->u_flags |= UVM_VNODE_IOSYNC; 849 rwsleep_nsec(&uvn->u_nio, uobj->vmobjlock, PVM, 850 "uvn_flush", INFSLP); 851 } 852 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 853 wakeup(&uvn->u_flags); 854 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 855 } 856 857 uvm_pglistfree(&dead); 858 859 return retval; 860} 861 862/* 863 * uvn_cluster 864 * 865 * we are about to do I/O in an object at offset. this function is called 866 * to establish a range of offsets around "offset" in which we can cluster 867 * I/O. 868 */ 869 870void 871uvn_cluster(struct uvm_object *uobj, voff_t offset, voff_t *loffset, 872 voff_t *hoffset) 873{ 874 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 875 *loffset = offset; 876 877 KASSERT(rw_write_held(uobj->vmobjlock)); 878 879 if (*loffset >= uvn->u_size) 880 panic("uvn_cluster: offset out of range"); 881 882 /* 883 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 884 */ 885 *hoffset = *loffset + MAXBSIZE; 886 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 887 *hoffset = round_page(uvn->u_size); 888} 889 890/* 891 * uvn_put: flush page data to backing store. 892 * 893 * => prefer map unlocked (not required) 894 * => flags: PGO_SYNCIO -- use sync. I/O 895 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 896 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 897 * [thus we never do async i/o! see iodone comment] 898 */ 899int 900uvn_put(struct uvm_object *uobj, struct vm_page **pps, int npages, int flags) 901{ 902 struct uvm_vnode *uvn = (struct uvm_vnode *)uobj; 903 int dying, retval; 904 905 KASSERT(rw_write_held(uobj->vmobjlock)); 906 907 /* 908 * Unless we're recycling this vnode, grab a reference to it 909 * to prevent it from being recycled from under our feet. 910 * This also makes sure we can don't panic if we end up in 911 * uvn_vnp_uncache() as a result of the I/O operation as that 912 * function assumes we hold a reference. 913 * 914 * If the vnode is in the process of being recycled by someone 915 * else, grabbing a reference will fail. In that case the 916 * pages will already be written out by whoever is cleaning 917 * the vnode, so simply return VM_PAGER_AGAIN such that we 918 * skip these pages. 919 */ 920 dying = (uvn->u_flags & UVM_VNODE_DYING); 921 if (!dying) { 922 if (vget(uvn->u_vnode, LK_NOWAIT)) 923 return VM_PAGER_AGAIN; 924 } 925 926 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 927 928 if (!dying) 929 vrele(uvn->u_vnode); 930 931 return retval; 932} 933 934/* 935 * uvn_get: get pages (synchronously) from backing store 936 * 937 * => prefer map unlocked (not required) 938 * => flags: PGO_ALLPAGES: get all of the pages 939 * PGO_LOCKED: fault data structures are locked 940 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 941 * => NOTE: caller must check for released pages!! 942 */ 943int 944uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, 945 int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags) 946{ 947 voff_t current_offset; 948 struct vm_page *ptmp; 949 int lcv, result, gotpages; 950 boolean_t done; 951 952 KASSERT(((flags & PGO_LOCKED) != 0 && rw_lock_held(uobj->vmobjlock)) || 953 (flags & PGO_LOCKED) == 0); 954 955 /* step 1: handled the case where fault data structures are locked. */ 956 if (flags & PGO_LOCKED) { 957 /* 958 * gotpages is the current number of pages we've gotten (which 959 * we pass back up to caller via *npagesp. 960 */ 961 gotpages = 0; 962 963 /* 964 * step 1a: get pages that are already resident. only do this 965 * if the data structures are locked (i.e. the first time 966 * through). 967 */ 968 done = TRUE; /* be optimistic */ 969 970 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 971 lcv++, current_offset += PAGE_SIZE) { 972 973 /* do we care about this page? if not, skip it */ 974 if (pps[lcv] == PGO_DONTCARE) 975 continue; 976 977 /* lookup page */ 978 ptmp = uvm_pagelookup(uobj, current_offset); 979 980 /* to be useful must get a non-busy, non-released pg */ 981 if (ptmp == NULL || 982 (ptmp->pg_flags & PG_BUSY) != 0) { 983 if (lcv == centeridx || (flags & PGO_ALLPAGES) 984 != 0) 985 done = FALSE; /* need to do a wait or I/O! */ 986 continue; 987 } 988 989 /* 990 * useful page: busy it and plug it in our 991 * result array 992 */ 993 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 994 UVM_PAGE_OWN(ptmp, "uvn_get1"); 995 pps[lcv] = ptmp; 996 gotpages++; 997 998 } 999 1000 /* 1001 * XXX: given the "advice", should we consider async read-ahead? 1002 * XXX: fault current does deactivate of pages behind us. is 1003 * this good (other callers might now). 1004 */ 1005 /* 1006 * XXX: read-ahead currently handled by buffer cache (bread) 1007 * level. 1008 * XXX: no async i/o available. 1009 * XXX: so we don't do anything now. 1010 */ 1011 1012 /* 1013 * step 1c: now we've either done everything needed or we to 1014 * unlock and do some waiting or I/O. 1015 */ 1016 1017 *npagesp = gotpages; /* let caller know */ 1018 if (done) 1019 return VM_PAGER_OK; /* bingo! */ 1020 else 1021 return VM_PAGER_UNLOCK; 1022 } 1023 1024 /* 1025 * step 2: get non-resident or busy pages. 1026 * data structures are unlocked. 1027 * 1028 * XXX: because we can't do async I/O at this level we get things 1029 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1030 * async-read-ahead for us at a lower level. 1031 */ 1032 for (lcv = 0, current_offset = offset; 1033 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1034 1035 /* skip over pages we've already gotten or don't want */ 1036 /* skip over pages we don't _have_ to get */ 1037 if (pps[lcv] != NULL || (lcv != centeridx && 1038 (flags & PGO_ALLPAGES) == 0)) 1039 continue; 1040 1041 /* 1042 * we have yet to locate the current page (pps[lcv]). we first 1043 * look for a page that is already at the current offset. if 1044 * we fine a page, we check to see if it is busy or released. 1045 * if that is the case, then we sleep on the page until it is 1046 * no longer busy or released and repeat the lookup. if the 1047 * page we found is neither busy nor released, then we busy it 1048 * (so we own it) and plug it into pps[lcv]. this breaks the 1049 * following while loop and indicates we are ready to move on 1050 * to the next page in the "lcv" loop above. 1051 * 1052 * if we exit the while loop with pps[lcv] still set to NULL, 1053 * then it means that we allocated a new busy/fake/clean page 1054 * ptmp in the object and we need to do I/O to fill in the data. 1055 */ 1056 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1057 /* look for a current page */ 1058 ptmp = uvm_pagelookup(uobj, current_offset); 1059 1060 /* nope? allocate one now (if we can) */ 1061 if (ptmp == NULL) { 1062 ptmp = uvm_pagealloc(uobj, current_offset, 1063 NULL, 0); 1064 1065 /* out of RAM? */ 1066 if (ptmp == NULL) { 1067 uvm_wait("uvn_getpage"); 1068 1069 /* goto top of pps while loop */ 1070 continue; 1071 } 1072 1073 /* 1074 * got new page ready for I/O. break pps 1075 * while loop. pps[lcv] is still NULL. 1076 */ 1077 break; 1078 } 1079 1080 /* page is there, see if we need to wait on it */ 1081 if ((ptmp->pg_flags & PG_BUSY) != 0) { 1082 uvm_pagewait(ptmp, uobj->vmobjlock, "uvn_get"); 1083 rw_enter(uobj->vmobjlock, RW_WRITE); 1084 continue; /* goto top of pps while loop */ 1085 } 1086 1087 /* 1088 * if we get here then the page has become resident 1089 * and unbusy between steps 1 and 2. we busy it 1090 * now (so we own it) and set pps[lcv] (so that we 1091 * exit the while loop). 1092 */ 1093 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1094 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1095 pps[lcv] = ptmp; 1096 } 1097 1098 /* 1099 * if we own the a valid page at the correct offset, pps[lcv] 1100 * will point to it. nothing more to do except go to the 1101 * next page. 1102 */ 1103 if (pps[lcv]) 1104 continue; /* next lcv */ 1105 1106 /* 1107 * we have a "fake/busy/clean" page that we just allocated. do 1108 * I/O to fill it with valid data. 1109 */ 1110 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1111 PGO_SYNCIO|PGO_NOWAIT, UIO_READ); 1112 1113 /* 1114 * I/O done. because we used syncio the result can not be 1115 * PEND or AGAIN. 1116 */ 1117 if (result != VM_PAGER_OK) { 1118 if (ptmp->pg_flags & PG_WANTED) 1119 wakeup(ptmp); 1120 1121 atomic_clearbits_int(&ptmp->pg_flags, 1122 PG_WANTED|PG_BUSY); 1123 UVM_PAGE_OWN(ptmp, NULL); 1124 uvm_lock_pageq(); 1125 uvm_pagefree(ptmp); 1126 uvm_unlock_pageq(); 1127 rw_exit(uobj->vmobjlock); 1128 return result; 1129 } 1130 1131 /* 1132 * we got the page! clear the fake flag (indicates valid 1133 * data now in page) and plug into our result array. note 1134 * that page is still busy. 1135 * 1136 * it is the callers job to: 1137 * => check if the page is released 1138 * => unbusy the page 1139 * => activate the page 1140 */ 1141 1142 /* data is valid ... */ 1143 atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE); 1144 pmap_clear_modify(ptmp); /* ... and clean */ 1145 pps[lcv] = ptmp; 1146 1147 } 1148 1149 1150 rw_exit(uobj->vmobjlock); 1151 return (VM_PAGER_OK); 1152} 1153 1154/* 1155 * uvn_io: do I/O to a vnode 1156 * 1157 * => prefer map unlocked (not required) 1158 * => flags: PGO_SYNCIO -- use sync. I/O 1159 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1160 * [thus we never do async i/o! see iodone comment] 1161 */ 1162 1163int 1164uvn_io(struct uvm_vnode *uvn, vm_page_t *pps, int npages, int flags, int rw) 1165{ 1166 struct uvm_object *uobj = &uvn->u_obj; 1167 struct vnode *vn; 1168 struct uio uio; 1169 struct iovec iov; 1170 vaddr_t kva; 1171 off_t file_offset; 1172 int waitf, result, mapinflags; 1173 size_t got, wanted; 1174 int vnlocked, netunlocked = 0; 1175 int lkflags = (flags & PGO_NOWAIT) ? LK_NOWAIT : 0; 1176 voff_t uvnsize; 1177 1178 KASSERT(rw_write_held(uobj->vmobjlock)); 1179 1180 /* init values */ 1181 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1182 vn = uvn->u_vnode; 1183 file_offset = pps[0]->offset; 1184 1185 /* check for sync'ing I/O. */ 1186 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1187 if (waitf == M_NOWAIT) { 1188 return VM_PAGER_AGAIN; 1189 } 1190 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1191 rwsleep_nsec(&uvn->u_flags, uobj->vmobjlock, PVM, "uvn_iosync", 1192 INFSLP); 1193 } 1194 1195 /* check size */ 1196 if (file_offset >= uvn->u_size) { 1197 return VM_PAGER_BAD; 1198 } 1199 1200 /* first try and map the pages in (without waiting) */ 1201 mapinflags = (rw == UIO_READ) ? 1202 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1203 1204 kva = uvm_pagermapin(pps, npages, mapinflags); 1205 if (kva == 0 && waitf == M_NOWAIT) { 1206 return VM_PAGER_AGAIN; 1207 } 1208 1209 /* 1210 * ok, now bump u_nio up. at this point we are done with uvn 1211 * and can unlock it. if we still don't have a kva, try again 1212 * (this time with sleep ok). 1213 */ 1214 uvn->u_nio++; /* we have an I/O in progress! */ 1215 vnlocked = (uvn->u_flags & UVM_VNODE_VNISLOCKED); 1216 uvnsize = uvn->u_size; 1217 rw_exit(uobj->vmobjlock); 1218 if (kva == 0) 1219 kva = uvm_pagermapin(pps, npages, 1220 mapinflags | UVMPAGER_MAPIN_WAITOK); 1221 1222 /* 1223 * ok, mapped in. our pages are PG_BUSY so they are not going to 1224 * get touched (so we can look at "offset" without having to lock 1225 * the object). set up for I/O. 1226 */ 1227 /* fill out uio/iov */ 1228 iov.iov_base = (caddr_t) kva; 1229 wanted = (size_t)npages << PAGE_SHIFT; 1230 if (file_offset + wanted > uvnsize) 1231 wanted = uvnsize - file_offset; /* XXX: needed? */ 1232 iov.iov_len = wanted; 1233 uio.uio_iov = &iov; 1234 uio.uio_iovcnt = 1; 1235 uio.uio_offset = file_offset; 1236 uio.uio_segflg = UIO_SYSSPACE; 1237 uio.uio_rw = rw; 1238 uio.uio_resid = wanted; 1239 uio.uio_procp = curproc; 1240 1241 /* 1242 * This process may already have the NET_LOCK(), if we 1243 * faulted in copyin() or copyout() in the network stack. 1244 */ 1245 if (rw_status(&netlock) == RW_WRITE) { 1246 NET_UNLOCK(); 1247 netunlocked = 1; 1248 } 1249 1250 /* do the I/O! (XXX: curproc?) */ 1251 /* 1252 * This process may already have this vnode locked, if we faulted in 1253 * copyin() or copyout() on a region backed by this vnode 1254 * while doing I/O to the vnode. If this is the case, don't 1255 * panic.. instead, return the error to the user. 1256 * 1257 * XXX this is a stopgap to prevent a panic. 1258 * Ideally, this kind of operation *should* work. 1259 */ 1260 result = 0; 1261 KERNEL_LOCK(); 1262 if (!vnlocked) 1263 result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL | lkflags); 1264 if (result == 0) { 1265 /* NOTE: vnode now locked! */ 1266 if (rw == UIO_READ) 1267 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1268 else 1269 result = VOP_WRITE(vn, &uio, 1270 (flags & PGO_PDFREECLUST) ? IO_NOCACHE : 0, 1271 curproc->p_ucred); 1272 1273 if (!vnlocked) 1274 VOP_UNLOCK(vn); 1275 1276 } 1277 KERNEL_UNLOCK(); 1278 1279 if (netunlocked) 1280 NET_LOCK(); 1281 1282 1283 /* NOTE: vnode now unlocked (unless vnislocked) */ 1284 /* 1285 * result == unix style errno (0 == OK!) 1286 * 1287 * zero out rest of buffer (if needed) 1288 */ 1289 if (result == 0) { 1290 got = wanted - uio.uio_resid; 1291 1292 if (wanted && got == 0) { 1293 result = EIO; /* XXX: error? */ 1294 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1295 memset((void *) (kva + got), 0, 1296 ((size_t)npages << PAGE_SHIFT) - got); 1297 } 1298 } 1299 1300 /* now remove pager mapping */ 1301 uvm_pagermapout(kva, npages); 1302 1303 /* now clean up the object (i.e. drop I/O count) */ 1304 rw_enter(uobj->vmobjlock, RW_WRITE); 1305 uvn->u_nio--; /* I/O DONE! */ 1306 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1307 wakeup(&uvn->u_nio); 1308 } 1309 1310 if (result == 0) { 1311 return VM_PAGER_OK; 1312 } else if (result == EBUSY) { 1313 KASSERT(flags & PGO_NOWAIT); 1314 return VM_PAGER_AGAIN; 1315 } else { 1316 if (rebooting) { 1317 KERNEL_LOCK(); 1318 while (rebooting) 1319 tsleep_nsec(&rebooting, PVM, "uvndead", INFSLP); 1320 KERNEL_UNLOCK(); 1321 } 1322 return VM_PAGER_ERROR; 1323 } 1324} 1325 1326/* 1327 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1328 * is gone we will kill the object (flushing dirty pages back to the vnode 1329 * if needed). 1330 * 1331 * => returns TRUE if there was no uvm_object attached or if there was 1332 * one and we killed it [i.e. if there is no active uvn] 1333 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1334 * needed] 1335 * 1336 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1337 * having to hold a reference on the vnode) and given a working 1338 * uvm_vnp_sync(), how does that effect the need for this function? 1339 * [XXXCDC: seems like it can die?] 1340 * 1341 * => XXX: this function should DIE once we merge the VM and buffer 1342 * cache. 1343 * 1344 * research shows that this is called in the following places: 1345 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1346 * changes sizes 1347 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1348 * are written to 1349 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1350 * is off 1351 * ffs_realloccg: when we can't extend the current block and have 1352 * to allocate a new one we call this [XXX: why?] 1353 * nfsrv_rename, rename_files: called when the target filename is there 1354 * and we want to remove it 1355 * nfsrv_remove, sys_unlink: called on file we are removing 1356 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1357 * then return "text busy" 1358 * nfs_open: seems to uncache any file opened with nfs 1359 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1360 * fusefs_open: uncaches any file that is opened 1361 * fusefs_write: uncaches on every write 1362 */ 1363 1364int 1365uvm_vnp_uncache(struct vnode *vp) 1366{ 1367 struct uvm_vnode *uvn = vp->v_uvm; 1368 struct uvm_object *uobj = &uvn->u_obj; 1369 1370 /* lock uvn part of the vnode and check if we need to do anything */ 1371 1372 rw_enter(uobj->vmobjlock, RW_WRITE); 1373 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1374 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1375 rw_exit(uobj->vmobjlock); 1376 return TRUE; 1377 } 1378 1379 /* 1380 * we have a valid, non-blocked uvn. clear persist flag. 1381 * if uvn is currently active we can return now. 1382 */ 1383 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1384 if (uvn->u_obj.uo_refs) { 1385 rw_exit(uobj->vmobjlock); 1386 return FALSE; 1387 } 1388 1389 /* 1390 * uvn is currently persisting! we have to gain a reference to 1391 * it so that we can call uvn_detach to kill the uvn. 1392 */ 1393 vref(vp); /* seems ok, even with VOP_LOCK */ 1394 uvn->u_obj.uo_refs++; /* value is now 1 */ 1395 rw_exit(uobj->vmobjlock); 1396 1397#ifdef VFSLCKDEBUG 1398 /* 1399 * carry over sanity check from old vnode pager: the vnode should 1400 * be VOP_LOCK'd, and we confirm it here. 1401 */ 1402 if ((vp->v_flag & VLOCKSWORK) && !VOP_ISLOCKED(vp)) 1403 panic("uvm_vnp_uncache: vnode not locked!"); 1404#endif 1405 1406 /* 1407 * now drop our reference to the vnode. if we have the sole 1408 * reference to the vnode then this will cause it to die [as we 1409 * just cleared the persist flag]. we have to unlock the vnode 1410 * while we are doing this as it may trigger I/O. 1411 * 1412 * XXX: it might be possible for uvn to get reclaimed while we are 1413 * unlocked causing us to return TRUE when we should not. we ignore 1414 * this as a false-positive return value doesn't hurt us. 1415 */ 1416 VOP_UNLOCK(vp); 1417 uvn_detach(&uvn->u_obj); 1418 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1419 1420 return TRUE; 1421} 1422 1423/* 1424 * uvm_vnp_setsize: grow or shrink a vnode uvn 1425 * 1426 * grow => just update size value 1427 * shrink => toss un-needed pages 1428 * 1429 * => we assume that the caller has a reference of some sort to the 1430 * vnode in question so that it will not be yanked out from under 1431 * us. 1432 * 1433 * called from: 1434 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos], 1435 * fusefs_setattr) 1436 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write 1437 * fusefs_write) 1438 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1439 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1440 * => union fs: union_newsize 1441 */ 1442 1443void 1444uvm_vnp_setsize(struct vnode *vp, off_t newsize) 1445{ 1446 struct uvm_vnode *uvn = vp->v_uvm; 1447 struct uvm_object *uobj = &uvn->u_obj; 1448 1449 KERNEL_ASSERT_LOCKED(); 1450 1451 rw_enter(uobj->vmobjlock, RW_WRITE); 1452 1453 /* lock uvn and check for valid object, and if valid: do it! */ 1454 if (uvn->u_flags & UVM_VNODE_VALID) { 1455 1456 /* 1457 * now check if the size has changed: if we shrink we had better 1458 * toss some pages... 1459 */ 1460 1461 if (uvn->u_size > newsize) { 1462 (void)uvn_flush(&uvn->u_obj, newsize, 1463 uvn->u_size, PGO_FREE); 1464 } 1465 uvn->u_size = newsize; 1466 } 1467 rw_exit(uobj->vmobjlock); 1468} 1469 1470/* 1471 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1472 * 1473 * => called from sys_sync with no VM structures locked 1474 * => only one process can do a sync at a time (because the uvn 1475 * structure only has one queue for sync'ing). we ensure this 1476 * by holding the uvn_sync_lock while the sync is in progress. 1477 * other processes attempting a sync will sleep on this lock 1478 * until we are done. 1479 */ 1480void 1481uvm_vnp_sync(struct mount *mp) 1482{ 1483 struct uvm_vnode *uvn; 1484 struct vnode *vp; 1485 1486 /* 1487 * step 1: ensure we are only ones using the uvn_sync_q by locking 1488 * our lock... 1489 */ 1490 rw_enter_write(&uvn_sync_lock); 1491 1492 /* 1493 * step 2: build up a simpleq of uvns of interest based on the 1494 * write list. we gain a reference to uvns of interest. 1495 */ 1496 SIMPLEQ_INIT(&uvn_sync_q); 1497 LIST_FOREACH(uvn, &uvn_wlist, u_wlist) { 1498 vp = uvn->u_vnode; 1499 if (mp && vp->v_mount != mp) 1500 continue; 1501 1502 /* 1503 * If the vnode is "blocked" it means it must be dying, which 1504 * in turn means its in the process of being flushed out so 1505 * we can safely skip it. 1506 * 1507 * note that uvn must already be valid because we found it on 1508 * the wlist (this also means it can't be ALOCK'd). 1509 */ 1510 if ((uvn->u_flags & UVM_VNODE_BLOCKED) != 0) 1511 continue; 1512 1513 /* 1514 * gain reference. watch out for persisting uvns (need to 1515 * regain vnode REF). 1516 */ 1517 if (uvn->u_obj.uo_refs == 0) 1518 vref(vp); 1519 uvn->u_obj.uo_refs++; 1520 1521 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 1522 } 1523 1524 /* step 3: we now have a list of uvn's that may need cleaning. */ 1525 SIMPLEQ_FOREACH(uvn, &uvn_sync_q, u_syncq) { 1526 rw_enter(uvn->u_obj.vmobjlock, RW_WRITE); 1527#ifdef DEBUG 1528 if (uvn->u_flags & UVM_VNODE_DYING) { 1529 printf("uvm_vnp_sync: dying vnode on sync list\n"); 1530 } 1531#endif 1532 uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 1533 1534 /* 1535 * if we have the only reference and we just cleaned the uvn, 1536 * then we can pull it out of the UVM_VNODE_WRITEABLE state 1537 * thus allowing us to avoid thinking about flushing it again 1538 * on later sync ops. 1539 */ 1540 if (uvn->u_obj.uo_refs == 1 && 1541 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 1542 LIST_REMOVE(uvn, u_wlist); 1543 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 1544 } 1545 rw_exit(uvn->u_obj.vmobjlock); 1546 1547 /* now drop our reference to the uvn */ 1548 uvn_detach(&uvn->u_obj); 1549 } 1550 1551 rw_exit_write(&uvn_sync_lock); 1552} 1553