vfs_bio.c revision 45685
1/* 2 * Copyright (c) 1994,1997 John S. Dyson 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice immediately at the beginning of the file, without modification, 10 * this list of conditions, and the following disclaimer. 11 * 2. Absolutely no warranty of function or purpose is made by the author 12 * John S. Dyson. 13 * 14 * $Id: vfs_bio.c,v 1.205 1999/04/07 02:41:54 alc Exp $ 15 */ 16 17/* 18 * this file contains a new buffer I/O scheme implementing a coherent 19 * VM object and buffer cache scheme. Pains have been taken to make 20 * sure that the performance degradation associated with schemes such 21 * as this is not realized. 22 * 23 * Author: John S. Dyson 24 * Significant help during the development and debugging phases 25 * had been provided by David Greenman, also of the FreeBSD core team. 26 * 27 * see man buf(9) for more info. 28 */ 29 30#define VMIO 31#include <sys/param.h> 32#include <sys/systm.h> 33#include <sys/sysproto.h> 34#include <sys/kernel.h> 35#include <sys/sysctl.h> 36#include <sys/proc.h> 37#include <sys/vnode.h> 38#include <sys/vmmeter.h> 39#include <sys/lock.h> 40#include <miscfs/specfs/specdev.h> 41#include <vm/vm.h> 42#include <vm/vm_param.h> 43#include <vm/vm_prot.h> 44#include <vm/vm_kern.h> 45#include <vm/vm_pageout.h> 46#include <vm/vm_page.h> 47#include <vm/vm_object.h> 48#include <vm/vm_extern.h> 49#include <vm/vm_map.h> 50#include <sys/buf.h> 51#include <sys/mount.h> 52#include <sys/malloc.h> 53#include <sys/resourcevar.h> 54 55static MALLOC_DEFINE(M_BIOBUF, "BIO buffer", "BIO buffer"); 56 57struct bio_ops bioops; /* I/O operation notification */ 58 59#if 0 /* replaced bu sched_sync */ 60static void vfs_update __P((void)); 61static struct proc *updateproc; 62static struct kproc_desc up_kp = { 63 "update", 64 vfs_update, 65 &updateproc 66}; 67SYSINIT_KT(update, SI_SUB_KTHREAD_UPDATE, SI_ORDER_FIRST, kproc_start, &up_kp) 68#endif 69 70struct buf *buf; /* buffer header pool */ 71struct swqueue bswlist; 72 73static void vm_hold_free_pages(struct buf * bp, vm_offset_t from, 74 vm_offset_t to); 75static void vm_hold_load_pages(struct buf * bp, vm_offset_t from, 76 vm_offset_t to); 77static void vfs_buf_set_valid(struct buf *bp, vm_ooffset_t foff, 78 vm_offset_t off, vm_offset_t size, 79 vm_page_t m); 80static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, 81 int pageno, vm_page_t m); 82static void vfs_clean_pages(struct buf * bp); 83static void vfs_setdirty(struct buf *bp); 84static void vfs_vmio_release(struct buf *bp); 85static void flushdirtybuffers(int slpflag, int slptimeo); 86static int flushbufqueues(void); 87 88/* 89 * Internal update daemon, process 3 90 * The variable vfs_update_wakeup allows for internal syncs. 91 */ 92int vfs_update_wakeup; 93 94/* 95 * bogus page -- for I/O to/from partially complete buffers 96 * this is a temporary solution to the problem, but it is not 97 * really that bad. it would be better to split the buffer 98 * for input in the case of buffers partially already in memory, 99 * but the code is intricate enough already. 100 */ 101vm_page_t bogus_page; 102int runningbufspace; 103static vm_offset_t bogus_offset; 104 105static int bufspace, maxbufspace, vmiospace, maxvmiobufspace, 106 bufmallocspace, maxbufmallocspace, hibufspace; 107static int needsbuffer; 108static int numdirtybuffers, lodirtybuffers, hidirtybuffers; 109static int numfreebuffers, lofreebuffers, hifreebuffers; 110static int kvafreespace; 111 112SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD, 113 &numdirtybuffers, 0, ""); 114SYSCTL_INT(_vfs, OID_AUTO, lodirtybuffers, CTLFLAG_RW, 115 &lodirtybuffers, 0, ""); 116SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW, 117 &hidirtybuffers, 0, ""); 118SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, 119 &numfreebuffers, 0, ""); 120SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW, 121 &lofreebuffers, 0, ""); 122SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW, 123 &hifreebuffers, 0, ""); 124SYSCTL_INT(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, 125 &runningbufspace, 0, ""); 126SYSCTL_INT(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RW, 127 &maxbufspace, 0, ""); 128SYSCTL_INT(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD, 129 &hibufspace, 0, ""); 130SYSCTL_INT(_vfs, OID_AUTO, bufspace, CTLFLAG_RD, 131 &bufspace, 0, ""); 132SYSCTL_INT(_vfs, OID_AUTO, maxvmiobufspace, CTLFLAG_RW, 133 &maxvmiobufspace, 0, ""); 134SYSCTL_INT(_vfs, OID_AUTO, vmiospace, CTLFLAG_RD, 135 &vmiospace, 0, ""); 136SYSCTL_INT(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, 137 &maxbufmallocspace, 0, ""); 138SYSCTL_INT(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, 139 &bufmallocspace, 0, ""); 140SYSCTL_INT(_vfs, OID_AUTO, kvafreespace, CTLFLAG_RD, 141 &kvafreespace, 0, ""); 142 143static LIST_HEAD(bufhashhdr, buf) bufhashtbl[BUFHSZ], invalhash; 144struct bqueues bufqueues[BUFFER_QUEUES] = { { 0 } }; 145 146extern int vm_swap_size; 147 148#define BUF_MAXUSE 24 149 150#define VFS_BIO_NEED_ANY 0x01 /* any freeable buffer */ 151#define VFS_BIO_NEED_RESERVED02 0x02 /* unused */ 152#define VFS_BIO_NEED_FREE 0x04 /* wait for free bufs, hi hysteresis */ 153#define VFS_BIO_NEED_BUFSPACE 0x08 /* wait for buf space, lo hysteresis */ 154#define VFS_BIO_NEED_KVASPACE 0x10 /* wait for buffer_map space, emerg */ 155 156/* 157 * kvaspacewakeup: 158 * 159 * Called when kva space is potential available for recovery or when 160 * kva space is recovered in the buffer_map. This function wakes up 161 * anyone waiting for buffer_map kva space. Even though the buffer_map 162 * is larger then maxbufspace, this situation will typically occur 163 * when the buffer_map gets fragmented. 164 */ 165 166static __inline void 167kvaspacewakeup(void) 168{ 169 /* 170 * If someone is waiting for KVA space, wake them up. Even 171 * though we haven't freed the kva space yet, the waiting 172 * process will be able to now. 173 */ 174 if (needsbuffer & VFS_BIO_NEED_KVASPACE) { 175 needsbuffer &= ~VFS_BIO_NEED_KVASPACE; 176 wakeup(&needsbuffer); 177 } 178} 179 180/* 181 * bufspacewakeup: 182 * 183 * Called when buffer space is potentially available for recovery or when 184 * buffer space is recovered. getnewbuf() will block on this flag when 185 * it is unable to free sufficient buffer space. Buffer space becomes 186 * recoverable when bp's get placed back in the queues. 187 */ 188 189static __inline void 190bufspacewakeup(void) 191{ 192 /* 193 * If someone is waiting for BUF space, wake them up. Even 194 * though we haven't freed the kva space yet, the waiting 195 * process will be able to now. 196 */ 197 if (needsbuffer & VFS_BIO_NEED_BUFSPACE) { 198 needsbuffer &= ~VFS_BIO_NEED_BUFSPACE; 199 wakeup(&needsbuffer); 200 } 201} 202 203/* 204 * bufcountwakeup: 205 * 206 * Called when a buffer has been added to one of the free queues to 207 * account for the buffer and to wakeup anyone waiting for free buffers. 208 * This typically occurs when large amounts of metadata are being handled 209 * by the buffer cache ( else buffer space runs out first, usually ). 210 */ 211 212static __inline void 213bufcountwakeup(void) 214{ 215 ++numfreebuffers; 216 if (needsbuffer) { 217 needsbuffer &= ~VFS_BIO_NEED_ANY; 218 if (numfreebuffers >= hifreebuffers) 219 needsbuffer &= ~VFS_BIO_NEED_FREE; 220 wakeup(&needsbuffer); 221 } 222} 223 224/* 225 * Initialize buffer headers and related structures. 226 */ 227void 228bufinit() 229{ 230 struct buf *bp; 231 int i; 232 233 TAILQ_INIT(&bswlist); 234 LIST_INIT(&invalhash); 235 236 /* first, make a null hash table */ 237 for (i = 0; i < BUFHSZ; i++) 238 LIST_INIT(&bufhashtbl[i]); 239 240 /* next, make a null set of free lists */ 241 for (i = 0; i < BUFFER_QUEUES; i++) 242 TAILQ_INIT(&bufqueues[i]); 243 244 /* finally, initialize each buffer header and stick on empty q */ 245 for (i = 0; i < nbuf; i++) { 246 bp = &buf[i]; 247 bzero(bp, sizeof *bp); 248 bp->b_flags = B_INVAL; /* we're just an empty header */ 249 bp->b_dev = NODEV; 250 bp->b_rcred = NOCRED; 251 bp->b_wcred = NOCRED; 252 bp->b_qindex = QUEUE_EMPTY; 253 bp->b_xflags = 0; 254 LIST_INIT(&bp->b_dep); 255 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist); 256 LIST_INSERT_HEAD(&invalhash, bp, b_hash); 257 } 258 259 /* 260 * maxbufspace is currently calculated to support all filesystem 261 * blocks to be 8K. If you happen to use a 16K filesystem, the size 262 * of the buffer cache is still the same as it would be for 8K 263 * filesystems. This keeps the size of the buffer cache "in check" 264 * for big block filesystems. 265 * 266 * maxbufspace is calculated as around 50% of the KVA available in 267 * the buffer_map ( DFLTSIZE vs BKVASIZE ), I presume to reduce the 268 * effect of fragmentation. 269 */ 270 maxbufspace = (nbuf + 8) * DFLTBSIZE; 271 if ((hibufspace = maxbufspace - MAXBSIZE * 5) <= MAXBSIZE) 272 hibufspace = 3 * maxbufspace / 4; 273/* 274 * reserve 1/3 of the buffers for metadata (VDIR) which might not be VMIO'ed 275 */ 276 maxvmiobufspace = 2 * hibufspace / 3; 277/* 278 * Limit the amount of malloc memory since it is wired permanently into 279 * the kernel space. Even though this is accounted for in the buffer 280 * allocation, we don't want the malloced region to grow uncontrolled. 281 * The malloc scheme improves memory utilization significantly on average 282 * (small) directories. 283 */ 284 maxbufmallocspace = hibufspace / 20; 285 286/* 287 * Reduce the chance of a deadlock occuring by limiting the number 288 * of delayed-write dirty buffers we allow to stack up. 289 */ 290 lodirtybuffers = nbuf / 16 + 10; 291 hidirtybuffers = nbuf / 8 + 20; 292 numdirtybuffers = 0; 293 294/* 295 * Try to keep the number of free buffers in the specified range, 296 * and give the syncer access to an emergency reserve. 297 */ 298 lofreebuffers = nbuf / 18 + 5; 299 hifreebuffers = 2 * lofreebuffers; 300 numfreebuffers = nbuf; 301 302 kvafreespace = 0; 303 304 bogus_offset = kmem_alloc_pageable(kernel_map, PAGE_SIZE); 305 bogus_page = vm_page_alloc(kernel_object, 306 ((bogus_offset - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT), 307 VM_ALLOC_NORMAL); 308 309} 310 311/* 312 * Free the kva allocation for a buffer 313 * Must be called only at splbio or higher, 314 * as this is the only locking for buffer_map. 315 */ 316static void 317bfreekva(struct buf * bp) 318{ 319 if (bp->b_kvasize) { 320 vm_map_delete(buffer_map, 321 (vm_offset_t) bp->b_kvabase, 322 (vm_offset_t) bp->b_kvabase + bp->b_kvasize 323 ); 324 bp->b_kvasize = 0; 325 kvaspacewakeup(); 326 } 327} 328 329/* 330 * bremfree: 331 * 332 * Remove the buffer from the appropriate free list. 333 */ 334void 335bremfree(struct buf * bp) 336{ 337 int s = splbio(); 338 int old_qindex = bp->b_qindex; 339 340 if (bp->b_qindex != QUEUE_NONE) { 341 if (bp->b_qindex == QUEUE_EMPTY) { 342 kvafreespace -= bp->b_kvasize; 343 } 344 TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist); 345 bp->b_qindex = QUEUE_NONE; 346 runningbufspace += bp->b_bufsize; 347 } else { 348#if !defined(MAX_PERF) 349 panic("bremfree: removing a buffer when not on a queue"); 350#endif 351 } 352 353 /* 354 * Fixup numfreebuffers count. If the buffer is invalid or not 355 * delayed-write, and it was on the EMPTY, LRU, or AGE queues, 356 * the buffer was free and we must decrement numfreebuffers. 357 */ 358 if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) { 359 switch(old_qindex) { 360 case QUEUE_EMPTY: 361 case QUEUE_LRU: 362 case QUEUE_AGE: 363 --numfreebuffers; 364 break; 365 default: 366 break; 367 } 368 } 369 splx(s); 370} 371 372 373/* 374 * Get a buffer with the specified data. Look in the cache first. 375 */ 376int 377bread(struct vnode * vp, daddr_t blkno, int size, struct ucred * cred, 378 struct buf ** bpp) 379{ 380 struct buf *bp; 381 382 bp = getblk(vp, blkno, size, 0, 0); 383 *bpp = bp; 384 385 /* if not found in cache, do some I/O */ 386 if ((bp->b_flags & B_CACHE) == 0) { 387 if (curproc != NULL) 388 curproc->p_stats->p_ru.ru_inblock++; 389 KASSERT(!(bp->b_flags & B_ASYNC), ("bread: illegal async bp %p", bp)); 390 bp->b_flags |= B_READ; 391 bp->b_flags &= ~(B_DONE | B_ERROR | B_INVAL); 392 if (bp->b_rcred == NOCRED) { 393 if (cred != NOCRED) 394 crhold(cred); 395 bp->b_rcred = cred; 396 } 397 vfs_busy_pages(bp, 0); 398 VOP_STRATEGY(vp, bp); 399 return (biowait(bp)); 400 } 401 return (0); 402} 403 404/* 405 * Operates like bread, but also starts asynchronous I/O on 406 * read-ahead blocks. 407 */ 408int 409breadn(struct vnode * vp, daddr_t blkno, int size, 410 daddr_t * rablkno, int *rabsize, 411 int cnt, struct ucred * cred, struct buf ** bpp) 412{ 413 struct buf *bp, *rabp; 414 int i; 415 int rv = 0, readwait = 0; 416 417 *bpp = bp = getblk(vp, blkno, size, 0, 0); 418 419 /* if not found in cache, do some I/O */ 420 if ((bp->b_flags & B_CACHE) == 0) { 421 if (curproc != NULL) 422 curproc->p_stats->p_ru.ru_inblock++; 423 bp->b_flags |= B_READ; 424 bp->b_flags &= ~(B_DONE | B_ERROR | B_INVAL); 425 if (bp->b_rcred == NOCRED) { 426 if (cred != NOCRED) 427 crhold(cred); 428 bp->b_rcred = cred; 429 } 430 vfs_busy_pages(bp, 0); 431 VOP_STRATEGY(vp, bp); 432 ++readwait; 433 } 434 435 for (i = 0; i < cnt; i++, rablkno++, rabsize++) { 436 if (inmem(vp, *rablkno)) 437 continue; 438 rabp = getblk(vp, *rablkno, *rabsize, 0, 0); 439 440 if ((rabp->b_flags & B_CACHE) == 0) { 441 if (curproc != NULL) 442 curproc->p_stats->p_ru.ru_inblock++; 443 rabp->b_flags |= B_READ | B_ASYNC; 444 rabp->b_flags &= ~(B_DONE | B_ERROR | B_INVAL); 445 if (rabp->b_rcred == NOCRED) { 446 if (cred != NOCRED) 447 crhold(cred); 448 rabp->b_rcred = cred; 449 } 450 vfs_busy_pages(rabp, 0); 451 VOP_STRATEGY(vp, rabp); 452 } else { 453 brelse(rabp); 454 } 455 } 456 457 if (readwait) { 458 rv = biowait(bp); 459 } 460 return (rv); 461} 462 463/* 464 * Write, release buffer on completion. (Done by iodone 465 * if async.) 466 */ 467int 468bwrite(struct buf * bp) 469{ 470 int oldflags, s; 471 struct vnode *vp; 472 struct mount *mp; 473 474 if (bp->b_flags & B_INVAL) { 475 brelse(bp); 476 return (0); 477 } 478 479 oldflags = bp->b_flags; 480 481#if !defined(MAX_PERF) 482 if ((bp->b_flags & B_BUSY) == 0) 483 panic("bwrite: buffer is not busy???"); 484#endif 485 s = splbio(); 486 bundirty(bp); 487 488 bp->b_flags &= ~(B_READ | B_DONE | B_ERROR); 489 bp->b_flags |= B_WRITEINPROG; 490 491 bp->b_vp->v_numoutput++; 492 vfs_busy_pages(bp, 1); 493 if (curproc != NULL) 494 curproc->p_stats->p_ru.ru_oublock++; 495 splx(s); 496 VOP_STRATEGY(bp->b_vp, bp); 497 498 /* 499 * Collect statistics on synchronous and asynchronous writes. 500 * Writes to block devices are charged to their associated 501 * filesystem (if any). 502 */ 503 if ((vp = bp->b_vp) != NULL) { 504 if (vp->v_type == VBLK) 505 mp = vp->v_specmountpoint; 506 else 507 mp = vp->v_mount; 508 if (mp != NULL) 509 if ((oldflags & B_ASYNC) == 0) 510 mp->mnt_stat.f_syncwrites++; 511 else 512 mp->mnt_stat.f_asyncwrites++; 513 } 514 515 if ((oldflags & B_ASYNC) == 0) { 516 int rtval = biowait(bp); 517 brelse(bp); 518 return (rtval); 519 } 520 521 return (0); 522} 523 524/* 525 * Delayed write. (Buffer is marked dirty). 526 */ 527void 528bdwrite(struct buf * bp) 529{ 530 struct vnode *vp; 531 532#if !defined(MAX_PERF) 533 if ((bp->b_flags & B_BUSY) == 0) { 534 panic("bdwrite: buffer is not busy"); 535 } 536#endif 537 538 if (bp->b_flags & B_INVAL) { 539 brelse(bp); 540 return; 541 } 542 bdirty(bp); 543 544 /* 545 * This bmap keeps the system from needing to do the bmap later, 546 * perhaps when the system is attempting to do a sync. Since it 547 * is likely that the indirect block -- or whatever other datastructure 548 * that the filesystem needs is still in memory now, it is a good 549 * thing to do this. Note also, that if the pageout daemon is 550 * requesting a sync -- there might not be enough memory to do 551 * the bmap then... So, this is important to do. 552 */ 553 if (bp->b_lblkno == bp->b_blkno) { 554 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL); 555 } 556 557 /* 558 * Set the *dirty* buffer range based upon the VM system dirty pages. 559 */ 560 vfs_setdirty(bp); 561 562 /* 563 * We need to do this here to satisfy the vnode_pager and the 564 * pageout daemon, so that it thinks that the pages have been 565 * "cleaned". Note that since the pages are in a delayed write 566 * buffer -- the VFS layer "will" see that the pages get written 567 * out on the next sync, or perhaps the cluster will be completed. 568 */ 569 vfs_clean_pages(bp); 570 bqrelse(bp); 571 572 /* 573 * XXX The soft dependency code is not prepared to 574 * have I/O done when a bdwrite is requested. For 575 * now we just let the write be delayed if it is 576 * requested by the soft dependency code. 577 */ 578 if ((vp = bp->b_vp) && 579 ((vp->v_type == VBLK && vp->v_specmountpoint && 580 (vp->v_specmountpoint->mnt_flag & MNT_SOFTDEP)) || 581 (vp->v_mount && (vp->v_mount->mnt_flag & MNT_SOFTDEP)))) 582 return; 583 584 if (numdirtybuffers >= hidirtybuffers) 585 flushdirtybuffers(0, 0); 586} 587 588/* 589 * bdirty: 590 * 591 * Turn buffer into delayed write request. We must clear B_READ and 592 * B_RELBUF, and we must set B_DELWRI. We reassign the buffer to 593 * itself to properly update it in the dirty/clean lists. We mark it 594 * B_DONE to ensure that any asynchronization of the buffer properly 595 * clears B_DONE ( else a panic will occur later ). Note that B_INVALID 596 * buffers are not considered dirty even if B_DELWRI is set. 597 * 598 * Since the buffer is not on a queue, we do not update the numfreebuffers 599 * count. 600 * 601 * Must be called at splbio(). 602 * The buffer must be on QUEUE_NONE. 603 */ 604void 605bdirty(bp) 606 struct buf *bp; 607{ 608 KASSERT(bp->b_qindex == QUEUE_NONE, ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex)); 609 bp->b_flags &= ~(B_READ|B_RELBUF); 610 611 if ((bp->b_flags & B_DELWRI) == 0) { 612 bp->b_flags |= B_DONE | B_DELWRI; 613 reassignbuf(bp, bp->b_vp); 614 ++numdirtybuffers; 615 } 616} 617 618/* 619 * bundirty: 620 * 621 * Clear B_DELWRI for buffer. 622 * 623 * Since the buffer is not on a queue, we do not update the numfreebuffers 624 * count. 625 * 626 * Must be called at splbio(). 627 * The buffer must be on QUEUE_NONE. 628 */ 629 630void 631bundirty(bp) 632 struct buf *bp; 633{ 634 KASSERT(bp->b_qindex == QUEUE_NONE, ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex)); 635 636 if (bp->b_flags & B_DELWRI) { 637 bp->b_flags &= ~B_DELWRI; 638 reassignbuf(bp, bp->b_vp); 639 --numdirtybuffers; 640 } 641} 642 643/* 644 * bawrite: 645 * 646 * Asynchronous write. Start output on a buffer, but do not wait for 647 * it to complete. The buffer is released when the output completes. 648 */ 649void 650bawrite(struct buf * bp) 651{ 652 bp->b_flags |= B_ASYNC; 653 (void) VOP_BWRITE(bp); 654} 655 656/* 657 * bowrite: 658 * 659 * Ordered write. Start output on a buffer, and flag it so that the 660 * device will write it in the order it was queued. The buffer is 661 * released when the output completes. 662 */ 663int 664bowrite(struct buf * bp) 665{ 666 bp->b_flags |= B_ORDERED | B_ASYNC; 667 return (VOP_BWRITE(bp)); 668} 669 670/* 671 * brelse: 672 * 673 * Release a busy buffer and, if requested, free its resources. The 674 * buffer will be stashed in the appropriate bufqueue[] allowing it 675 * to be accessed later as a cache entity or reused for other purposes. 676 */ 677void 678brelse(struct buf * bp) 679{ 680 int s; 681 682 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp)); 683 684#if 0 685 if (bp->b_flags & B_CLUSTER) { 686 relpbuf(bp, NULL); 687 return; 688 } 689#endif 690 691 s = splbio(); 692 693 if (bp->b_flags & B_LOCKED) 694 bp->b_flags &= ~B_ERROR; 695 696 if ((bp->b_flags & (B_READ | B_ERROR)) == B_ERROR) { 697 bp->b_flags &= ~B_ERROR; 698 bdirty(bp); 699 } else if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_ERROR | B_FREEBUF)) || 700 (bp->b_bufsize <= 0)) { 701 bp->b_flags |= B_INVAL; 702 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 703 (*bioops.io_deallocate)(bp); 704 if (bp->b_flags & B_DELWRI) 705 --numdirtybuffers; 706 bp->b_flags &= ~(B_DELWRI | B_CACHE | B_FREEBUF); 707 if ((bp->b_flags & B_VMIO) == 0) { 708 if (bp->b_bufsize) 709 allocbuf(bp, 0); 710 if (bp->b_vp) 711 brelvp(bp); 712 } 713 } 714 715 /* 716 * We must clear B_RELBUF if B_DELWRI is set. If vfs_vmio_release() 717 * is called with B_DELWRI set, the underlying pages may wind up 718 * getting freed causing a previous write (bdwrite()) to get 'lost' 719 * because pages associated with a B_DELWRI bp are marked clean. 720 * 721 * We still allow the B_INVAL case to call vfs_vmio_release(), even 722 * if B_DELWRI is set. 723 */ 724 725 if (bp->b_flags & B_DELWRI) 726 bp->b_flags &= ~B_RELBUF; 727 728 /* 729 * VMIO buffer rundown. It is not very necessary to keep a VMIO buffer 730 * constituted, so the B_INVAL flag is used to *invalidate* the buffer, 731 * but the VM object is kept around. The B_NOCACHE flag is used to 732 * invalidate the pages in the VM object. 733 * 734 * The b_{validoff,validend,dirtyoff,dirtyend} values are relative 735 * to b_offset and currently have byte granularity, whereas the 736 * valid flags in the vm_pages have only DEV_BSIZE resolution. 737 * The byte resolution fields are used to avoid unnecessary re-reads 738 * of the buffer but the code really needs to be genericized so 739 * other filesystem modules can take advantage of these fields. 740 * 741 * XXX this seems to cause performance problems. 742 */ 743 if ((bp->b_flags & B_VMIO) 744 && !(bp->b_vp->v_tag == VT_NFS && 745 bp->b_vp->v_type != VBLK && 746 (bp->b_flags & B_DELWRI) != 0) 747#ifdef notdef 748 && (bp->b_vp->v_tag != VT_NFS 749 || bp->b_vp->v_type == VBLK 750 || (bp->b_flags & (B_NOCACHE | B_INVAL | B_ERROR)) 751 || bp->b_validend == 0 752 || (bp->b_validoff == 0 753 && bp->b_validend == bp->b_bufsize)) 754#endif 755 ) { 756 757 int i, j, resid; 758 vm_page_t m; 759 off_t foff; 760 vm_pindex_t poff; 761 vm_object_t obj; 762 struct vnode *vp; 763 764 vp = bp->b_vp; 765 766 /* 767 * Get the base offset and length of the buffer. Note that 768 * for block sizes that are less then PAGE_SIZE, the b_data 769 * base of the buffer does not represent exactly b_offset and 770 * neither b_offset nor b_size are necessarily page aligned. 771 * Instead, the starting position of b_offset is: 772 * 773 * b_data + (b_offset & PAGE_MASK) 774 * 775 * block sizes less then DEV_BSIZE (usually 512) are not 776 * supported due to the page granularity bits (m->valid, 777 * m->dirty, etc...). 778 * 779 * See man buf(9) for more information 780 */ 781 782 resid = bp->b_bufsize; 783 foff = bp->b_offset; 784 785 for (i = 0; i < bp->b_npages; i++) { 786 m = bp->b_pages[i]; 787 vm_page_flag_clear(m, PG_ZERO); 788 if (m == bogus_page) { 789 790 obj = (vm_object_t) vp->v_object; 791 poff = OFF_TO_IDX(bp->b_offset); 792 793 for (j = i; j < bp->b_npages; j++) { 794 m = bp->b_pages[j]; 795 if (m == bogus_page) { 796 m = vm_page_lookup(obj, poff + j); 797#if !defined(MAX_PERF) 798 if (!m) { 799 panic("brelse: page missing\n"); 800 } 801#endif 802 bp->b_pages[j] = m; 803 } 804 } 805 806 if ((bp->b_flags & B_INVAL) == 0) { 807 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages); 808 } 809 } 810 if (bp->b_flags & (B_NOCACHE|B_ERROR)) { 811 int poffset = foff & PAGE_MASK; 812 int presid = resid > (PAGE_SIZE - poffset) ? 813 (PAGE_SIZE - poffset) : resid; 814 815 KASSERT(presid >= 0, ("brelse: extra page")); 816 vm_page_set_invalid(m, poffset, presid); 817 } 818 resid -= PAGE_SIZE - (foff & PAGE_MASK); 819 foff = (foff + PAGE_SIZE) & ~PAGE_MASK; 820 } 821 822 if (bp->b_flags & (B_INVAL | B_RELBUF)) 823 vfs_vmio_release(bp); 824 825 } else if (bp->b_flags & B_VMIO) { 826 827 if (bp->b_flags & (B_INVAL | B_RELBUF)) 828 vfs_vmio_release(bp); 829 830 } 831 832#if !defined(MAX_PERF) 833 if (bp->b_qindex != QUEUE_NONE) 834 panic("brelse: free buffer onto another queue???"); 835#endif 836 /* enqueue */ 837 838 /* buffers with no memory */ 839 if (bp->b_bufsize == 0) { 840 bp->b_flags |= B_INVAL; 841 bp->b_qindex = QUEUE_EMPTY; 842 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_EMPTY], bp, b_freelist); 843 LIST_REMOVE(bp, b_hash); 844 LIST_INSERT_HEAD(&invalhash, bp, b_hash); 845 bp->b_dev = NODEV; 846 kvafreespace += bp->b_kvasize; 847 if (bp->b_kvasize) 848 kvaspacewakeup(); 849 /* buffers with junk contents */ 850 } else if (bp->b_flags & (B_ERROR | B_INVAL | B_NOCACHE | B_RELBUF)) { 851 bp->b_flags |= B_INVAL; 852 bp->b_qindex = QUEUE_AGE; 853 TAILQ_INSERT_HEAD(&bufqueues[QUEUE_AGE], bp, b_freelist); 854 LIST_REMOVE(bp, b_hash); 855 LIST_INSERT_HEAD(&invalhash, bp, b_hash); 856 bp->b_dev = NODEV; 857 858 /* buffers that are locked */ 859 } else if (bp->b_flags & B_LOCKED) { 860 bp->b_qindex = QUEUE_LOCKED; 861 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist); 862 863 /* buffers with stale but valid contents */ 864 } else if (bp->b_flags & B_AGE) { 865 bp->b_qindex = QUEUE_AGE; 866 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_AGE], bp, b_freelist); 867 868 /* buffers with valid and quite potentially reuseable contents */ 869 } else { 870 bp->b_qindex = QUEUE_LRU; 871 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LRU], bp, b_freelist); 872 } 873 874 /* 875 * If B_INVAL, clear B_DELWRI. 876 */ 877 if ((bp->b_flags & (B_INVAL|B_DELWRI)) == (B_INVAL|B_DELWRI)) { 878 bp->b_flags &= ~B_DELWRI; 879 --numdirtybuffers; 880 } 881 882 runningbufspace -= bp->b_bufsize; 883 884 /* 885 * Fixup numfreebuffers count. The bp is on an appropriate queue 886 * unless locked. We then bump numfreebuffers if it is not B_DELWRI. 887 * We've already handled the B_INVAL case ( B_DELWRI will be clear 888 * if B_INVAL is set ). 889 */ 890 891 if ((bp->b_flags & B_LOCKED) == 0 && !(bp->b_flags & B_DELWRI)) 892 bufcountwakeup(); 893 894 /* 895 * Something we can maybe free. 896 */ 897 898 if (bp->b_bufsize) 899 bufspacewakeup(); 900 901 if (bp->b_flags & B_WANTED) { 902 bp->b_flags &= ~(B_WANTED | B_AGE); 903 wakeup(bp); 904 } 905 906 /* unlock */ 907 bp->b_flags &= ~(B_ORDERED | B_WANTED | B_BUSY | 908 B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF); 909 splx(s); 910} 911 912/* 913 * Release a buffer back to the appropriate queue but do not try to free 914 * it. 915 */ 916void 917bqrelse(struct buf * bp) 918{ 919 int s; 920 921 s = splbio(); 922 923 KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)), ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp)); 924 925#if !defined(MAX_PERF) 926 if (bp->b_qindex != QUEUE_NONE) 927 panic("bqrelse: free buffer onto another queue???"); 928#endif 929 if (bp->b_flags & B_LOCKED) { 930 bp->b_flags &= ~B_ERROR; 931 bp->b_qindex = QUEUE_LOCKED; 932 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LOCKED], bp, b_freelist); 933 /* buffers with stale but valid contents */ 934 } else { 935 bp->b_qindex = QUEUE_LRU; 936 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LRU], bp, b_freelist); 937 } 938 939 runningbufspace -= bp->b_bufsize; 940 941 if ((bp->b_flags & B_LOCKED) == 0 && 942 ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI)) 943 ) { 944 bufcountwakeup(); 945 } 946 947 /* 948 * Something we can maybe wakeup 949 */ 950 if (bp->b_bufsize) 951 bufspacewakeup(); 952 953 /* anyone need this block? */ 954 if (bp->b_flags & B_WANTED) { 955 bp->b_flags &= ~(B_WANTED | B_AGE); 956 wakeup(bp); 957 } 958 959 /* unlock */ 960 bp->b_flags &= ~(B_ORDERED | B_WANTED | B_BUSY | 961 B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF); 962 splx(s); 963} 964 965static void 966vfs_vmio_release(bp) 967 struct buf *bp; 968{ 969 int i, s; 970 vm_page_t m; 971 972 s = splvm(); 973 for (i = 0; i < bp->b_npages; i++) { 974 m = bp->b_pages[i]; 975 bp->b_pages[i] = NULL; 976 /* 977 * In order to keep page LRU ordering consistent, put 978 * everything on the inactive queue. 979 */ 980 vm_page_unwire(m, 0); 981 /* 982 * We don't mess with busy pages, it is 983 * the responsibility of the process that 984 * busied the pages to deal with them. 985 */ 986 if ((m->flags & PG_BUSY) || (m->busy != 0)) 987 continue; 988 989 if (m->wire_count == 0) { 990 vm_page_flag_clear(m, PG_ZERO); 991 /* 992 * Might as well free the page if we can and it has 993 * no valid data. 994 */ 995 if ((bp->b_flags & B_ASYNC) == 0 && !m->valid && m->hold_count == 0) { 996 vm_page_busy(m); 997 vm_page_protect(m, VM_PROT_NONE); 998 vm_page_free(m); 999 } 1000 } 1001 } 1002 bufspace -= bp->b_bufsize; 1003 vmiospace -= bp->b_bufsize; 1004 runningbufspace -= bp->b_bufsize; 1005 splx(s); 1006 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages); 1007 if (bp->b_bufsize) 1008 bufspacewakeup(); 1009 bp->b_npages = 0; 1010 bp->b_bufsize = 0; 1011 bp->b_flags &= ~B_VMIO; 1012 if (bp->b_vp) 1013 brelvp(bp); 1014} 1015 1016/* 1017 * Check to see if a block is currently memory resident. 1018 */ 1019struct buf * 1020gbincore(struct vnode * vp, daddr_t blkno) 1021{ 1022 struct buf *bp; 1023 struct bufhashhdr *bh; 1024 1025 bh = BUFHASH(vp, blkno); 1026 bp = bh->lh_first; 1027 1028 /* Search hash chain */ 1029 while (bp != NULL) { 1030 /* hit */ 1031 if (bp->b_vp == vp && bp->b_lblkno == blkno && 1032 (bp->b_flags & B_INVAL) == 0) { 1033 break; 1034 } 1035 bp = bp->b_hash.le_next; 1036 } 1037 return (bp); 1038} 1039 1040/* 1041 * this routine implements clustered async writes for 1042 * clearing out B_DELWRI buffers... This is much better 1043 * than the old way of writing only one buffer at a time. 1044 */ 1045int 1046vfs_bio_awrite(struct buf * bp) 1047{ 1048 int i; 1049 daddr_t lblkno = bp->b_lblkno; 1050 struct vnode *vp = bp->b_vp; 1051 int s; 1052 int ncl; 1053 struct buf *bpa; 1054 int nwritten; 1055 int size; 1056 int maxcl; 1057 1058 s = splbio(); 1059 /* 1060 * right now we support clustered writing only to regular files, and 1061 * then only if our I/O system is not saturated. 1062 */ 1063 if ((vp->v_type == VREG) && 1064 (vp->v_mount != 0) && /* Only on nodes that have the size info */ 1065 (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) { 1066 1067 size = vp->v_mount->mnt_stat.f_iosize; 1068 maxcl = MAXPHYS / size; 1069 1070 for (i = 1; i < maxcl; i++) { 1071 if ((bpa = gbincore(vp, lblkno + i)) && 1072 ((bpa->b_flags & (B_BUSY | B_DELWRI | B_CLUSTEROK | B_INVAL)) == 1073 (B_DELWRI | B_CLUSTEROK)) && 1074 (bpa->b_bufsize == size)) { 1075 if ((bpa->b_blkno == bpa->b_lblkno) || 1076 (bpa->b_blkno != bp->b_blkno + ((i * size) >> DEV_BSHIFT))) 1077 break; 1078 } else { 1079 break; 1080 } 1081 } 1082 ncl = i; 1083 /* 1084 * this is a possible cluster write 1085 */ 1086 if (ncl != 1) { 1087 nwritten = cluster_wbuild(vp, size, lblkno, ncl); 1088 splx(s); 1089 return nwritten; 1090 } 1091 } 1092 1093 bremfree(bp); 1094 bp->b_flags |= B_BUSY | B_ASYNC; 1095 1096 splx(s); 1097 /* 1098 * default (old) behavior, writing out only one block 1099 */ 1100 nwritten = bp->b_bufsize; 1101 (void) VOP_BWRITE(bp); 1102 1103 return nwritten; 1104} 1105 1106/* 1107 * getnewbuf: 1108 * 1109 * Find and initialize a new buffer header, freeing up existing buffers 1110 * in the bufqueues as necessary. 1111 * 1112 * We block if: 1113 * We have insufficient buffer headers 1114 * We have insufficient buffer space 1115 * buffer_map is too fragmented ( space reservation fails ) 1116 * 1117 * We do *not* attempt to flush dirty buffers more then one level deep. 1118 * I.e., if P_FLSINPROG is set we do not flush dirty buffers at all. 1119 * 1120 * If P_FLSINPROG is set, we are allowed to dip into our emergency 1121 * reserve. 1122 */ 1123static struct buf * 1124getnewbuf(struct vnode *vp, daddr_t blkno, 1125 int slpflag, int slptimeo, int size, int maxsize) 1126{ 1127 struct buf *bp; 1128 struct buf *nbp; 1129 int outofspace; 1130 int nqindex; 1131 int defrag = 0; 1132 int countawrites = 0; 1133 1134restart: 1135 /* 1136 * Calculate whether we are out of buffer space. This state is 1137 * recalculated on every restart. If we are out of space, we 1138 * have to turn off defragmentation. The outofspace code will 1139 * defragment too, but the looping conditionals will be messed up 1140 * if both outofspace and defrag are on. 1141 */ 1142 1143 outofspace = 0; 1144 if (bufspace >= hibufspace) { 1145 if ((curproc->p_flag & P_FLSINPROG) == 0 || 1146 bufspace >= maxbufspace 1147 ) { 1148 outofspace = 1; 1149 defrag = 0; 1150 } 1151 } 1152 1153 /* 1154 * defrag state is semi-persistant. 1 means we are flagged for 1155 * defragging. -1 means we actually defragged something. 1156 */ 1157 /* nop */ 1158 1159 /* 1160 * Setup for scan. If we do not have enough free buffers, 1161 * we setup a degenerate case that falls through the while. 1162 * 1163 * If we are in the middle of a flush, we can dip into the 1164 * emergency reserve. 1165 * 1166 * If we are out of space, we skip trying to scan QUEUE_EMPTY 1167 * because those buffers are, well, empty. 1168 */ 1169 1170 if ((curproc->p_flag & P_FLSINPROG) == 0 && 1171 numfreebuffers < lofreebuffers 1172 ) { 1173 nqindex = QUEUE_LRU; 1174 nbp = NULL; 1175 } else { 1176 nqindex = QUEUE_EMPTY; 1177 if (outofspace || 1178 (nbp = TAILQ_FIRST(&bufqueues[QUEUE_EMPTY])) == NULL 1179 ) { 1180 nqindex = QUEUE_AGE; 1181 nbp = TAILQ_FIRST(&bufqueues[QUEUE_AGE]); 1182 if (nbp == NULL) { 1183 nqindex = QUEUE_LRU; 1184 nbp = TAILQ_FIRST(&bufqueues[QUEUE_LRU]); 1185 } 1186 } 1187 } 1188 1189 /* 1190 * Run scan, possibly freeing data and/or kva mappings on the fly 1191 * depending. 1192 */ 1193 1194 while ((bp = nbp) != NULL) { 1195 int qindex = nqindex; 1196 /* 1197 * Calculate next bp ( we can only use it if we do not block 1198 * or do other fancy things ). 1199 */ 1200 if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) { 1201 switch(qindex) { 1202 case QUEUE_EMPTY: 1203 nqindex = QUEUE_AGE; 1204 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_AGE]))) 1205 break; 1206 /* fall through */ 1207 case QUEUE_AGE: 1208 nqindex = QUEUE_LRU; 1209 if ((nbp = TAILQ_FIRST(&bufqueues[QUEUE_LRU]))) 1210 break; 1211 /* fall through */ 1212 case QUEUE_LRU: 1213 /* 1214 * nbp is NULL. 1215 */ 1216 break; 1217 } 1218 } 1219 1220 /* 1221 * Sanity Checks 1222 */ 1223 KASSERT(!(bp->b_flags & B_BUSY), ("getnewbuf: busy buffer %p on free list", bp)); 1224 KASSERT(bp->b_qindex == qindex, ("getnewbuf: inconsistant queue %d bp %p", qindex, bp)); 1225 1226 /* 1227 * Here we try to move NON VMIO buffers to the end of the 1228 * LRU queue in order to make VMIO buffers more readily 1229 * freeable. We also try to move buffers with a positive 1230 * usecount to the end. 1231 * 1232 * Note that by moving the bp to the end, we setup a following 1233 * loop. Since we continue to decrement b_usecount this 1234 * is ok and, in fact, desireable. 1235 * 1236 * If we are at the end of the list, we move ourself to the 1237 * same place and need to fixup nbp and nqindex to handle 1238 * the following case. 1239 */ 1240 1241 if ((qindex == QUEUE_LRU) && bp->b_usecount > 0) { 1242 if ((bp->b_flags & B_VMIO) == 0 || 1243 (vmiospace < maxvmiobufspace) 1244 ) { 1245 --bp->b_usecount; 1246 TAILQ_REMOVE(&bufqueues[QUEUE_LRU], bp, b_freelist); 1247 TAILQ_INSERT_TAIL(&bufqueues[QUEUE_LRU], bp, b_freelist); 1248 if (nbp == NULL) { 1249 nqindex = qindex; 1250 nbp = bp; 1251 } 1252 continue; 1253 } 1254 } 1255 1256 /* 1257 * If we come across a delayed write and numdirtybuffers should 1258 * be flushed, try to write it out. Only if P_FLSINPROG is 1259 * not set. We can't afford to recursively stack more then 1260 * one deep due to the possibility of having deep VFS call 1261 * stacks. 1262 * 1263 * Limit the number of dirty buffers we are willing to try 1264 * to recover since it really isn't our job here. 1265 */ 1266 if ((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) { 1267 if ((curproc->p_flag & P_FLSINPROG) || 1268 numdirtybuffers < hidirtybuffers || 1269 countawrites > 16 1270 ) { 1271 continue; 1272 } 1273 curproc->p_flag |= P_FLSINPROG; 1274 vfs_bio_awrite(bp); 1275 curproc->p_flag &= ~P_FLSINPROG; 1276 ++countawrites; 1277 goto restart; 1278 } 1279 1280 if (defrag > 0 && bp->b_kvasize == 0) 1281 continue; 1282 if (outofspace > 0 && bp->b_bufsize == 0) 1283 continue; 1284 1285 /* 1286 * Start freeing the bp. This is somewhat involved. nbp 1287 * remains valid only for QUEUE_EMPTY bp's. 1288 */ 1289 1290 bremfree(bp); 1291 bp->b_flags |= B_BUSY; 1292 1293 if (qindex == QUEUE_LRU || qindex == QUEUE_AGE) { 1294 if (bp->b_flags & B_VMIO) { 1295 bp->b_flags &= ~B_ASYNC; 1296 vfs_vmio_release(bp); 1297 } 1298 if (bp->b_vp) 1299 brelvp(bp); 1300 } 1301 1302 if (bp->b_flags & B_WANTED) { 1303 bp->b_flags &= ~B_WANTED; 1304 wakeup(bp); 1305 } 1306 1307 /* 1308 * NOTE: nbp is now entirely invalid. We can only restart 1309 * the scan from this point on. 1310 * 1311 * Get the rest of the buffer freed up. b_kva* is still 1312 * valid after this operation. 1313 */ 1314 1315 if (bp->b_rcred != NOCRED) { 1316 crfree(bp->b_rcred); 1317 bp->b_rcred = NOCRED; 1318 } 1319 if (bp->b_wcred != NOCRED) { 1320 crfree(bp->b_wcred); 1321 bp->b_wcred = NOCRED; 1322 } 1323 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 1324 (*bioops.io_deallocate)(bp); 1325 1326 LIST_REMOVE(bp, b_hash); 1327 LIST_INSERT_HEAD(&invalhash, bp, b_hash); 1328 1329 if (bp->b_bufsize) 1330 allocbuf(bp, 0); 1331 1332 bp->b_flags = B_BUSY; 1333 bp->b_dev = NODEV; 1334 bp->b_vp = NULL; 1335 bp->b_blkno = bp->b_lblkno = 0; 1336 bp->b_offset = NOOFFSET; 1337 bp->b_iodone = 0; 1338 bp->b_error = 0; 1339 bp->b_resid = 0; 1340 bp->b_bcount = 0; 1341 bp->b_npages = 0; 1342 bp->b_dirtyoff = bp->b_dirtyend = 0; 1343 bp->b_validoff = bp->b_validend = 0; 1344 bp->b_usecount = 5; 1345 1346 LIST_INIT(&bp->b_dep); 1347 1348 /* 1349 * Ok, now that we have a free buffer, if we are defragging 1350 * we have to recover the kvaspace. 1351 */ 1352 1353 if (defrag > 0) { 1354 defrag = -1; 1355 bp->b_flags |= B_INVAL; 1356 bfreekva(bp); 1357 brelse(bp); 1358 goto restart; 1359 } 1360 1361 if (outofspace > 0) { 1362 outofspace = -1; 1363 bp->b_flags |= B_INVAL; 1364 bfreekva(bp); 1365 brelse(bp); 1366 goto restart; 1367 } 1368 1369 /* 1370 * We are done 1371 */ 1372 break; 1373 } 1374 1375 /* 1376 * If we exhausted our list, sleep as appropriate. 1377 */ 1378 1379 if (bp == NULL) { 1380 int flags; 1381 1382dosleep: 1383 if (defrag > 0) 1384 flags = VFS_BIO_NEED_KVASPACE; 1385 else if (outofspace > 0) 1386 flags = VFS_BIO_NEED_BUFSPACE; 1387 else 1388 flags = VFS_BIO_NEED_ANY; 1389 1390 if (rushjob < syncdelay / 2) 1391 ++rushjob; 1392 needsbuffer |= flags; 1393 while (needsbuffer & flags) { 1394 if (tsleep(&needsbuffer, (PRIBIO + 4) | slpflag, 1395 "newbuf", slptimeo)) 1396 return (NULL); 1397 } 1398 } else { 1399 /* 1400 * We finally have a valid bp. We aren't quite out of the 1401 * woods, we still have to reserve kva space. 1402 */ 1403 vm_offset_t addr = 0; 1404 1405 maxsize = (maxsize + PAGE_MASK) & ~PAGE_MASK; 1406 1407 if (maxsize != bp->b_kvasize) { 1408 bfreekva(bp); 1409 1410 if (vm_map_findspace(buffer_map, 1411 vm_map_min(buffer_map), maxsize, &addr) 1412 ) { 1413 /* 1414 * Uh oh. Buffer map is to fragmented. Try 1415 * to defragment. 1416 */ 1417 if (defrag <= 0) { 1418 defrag = 1; 1419 bp->b_flags |= B_INVAL; 1420 brelse(bp); 1421 goto restart; 1422 } 1423 /* 1424 * Uh oh. We couldn't seem to defragment 1425 */ 1426 bp = NULL; 1427 goto dosleep; 1428 } 1429 } 1430 if (addr) { 1431 vm_map_insert(buffer_map, NULL, 0, 1432 addr, addr + maxsize, 1433 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 1434 1435 bp->b_kvabase = (caddr_t) addr; 1436 bp->b_kvasize = maxsize; 1437 } 1438 bp->b_data = bp->b_kvabase; 1439 } 1440 1441 return (bp); 1442} 1443 1444/* 1445 * waitfreebuffers: 1446 * 1447 * Wait for sufficient free buffers. This routine is not called if 1448 * curproc is the update process so we do not have to do anything 1449 * fancy. 1450 */ 1451 1452static void 1453waitfreebuffers(int slpflag, int slptimeo) 1454{ 1455 while (numfreebuffers < hifreebuffers) { 1456 flushdirtybuffers(slpflag, slptimeo); 1457 if (numfreebuffers < hifreebuffers) 1458 break; 1459 needsbuffer |= VFS_BIO_NEED_FREE; 1460 if (tsleep(&needsbuffer, (PRIBIO + 4)|slpflag, "biofre", slptimeo)) 1461 break; 1462 } 1463} 1464 1465/* 1466 * flushdirtybuffers: 1467 * 1468 * This routine is called when we get too many dirty buffers. 1469 * 1470 * We have to protect ourselves from recursion, but we also do not want 1471 * other process's flushdirtybuffers() to interfere with the syncer if 1472 * it decides to flushdirtybuffers(). 1473 * 1474 * In order to maximize operations, we allow any process to flush 1475 * dirty buffers and use P_FLSINPROG to prevent recursion. 1476 */ 1477 1478static void 1479flushdirtybuffers(int slpflag, int slptimeo) 1480{ 1481 int s; 1482 1483 s = splbio(); 1484 1485 if (curproc->p_flag & P_FLSINPROG) { 1486 splx(s); 1487 return; 1488 } 1489 curproc->p_flag |= P_FLSINPROG; 1490 1491 while (numdirtybuffers > lodirtybuffers) { 1492 if (flushbufqueues() == 0) 1493 break; 1494 } 1495 1496 curproc->p_flag &= ~P_FLSINPROG; 1497 1498 splx(s); 1499} 1500 1501static int 1502flushbufqueues(void) 1503{ 1504 struct buf *bp; 1505 int qindex; 1506 int r = 0; 1507 1508 qindex = QUEUE_AGE; 1509 bp = TAILQ_FIRST(&bufqueues[QUEUE_AGE]); 1510 1511 for (;;) { 1512 if (bp == NULL) { 1513 if (qindex == QUEUE_LRU) 1514 break; 1515 qindex = QUEUE_LRU; 1516 if ((bp = TAILQ_FIRST(&bufqueues[QUEUE_LRU])) == NULL) 1517 break; 1518 } 1519 1520 /* 1521 * XXX NFS does weird things with B_INVAL bps if we bwrite 1522 * them ( vfs_bio_awrite/bawrite/bdwrite/etc ) Why? 1523 * 1524 */ 1525 if ((bp->b_flags & B_DELWRI) != 0) { 1526 if (bp->b_flags & B_INVAL) { 1527 bremfree(bp); 1528 bp->b_flags |= B_BUSY; 1529 brelse(bp); 1530 } else { 1531 vfs_bio_awrite(bp); 1532 } 1533 ++r; 1534 break; 1535 } 1536 bp = TAILQ_NEXT(bp, b_freelist); 1537 } 1538 return(r); 1539} 1540 1541/* 1542 * Check to see if a block is currently memory resident. 1543 */ 1544struct buf * 1545incore(struct vnode * vp, daddr_t blkno) 1546{ 1547 struct buf *bp; 1548 1549 int s = splbio(); 1550 bp = gbincore(vp, blkno); 1551 splx(s); 1552 return (bp); 1553} 1554 1555/* 1556 * Returns true if no I/O is needed to access the 1557 * associated VM object. This is like incore except 1558 * it also hunts around in the VM system for the data. 1559 */ 1560 1561int 1562inmem(struct vnode * vp, daddr_t blkno) 1563{ 1564 vm_object_t obj; 1565 vm_offset_t toff, tinc, size; 1566 vm_page_t m; 1567 vm_ooffset_t off; 1568 1569 if (incore(vp, blkno)) 1570 return 1; 1571 if (vp->v_mount == NULL) 1572 return 0; 1573 if ((vp->v_object == NULL) || (vp->v_flag & VOBJBUF) == 0) 1574 return 0; 1575 1576 obj = vp->v_object; 1577 size = PAGE_SIZE; 1578 if (size > vp->v_mount->mnt_stat.f_iosize) 1579 size = vp->v_mount->mnt_stat.f_iosize; 1580 off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize; 1581 1582 for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) { 1583 m = vm_page_lookup(obj, OFF_TO_IDX(off + toff)); 1584 if (!m) 1585 return 0; 1586 tinc = size; 1587 if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK)) 1588 tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK); 1589 if (vm_page_is_valid(m, 1590 (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0) 1591 return 0; 1592 } 1593 return 1; 1594} 1595 1596/* 1597 * now we set the dirty range for the buffer -- 1598 * for NFS -- if the file is mapped and pages have 1599 * been written to, let it know. We want the 1600 * entire range of the buffer to be marked dirty if 1601 * any of the pages have been written to for consistancy 1602 * with the b_validoff, b_validend set in the nfs write 1603 * code, and used by the nfs read code. 1604 */ 1605static void 1606vfs_setdirty(struct buf *bp) 1607{ 1608 int i; 1609 vm_object_t object; 1610 vm_offset_t boffset; 1611 1612 /* 1613 * We qualify the scan for modified pages on whether the 1614 * object has been flushed yet. The OBJ_WRITEABLE flag 1615 * is not cleared simply by protecting pages off. 1616 */ 1617 1618 if ((bp->b_flags & B_VMIO) == 0) 1619 return; 1620 1621 object = bp->b_pages[0]->object; 1622 1623 if ((object->flags & OBJ_WRITEABLE) && !(object->flags & OBJ_MIGHTBEDIRTY)) 1624 printf("Warning: object %p writeable but not mightbedirty\n", object); 1625 if (!(object->flags & OBJ_WRITEABLE) && (object->flags & OBJ_MIGHTBEDIRTY)) 1626 printf("Warning: object %p mightbedirty but not writeable\n", object); 1627 1628 if (object->flags & (OBJ_MIGHTBEDIRTY|OBJ_CLEANING)) { 1629 /* 1630 * test the pages to see if they have been modified directly 1631 * by users through the VM system. 1632 */ 1633 for (i = 0; i < bp->b_npages; i++) { 1634 vm_page_flag_clear(bp->b_pages[i], PG_ZERO); 1635 vm_page_test_dirty(bp->b_pages[i]); 1636 } 1637 1638 /* 1639 * scan forwards for the first page modified 1640 */ 1641 for (i = 0; i < bp->b_npages; i++) { 1642 if (bp->b_pages[i]->dirty) { 1643 break; 1644 } 1645 } 1646 1647 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); 1648 if (boffset < bp->b_dirtyoff) { 1649 bp->b_dirtyoff = max(boffset, 0); 1650 } 1651 1652 /* 1653 * scan backwards for the last page modified 1654 */ 1655 for (i = bp->b_npages - 1; i >= 0; --i) { 1656 if (bp->b_pages[i]->dirty) { 1657 break; 1658 } 1659 } 1660 boffset = (i + 1); 1661#if 0 1662 offset = boffset + bp->b_pages[0]->pindex; 1663 if (offset >= object->size) 1664 boffset = object->size - bp->b_pages[0]->pindex; 1665#endif 1666 boffset = (boffset << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK); 1667 if (bp->b_dirtyend < boffset) 1668 bp->b_dirtyend = min(boffset, bp->b_bufsize); 1669 } 1670} 1671 1672/* 1673 * Get a block given a specified block and offset into a file/device. 1674 */ 1675struct buf * 1676getblk(struct vnode * vp, daddr_t blkno, int size, int slpflag, int slptimeo) 1677{ 1678 struct buf *bp; 1679 int i, s; 1680 struct bufhashhdr *bh; 1681 1682#if !defined(MAX_PERF) 1683 if (size > MAXBSIZE) 1684 panic("getblk: size(%d) > MAXBSIZE(%d)\n", size, MAXBSIZE); 1685#endif 1686 1687 s = splbio(); 1688loop: 1689 /* 1690 * Block if we are low on buffers. The syncer is allowed more 1691 * buffers in order to avoid a deadlock. 1692 */ 1693 if (curproc == updateproc && numfreebuffers == 0) { 1694 needsbuffer |= VFS_BIO_NEED_ANY; 1695 tsleep(&needsbuffer, (PRIBIO + 4) | slpflag, "newbuf", 1696 slptimeo); 1697 } else if (curproc != updateproc && numfreebuffers < lofreebuffers) { 1698 waitfreebuffers(slpflag, slptimeo); 1699 } 1700 1701 if ((bp = gbincore(vp, blkno))) { 1702 if (bp->b_flags & B_BUSY) { 1703 bp->b_flags |= B_WANTED; 1704 if (bp->b_usecount < BUF_MAXUSE) 1705 ++bp->b_usecount; 1706 1707 if (!tsleep(bp, 1708 (PRIBIO + 4) | slpflag, "getblk", slptimeo)) { 1709 goto loop; 1710 } 1711 1712 splx(s); 1713 return (struct buf *) NULL; 1714 } 1715 bp->b_flags |= B_BUSY | B_CACHE; 1716 bremfree(bp); 1717 1718 /* 1719 * check for size inconsistancies for non-VMIO case. 1720 */ 1721 1722 if (bp->b_bcount != size) { 1723 if ((bp->b_flags & B_VMIO) == 0 || 1724 (size > bp->b_kvasize) 1725 ) { 1726 if (bp->b_flags & B_DELWRI) { 1727 bp->b_flags |= B_NOCACHE; 1728 VOP_BWRITE(bp); 1729 } else { 1730 if ((bp->b_flags & B_VMIO) && 1731 (LIST_FIRST(&bp->b_dep) == NULL)) { 1732 bp->b_flags |= B_RELBUF; 1733 brelse(bp); 1734 } else { 1735 bp->b_flags |= B_NOCACHE; 1736 VOP_BWRITE(bp); 1737 } 1738 } 1739 goto loop; 1740 } 1741 } 1742 1743 /* 1744 * If the size is inconsistant in the VMIO case, we can resize 1745 * the buffer. This might lead to B_CACHE getting cleared. 1746 */ 1747 1748 if (bp->b_bcount != size) 1749 allocbuf(bp, size); 1750 1751 KASSERT(bp->b_offset != NOOFFSET, 1752 ("getblk: no buffer offset")); 1753 1754 /* 1755 * Check that the constituted buffer really deserves for the 1756 * B_CACHE bit to be set. B_VMIO type buffers might not 1757 * contain fully valid pages. Normal (old-style) buffers 1758 * should be fully valid. This might also lead to B_CACHE 1759 * getting clear. 1760 * 1761 * If B_CACHE is already clear, don't bother checking to see 1762 * if we have to clear it again. 1763 * 1764 * XXX this code should not be necessary unless the B_CACHE 1765 * handling is broken elsewhere in the kernel. We need to 1766 * check the cases and then turn the clearing part of this 1767 * code into a panic. 1768 */ 1769 if ( 1770 (bp->b_flags & (B_VMIO|B_CACHE)) == (B_VMIO|B_CACHE) && 1771 (bp->b_vp->v_tag != VT_NFS || bp->b_validend <= 0) 1772 ) { 1773 int checksize = bp->b_bufsize; 1774 int poffset = bp->b_offset & PAGE_MASK; 1775 int resid; 1776 for (i = 0; i < bp->b_npages; i++) { 1777 resid = (checksize > (PAGE_SIZE - poffset)) ? 1778 (PAGE_SIZE - poffset) : checksize; 1779 if (!vm_page_is_valid(bp->b_pages[i], poffset, resid)) { 1780 bp->b_flags &= ~(B_CACHE | B_DONE); 1781 break; 1782 } 1783 checksize -= resid; 1784 poffset = 0; 1785 } 1786 } 1787 1788 /* 1789 * If B_DELWRI is set and B_CACHE got cleared ( or was 1790 * already clear ), we have to commit the write and 1791 * retry. The NFS code absolutely depends on this, 1792 * and so might the FFS code. In anycase, it formalizes 1793 * the B_CACHE rules. See sys/buf.h. 1794 */ 1795 1796 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) { 1797 VOP_BWRITE(bp); 1798 goto loop; 1799 } 1800 1801 if (bp->b_usecount < BUF_MAXUSE) 1802 ++bp->b_usecount; 1803 splx(s); 1804 return (bp); 1805 } else { 1806 int bsize, maxsize, vmio; 1807 off_t offset; 1808 1809 if (vp->v_type == VBLK) 1810 bsize = DEV_BSIZE; 1811 else if (vp->v_mountedhere) 1812 bsize = vp->v_mountedhere->mnt_stat.f_iosize; 1813 else if (vp->v_mount) 1814 bsize = vp->v_mount->mnt_stat.f_iosize; 1815 else 1816 bsize = size; 1817 1818 offset = (off_t)blkno * bsize; 1819 vmio = (vp->v_object != 0) && (vp->v_flag & VOBJBUF); 1820 maxsize = vmio ? size + (offset & PAGE_MASK) : size; 1821 maxsize = imax(maxsize, bsize); 1822 1823 if ((bp = getnewbuf(vp, blkno, 1824 slpflag, slptimeo, size, maxsize)) == 0) { 1825 if (slpflag || slptimeo) { 1826 splx(s); 1827 return NULL; 1828 } 1829 goto loop; 1830 } 1831 1832 /* 1833 * This code is used to make sure that a buffer is not 1834 * created while the getnewbuf routine is blocked. 1835 * This can be a problem whether the vnode is locked or not. 1836 */ 1837 if (gbincore(vp, blkno)) { 1838 bp->b_flags |= B_INVAL; 1839 brelse(bp); 1840 goto loop; 1841 } 1842 1843 /* 1844 * Insert the buffer into the hash, so that it can 1845 * be found by incore. 1846 */ 1847 bp->b_blkno = bp->b_lblkno = blkno; 1848 bp->b_offset = offset; 1849 1850 bgetvp(vp, bp); 1851 LIST_REMOVE(bp, b_hash); 1852 bh = BUFHASH(vp, blkno); 1853 LIST_INSERT_HEAD(bh, bp, b_hash); 1854 1855 if (vmio) { 1856 bp->b_flags |= (B_VMIO | B_CACHE); 1857#if defined(VFS_BIO_DEBUG) 1858 if (vp->v_type != VREG && vp->v_type != VBLK) 1859 printf("getblk: vmioing file type %d???\n", vp->v_type); 1860#endif 1861 } else { 1862 bp->b_flags &= ~B_VMIO; 1863 } 1864 1865 allocbuf(bp, size); 1866 1867 splx(s); 1868 return (bp); 1869 } 1870} 1871 1872/* 1873 * Get an empty, disassociated buffer of given size. 1874 */ 1875struct buf * 1876geteblk(int size) 1877{ 1878 struct buf *bp; 1879 int s; 1880 1881 s = splbio(); 1882 while ((bp = getnewbuf(0, (daddr_t) 0, 0, 0, size, MAXBSIZE)) == 0); 1883 splx(s); 1884 allocbuf(bp, size); 1885 bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */ 1886 return (bp); 1887} 1888 1889 1890/* 1891 * This code constitutes the buffer memory from either anonymous system 1892 * memory (in the case of non-VMIO operations) or from an associated 1893 * VM object (in the case of VMIO operations). This code is able to 1894 * resize a buffer up or down. 1895 * 1896 * Note that this code is tricky, and has many complications to resolve 1897 * deadlock or inconsistant data situations. Tread lightly!!! 1898 * There are B_CACHE and B_DELWRI interactions that must be dealt with by 1899 * the caller. Calling this code willy nilly can result in the loss of data. 1900 */ 1901 1902int 1903allocbuf(struct buf *bp, int size) 1904{ 1905 int newbsize, mbsize; 1906 int i; 1907 1908#if !defined(MAX_PERF) 1909 if (!(bp->b_flags & B_BUSY)) 1910 panic("allocbuf: buffer not busy"); 1911 1912 if (bp->b_kvasize < size) 1913 panic("allocbuf: buffer too small"); 1914#endif 1915 1916 if ((bp->b_flags & B_VMIO) == 0) { 1917 caddr_t origbuf; 1918 int origbufsize; 1919 /* 1920 * Just get anonymous memory from the kernel 1921 */ 1922 mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1); 1923#if !defined(NO_B_MALLOC) 1924 if (bp->b_flags & B_MALLOC) 1925 newbsize = mbsize; 1926 else 1927#endif 1928 newbsize = round_page(size); 1929 1930 if (newbsize < bp->b_bufsize) { 1931#if !defined(NO_B_MALLOC) 1932 /* 1933 * malloced buffers are not shrunk 1934 */ 1935 if (bp->b_flags & B_MALLOC) { 1936 if (newbsize) { 1937 bp->b_bcount = size; 1938 } else { 1939 free(bp->b_data, M_BIOBUF); 1940 bufspace -= bp->b_bufsize; 1941 bufmallocspace -= bp->b_bufsize; 1942 runningbufspace -= bp->b_bufsize; 1943 if (bp->b_bufsize) 1944 bufspacewakeup(); 1945 bp->b_data = bp->b_kvabase; 1946 bp->b_bufsize = 0; 1947 bp->b_bcount = 0; 1948 bp->b_flags &= ~B_MALLOC; 1949 } 1950 return 1; 1951 } 1952#endif 1953 vm_hold_free_pages( 1954 bp, 1955 (vm_offset_t) bp->b_data + newbsize, 1956 (vm_offset_t) bp->b_data + bp->b_bufsize); 1957 } else if (newbsize > bp->b_bufsize) { 1958#if !defined(NO_B_MALLOC) 1959 /* 1960 * We only use malloced memory on the first allocation. 1961 * and revert to page-allocated memory when the buffer grows. 1962 */ 1963 if ( (bufmallocspace < maxbufmallocspace) && 1964 (bp->b_bufsize == 0) && 1965 (mbsize <= PAGE_SIZE/2)) { 1966 1967 bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK); 1968 bp->b_bufsize = mbsize; 1969 bp->b_bcount = size; 1970 bp->b_flags |= B_MALLOC; 1971 bufspace += mbsize; 1972 bufmallocspace += mbsize; 1973 runningbufspace += bp->b_bufsize; 1974 return 1; 1975 } 1976#endif 1977 origbuf = NULL; 1978 origbufsize = 0; 1979#if !defined(NO_B_MALLOC) 1980 /* 1981 * If the buffer is growing on its other-than-first allocation, 1982 * then we revert to the page-allocation scheme. 1983 */ 1984 if (bp->b_flags & B_MALLOC) { 1985 origbuf = bp->b_data; 1986 origbufsize = bp->b_bufsize; 1987 bp->b_data = bp->b_kvabase; 1988 bufspace -= bp->b_bufsize; 1989 bufmallocspace -= bp->b_bufsize; 1990 runningbufspace -= bp->b_bufsize; 1991 if (bp->b_bufsize) 1992 bufspacewakeup(); 1993 bp->b_bufsize = 0; 1994 bp->b_flags &= ~B_MALLOC; 1995 newbsize = round_page(newbsize); 1996 } 1997#endif 1998 vm_hold_load_pages( 1999 bp, 2000 (vm_offset_t) bp->b_data + bp->b_bufsize, 2001 (vm_offset_t) bp->b_data + newbsize); 2002#if !defined(NO_B_MALLOC) 2003 if (origbuf) { 2004 bcopy(origbuf, bp->b_data, origbufsize); 2005 free(origbuf, M_BIOBUF); 2006 } 2007#endif 2008 } 2009 } else { 2010 vm_page_t m; 2011 int desiredpages; 2012 2013 newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1); 2014 desiredpages = (size == 0) ? 0 : 2015 num_pages((bp->b_offset & PAGE_MASK) + newbsize); 2016 2017#if !defined(NO_B_MALLOC) 2018 if (bp->b_flags & B_MALLOC) 2019 panic("allocbuf: VMIO buffer can't be malloced"); 2020#endif 2021 2022 if (newbsize < bp->b_bufsize) { 2023 if (desiredpages < bp->b_npages) { 2024 for (i = desiredpages; i < bp->b_npages; i++) { 2025 /* 2026 * the page is not freed here -- it 2027 * is the responsibility of vnode_pager_setsize 2028 */ 2029 m = bp->b_pages[i]; 2030 KASSERT(m != bogus_page, 2031 ("allocbuf: bogus page found")); 2032 while (vm_page_sleep_busy(m, TRUE, "biodep")) 2033 ; 2034 2035 bp->b_pages[i] = NULL; 2036 vm_page_unwire(m, 0); 2037 } 2038 pmap_qremove((vm_offset_t) trunc_page((vm_offset_t)bp->b_data) + 2039 (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages)); 2040 bp->b_npages = desiredpages; 2041 } 2042 } else if (newbsize > bp->b_bufsize) { 2043 vm_object_t obj; 2044 vm_offset_t tinc, toff; 2045 vm_ooffset_t off; 2046 vm_pindex_t objoff; 2047 int pageindex, curbpnpages; 2048 struct vnode *vp; 2049 int bsize; 2050 int orig_validoff = bp->b_validoff; 2051 int orig_validend = bp->b_validend; 2052 2053 vp = bp->b_vp; 2054 2055 if (vp->v_type == VBLK) 2056 bsize = DEV_BSIZE; 2057 else 2058 bsize = vp->v_mount->mnt_stat.f_iosize; 2059 2060 if (bp->b_npages < desiredpages) { 2061 obj = vp->v_object; 2062 tinc = PAGE_SIZE; 2063 2064 off = bp->b_offset; 2065 KASSERT(bp->b_offset != NOOFFSET, 2066 ("allocbuf: no buffer offset")); 2067 curbpnpages = bp->b_npages; 2068 doretry: 2069 bp->b_validoff = orig_validoff; 2070 bp->b_validend = orig_validend; 2071 bp->b_flags |= B_CACHE; 2072 for (toff = 0; toff < newbsize; toff += tinc) { 2073 objoff = OFF_TO_IDX(off + toff); 2074 pageindex = objoff - OFF_TO_IDX(off); 2075 tinc = PAGE_SIZE - ((off + toff) & PAGE_MASK); 2076 if (pageindex < curbpnpages) { 2077 2078 m = bp->b_pages[pageindex]; 2079#ifdef VFS_BIO_DIAG 2080 if (m->pindex != objoff) 2081 panic("allocbuf: page changed offset?!!!?"); 2082#endif 2083 if (tinc > (newbsize - toff)) 2084 tinc = newbsize - toff; 2085 if (bp->b_flags & B_CACHE) 2086 vfs_buf_set_valid(bp, off, toff, tinc, m); 2087 continue; 2088 } 2089 m = vm_page_lookup(obj, objoff); 2090 if (!m) { 2091 m = vm_page_alloc(obj, objoff, VM_ALLOC_NORMAL); 2092 if (!m) { 2093 VM_WAIT; 2094 vm_pageout_deficit += (desiredpages - curbpnpages); 2095 goto doretry; 2096 } 2097 2098 vm_page_wire(m); 2099 vm_page_wakeup(m); 2100 bp->b_flags &= ~B_CACHE; 2101 2102 } else if (vm_page_sleep_busy(m, FALSE, "pgtblk")) { 2103 /* 2104 * If we had to sleep, retry. 2105 * 2106 * Also note that we only test 2107 * PG_BUSY here, not m->busy. 2108 * 2109 * We cannot sleep on m->busy 2110 * here because a vm_fault -> 2111 * getpages -> cluster-read -> 2112 * ...-> allocbuf sequence 2113 * will convert PG_BUSY to 2114 * m->busy so we have to let 2115 * m->busy through if we do 2116 * not want to deadlock. 2117 */ 2118 goto doretry; 2119 } else { 2120 if ((curproc != pageproc) && 2121 ((m->queue - m->pc) == PQ_CACHE) && 2122 ((cnt.v_free_count + cnt.v_cache_count) < 2123 (cnt.v_free_min + cnt.v_cache_min))) { 2124 pagedaemon_wakeup(); 2125 } 2126 if (tinc > (newbsize - toff)) 2127 tinc = newbsize - toff; 2128 if (bp->b_flags & B_CACHE) 2129 vfs_buf_set_valid(bp, off, toff, tinc, m); 2130 vm_page_flag_clear(m, PG_ZERO); 2131 vm_page_wire(m); 2132 } 2133 bp->b_pages[pageindex] = m; 2134 curbpnpages = pageindex + 1; 2135 } 2136 if (vp->v_tag == VT_NFS && 2137 vp->v_type != VBLK) { 2138 if (bp->b_dirtyend > 0) { 2139 bp->b_validoff = min(bp->b_validoff, bp->b_dirtyoff); 2140 bp->b_validend = max(bp->b_validend, bp->b_dirtyend); 2141 } 2142 if (bp->b_validend == 0) 2143 bp->b_flags &= ~B_CACHE; 2144 } 2145 bp->b_data = (caddr_t) trunc_page((vm_offset_t)bp->b_data); 2146 bp->b_npages = curbpnpages; 2147 pmap_qenter((vm_offset_t) bp->b_data, 2148 bp->b_pages, bp->b_npages); 2149 ((vm_offset_t) bp->b_data) |= off & PAGE_MASK; 2150 } 2151 } 2152 } 2153 if (bp->b_flags & B_VMIO) 2154 vmiospace += (newbsize - bp->b_bufsize); 2155 bufspace += (newbsize - bp->b_bufsize); 2156 runningbufspace += (newbsize - bp->b_bufsize); 2157 if (newbsize < bp->b_bufsize) 2158 bufspacewakeup(); 2159 bp->b_bufsize = newbsize; 2160 bp->b_bcount = size; 2161 return 1; 2162} 2163 2164/* 2165 * Wait for buffer I/O completion, returning error status. 2166 */ 2167int 2168biowait(register struct buf * bp) 2169{ 2170 int s; 2171 2172 s = splbio(); 2173 while ((bp->b_flags & B_DONE) == 0) 2174#if defined(NO_SCHEDULE_MODS) 2175 tsleep(bp, PRIBIO, "biowait", 0); 2176#else 2177 if (bp->b_flags & B_READ) 2178 tsleep(bp, PRIBIO, "biord", 0); 2179 else 2180 tsleep(bp, PRIBIO, "biowr", 0); 2181#endif 2182 splx(s); 2183 if (bp->b_flags & B_EINTR) { 2184 bp->b_flags &= ~B_EINTR; 2185 return (EINTR); 2186 } 2187 if (bp->b_flags & B_ERROR) { 2188 return (bp->b_error ? bp->b_error : EIO); 2189 } else { 2190 return (0); 2191 } 2192} 2193 2194/* 2195 * Finish I/O on a buffer, calling an optional function. 2196 * This is usually called from interrupt level, so process blocking 2197 * is not *a good idea*. 2198 */ 2199void 2200biodone(register struct buf * bp) 2201{ 2202 int s; 2203 2204 s = splbio(); 2205 2206 KASSERT((bp->b_flags & B_BUSY), ("biodone: bp %p not busy", bp)); 2207 KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp)); 2208 2209 bp->b_flags |= B_DONE; 2210 2211 if (bp->b_flags & B_FREEBUF) { 2212 brelse(bp); 2213 splx(s); 2214 return; 2215 } 2216 2217 if ((bp->b_flags & B_READ) == 0) { 2218 vwakeup(bp); 2219 } 2220 2221 /* call optional completion function if requested */ 2222 if (bp->b_flags & B_CALL) { 2223 bp->b_flags &= ~B_CALL; 2224 (*bp->b_iodone) (bp); 2225 splx(s); 2226 return; 2227 } 2228 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete) 2229 (*bioops.io_complete)(bp); 2230 2231 if (bp->b_flags & B_VMIO) { 2232 int i, resid; 2233 vm_ooffset_t foff; 2234 vm_page_t m; 2235 vm_object_t obj; 2236 int iosize; 2237 struct vnode *vp = bp->b_vp; 2238 2239 obj = vp->v_object; 2240 2241#if defined(VFS_BIO_DEBUG) 2242 if (vp->v_usecount == 0) { 2243 panic("biodone: zero vnode ref count"); 2244 } 2245 2246 if (vp->v_object == NULL) { 2247 panic("biodone: missing VM object"); 2248 } 2249 2250 if ((vp->v_flag & VOBJBUF) == 0) { 2251 panic("biodone: vnode is not setup for merged cache"); 2252 } 2253#endif 2254 2255 foff = bp->b_offset; 2256 KASSERT(bp->b_offset != NOOFFSET, 2257 ("biodone: no buffer offset")); 2258 2259#if !defined(MAX_PERF) 2260 if (!obj) { 2261 panic("biodone: no object"); 2262 } 2263#endif 2264#if defined(VFS_BIO_DEBUG) 2265 if (obj->paging_in_progress < bp->b_npages) { 2266 printf("biodone: paging in progress(%d) < bp->b_npages(%d)\n", 2267 obj->paging_in_progress, bp->b_npages); 2268 } 2269#endif 2270 iosize = bp->b_bufsize; 2271 for (i = 0; i < bp->b_npages; i++) { 2272 int bogusflag = 0; 2273 m = bp->b_pages[i]; 2274 if (m == bogus_page) { 2275 bogusflag = 1; 2276 m = vm_page_lookup(obj, OFF_TO_IDX(foff)); 2277 if (!m) { 2278#if defined(VFS_BIO_DEBUG) 2279 printf("biodone: page disappeared\n"); 2280#endif 2281 vm_object_pip_subtract(obj, 1); 2282 continue; 2283 } 2284 bp->b_pages[i] = m; 2285 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages); 2286 } 2287#if defined(VFS_BIO_DEBUG) 2288 if (OFF_TO_IDX(foff) != m->pindex) { 2289 printf( 2290"biodone: foff(%lu)/m->pindex(%d) mismatch\n", 2291 (unsigned long)foff, m->pindex); 2292 } 2293#endif 2294 resid = IDX_TO_OFF(m->pindex + 1) - foff; 2295 if (resid > iosize) 2296 resid = iosize; 2297 2298 /* 2299 * In the write case, the valid and clean bits are 2300 * already changed correctly, so we only need to do this 2301 * here in the read case. 2302 */ 2303 if ((bp->b_flags & B_READ) && !bogusflag && resid > 0) { 2304 vfs_page_set_valid(bp, foff, i, m); 2305 } 2306 vm_page_flag_clear(m, PG_ZERO); 2307 2308 /* 2309 * when debugging new filesystems or buffer I/O methods, this 2310 * is the most common error that pops up. if you see this, you 2311 * have not set the page busy flag correctly!!! 2312 */ 2313 if (m->busy == 0) { 2314#if !defined(MAX_PERF) 2315 printf("biodone: page busy < 0, " 2316 "pindex: %d, foff: 0x(%x,%x), " 2317 "resid: %d, index: %d\n", 2318 (int) m->pindex, (int)(foff >> 32), 2319 (int) foff & 0xffffffff, resid, i); 2320#endif 2321 if (vp->v_type != VBLK) 2322#if !defined(MAX_PERF) 2323 printf(" iosize: %ld, lblkno: %d, flags: 0x%lx, npages: %d\n", 2324 bp->b_vp->v_mount->mnt_stat.f_iosize, 2325 (int) bp->b_lblkno, 2326 bp->b_flags, bp->b_npages); 2327 else 2328 printf(" VDEV, lblkno: %d, flags: 0x%lx, npages: %d\n", 2329 (int) bp->b_lblkno, 2330 bp->b_flags, bp->b_npages); 2331 printf(" valid: 0x%x, dirty: 0x%x, wired: %d\n", 2332 m->valid, m->dirty, m->wire_count); 2333#endif 2334 panic("biodone: page busy < 0\n"); 2335 } 2336 vm_page_io_finish(m); 2337 vm_object_pip_subtract(obj, 1); 2338 foff += resid; 2339 iosize -= resid; 2340 } 2341 if (obj) 2342 vm_object_pip_wakeupn(obj, 0); 2343 } 2344 /* 2345 * For asynchronous completions, release the buffer now. The brelse 2346 * checks for B_WANTED and will do the wakeup there if necessary - so 2347 * no need to do a wakeup here in the async case. 2348 */ 2349 2350 if (bp->b_flags & B_ASYNC) { 2351 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_ERROR | B_RELBUF)) != 0) 2352 brelse(bp); 2353 else 2354 bqrelse(bp); 2355 } else { 2356 bp->b_flags &= ~B_WANTED; 2357 wakeup(bp); 2358 } 2359 splx(s); 2360} 2361 2362#if 0 /* not with kirks code */ 2363static int vfs_update_interval = 30; 2364 2365static void 2366vfs_update() 2367{ 2368 while (1) { 2369 tsleep(&vfs_update_wakeup, PUSER, "update", 2370 hz * vfs_update_interval); 2371 vfs_update_wakeup = 0; 2372 sync(curproc, NULL); 2373 } 2374} 2375 2376static int 2377sysctl_kern_updateinterval SYSCTL_HANDLER_ARGS 2378{ 2379 int error = sysctl_handle_int(oidp, 2380 oidp->oid_arg1, oidp->oid_arg2, req); 2381 if (!error) 2382 wakeup(&vfs_update_wakeup); 2383 return error; 2384} 2385 2386SYSCTL_PROC(_kern, KERN_UPDATEINTERVAL, update, CTLTYPE_INT|CTLFLAG_RW, 2387 &vfs_update_interval, 0, sysctl_kern_updateinterval, "I", ""); 2388 2389#endif 2390 2391 2392/* 2393 * This routine is called in lieu of iodone in the case of 2394 * incomplete I/O. This keeps the busy status for pages 2395 * consistant. 2396 */ 2397void 2398vfs_unbusy_pages(struct buf * bp) 2399{ 2400 int i; 2401 2402 if (bp->b_flags & B_VMIO) { 2403 struct vnode *vp = bp->b_vp; 2404 vm_object_t obj = vp->v_object; 2405 2406 for (i = 0; i < bp->b_npages; i++) { 2407 vm_page_t m = bp->b_pages[i]; 2408 2409 if (m == bogus_page) { 2410 m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i); 2411#if !defined(MAX_PERF) 2412 if (!m) { 2413 panic("vfs_unbusy_pages: page missing\n"); 2414 } 2415#endif 2416 bp->b_pages[i] = m; 2417 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages); 2418 } 2419 vm_object_pip_subtract(obj, 1); 2420 vm_page_flag_clear(m, PG_ZERO); 2421 vm_page_io_finish(m); 2422 } 2423 vm_object_pip_wakeupn(obj, 0); 2424 } 2425} 2426 2427/* 2428 * Set NFS' b_validoff and b_validend fields from the valid bits 2429 * of a page. If the consumer is not NFS, and the page is not 2430 * valid for the entire range, clear the B_CACHE flag to force 2431 * the consumer to re-read the page. 2432 * 2433 * B_CACHE interaction is especially tricky. 2434 */ 2435static void 2436vfs_buf_set_valid(struct buf *bp, 2437 vm_ooffset_t foff, vm_offset_t off, vm_offset_t size, 2438 vm_page_t m) 2439{ 2440 if (bp->b_vp->v_tag == VT_NFS && bp->b_vp->v_type != VBLK) { 2441 vm_offset_t svalid, evalid; 2442 int validbits = m->valid >> (((foff+off)&PAGE_MASK)/DEV_BSIZE); 2443 2444 /* 2445 * This only bothers with the first valid range in the 2446 * page. 2447 */ 2448 svalid = off; 2449 while (validbits && !(validbits & 1)) { 2450 svalid += DEV_BSIZE; 2451 validbits >>= 1; 2452 } 2453 evalid = svalid; 2454 while (validbits & 1) { 2455 evalid += DEV_BSIZE; 2456 validbits >>= 1; 2457 } 2458 evalid = min(evalid, off + size); 2459 /* 2460 * We can only set b_validoff/end if this range is contiguous 2461 * with the range built up already. If we cannot set 2462 * b_validoff/end, we must clear B_CACHE to force an update 2463 * to clean the bp up. 2464 */ 2465 if (svalid == bp->b_validend) { 2466 bp->b_validoff = min(bp->b_validoff, svalid); 2467 bp->b_validend = max(bp->b_validend, evalid); 2468 } else { 2469 bp->b_flags &= ~B_CACHE; 2470 } 2471 } else if (!vm_page_is_valid(m, 2472 (vm_offset_t) ((foff + off) & PAGE_MASK), 2473 size)) { 2474 bp->b_flags &= ~B_CACHE; 2475 } 2476} 2477 2478/* 2479 * Set the valid bits in a page, taking care of the b_validoff, 2480 * b_validend fields which NFS uses to optimise small reads. Off is 2481 * the offset within the file and pageno is the page index within the buf. 2482 * 2483 * XXX we have to set the valid & clean bits for all page fragments 2484 * touched by b_validoff/validend, even if the page fragment goes somewhat 2485 * beyond b_validoff/validend due to alignment. 2486 */ 2487static void 2488vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, int pageno, vm_page_t m) 2489{ 2490 struct vnode *vp = bp->b_vp; 2491 vm_ooffset_t soff, eoff; 2492 2493 /* 2494 * Start and end offsets in buffer. eoff - soff may not cross a 2495 * page boundry or cross the end of the buffer. 2496 */ 2497 soff = off; 2498 eoff = (off + PAGE_SIZE) & ~PAGE_MASK; 2499 if (eoff > bp->b_offset + bp->b_bufsize) 2500 eoff = bp->b_offset + bp->b_bufsize; 2501 2502 if (vp->v_tag == VT_NFS && vp->v_type != VBLK) { 2503 vm_ooffset_t sv, ev; 2504 vm_page_set_invalid(m, 2505 (vm_offset_t) (soff & PAGE_MASK), 2506 (vm_offset_t) (eoff - soff)); 2507 /* 2508 * bp->b_validoff and bp->b_validend restrict the valid range 2509 * that we can set. Note that these offsets are not DEV_BSIZE 2510 * aligned. vm_page_set_validclean() must know what 2511 * sub-DEV_BSIZE ranges to clear. 2512 */ 2513#if 0 2514 sv = (bp->b_offset + bp->b_validoff + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1); 2515 ev = (bp->b_offset + bp->b_validend + (DEV_BSIZE - 1)) & 2516 ~(DEV_BSIZE - 1); 2517#endif 2518 sv = bp->b_offset + bp->b_validoff; 2519 ev = bp->b_offset + bp->b_validend; 2520 soff = qmax(sv, soff); 2521 eoff = qmin(ev, eoff); 2522 } 2523 2524 if (eoff > soff) 2525 vm_page_set_validclean(m, 2526 (vm_offset_t) (soff & PAGE_MASK), 2527 (vm_offset_t) (eoff - soff)); 2528} 2529 2530/* 2531 * This routine is called before a device strategy routine. 2532 * It is used to tell the VM system that paging I/O is in 2533 * progress, and treat the pages associated with the buffer 2534 * almost as being PG_BUSY. Also the object paging_in_progress 2535 * flag is handled to make sure that the object doesn't become 2536 * inconsistant. 2537 */ 2538void 2539vfs_busy_pages(struct buf * bp, int clear_modify) 2540{ 2541 int i, bogus; 2542 2543 if (bp->b_flags & B_VMIO) { 2544 struct vnode *vp = bp->b_vp; 2545 vm_object_t obj = vp->v_object; 2546 vm_ooffset_t foff; 2547 2548 foff = bp->b_offset; 2549 KASSERT(bp->b_offset != NOOFFSET, 2550 ("vfs_busy_pages: no buffer offset")); 2551 vfs_setdirty(bp); 2552 2553retry: 2554 for (i = 0; i < bp->b_npages; i++) { 2555 vm_page_t m = bp->b_pages[i]; 2556 if (vm_page_sleep_busy(m, FALSE, "vbpage")) 2557 goto retry; 2558 } 2559 2560 bogus = 0; 2561 for (i = 0; i < bp->b_npages; i++) { 2562 vm_page_t m = bp->b_pages[i]; 2563 2564 vm_page_flag_clear(m, PG_ZERO); 2565 if ((bp->b_flags & B_CLUSTER) == 0) { 2566 vm_object_pip_add(obj, 1); 2567 vm_page_io_start(m); 2568 } 2569 2570 vm_page_protect(m, VM_PROT_NONE); 2571 if (clear_modify) 2572 vfs_page_set_valid(bp, foff, i, m); 2573 else if (m->valid == VM_PAGE_BITS_ALL && 2574 (bp->b_flags & B_CACHE) == 0) { 2575 bp->b_pages[i] = bogus_page; 2576 bogus++; 2577 } 2578 foff = (foff + PAGE_SIZE) & ~PAGE_MASK; 2579 } 2580 if (bogus) 2581 pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages); 2582 } 2583} 2584 2585/* 2586 * Tell the VM system that the pages associated with this buffer 2587 * are clean. This is used for delayed writes where the data is 2588 * going to go to disk eventually without additional VM intevention. 2589 */ 2590void 2591vfs_clean_pages(struct buf * bp) 2592{ 2593 int i; 2594 2595 if (bp->b_flags & B_VMIO) { 2596 vm_ooffset_t foff; 2597 foff = bp->b_offset; 2598 KASSERT(bp->b_offset != NOOFFSET, 2599 ("vfs_clean_pages: no buffer offset")); 2600 for (i = 0; i < bp->b_npages; i++) { 2601 vm_page_t m = bp->b_pages[i]; 2602 vfs_page_set_valid(bp, foff, i, m); 2603 foff = (foff + PAGE_SIZE) & ~PAGE_MASK; 2604 } 2605 } 2606} 2607 2608void 2609vfs_bio_clrbuf(struct buf *bp) { 2610 int i, mask = 0; 2611 caddr_t sa, ea; 2612 if ((bp->b_flags & (B_VMIO | B_MALLOC)) == B_VMIO) { 2613 if( (bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && 2614 (bp->b_offset & PAGE_MASK) == 0) { 2615 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1; 2616 if (((bp->b_pages[0]->flags & PG_ZERO) == 0) && 2617 ((bp->b_pages[0]->valid & mask) != mask)) { 2618 bzero(bp->b_data, bp->b_bufsize); 2619 } 2620 bp->b_pages[0]->valid |= mask; 2621 bp->b_resid = 0; 2622 return; 2623 } 2624 ea = sa = bp->b_data; 2625 for(i=0;i<bp->b_npages;i++,sa=ea) { 2626 int j = ((u_long)sa & PAGE_MASK) / DEV_BSIZE; 2627 ea = (caddr_t)trunc_page((vm_offset_t)sa + PAGE_SIZE); 2628 ea = (caddr_t)ulmin((u_long)ea, 2629 (u_long)bp->b_data + bp->b_bufsize); 2630 mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j; 2631 if ((bp->b_pages[i]->valid & mask) == mask) 2632 continue; 2633 if ((bp->b_pages[i]->valid & mask) == 0) { 2634 if ((bp->b_pages[i]->flags & PG_ZERO) == 0) { 2635 bzero(sa, ea - sa); 2636 } 2637 } else { 2638 for (; sa < ea; sa += DEV_BSIZE, j++) { 2639 if (((bp->b_pages[i]->flags & PG_ZERO) == 0) && 2640 (bp->b_pages[i]->valid & (1<<j)) == 0) 2641 bzero(sa, DEV_BSIZE); 2642 } 2643 } 2644 bp->b_pages[i]->valid |= mask; 2645 vm_page_flag_clear(bp->b_pages[i], PG_ZERO); 2646 } 2647 bp->b_resid = 0; 2648 } else { 2649 clrbuf(bp); 2650 } 2651} 2652 2653/* 2654 * vm_hold_load_pages and vm_hold_unload pages get pages into 2655 * a buffers address space. The pages are anonymous and are 2656 * not associated with a file object. 2657 */ 2658void 2659vm_hold_load_pages(struct buf * bp, vm_offset_t from, vm_offset_t to) 2660{ 2661 vm_offset_t pg; 2662 vm_page_t p; 2663 int index; 2664 2665 to = round_page(to); 2666 from = round_page(from); 2667 index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT; 2668 2669 for (pg = from; pg < to; pg += PAGE_SIZE, index++) { 2670 2671tryagain: 2672 2673 p = vm_page_alloc(kernel_object, 2674 ((pg - VM_MIN_KERNEL_ADDRESS) >> PAGE_SHIFT), 2675 VM_ALLOC_NORMAL); 2676 if (!p) { 2677 vm_pageout_deficit += (to - from) >> PAGE_SHIFT; 2678 VM_WAIT; 2679 goto tryagain; 2680 } 2681 vm_page_wire(p); 2682 p->valid = VM_PAGE_BITS_ALL; 2683 vm_page_flag_clear(p, PG_ZERO); 2684 pmap_kenter(pg, VM_PAGE_TO_PHYS(p)); 2685 bp->b_pages[index] = p; 2686 vm_page_wakeup(p); 2687 } 2688 bp->b_npages = index; 2689} 2690 2691void 2692vm_hold_free_pages(struct buf * bp, vm_offset_t from, vm_offset_t to) 2693{ 2694 vm_offset_t pg; 2695 vm_page_t p; 2696 int index, newnpages; 2697 2698 from = round_page(from); 2699 to = round_page(to); 2700 newnpages = index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT; 2701 2702 for (pg = from; pg < to; pg += PAGE_SIZE, index++) { 2703 p = bp->b_pages[index]; 2704 if (p && (index < bp->b_npages)) { 2705#if !defined(MAX_PERF) 2706 if (p->busy) { 2707 printf("vm_hold_free_pages: blkno: %d, lblkno: %d\n", 2708 bp->b_blkno, bp->b_lblkno); 2709 } 2710#endif 2711 bp->b_pages[index] = NULL; 2712 pmap_kremove(pg); 2713 vm_page_busy(p); 2714 vm_page_unwire(p, 0); 2715 vm_page_free(p); 2716 } 2717 } 2718 bp->b_npages = newnpages; 2719} 2720 2721 2722#include "opt_ddb.h" 2723#ifdef DDB 2724#include <ddb/ddb.h> 2725 2726DB_SHOW_COMMAND(buffer, db_show_buffer) 2727{ 2728 /* get args */ 2729 struct buf *bp = (struct buf *)addr; 2730 2731 if (!have_addr) { 2732 db_printf("usage: show buffer <addr>\n"); 2733 return; 2734 } 2735 2736 db_printf("b_proc = %p,\nb_flags = 0x%b\n", (void *)bp->b_proc, 2737 (u_int)bp->b_flags, PRINT_BUF_FLAGS); 2738 db_printf("b_error = %d, b_bufsize = %ld, b_bcount = %ld, " 2739 "b_resid = %ld\nb_dev = 0x%x, b_data = %p, " 2740 "b_blkno = %d, b_pblkno = %d\n", 2741 bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid, 2742 bp->b_dev, bp->b_data, bp->b_blkno, bp->b_pblkno); 2743 if (bp->b_npages) { 2744 int i; 2745 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages); 2746 for (i = 0; i < bp->b_npages; i++) { 2747 vm_page_t m; 2748 m = bp->b_pages[i]; 2749 db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object, 2750 (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m)); 2751 if ((i + 1) < bp->b_npages) 2752 db_printf(","); 2753 } 2754 db_printf("\n"); 2755 } 2756} 2757#endif /* DDB */ 2758