vm_phys.c revision 276546
1/*- 2 * Copyright (c) 2002-2006 Rice University 3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4 * All rights reserved. 5 * 6 * This software was developed for the FreeBSD Project by Alan L. Cox, 7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 25 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 28 * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Physical memory system implementation 34 * 35 * Any external functions defined by this module are only to be used by the 36 * virtual memory system. 37 */ 38 39#include <sys/cdefs.h> 40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 276546 2015-01-02 17:45:52Z alc $"); 41 42#include "opt_ddb.h" 43#include "opt_vm.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/lock.h> 48#include <sys/kernel.h> 49#include <sys/malloc.h> 50#include <sys/mutex.h> 51#if MAXMEMDOM > 1 52#include <sys/proc.h> 53#endif 54#include <sys/queue.h> 55#include <sys/sbuf.h> 56#include <sys/sysctl.h> 57#include <sys/vmmeter.h> 58 59#include <ddb/ddb.h> 60 61#include <vm/vm.h> 62#include <vm/vm_param.h> 63#include <vm/vm_kern.h> 64#include <vm/vm_object.h> 65#include <vm/vm_page.h> 66#include <vm/vm_phys.h> 67 68_Static_assert(sizeof(long) * NBBY >= VM_PHYSSEG_MAX, 69 "Too many physsegs."); 70 71struct mem_affinity *mem_affinity; 72 73int vm_ndomains = 1; 74 75struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX]; 76int vm_phys_nsegs; 77 78#define VM_PHYS_FICTITIOUS_NSEGS 8 79static struct vm_phys_fictitious_seg { 80 vm_paddr_t start; 81 vm_paddr_t end; 82 vm_page_t first_page; 83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS]; 84static struct mtx vm_phys_fictitious_reg_mtx; 85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); 86 87static struct vm_freelist 88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 89 90static int vm_nfreelists = VM_FREELIST_DEFAULT + 1; 91 92static int cnt_prezero; 93SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD, 94 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time"); 95 96static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 97SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 98 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 99 100static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 101SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 102 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 103 104SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, 105 &vm_ndomains, 0, "Number of physical memory domains available."); 106 107static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, 108 int order); 109static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, 110 int domain); 111static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind); 112static int vm_phys_paddr_to_segind(vm_paddr_t pa); 113static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 114 int order); 115 116static __inline int 117vm_rr_selectdomain(void) 118{ 119#if MAXMEMDOM > 1 120 struct thread *td; 121 122 td = curthread; 123 124 td->td_dom_rr_idx++; 125 td->td_dom_rr_idx %= vm_ndomains; 126 return (td->td_dom_rr_idx); 127#else 128 return (0); 129#endif 130} 131 132boolean_t 133vm_phys_domain_intersects(long mask, vm_paddr_t low, vm_paddr_t high) 134{ 135 struct vm_phys_seg *s; 136 int idx; 137 138 while ((idx = ffsl(mask)) != 0) { 139 idx--; /* ffsl counts from 1 */ 140 mask &= ~(1UL << idx); 141 s = &vm_phys_segs[idx]; 142 if (low < s->end && high > s->start) 143 return (TRUE); 144 } 145 return (FALSE); 146} 147 148/* 149 * Outputs the state of the physical memory allocator, specifically, 150 * the amount of physical memory in each free list. 151 */ 152static int 153sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS) 154{ 155 struct sbuf sbuf; 156 struct vm_freelist *fl; 157 int dom, error, flind, oind, pind; 158 159 error = sysctl_wire_old_buffer(req, 0); 160 if (error != 0) 161 return (error); 162 sbuf_new_for_sysctl(&sbuf, NULL, 128 * vm_ndomains, req); 163 for (dom = 0; dom < vm_ndomains; dom++) { 164 sbuf_printf(&sbuf,"\nDOMAIN %d:\n", dom); 165 for (flind = 0; flind < vm_nfreelists; flind++) { 166 sbuf_printf(&sbuf, "\nFREE LIST %d:\n" 167 "\n ORDER (SIZE) | NUMBER" 168 "\n ", flind); 169 for (pind = 0; pind < VM_NFREEPOOL; pind++) 170 sbuf_printf(&sbuf, " | POOL %d", pind); 171 sbuf_printf(&sbuf, "\n-- "); 172 for (pind = 0; pind < VM_NFREEPOOL; pind++) 173 sbuf_printf(&sbuf, "-- -- "); 174 sbuf_printf(&sbuf, "--\n"); 175 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 176 sbuf_printf(&sbuf, " %2d (%6dK)", oind, 177 1 << (PAGE_SHIFT - 10 + oind)); 178 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 179 fl = vm_phys_free_queues[dom][flind][pind]; 180 sbuf_printf(&sbuf, " | %6d", 181 fl[oind].lcnt); 182 } 183 sbuf_printf(&sbuf, "\n"); 184 } 185 } 186 } 187 error = sbuf_finish(&sbuf); 188 sbuf_delete(&sbuf); 189 return (error); 190} 191 192/* 193 * Outputs the set of physical memory segments. 194 */ 195static int 196sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS) 197{ 198 struct sbuf sbuf; 199 struct vm_phys_seg *seg; 200 int error, segind; 201 202 error = sysctl_wire_old_buffer(req, 0); 203 if (error != 0) 204 return (error); 205 sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 206 for (segind = 0; segind < vm_phys_nsegs; segind++) { 207 sbuf_printf(&sbuf, "\nSEGMENT %d:\n\n", segind); 208 seg = &vm_phys_segs[segind]; 209 sbuf_printf(&sbuf, "start: %#jx\n", 210 (uintmax_t)seg->start); 211 sbuf_printf(&sbuf, "end: %#jx\n", 212 (uintmax_t)seg->end); 213 sbuf_printf(&sbuf, "domain: %d\n", seg->domain); 214 sbuf_printf(&sbuf, "free list: %p\n", seg->free_queues); 215 } 216 error = sbuf_finish(&sbuf); 217 sbuf_delete(&sbuf); 218 return (error); 219} 220 221static void 222vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int tail) 223{ 224 225 m->order = order; 226 if (tail) 227 TAILQ_INSERT_TAIL(&fl[order].pl, m, plinks.q); 228 else 229 TAILQ_INSERT_HEAD(&fl[order].pl, m, plinks.q); 230 fl[order].lcnt++; 231} 232 233static void 234vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order) 235{ 236 237 TAILQ_REMOVE(&fl[order].pl, m, plinks.q); 238 fl[order].lcnt--; 239 m->order = VM_NFREEORDER; 240} 241 242/* 243 * Create a physical memory segment. 244 */ 245static void 246_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain) 247{ 248 struct vm_phys_seg *seg; 249 250 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 251 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 252 KASSERT(domain < vm_ndomains, 253 ("vm_phys_create_seg: invalid domain provided")); 254 seg = &vm_phys_segs[vm_phys_nsegs++]; 255 while (seg > vm_phys_segs && (seg - 1)->start >= end) { 256 *seg = *(seg - 1); 257 seg--; 258 } 259 seg->start = start; 260 seg->end = end; 261 seg->domain = domain; 262 seg->free_queues = &vm_phys_free_queues[domain][flind]; 263} 264 265static void 266vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind) 267{ 268 int i; 269 270 if (mem_affinity == NULL) { 271 _vm_phys_create_seg(start, end, flind, 0); 272 return; 273 } 274 275 for (i = 0;; i++) { 276 if (mem_affinity[i].end == 0) 277 panic("Reached end of affinity info"); 278 if (mem_affinity[i].end <= start) 279 continue; 280 if (mem_affinity[i].start > start) 281 panic("No affinity info for start %jx", 282 (uintmax_t)start); 283 if (mem_affinity[i].end >= end) { 284 _vm_phys_create_seg(start, end, flind, 285 mem_affinity[i].domain); 286 break; 287 } 288 _vm_phys_create_seg(start, mem_affinity[i].end, flind, 289 mem_affinity[i].domain); 290 start = mem_affinity[i].end; 291 } 292} 293 294/* 295 * Add a physical memory segment. 296 */ 297void 298vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end) 299{ 300 301 KASSERT((start & PAGE_MASK) == 0, 302 ("vm_phys_define_seg: start is not page aligned")); 303 KASSERT((end & PAGE_MASK) == 0, 304 ("vm_phys_define_seg: end is not page aligned")); 305#ifdef VM_FREELIST_ISADMA 306 if (start < 16777216) { 307 if (end > 16777216) { 308 vm_phys_create_seg(start, 16777216, 309 VM_FREELIST_ISADMA); 310 vm_phys_create_seg(16777216, end, VM_FREELIST_DEFAULT); 311 } else 312 vm_phys_create_seg(start, end, VM_FREELIST_ISADMA); 313 if (VM_FREELIST_ISADMA >= vm_nfreelists) 314 vm_nfreelists = VM_FREELIST_ISADMA + 1; 315 } else 316#endif 317#ifdef VM_FREELIST_HIGHMEM 318 if (end > VM_HIGHMEM_ADDRESS) { 319 if (start < VM_HIGHMEM_ADDRESS) { 320 vm_phys_create_seg(start, VM_HIGHMEM_ADDRESS, 321 VM_FREELIST_DEFAULT); 322 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, end, 323 VM_FREELIST_HIGHMEM); 324 } else 325 vm_phys_create_seg(start, end, VM_FREELIST_HIGHMEM); 326 if (VM_FREELIST_HIGHMEM >= vm_nfreelists) 327 vm_nfreelists = VM_FREELIST_HIGHMEM + 1; 328 } else 329#endif 330 vm_phys_create_seg(start, end, VM_FREELIST_DEFAULT); 331} 332 333/* 334 * Initialize the physical memory allocator. 335 */ 336void 337vm_phys_init(void) 338{ 339 struct vm_freelist *fl; 340 struct vm_phys_seg *seg; 341#ifdef VM_PHYSSEG_SPARSE 342 long pages; 343#endif 344 int dom, flind, oind, pind, segind; 345 346#ifdef VM_PHYSSEG_SPARSE 347 pages = 0; 348#endif 349 for (segind = 0; segind < vm_phys_nsegs; segind++) { 350 seg = &vm_phys_segs[segind]; 351#ifdef VM_PHYSSEG_SPARSE 352 seg->first_page = &vm_page_array[pages]; 353 pages += atop(seg->end - seg->start); 354#else 355 seg->first_page = PHYS_TO_VM_PAGE(seg->start); 356#endif 357 } 358 for (dom = 0; dom < vm_ndomains; dom++) { 359 for (flind = 0; flind < vm_nfreelists; flind++) { 360 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 361 fl = vm_phys_free_queues[dom][flind][pind]; 362 for (oind = 0; oind < VM_NFREEORDER; oind++) 363 TAILQ_INIT(&fl[oind].pl); 364 } 365 } 366 } 367 mtx_init(&vm_phys_fictitious_reg_mtx, "vmfctr", NULL, MTX_DEF); 368} 369 370/* 371 * Split a contiguous, power of two-sized set of physical pages. 372 */ 373static __inline void 374vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order) 375{ 376 vm_page_t m_buddy; 377 378 while (oind > order) { 379 oind--; 380 m_buddy = &m[1 << oind]; 381 KASSERT(m_buddy->order == VM_NFREEORDER, 382 ("vm_phys_split_pages: page %p has unexpected order %d", 383 m_buddy, m_buddy->order)); 384 vm_freelist_add(fl, m_buddy, oind, 0); 385 } 386} 387 388/* 389 * Initialize a physical page and add it to the free lists. 390 */ 391void 392vm_phys_add_page(vm_paddr_t pa) 393{ 394 vm_page_t m; 395 struct vm_domain *vmd; 396 397 cnt.v_page_count++; 398 m = vm_phys_paddr_to_vm_page(pa); 399 m->phys_addr = pa; 400 m->queue = PQ_NONE; 401 m->segind = vm_phys_paddr_to_segind(pa); 402 vmd = vm_phys_domain(m); 403 vmd->vmd_page_count++; 404 vmd->vmd_segs |= 1UL << m->segind; 405 m->flags = PG_FREE; 406 KASSERT(m->order == VM_NFREEORDER, 407 ("vm_phys_add_page: page %p has unexpected order %d", 408 m, m->order)); 409 m->pool = VM_FREEPOOL_DEFAULT; 410 pmap_page_init(m); 411 mtx_lock(&vm_page_queue_free_mtx); 412 vm_phys_freecnt_adj(m, 1); 413 vm_phys_free_pages(m, 0); 414 mtx_unlock(&vm_page_queue_free_mtx); 415} 416 417/* 418 * Allocate a contiguous, power of two-sized set of physical pages 419 * from the free lists. 420 * 421 * The free page queues must be locked. 422 */ 423vm_page_t 424vm_phys_alloc_pages(int pool, int order) 425{ 426 vm_page_t m; 427 int dom, domain, flind; 428 429 KASSERT(pool < VM_NFREEPOOL, 430 ("vm_phys_alloc_pages: pool %d is out of range", pool)); 431 KASSERT(order < VM_NFREEORDER, 432 ("vm_phys_alloc_pages: order %d is out of range", order)); 433 434 for (dom = 0; dom < vm_ndomains; dom++) { 435 domain = vm_rr_selectdomain(); 436 for (flind = 0; flind < vm_nfreelists; flind++) { 437 m = vm_phys_alloc_domain_pages(domain, flind, pool, 438 order); 439 if (m != NULL) 440 return (m); 441 } 442 } 443 return (NULL); 444} 445 446/* 447 * Find and dequeue a free page on the given free list, with the 448 * specified pool and order 449 */ 450vm_page_t 451vm_phys_alloc_freelist_pages(int flind, int pool, int order) 452{ 453 vm_page_t m; 454 int dom, domain; 455 456 KASSERT(flind < VM_NFREELIST, 457 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind)); 458 KASSERT(pool < VM_NFREEPOOL, 459 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); 460 KASSERT(order < VM_NFREEORDER, 461 ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); 462 463 for (dom = 0; dom < vm_ndomains; dom++) { 464 domain = vm_rr_selectdomain(); 465 m = vm_phys_alloc_domain_pages(domain, flind, pool, order); 466 if (m != NULL) 467 return (m); 468 } 469 return (NULL); 470} 471 472static vm_page_t 473vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order) 474{ 475 struct vm_freelist *fl; 476 struct vm_freelist *alt; 477 int oind, pind; 478 vm_page_t m; 479 480 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 481 fl = &vm_phys_free_queues[domain][flind][pool][0]; 482 for (oind = order; oind < VM_NFREEORDER; oind++) { 483 m = TAILQ_FIRST(&fl[oind].pl); 484 if (m != NULL) { 485 vm_freelist_rem(fl, m, oind); 486 vm_phys_split_pages(m, oind, fl, order); 487 return (m); 488 } 489 } 490 491 /* 492 * The given pool was empty. Find the largest 493 * contiguous, power-of-two-sized set of pages in any 494 * pool. Transfer these pages to the given pool, and 495 * use them to satisfy the allocation. 496 */ 497 for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { 498 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 499 alt = &vm_phys_free_queues[domain][flind][pind][0]; 500 m = TAILQ_FIRST(&alt[oind].pl); 501 if (m != NULL) { 502 vm_freelist_rem(alt, m, oind); 503 vm_phys_set_pool(pool, m, oind); 504 vm_phys_split_pages(m, oind, fl, order); 505 return (m); 506 } 507 } 508 } 509 return (NULL); 510} 511 512/* 513 * Find the vm_page corresponding to the given physical address. 514 */ 515vm_page_t 516vm_phys_paddr_to_vm_page(vm_paddr_t pa) 517{ 518 struct vm_phys_seg *seg; 519 int segind; 520 521 for (segind = 0; segind < vm_phys_nsegs; segind++) { 522 seg = &vm_phys_segs[segind]; 523 if (pa >= seg->start && pa < seg->end) 524 return (&seg->first_page[atop(pa - seg->start)]); 525 } 526 return (NULL); 527} 528 529vm_page_t 530vm_phys_fictitious_to_vm_page(vm_paddr_t pa) 531{ 532 struct vm_phys_fictitious_seg *seg; 533 vm_page_t m; 534 int segind; 535 536 m = NULL; 537 for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { 538 seg = &vm_phys_fictitious_segs[segind]; 539 if (pa >= seg->start && pa < seg->end) { 540 m = &seg->first_page[atop(pa - seg->start)]; 541 KASSERT((m->flags & PG_FICTITIOUS) != 0, 542 ("%p not fictitious", m)); 543 break; 544 } 545 } 546 return (m); 547} 548 549int 550vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, 551 vm_memattr_t memattr) 552{ 553 struct vm_phys_fictitious_seg *seg; 554 vm_page_t fp; 555 long i, page_count; 556 int segind; 557#ifdef VM_PHYSSEG_DENSE 558 long pi; 559 boolean_t malloced; 560#endif 561 562 page_count = (end - start) / PAGE_SIZE; 563 564#ifdef VM_PHYSSEG_DENSE 565 pi = atop(start); 566 if (pi >= first_page && pi < vm_page_array_size + first_page) { 567 if (atop(end) >= vm_page_array_size + first_page) 568 return (EINVAL); 569 fp = &vm_page_array[pi - first_page]; 570 malloced = FALSE; 571 } else 572#endif 573 { 574 fp = malloc(page_count * sizeof(struct vm_page), M_FICT_PAGES, 575 M_WAITOK | M_ZERO); 576#ifdef VM_PHYSSEG_DENSE 577 malloced = TRUE; 578#endif 579 } 580 for (i = 0; i < page_count; i++) { 581 vm_page_initfake(&fp[i], start + PAGE_SIZE * i, memattr); 582 fp[i].oflags &= ~VPO_UNMANAGED; 583 fp[i].busy_lock = VPB_UNBUSIED; 584 } 585 mtx_lock(&vm_phys_fictitious_reg_mtx); 586 for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { 587 seg = &vm_phys_fictitious_segs[segind]; 588 if (seg->start == 0 && seg->end == 0) { 589 seg->start = start; 590 seg->end = end; 591 seg->first_page = fp; 592 mtx_unlock(&vm_phys_fictitious_reg_mtx); 593 return (0); 594 } 595 } 596 mtx_unlock(&vm_phys_fictitious_reg_mtx); 597#ifdef VM_PHYSSEG_DENSE 598 if (malloced) 599#endif 600 free(fp, M_FICT_PAGES); 601 return (EBUSY); 602} 603 604void 605vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end) 606{ 607 struct vm_phys_fictitious_seg *seg; 608 vm_page_t fp; 609 int segind; 610#ifdef VM_PHYSSEG_DENSE 611 long pi; 612#endif 613 614#ifdef VM_PHYSSEG_DENSE 615 pi = atop(start); 616#endif 617 618 mtx_lock(&vm_phys_fictitious_reg_mtx); 619 for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { 620 seg = &vm_phys_fictitious_segs[segind]; 621 if (seg->start == start && seg->end == end) { 622 seg->start = seg->end = 0; 623 fp = seg->first_page; 624 seg->first_page = NULL; 625 mtx_unlock(&vm_phys_fictitious_reg_mtx); 626#ifdef VM_PHYSSEG_DENSE 627 if (pi < first_page || atop(end) >= vm_page_array_size) 628#endif 629 free(fp, M_FICT_PAGES); 630 return; 631 } 632 } 633 mtx_unlock(&vm_phys_fictitious_reg_mtx); 634 KASSERT(0, ("Unregistering not registered fictitious range")); 635} 636 637/* 638 * Find the segment containing the given physical address. 639 */ 640static int 641vm_phys_paddr_to_segind(vm_paddr_t pa) 642{ 643 struct vm_phys_seg *seg; 644 int segind; 645 646 for (segind = 0; segind < vm_phys_nsegs; segind++) { 647 seg = &vm_phys_segs[segind]; 648 if (pa >= seg->start && pa < seg->end) 649 return (segind); 650 } 651 panic("vm_phys_paddr_to_segind: paddr %#jx is not in any segment" , 652 (uintmax_t)pa); 653} 654 655/* 656 * Free a contiguous, power of two-sized set of physical pages. 657 * 658 * The free page queues must be locked. 659 */ 660void 661vm_phys_free_pages(vm_page_t m, int order) 662{ 663 struct vm_freelist *fl; 664 struct vm_phys_seg *seg; 665 vm_paddr_t pa; 666 vm_page_t m_buddy; 667 668 KASSERT(m->order == VM_NFREEORDER, 669 ("vm_phys_free_pages: page %p has unexpected order %d", 670 m, m->order)); 671 KASSERT(m->pool < VM_NFREEPOOL, 672 ("vm_phys_free_pages: page %p has unexpected pool %d", 673 m, m->pool)); 674 KASSERT(order < VM_NFREEORDER, 675 ("vm_phys_free_pages: order %d is out of range", order)); 676 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 677 seg = &vm_phys_segs[m->segind]; 678 if (order < VM_NFREEORDER - 1) { 679 pa = VM_PAGE_TO_PHYS(m); 680 do { 681 pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order)); 682 if (pa < seg->start || pa >= seg->end) 683 break; 684 m_buddy = &seg->first_page[atop(pa - seg->start)]; 685 if (m_buddy->order != order) 686 break; 687 fl = (*seg->free_queues)[m_buddy->pool]; 688 vm_freelist_rem(fl, m_buddy, order); 689 if (m_buddy->pool != m->pool) 690 vm_phys_set_pool(m->pool, m_buddy, order); 691 order++; 692 pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1); 693 m = &seg->first_page[atop(pa - seg->start)]; 694 } while (order < VM_NFREEORDER - 1); 695 } 696 fl = (*seg->free_queues)[m->pool]; 697 vm_freelist_add(fl, m, order, 1); 698} 699 700/* 701 * Free a contiguous, arbitrarily sized set of physical pages. 702 * 703 * The free page queues must be locked. 704 */ 705void 706vm_phys_free_contig(vm_page_t m, u_long npages) 707{ 708 u_int n; 709 int order; 710 711 /* 712 * Avoid unnecessary coalescing by freeing the pages in the largest 713 * possible power-of-two-sized subsets. 714 */ 715 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 716 for (;; npages -= n) { 717 /* 718 * Unsigned "min" is used here so that "order" is assigned 719 * "VM_NFREEORDER - 1" when "m"'s physical address is zero 720 * or the low-order bits of its physical address are zero 721 * because the size of a physical address exceeds the size of 722 * a long. 723 */ 724 order = min(ffsl(VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) - 1, 725 VM_NFREEORDER - 1); 726 n = 1 << order; 727 if (npages < n) 728 break; 729 vm_phys_free_pages(m, order); 730 m += n; 731 } 732 /* The residual "npages" is less than "1 << (VM_NFREEORDER - 1)". */ 733 for (; npages > 0; npages -= n) { 734 order = flsl(npages) - 1; 735 n = 1 << order; 736 vm_phys_free_pages(m, order); 737 m += n; 738 } 739} 740 741/* 742 * Set the pool for a contiguous, power of two-sized set of physical pages. 743 */ 744void 745vm_phys_set_pool(int pool, vm_page_t m, int order) 746{ 747 vm_page_t m_tmp; 748 749 for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) 750 m_tmp->pool = pool; 751} 752 753/* 754 * Search for the given physical page "m" in the free lists. If the search 755 * succeeds, remove "m" from the free lists and return TRUE. Otherwise, return 756 * FALSE, indicating that "m" is not in the free lists. 757 * 758 * The free page queues must be locked. 759 */ 760boolean_t 761vm_phys_unfree_page(vm_page_t m) 762{ 763 struct vm_freelist *fl; 764 struct vm_phys_seg *seg; 765 vm_paddr_t pa, pa_half; 766 vm_page_t m_set, m_tmp; 767 int order; 768 769 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 770 771 /* 772 * First, find the contiguous, power of two-sized set of free 773 * physical pages containing the given physical page "m" and 774 * assign it to "m_set". 775 */ 776 seg = &vm_phys_segs[m->segind]; 777 for (m_set = m, order = 0; m_set->order == VM_NFREEORDER && 778 order < VM_NFREEORDER - 1; ) { 779 order++; 780 pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); 781 if (pa >= seg->start) 782 m_set = &seg->first_page[atop(pa - seg->start)]; 783 else 784 return (FALSE); 785 } 786 if (m_set->order < order) 787 return (FALSE); 788 if (m_set->order == VM_NFREEORDER) 789 return (FALSE); 790 KASSERT(m_set->order < VM_NFREEORDER, 791 ("vm_phys_unfree_page: page %p has unexpected order %d", 792 m_set, m_set->order)); 793 794 /* 795 * Next, remove "m_set" from the free lists. Finally, extract 796 * "m" from "m_set" using an iterative algorithm: While "m_set" 797 * is larger than a page, shrink "m_set" by returning the half 798 * of "m_set" that does not contain "m" to the free lists. 799 */ 800 fl = (*seg->free_queues)[m_set->pool]; 801 order = m_set->order; 802 vm_freelist_rem(fl, m_set, order); 803 while (order > 0) { 804 order--; 805 pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); 806 if (m->phys_addr < pa_half) 807 m_tmp = &seg->first_page[atop(pa_half - seg->start)]; 808 else { 809 m_tmp = m_set; 810 m_set = &seg->first_page[atop(pa_half - seg->start)]; 811 } 812 vm_freelist_add(fl, m_tmp, order, 0); 813 } 814 KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); 815 return (TRUE); 816} 817 818/* 819 * Try to zero one physical page. Used by an idle priority thread. 820 */ 821boolean_t 822vm_phys_zero_pages_idle(void) 823{ 824 static struct vm_freelist *fl; 825 static int flind, oind, pind; 826 vm_page_t m, m_tmp; 827 int domain; 828 829 domain = vm_rr_selectdomain(); 830 fl = vm_phys_free_queues[domain][0][0]; 831 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 832 for (;;) { 833 TAILQ_FOREACH_REVERSE(m, &fl[oind].pl, pglist, plinks.q) { 834 for (m_tmp = m; m_tmp < &m[1 << oind]; m_tmp++) { 835 if ((m_tmp->flags & (PG_CACHED | PG_ZERO)) == 0) { 836 vm_phys_unfree_page(m_tmp); 837 vm_phys_freecnt_adj(m, -1); 838 mtx_unlock(&vm_page_queue_free_mtx); 839 pmap_zero_page_idle(m_tmp); 840 m_tmp->flags |= PG_ZERO; 841 mtx_lock(&vm_page_queue_free_mtx); 842 vm_phys_freecnt_adj(m, 1); 843 vm_phys_free_pages(m_tmp, 0); 844 vm_page_zero_count++; 845 cnt_prezero++; 846 return (TRUE); 847 } 848 } 849 } 850 oind++; 851 if (oind == VM_NFREEORDER) { 852 oind = 0; 853 pind++; 854 if (pind == VM_NFREEPOOL) { 855 pind = 0; 856 flind++; 857 if (flind == vm_nfreelists) 858 flind = 0; 859 } 860 fl = vm_phys_free_queues[domain][flind][pind]; 861 } 862 } 863} 864 865/* 866 * Allocate a contiguous set of physical pages of the given size 867 * "npages" from the free lists. All of the physical pages must be at 868 * or above the given physical address "low" and below the given 869 * physical address "high". The given value "alignment" determines the 870 * alignment of the first physical page in the set. If the given value 871 * "boundary" is non-zero, then the set of physical pages cannot cross 872 * any physical address boundary that is a multiple of that value. Both 873 * "alignment" and "boundary" must be a power of two. 874 */ 875vm_page_t 876vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, 877 u_long alignment, vm_paddr_t boundary) 878{ 879 struct vm_freelist *fl; 880 struct vm_phys_seg *seg; 881 vm_paddr_t pa, pa_last, size; 882 vm_page_t m, m_ret; 883 u_long npages_end; 884 int dom, domain, flind, oind, order, pind; 885 886 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 887 size = npages << PAGE_SHIFT; 888 KASSERT(size != 0, 889 ("vm_phys_alloc_contig: size must not be 0")); 890 KASSERT((alignment & (alignment - 1)) == 0, 891 ("vm_phys_alloc_contig: alignment must be a power of 2")); 892 KASSERT((boundary & (boundary - 1)) == 0, 893 ("vm_phys_alloc_contig: boundary must be a power of 2")); 894 /* Compute the queue that is the best fit for npages. */ 895 for (order = 0; (1 << order) < npages; order++); 896 dom = 0; 897restartdom: 898 domain = vm_rr_selectdomain(); 899 for (flind = 0; flind < vm_nfreelists; flind++) { 900 for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) { 901 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 902 fl = &vm_phys_free_queues[domain][flind][pind][0]; 903 TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { 904 /* 905 * A free list may contain physical pages 906 * from one or more segments. 907 */ 908 seg = &vm_phys_segs[m_ret->segind]; 909 if (seg->start > high || 910 low >= seg->end) 911 continue; 912 913 /* 914 * Is the size of this allocation request 915 * larger than the largest block size? 916 */ 917 if (order >= VM_NFREEORDER) { 918 /* 919 * Determine if a sufficient number 920 * of subsequent blocks to satisfy 921 * the allocation request are free. 922 */ 923 pa = VM_PAGE_TO_PHYS(m_ret); 924 pa_last = pa + size; 925 for (;;) { 926 pa += 1 << (PAGE_SHIFT + VM_NFREEORDER - 1); 927 if (pa >= pa_last) 928 break; 929 if (pa < seg->start || 930 pa >= seg->end) 931 break; 932 m = &seg->first_page[atop(pa - seg->start)]; 933 if (m->order != VM_NFREEORDER - 1) 934 break; 935 } 936 /* If not, continue to the next block. */ 937 if (pa < pa_last) 938 continue; 939 } 940 941 /* 942 * Determine if the blocks are within the given range, 943 * satisfy the given alignment, and do not cross the 944 * given boundary. 945 */ 946 pa = VM_PAGE_TO_PHYS(m_ret); 947 if (pa >= low && 948 pa + size <= high && 949 (pa & (alignment - 1)) == 0 && 950 ((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0) 951 goto done; 952 } 953 } 954 } 955 } 956 if (++dom < vm_ndomains) 957 goto restartdom; 958 return (NULL); 959done: 960 for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { 961 fl = (*seg->free_queues)[m->pool]; 962 vm_freelist_rem(fl, m, m->order); 963 } 964 if (m_ret->pool != VM_FREEPOOL_DEFAULT) 965 vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); 966 fl = (*seg->free_queues)[m_ret->pool]; 967 vm_phys_split_pages(m_ret, oind, fl, order); 968 /* Return excess pages to the free lists. */ 969 npages_end = roundup2(npages, 1 << imin(oind, order)); 970 if (npages < npages_end) 971 vm_phys_free_contig(&m_ret[npages], npages_end - npages); 972 return (m_ret); 973} 974 975#ifdef DDB 976/* 977 * Show the number of physical pages in each of the free lists. 978 */ 979DB_SHOW_COMMAND(freepages, db_show_freepages) 980{ 981 struct vm_freelist *fl; 982 int flind, oind, pind, dom; 983 984 for (dom = 0; dom < vm_ndomains; dom++) { 985 db_printf("DOMAIN: %d\n", dom); 986 for (flind = 0; flind < vm_nfreelists; flind++) { 987 db_printf("FREE LIST %d:\n" 988 "\n ORDER (SIZE) | NUMBER" 989 "\n ", flind); 990 for (pind = 0; pind < VM_NFREEPOOL; pind++) 991 db_printf(" | POOL %d", pind); 992 db_printf("\n-- "); 993 for (pind = 0; pind < VM_NFREEPOOL; pind++) 994 db_printf("-- -- "); 995 db_printf("--\n"); 996 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 997 db_printf(" %2.2d (%6.6dK)", oind, 998 1 << (PAGE_SHIFT - 10 + oind)); 999 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 1000 fl = vm_phys_free_queues[dom][flind][pind]; 1001 db_printf(" | %6.6d", fl[oind].lcnt); 1002 } 1003 db_printf("\n"); 1004 } 1005 db_printf("\n"); 1006 } 1007 db_printf("\n"); 1008 } 1009} 1010#endif 1011