1/* 2 * Copyright (c) Red Hat Inc. 3 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sub license, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the 12 * next paragraph) shall be included in all copies or substantial portions 13 * of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: Dave Airlie <airlied@redhat.com> 24 * Jerome Glisse <jglisse@redhat.com> 25 * Pauli Nieminen <suokkos@gmail.com> 26 */ 27/* 28 * Copyright (c) 2013 The FreeBSD Foundation 29 * All rights reserved. 30 * 31 * Portions of this software were developed by Konstantin Belousov 32 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 33 */ 34 35/* simple list based uncached page pool 36 * - Pool collects resently freed pages for reuse 37 * - Use page->lru to keep a free list 38 * - doesn't track currently in use pages 39 */ 40 41#include <sys/cdefs.h> 42#include <dev/drm2/drmP.h> 43#include <dev/drm2/ttm/ttm_bo_driver.h> 44#include <dev/drm2/ttm/ttm_page_alloc.h> 45#include <sys/eventhandler.h> 46#include <vm/vm_pageout.h> 47 48#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(vm_page_t)) 49#define SMALL_ALLOCATION 16 50#define FREE_ALL_PAGES (~0U) 51/* times are in msecs */ 52#define PAGE_FREE_INTERVAL 1000 53 54/** 55 * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. 56 * 57 * @lock: Protects the shared pool from concurrnet access. Must be used with 58 * irqsave/irqrestore variants because pool allocator maybe called from 59 * delayed work. 60 * @fill_lock: Prevent concurrent calls to fill. 61 * @list: Pool of free uc/wc pages for fast reuse. 62 * @gfp_flags: Flags to pass for alloc_page. 63 * @npages: Number of pages in pool. 64 */ 65struct ttm_page_pool { 66 struct mtx lock; 67 bool fill_lock; 68 bool dma32; 69 struct pglist list; 70 int ttm_page_alloc_flags; 71 unsigned npages; 72 char *name; 73 unsigned long nfrees; 74 unsigned long nrefills; 75}; 76 77/** 78 * Limits for the pool. They are handled without locks because only place where 79 * they may change is in sysfs store. They won't have immediate effect anyway 80 * so forcing serialization to access them is pointless. 81 */ 82 83struct ttm_pool_opts { 84 unsigned alloc_size; 85 unsigned max_size; 86 unsigned small; 87}; 88 89#define NUM_POOLS 4 90 91/** 92 * struct ttm_pool_manager - Holds memory pools for fst allocation 93 * 94 * Manager is read only object for pool code so it doesn't need locking. 95 * 96 * @free_interval: minimum number of jiffies between freeing pages from pool. 97 * @page_alloc_inited: reference counting for pool allocation. 98 * @work: Work that is used to shrink the pool. Work is only run when there is 99 * some pages to free. 100 * @small_allocation: Limit in number of pages what is small allocation. 101 * 102 * @pools: All pool objects in use. 103 **/ 104struct ttm_pool_manager { 105 unsigned int kobj_ref; 106 eventhandler_tag lowmem_handler; 107 struct ttm_pool_opts options; 108 109 union { 110 struct ttm_page_pool u_pools[NUM_POOLS]; 111 struct _utag { 112 struct ttm_page_pool u_wc_pool; 113 struct ttm_page_pool u_uc_pool; 114 struct ttm_page_pool u_wc_pool_dma32; 115 struct ttm_page_pool u_uc_pool_dma32; 116 } _ut; 117 } _u; 118}; 119 120#define pools _u.u_pools 121#define wc_pool _u._ut.u_wc_pool 122#define uc_pool _u._ut.u_uc_pool 123#define wc_pool_dma32 _u._ut.u_wc_pool_dma32 124#define uc_pool_dma32 _u._ut.u_uc_pool_dma32 125 126MALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager"); 127 128static void 129ttm_vm_page_free(vm_page_t m) 130{ 131 132 KASSERT(m->object == NULL, ("ttm page %p is owned", m)); 133 KASSERT(vm_page_wired(m), ("ttm lost wire %p", m)); 134 KASSERT((m->flags & PG_FICTITIOUS) != 0, ("ttm lost fictitious %p", m)); 135 KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m)); 136 m->flags &= ~PG_FICTITIOUS; 137 m->oflags |= VPO_UNMANAGED; 138 vm_page_unwire_noq(m); 139 vm_page_free(m); 140} 141 142static vm_memattr_t 143ttm_caching_state_to_vm(enum ttm_caching_state cstate) 144{ 145 146 switch (cstate) { 147 case tt_uncached: 148 return (VM_MEMATTR_UNCACHEABLE); 149 case tt_wc: 150 return (VM_MEMATTR_WRITE_COMBINING); 151 case tt_cached: 152 return (VM_MEMATTR_WRITE_BACK); 153 } 154 panic("caching state %d\n", cstate); 155} 156 157static vm_page_t 158ttm_vm_page_alloc_dma32(int req, vm_memattr_t memattr) 159{ 160 vm_page_t p; 161 int err, tries; 162 163 for (tries = 0; ; tries++) { 164 p = vm_page_alloc_noobj_contig(req, 1, 0, 0xffffffff, PAGE_SIZE, 165 0, memattr); 166 if (p != NULL || tries > 2) 167 return (p); 168 err = vm_page_reclaim_contig(req, 1, 0, 0xffffffff, 169 PAGE_SIZE, 0); 170 if (err == ENOMEM) 171 vm_wait(NULL); 172 else if (err != 0) 173 return (NULL); 174 } 175} 176 177static vm_page_t 178ttm_vm_page_alloc_any(int req, vm_memattr_t memattr) 179{ 180 vm_page_t p; 181 182 p = vm_page_alloc_noobj(req | VM_ALLOC_WAITOK); 183 pmap_page_set_memattr(p, memattr); 184 return (p); 185} 186 187static vm_page_t 188ttm_vm_page_alloc(int flags, enum ttm_caching_state cstate) 189{ 190 vm_page_t p; 191 vm_memattr_t memattr; 192 int req; 193 194 memattr = ttm_caching_state_to_vm(cstate); 195 req = VM_ALLOC_WIRED; 196 if ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0) 197 req |= VM_ALLOC_ZERO; 198 199 if ((flags & TTM_PAGE_FLAG_DMA32) != 0) 200 p = ttm_vm_page_alloc_dma32(req, memattr); 201 else 202 p = ttm_vm_page_alloc_any(req, memattr); 203 204 if (p != NULL) { 205 p->oflags &= ~VPO_UNMANAGED; 206 p->flags |= PG_FICTITIOUS; 207 } 208 return (p); 209} 210 211static void ttm_pool_kobj_release(struct ttm_pool_manager *m) 212{ 213 214 free(m, M_TTM_POOLMGR); 215} 216 217#if 0 218/* XXXKIB sysctl */ 219static ssize_t ttm_pool_store(struct ttm_pool_manager *m, 220 struct attribute *attr, const char *buffer, size_t size) 221{ 222 int chars; 223 unsigned val; 224 chars = sscanf(buffer, "%u", &val); 225 if (chars == 0) 226 return size; 227 228 /* Convert kb to number of pages */ 229 val = val / (PAGE_SIZE >> 10); 230 231 if (attr == &ttm_page_pool_max) 232 m->options.max_size = val; 233 else if (attr == &ttm_page_pool_small) 234 m->options.small = val; 235 else if (attr == &ttm_page_pool_alloc_size) { 236 if (val > NUM_PAGES_TO_ALLOC*8) { 237 pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", 238 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), 239 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 240 return size; 241 } else if (val > NUM_PAGES_TO_ALLOC) { 242 pr_warn("Setting allocation size to larger than %lu is not recommended\n", 243 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 244 } 245 m->options.alloc_size = val; 246 } 247 248 return size; 249} 250 251static ssize_t ttm_pool_show(struct ttm_pool_manager *m, 252 struct attribute *attr, char *buffer) 253{ 254 unsigned val = 0; 255 256 if (attr == &ttm_page_pool_max) 257 val = m->options.max_size; 258 else if (attr == &ttm_page_pool_small) 259 val = m->options.small; 260 else if (attr == &ttm_page_pool_alloc_size) 261 val = m->options.alloc_size; 262 263 val = val * (PAGE_SIZE >> 10); 264 265 return snprintf(buffer, PAGE_SIZE, "%u\n", val); 266} 267#endif 268 269static struct ttm_pool_manager *_manager; 270 271static int set_pages_array_wb(vm_page_t *pages, int addrinarray) 272{ 273#ifdef TTM_HAS_AGP 274 int i; 275 276 for (i = 0; i < addrinarray; i++) 277 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_BACK); 278#endif 279 return 0; 280} 281 282static int set_pages_array_wc(vm_page_t *pages, int addrinarray) 283{ 284#ifdef TTM_HAS_AGP 285 int i; 286 287 for (i = 0; i < addrinarray; i++) 288 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_COMBINING); 289#endif 290 return 0; 291} 292 293static int set_pages_array_uc(vm_page_t *pages, int addrinarray) 294{ 295#ifdef TTM_HAS_AGP 296 int i; 297 298 for (i = 0; i < addrinarray; i++) 299 pmap_page_set_memattr(pages[i], VM_MEMATTR_UNCACHEABLE); 300#endif 301 return 0; 302} 303 304/** 305 * Select the right pool or requested caching state and ttm flags. */ 306static struct ttm_page_pool *ttm_get_pool(int flags, 307 enum ttm_caching_state cstate) 308{ 309 int pool_index; 310 311 if (cstate == tt_cached) 312 return NULL; 313 314 if (cstate == tt_wc) 315 pool_index = 0x0; 316 else 317 pool_index = 0x1; 318 319 if (flags & TTM_PAGE_FLAG_DMA32) 320 pool_index |= 0x2; 321 322 return &_manager->pools[pool_index]; 323} 324 325/* set memory back to wb and free the pages. */ 326static void ttm_pages_put(vm_page_t *pages, unsigned npages) 327{ 328 unsigned i; 329 330 /* Our VM handles vm memattr automatically on the page free. */ 331 if (set_pages_array_wb(pages, npages)) 332 printf("[TTM] Failed to set %d pages to wb!\n", npages); 333 for (i = 0; i < npages; ++i) 334 ttm_vm_page_free(pages[i]); 335} 336 337static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, 338 unsigned freed_pages) 339{ 340 pool->npages -= freed_pages; 341 pool->nfrees += freed_pages; 342} 343 344/** 345 * Free pages from pool. 346 * 347 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC 348 * number of pages in one go. 349 * 350 * @pool: to free the pages from 351 * @free_all: If set to true will free all pages in pool 352 **/ 353static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) 354{ 355 vm_page_t p, p1; 356 vm_page_t *pages_to_free; 357 unsigned freed_pages = 0, 358 npages_to_free = nr_free; 359 unsigned i; 360 361 if (NUM_PAGES_TO_ALLOC < nr_free) 362 npages_to_free = NUM_PAGES_TO_ALLOC; 363 364 pages_to_free = malloc(npages_to_free * sizeof(vm_page_t), 365 M_TEMP, M_WAITOK | M_ZERO); 366 367restart: 368 mtx_lock(&pool->lock); 369 370 TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) { 371 if (freed_pages >= npages_to_free) 372 break; 373 374 pages_to_free[freed_pages++] = p; 375 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ 376 if (freed_pages >= NUM_PAGES_TO_ALLOC) { 377 /* remove range of pages from the pool */ 378 for (i = 0; i < freed_pages; i++) 379 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 380 381 ttm_pool_update_free_locked(pool, freed_pages); 382 /** 383 * Because changing page caching is costly 384 * we unlock the pool to prevent stalling. 385 */ 386 mtx_unlock(&pool->lock); 387 388 ttm_pages_put(pages_to_free, freed_pages); 389 if (likely(nr_free != FREE_ALL_PAGES)) 390 nr_free -= freed_pages; 391 392 if (NUM_PAGES_TO_ALLOC >= nr_free) 393 npages_to_free = nr_free; 394 else 395 npages_to_free = NUM_PAGES_TO_ALLOC; 396 397 freed_pages = 0; 398 399 /* free all so restart the processing */ 400 if (nr_free) 401 goto restart; 402 403 /* Not allowed to fall through or break because 404 * following context is inside spinlock while we are 405 * outside here. 406 */ 407 goto out; 408 409 } 410 } 411 412 /* remove range of pages from the pool */ 413 if (freed_pages) { 414 for (i = 0; i < freed_pages; i++) 415 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 416 417 ttm_pool_update_free_locked(pool, freed_pages); 418 nr_free -= freed_pages; 419 } 420 421 mtx_unlock(&pool->lock); 422 423 if (freed_pages) 424 ttm_pages_put(pages_to_free, freed_pages); 425out: 426 free(pages_to_free, M_TEMP); 427 return nr_free; 428} 429 430/* Get good estimation how many pages are free in pools */ 431static int ttm_pool_get_num_unused_pages(void) 432{ 433 unsigned i; 434 int total = 0; 435 for (i = 0; i < NUM_POOLS; ++i) 436 total += _manager->pools[i].npages; 437 438 return total; 439} 440 441/** 442 * Callback for mm to request pool to reduce number of page held. 443 */ 444static int ttm_pool_mm_shrink(void *arg) 445{ 446 static unsigned int start_pool = 0; 447 unsigned i; 448 unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1); 449 struct ttm_page_pool *pool; 450 int shrink_pages = 100; /* XXXKIB */ 451 452 pool_offset = pool_offset % NUM_POOLS; 453 /* select start pool in round robin fashion */ 454 for (i = 0; i < NUM_POOLS; ++i) { 455 unsigned nr_free = shrink_pages; 456 if (shrink_pages == 0) 457 break; 458 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; 459 shrink_pages = ttm_page_pool_free(pool, nr_free); 460 } 461 /* return estimated number of unused pages in pool */ 462 return ttm_pool_get_num_unused_pages(); 463} 464 465static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) 466{ 467 468 manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem, 469 ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY); 470} 471 472static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) 473{ 474 475 EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); 476} 477 478static int ttm_set_pages_caching(vm_page_t *pages, 479 enum ttm_caching_state cstate, unsigned cpages) 480{ 481 int r = 0; 482 /* Set page caching */ 483 switch (cstate) { 484 case tt_uncached: 485 r = set_pages_array_uc(pages, cpages); 486 if (r) 487 printf("[TTM] Failed to set %d pages to uc!\n", cpages); 488 break; 489 case tt_wc: 490 r = set_pages_array_wc(pages, cpages); 491 if (r) 492 printf("[TTM] Failed to set %d pages to wc!\n", cpages); 493 break; 494 default: 495 break; 496 } 497 return r; 498} 499 500/** 501 * Free pages the pages that failed to change the caching state. If there is 502 * any pages that have changed their caching state already put them to the 503 * pool. 504 */ 505static void ttm_handle_caching_state_failure(struct pglist *pages, 506 int ttm_flags, enum ttm_caching_state cstate, 507 vm_page_t *failed_pages, unsigned cpages) 508{ 509 unsigned i; 510 /* Failed pages have to be freed */ 511 for (i = 0; i < cpages; ++i) { 512 TAILQ_REMOVE(pages, failed_pages[i], plinks.q); 513 ttm_vm_page_free(failed_pages[i]); 514 } 515} 516 517/** 518 * Allocate new pages with correct caching. 519 * 520 * This function is reentrant if caller updates count depending on number of 521 * pages returned in pages array. 522 */ 523static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags, 524 int ttm_flags, enum ttm_caching_state cstate, unsigned count) 525{ 526 vm_page_t *caching_array; 527 vm_page_t p; 528 int r = 0; 529 unsigned i, cpages; 530 unsigned max_cpages = min(count, 531 (unsigned)(PAGE_SIZE/sizeof(vm_page_t))); 532 533 /* allocate array for page caching change */ 534 caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP, 535 M_WAITOK | M_ZERO); 536 537 for (i = 0, cpages = 0; i < count; ++i) { 538 p = ttm_vm_page_alloc(ttm_alloc_flags, cstate); 539 if (!p) { 540 printf("[TTM] Unable to get page %u\n", i); 541 542 /* store already allocated pages in the pool after 543 * setting the caching state */ 544 if (cpages) { 545 r = ttm_set_pages_caching(caching_array, 546 cstate, cpages); 547 if (r) 548 ttm_handle_caching_state_failure(pages, 549 ttm_flags, cstate, 550 caching_array, cpages); 551 } 552 r = -ENOMEM; 553 goto out; 554 } 555 556#ifdef CONFIG_HIGHMEM /* KIB: nop */ 557 /* gfp flags of highmem page should never be dma32 so we 558 * we should be fine in such case 559 */ 560 if (!PageHighMem(p)) 561#endif 562 { 563 caching_array[cpages++] = p; 564 if (cpages == max_cpages) { 565 566 r = ttm_set_pages_caching(caching_array, 567 cstate, cpages); 568 if (r) { 569 ttm_handle_caching_state_failure(pages, 570 ttm_flags, cstate, 571 caching_array, cpages); 572 goto out; 573 } 574 cpages = 0; 575 } 576 } 577 578 TAILQ_INSERT_HEAD(pages, p, plinks.q); 579 } 580 581 if (cpages) { 582 r = ttm_set_pages_caching(caching_array, cstate, cpages); 583 if (r) 584 ttm_handle_caching_state_failure(pages, 585 ttm_flags, cstate, 586 caching_array, cpages); 587 } 588out: 589 free(caching_array, M_TEMP); 590 591 return r; 592} 593 594/** 595 * Fill the given pool if there aren't enough pages and the requested number of 596 * pages is small. 597 */ 598static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, 599 int ttm_flags, enum ttm_caching_state cstate, unsigned count) 600{ 601 vm_page_t p; 602 int r; 603 unsigned cpages = 0; 604 /** 605 * Only allow one pool fill operation at a time. 606 * If pool doesn't have enough pages for the allocation new pages are 607 * allocated from outside of pool. 608 */ 609 if (pool->fill_lock) 610 return; 611 612 pool->fill_lock = true; 613 614 /* If allocation request is small and there are not enough 615 * pages in a pool we fill the pool up first. */ 616 if (count < _manager->options.small 617 && count > pool->npages) { 618 struct pglist new_pages; 619 unsigned alloc_size = _manager->options.alloc_size; 620 621 /** 622 * Can't change page caching if in irqsave context. We have to 623 * drop the pool->lock. 624 */ 625 mtx_unlock(&pool->lock); 626 627 TAILQ_INIT(&new_pages); 628 r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags, 629 ttm_flags, cstate, alloc_size); 630 mtx_lock(&pool->lock); 631 632 if (!r) { 633 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 634 ++pool->nrefills; 635 pool->npages += alloc_size; 636 } else { 637 printf("[TTM] Failed to fill pool (%p)\n", pool); 638 /* If we have any pages left put them to the pool. */ 639 TAILQ_FOREACH(p, &pool->list, plinks.q) { 640 ++cpages; 641 } 642 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 643 pool->npages += cpages; 644 } 645 646 } 647 pool->fill_lock = false; 648} 649 650/** 651 * Cut 'count' number of pages from the pool and put them on the return list. 652 * 653 * @return count of pages still required to fulfill the request. 654 */ 655static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, 656 struct pglist *pages, 657 int ttm_flags, 658 enum ttm_caching_state cstate, 659 unsigned count) 660{ 661 vm_page_t p; 662 unsigned i; 663 664 mtx_lock(&pool->lock); 665 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count); 666 667 if (count >= pool->npages) { 668 /* take all pages from the pool */ 669 TAILQ_CONCAT(pages, &pool->list, plinks.q); 670 count -= pool->npages; 671 pool->npages = 0; 672 goto out; 673 } 674 for (i = 0; i < count; i++) { 675 p = TAILQ_FIRST(&pool->list); 676 TAILQ_REMOVE(&pool->list, p, plinks.q); 677 TAILQ_INSERT_TAIL(pages, p, plinks.q); 678 } 679 pool->npages -= count; 680 count = 0; 681out: 682 mtx_unlock(&pool->lock); 683 return count; 684} 685 686/* Put all pages in pages list to correct pool to wait for reuse */ 687static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags, 688 enum ttm_caching_state cstate) 689{ 690 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 691 unsigned i; 692 693 if (pool == NULL) { 694 /* No pool for this memory type so free the pages */ 695 for (i = 0; i < npages; i++) { 696 if (pages[i]) { 697 ttm_vm_page_free(pages[i]); 698 pages[i] = NULL; 699 } 700 } 701 return; 702 } 703 704 mtx_lock(&pool->lock); 705 for (i = 0; i < npages; i++) { 706 if (pages[i]) { 707 TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q); 708 pages[i] = NULL; 709 pool->npages++; 710 } 711 } 712 /* Check that we don't go over the pool limit */ 713 npages = 0; 714 if (pool->npages > _manager->options.max_size) { 715 npages = pool->npages - _manager->options.max_size; 716 /* free at least NUM_PAGES_TO_ALLOC number of pages 717 * to reduce calls to set_memory_wb */ 718 if (npages < NUM_PAGES_TO_ALLOC) 719 npages = NUM_PAGES_TO_ALLOC; 720 } 721 mtx_unlock(&pool->lock); 722 if (npages) 723 ttm_page_pool_free(pool, npages); 724} 725 726/* 727 * On success pages list will hold count number of correctly 728 * cached pages. 729 */ 730static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags, 731 enum ttm_caching_state cstate) 732{ 733 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 734 struct pglist plist; 735 vm_page_t p = NULL; 736 int gfp_flags; 737 unsigned count; 738 int r; 739 740 /* No pool for cached pages */ 741 if (pool == NULL) { 742 for (r = 0; r < npages; ++r) { 743 p = ttm_vm_page_alloc(flags, cstate); 744 if (!p) { 745 printf("[TTM] Unable to allocate page\n"); 746 return -ENOMEM; 747 } 748 pages[r] = p; 749 } 750 return 0; 751 } 752 753 /* combine zero flag to pool flags */ 754 gfp_flags = flags | pool->ttm_page_alloc_flags; 755 756 /* First we take pages from the pool */ 757 TAILQ_INIT(&plist); 758 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); 759 count = 0; 760 TAILQ_FOREACH(p, &plist, plinks.q) { 761 pages[count++] = p; 762 } 763 764 /* clear the pages coming from the pool if requested */ 765 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { 766 TAILQ_FOREACH(p, &plist, plinks.q) { 767 pmap_zero_page(p); 768 } 769 } 770 771 /* If pool didn't have enough pages allocate new one. */ 772 if (npages > 0) { 773 /* ttm_alloc_new_pages doesn't reference pool so we can run 774 * multiple requests in parallel. 775 **/ 776 TAILQ_INIT(&plist); 777 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, 778 npages); 779 TAILQ_FOREACH(p, &plist, plinks.q) { 780 pages[count++] = p; 781 } 782 if (r) { 783 /* If there is any pages in the list put them back to 784 * the pool. */ 785 printf("[TTM] Failed to allocate extra pages for large request\n"); 786 ttm_put_pages(pages, count, flags, cstate); 787 return r; 788 } 789 } 790 791 return 0; 792} 793 794static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, 795 char *name) 796{ 797 mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF); 798 pool->fill_lock = false; 799 TAILQ_INIT(&pool->list); 800 pool->npages = pool->nfrees = 0; 801 pool->ttm_page_alloc_flags = flags; 802 pool->name = name; 803} 804 805int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) 806{ 807 808 if (_manager != NULL) 809 printf("[TTM] manager != NULL\n"); 810 printf("[TTM] Initializing pool allocator\n"); 811 812 _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO); 813 814 ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc"); 815 ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc"); 816 ttm_page_pool_init_locked(&_manager->wc_pool_dma32, 817 TTM_PAGE_FLAG_DMA32, "wc dma"); 818 ttm_page_pool_init_locked(&_manager->uc_pool_dma32, 819 TTM_PAGE_FLAG_DMA32, "uc dma"); 820 821 _manager->options.max_size = max_pages; 822 _manager->options.small = SMALL_ALLOCATION; 823 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; 824 825 refcount_init(&_manager->kobj_ref, 1); 826 ttm_pool_mm_shrink_init(_manager); 827 828 return 0; 829} 830 831void ttm_page_alloc_fini(void) 832{ 833 int i; 834 835 printf("[TTM] Finalizing pool allocator\n"); 836 ttm_pool_mm_shrink_fini(_manager); 837 838 for (i = 0; i < NUM_POOLS; ++i) 839 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); 840 841 if (refcount_release(&_manager->kobj_ref)) 842 ttm_pool_kobj_release(_manager); 843 _manager = NULL; 844} 845 846int ttm_pool_populate(struct ttm_tt *ttm) 847{ 848 struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; 849 unsigned i; 850 int ret; 851 852 if (ttm->state != tt_unpopulated) 853 return 0; 854 855 for (i = 0; i < ttm->num_pages; ++i) { 856 ret = ttm_get_pages(&ttm->pages[i], 1, 857 ttm->page_flags, 858 ttm->caching_state); 859 if (ret != 0) { 860 ttm_pool_unpopulate(ttm); 861 return -ENOMEM; 862 } 863 864 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], 865 false, false); 866 if (unlikely(ret != 0)) { 867 ttm_pool_unpopulate(ttm); 868 return -ENOMEM; 869 } 870 } 871 872 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 873 ret = ttm_tt_swapin(ttm); 874 if (unlikely(ret != 0)) { 875 ttm_pool_unpopulate(ttm); 876 return ret; 877 } 878 } 879 880 ttm->state = tt_unbound; 881 return 0; 882} 883 884void ttm_pool_unpopulate(struct ttm_tt *ttm) 885{ 886 unsigned i; 887 888 for (i = 0; i < ttm->num_pages; ++i) { 889 if (ttm->pages[i]) { 890 ttm_mem_global_free_page(ttm->glob->mem_glob, 891 ttm->pages[i]); 892 ttm_put_pages(&ttm->pages[i], 1, 893 ttm->page_flags, 894 ttm->caching_state); 895 } 896 } 897 ttm->state = tt_unpopulated; 898} 899 900#if 0 901/* XXXKIB sysctl */ 902int ttm_page_alloc_debugfs(struct seq_file *m, void *data) 903{ 904 struct ttm_page_pool *p; 905 unsigned i; 906 char *h[] = {"pool", "refills", "pages freed", "size"}; 907 if (!_manager) { 908 seq_printf(m, "No pool allocator running.\n"); 909 return 0; 910 } 911 seq_printf(m, "%6s %12s %13s %8s\n", 912 h[0], h[1], h[2], h[3]); 913 for (i = 0; i < NUM_POOLS; ++i) { 914 p = &_manager->pools[i]; 915 916 seq_printf(m, "%6s %12ld %13ld %8d\n", 917 p->name, p->nrefills, 918 p->nfrees, p->npages); 919 } 920 return 0; 921} 922#endif 923