1/* $OpenBSD: uvm_amap.c,v 1.95 2024/05/20 17:03:36 dv Exp $ */ 2/* $NetBSD: uvm_amap.c,v 1.27 2000/11/25 06:27:59 chs Exp $ */ 3 4/* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * uvm_amap.c: amap operations 31 * 32 * this file contains functions that perform operations on amaps. see 33 * uvm_amap.h for a brief explanation of the role of amaps in uvm. 34 */ 35 36#include <sys/param.h> 37#include <sys/systm.h> 38#include <sys/malloc.h> 39#include <sys/kernel.h> 40#include <sys/pool.h> 41#include <sys/atomic.h> 42 43#include <uvm/uvm.h> 44#include <uvm/uvm_swap.h> 45 46/* 47 * pools for allocation of vm_amap structures. note that in order to 48 * avoid an endless loop, the amap pool's allocator cannot allocate 49 * memory from an amap (it currently goes through the kernel uobj, so 50 * we are ok). 51 */ 52 53struct pool uvm_amap_pool; 54struct pool uvm_small_amap_pool[UVM_AMAP_CHUNK]; 55struct pool uvm_amap_chunk_pool; 56 57LIST_HEAD(, vm_amap) amap_list; 58struct rwlock amap_list_lock = RWLOCK_INITIALIZER("amaplstlk"); 59#define amap_lock_list() rw_enter_write(&amap_list_lock) 60#define amap_unlock_list() rw_exit_write(&amap_list_lock) 61 62static char amap_small_pool_names[UVM_AMAP_CHUNK][9]; 63 64/* 65 * local functions 66 */ 67 68static struct vm_amap *amap_alloc1(int, int, int); 69static inline void amap_list_insert(struct vm_amap *); 70static inline void amap_list_remove(struct vm_amap *); 71 72struct vm_amap_chunk *amap_chunk_get(struct vm_amap *, int, int, int); 73void amap_chunk_free(struct vm_amap *, struct vm_amap_chunk *); 74 75/* 76 * if we enable PPREF, then we have a couple of extra functions that 77 * we need to prototype here... 78 */ 79 80#ifdef UVM_AMAP_PPREF 81 82#define PPREF_NONE ((int *) -1) /* not using ppref */ 83 84void amap_pp_adjref(struct vm_amap *, int, vsize_t, int); 85void amap_pp_establish(struct vm_amap *); 86void amap_wiperange_chunk(struct vm_amap *, struct vm_amap_chunk *, int, 87 int); 88void amap_wiperange(struct vm_amap *, int, int); 89 90#endif /* UVM_AMAP_PPREF */ 91 92static inline void 93amap_list_insert(struct vm_amap *amap) 94{ 95 amap_lock_list(); 96 LIST_INSERT_HEAD(&amap_list, amap, am_list); 97 amap_unlock_list(); 98} 99 100static inline void 101amap_list_remove(struct vm_amap *amap) 102{ 103 amap_lock_list(); 104 LIST_REMOVE(amap, am_list); 105 amap_unlock_list(); 106} 107 108/* 109 * amap_chunk_get: lookup a chunk for slot. if create is non-zero, 110 * the chunk is created if it does not yet exist. 111 * 112 * => returns the chunk on success or NULL on error 113 */ 114struct vm_amap_chunk * 115amap_chunk_get(struct vm_amap *amap, int slot, int create, int waitf) 116{ 117 int bucket = UVM_AMAP_BUCKET(amap, slot); 118 int baseslot = AMAP_BASE_SLOT(slot); 119 int n; 120 struct vm_amap_chunk *chunk, *newchunk, *pchunk = NULL; 121 122 if (UVM_AMAP_SMALL(amap)) 123 return &amap->am_small; 124 125 for (chunk = amap->am_buckets[bucket]; chunk != NULL; 126 chunk = TAILQ_NEXT(chunk, ac_list)) { 127 if (UVM_AMAP_BUCKET(amap, chunk->ac_baseslot) != bucket) 128 break; 129 if (chunk->ac_baseslot == baseslot) 130 return chunk; 131 pchunk = chunk; 132 } 133 if (!create) 134 return NULL; 135 136 if (amap->am_nslot - baseslot >= UVM_AMAP_CHUNK) 137 n = UVM_AMAP_CHUNK; 138 else 139 n = amap->am_nslot - baseslot; 140 141 newchunk = pool_get(&uvm_amap_chunk_pool, waitf | PR_ZERO); 142 if (newchunk == NULL) 143 return NULL; 144 145 if (pchunk == NULL) { 146 TAILQ_INSERT_TAIL(&amap->am_chunks, newchunk, ac_list); 147 KASSERT(amap->am_buckets[bucket] == NULL); 148 amap->am_buckets[bucket] = newchunk; 149 } else 150 TAILQ_INSERT_AFTER(&amap->am_chunks, pchunk, newchunk, 151 ac_list); 152 153 amap->am_ncused++; 154 newchunk->ac_baseslot = baseslot; 155 newchunk->ac_nslot = n; 156 return newchunk; 157} 158 159void 160amap_chunk_free(struct vm_amap *amap, struct vm_amap_chunk *chunk) 161{ 162 int bucket = UVM_AMAP_BUCKET(amap, chunk->ac_baseslot); 163 struct vm_amap_chunk *nchunk; 164 165 if (UVM_AMAP_SMALL(amap)) 166 return; 167 168 nchunk = TAILQ_NEXT(chunk, ac_list); 169 TAILQ_REMOVE(&amap->am_chunks, chunk, ac_list); 170 if (amap->am_buckets[bucket] == chunk) { 171 if (nchunk != NULL && 172 UVM_AMAP_BUCKET(amap, nchunk->ac_baseslot) == bucket) 173 amap->am_buckets[bucket] = nchunk; 174 else 175 amap->am_buckets[bucket] = NULL; 176 177 } 178 pool_put(&uvm_amap_chunk_pool, chunk); 179 amap->am_ncused--; 180} 181 182#ifdef UVM_AMAP_PPREF 183/* 184 * what is ppref? ppref is an _optional_ amap feature which is used 185 * to keep track of reference counts on a per-page basis. it is enabled 186 * when UVM_AMAP_PPREF is defined. 187 * 188 * when enabled, an array of ints is allocated for the pprefs. this 189 * array is allocated only when a partial reference is added to the 190 * map (either by unmapping part of the amap, or gaining a reference 191 * to only a part of an amap). if the allocation of the array fails 192 * (M_NOWAIT), then we set the array pointer to PPREF_NONE to indicate 193 * that we tried to do ppref's but couldn't alloc the array so just 194 * give up (after all, this is an optional feature!). 195 * 196 * the array is divided into page sized "chunks." for chunks of length 1, 197 * the chunk reference count plus one is stored in that chunk's slot. 198 * for chunks of length > 1 the first slot contains (the reference count 199 * plus one) * -1. [the negative value indicates that the length is 200 * greater than one.] the second slot of the chunk contains the length 201 * of the chunk. here is an example: 202 * 203 * actual REFS: 2 2 2 2 3 1 1 0 0 0 4 4 0 1 1 1 204 * ppref: -3 4 x x 4 -2 2 -1 3 x -5 2 1 -2 3 x 205 * <----------><-><----><-------><----><-><-------> 206 * (x = don't care) 207 * 208 * this allows us to allow one int to contain the ref count for the whole 209 * chunk. note that the "plus one" part is needed because a reference 210 * count of zero is neither positive or negative (need a way to tell 211 * if we've got one zero or a bunch of them). 212 * 213 * here are some in-line functions to help us. 214 */ 215 216/* 217 * pp_getreflen: get the reference and length for a specific offset 218 * 219 * => ppref's amap must be locked 220 */ 221static inline void 222pp_getreflen(int *ppref, int offset, int *refp, int *lenp) 223{ 224 225 if (ppref[offset] > 0) { /* chunk size must be 1 */ 226 *refp = ppref[offset] - 1; /* don't forget to adjust */ 227 *lenp = 1; 228 } else { 229 *refp = (ppref[offset] * -1) - 1; 230 *lenp = ppref[offset+1]; 231 } 232} 233 234/* 235 * pp_setreflen: set the reference and length for a specific offset 236 * 237 * => ppref's amap must be locked 238 */ 239static inline void 240pp_setreflen(int *ppref, int offset, int ref, int len) 241{ 242 if (len == 1) { 243 ppref[offset] = ref + 1; 244 } else { 245 ppref[offset] = (ref + 1) * -1; 246 ppref[offset+1] = len; 247 } 248} 249#endif /* UVM_AMAP_PPREF */ 250 251/* 252 * amap_init: called at boot time to init global amap data structures 253 */ 254 255void 256amap_init(void) 257{ 258 int i; 259 size_t size; 260 261 /* Initialize the vm_amap pool. */ 262 pool_init(&uvm_amap_pool, sizeof(struct vm_amap), 263 0, IPL_MPFLOOR, PR_WAITOK, "amappl", NULL); 264 pool_sethiwat(&uvm_amap_pool, 4096); 265 266 /* initialize small amap pools */ 267 for (i = 0; i < nitems(uvm_small_amap_pool); i++) { 268 snprintf(amap_small_pool_names[i], 269 sizeof(amap_small_pool_names[0]), "amappl%d", i + 1); 270 size = offsetof(struct vm_amap, am_small.ac_anon) + 271 (i + 1) * sizeof(struct vm_anon *); 272 pool_init(&uvm_small_amap_pool[i], size, 0, IPL_MPFLOOR, 273 PR_WAITOK, amap_small_pool_names[i], NULL); 274 } 275 276 pool_init(&uvm_amap_chunk_pool, sizeof(struct vm_amap_chunk) + 277 UVM_AMAP_CHUNK * sizeof(struct vm_anon *), 278 0, IPL_MPFLOOR, PR_WAITOK, "amapchunkpl", NULL); 279 pool_sethiwat(&uvm_amap_chunk_pool, 4096); 280} 281 282/* 283 * amap_alloc1: allocate an amap, but do not initialise the overlay. 284 * 285 * => Note: lock is not set. 286 */ 287static inline struct vm_amap * 288amap_alloc1(int slots, int waitf, int lazyalloc) 289{ 290 struct vm_amap *amap; 291 struct vm_amap_chunk *chunk, *tmp; 292 int chunks, log_chunks, chunkperbucket = 1, hashshift = 0; 293 int buckets, i, n; 294 int pwaitf = (waitf & M_WAITOK) ? PR_WAITOK : PR_NOWAIT; 295 296 KASSERT(slots > 0); 297 298 /* 299 * Cast to unsigned so that rounding up cannot cause integer overflow 300 * if slots is large. 301 */ 302 chunks = roundup((unsigned int)slots, UVM_AMAP_CHUNK) / UVM_AMAP_CHUNK; 303 304 if (lazyalloc) { 305 /* 306 * Basically, the amap is a hash map where the number of 307 * buckets is fixed. We select the number of buckets using the 308 * following strategy: 309 * 310 * 1. The maximal number of entries to search in a bucket upon 311 * a collision should be less than or equal to 312 * log2(slots / UVM_AMAP_CHUNK). This is the worst-case number 313 * of lookups we would have if we could chunk the amap. The 314 * log2(n) comes from the fact that amaps are chunked by 315 * splitting up their vm_map_entries and organizing those 316 * in a binary search tree. 317 * 318 * 2. The maximal number of entries in a bucket must be a 319 * power of two. 320 * 321 * The maximal number of entries per bucket is used to hash 322 * a slot to a bucket. 323 * 324 * In the future, this strategy could be refined to make it 325 * even harder/impossible that the total amount of KVA needed 326 * for the hash buckets of all amaps to exceed the maximal 327 * amount of KVA memory reserved for amaps. 328 */ 329 for (log_chunks = 1; (chunks >> log_chunks) > 0; log_chunks++) 330 continue; 331 332 chunkperbucket = 1 << hashshift; 333 while (chunkperbucket + 1 < log_chunks) { 334 hashshift++; 335 chunkperbucket = 1 << hashshift; 336 } 337 } 338 339 if (slots > UVM_AMAP_CHUNK) 340 amap = pool_get(&uvm_amap_pool, pwaitf); 341 else 342 amap = pool_get(&uvm_small_amap_pool[slots - 1], 343 pwaitf | PR_ZERO); 344 if (amap == NULL) 345 return NULL; 346 347 amap->am_lock = NULL; 348 amap->am_ref = 1; 349 amap->am_flags = 0; 350#ifdef UVM_AMAP_PPREF 351 amap->am_ppref = NULL; 352#endif 353 amap->am_nslot = slots; 354 amap->am_nused = 0; 355 356 if (UVM_AMAP_SMALL(amap)) { 357 amap->am_small.ac_nslot = slots; 358 return amap; 359 } 360 361 amap->am_ncused = 0; 362 TAILQ_INIT(&amap->am_chunks); 363 amap->am_hashshift = hashshift; 364 amap->am_buckets = NULL; 365 366 buckets = howmany(chunks, chunkperbucket); 367 amap->am_buckets = mallocarray(buckets, sizeof(*amap->am_buckets), 368 M_UVMAMAP, waitf | (lazyalloc ? M_ZERO : 0)); 369 if (amap->am_buckets == NULL) 370 goto fail1; 371 amap->am_nbuckets = buckets; 372 373 if (!lazyalloc) { 374 for (i = 0; i < buckets; i++) { 375 if (i == buckets - 1) { 376 n = slots % UVM_AMAP_CHUNK; 377 if (n == 0) 378 n = UVM_AMAP_CHUNK; 379 } else 380 n = UVM_AMAP_CHUNK; 381 382 chunk = pool_get(&uvm_amap_chunk_pool, 383 PR_ZERO | pwaitf); 384 if (chunk == NULL) 385 goto fail1; 386 387 amap->am_buckets[i] = chunk; 388 amap->am_ncused++; 389 chunk->ac_baseslot = i * UVM_AMAP_CHUNK; 390 chunk->ac_nslot = n; 391 TAILQ_INSERT_TAIL(&amap->am_chunks, chunk, ac_list); 392 } 393 } 394 395 return amap; 396 397fail1: 398 free(amap->am_buckets, M_UVMAMAP, buckets * sizeof(*amap->am_buckets)); 399 TAILQ_FOREACH_SAFE(chunk, &amap->am_chunks, ac_list, tmp) 400 pool_put(&uvm_amap_chunk_pool, chunk); 401 pool_put(&uvm_amap_pool, amap); 402 return NULL; 403} 404 405static void 406amap_lock_alloc(struct vm_amap *amap) 407{ 408 rw_obj_alloc(&amap->am_lock, "amaplk"); 409} 410 411/* 412 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM 413 * 414 * => caller should ensure sz is a multiple of PAGE_SIZE 415 * => reference count to new amap is set to one 416 * => new amap is returned unlocked 417 */ 418struct vm_amap * 419amap_alloc(vaddr_t sz, int waitf, int lazyalloc) 420{ 421 struct vm_amap *amap; 422 size_t slots; 423 424 AMAP_B2SLOT(slots, sz); /* load slots */ 425 if (slots > INT_MAX) 426 return NULL; 427 428 amap = amap_alloc1(slots, waitf, lazyalloc); 429 if (amap != NULL) { 430 amap_lock_alloc(amap); 431 amap_list_insert(amap); 432 } 433 434 return amap; 435} 436 437 438/* 439 * amap_free: free an amap 440 * 441 * => the amap must be unlocked 442 * => the amap should have a zero reference count and be empty 443 */ 444void 445amap_free(struct vm_amap *amap) 446{ 447 struct vm_amap_chunk *chunk, *tmp; 448 449 KASSERT(amap->am_ref == 0 && amap->am_nused == 0); 450 KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0); 451 452 if (amap->am_lock != NULL) { 453 KASSERT(!rw_write_held(amap->am_lock)); 454 rw_obj_free(amap->am_lock); 455 } 456 457#ifdef UVM_AMAP_PPREF 458 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 459 free(amap->am_ppref, M_UVMAMAP, amap->am_nslot * sizeof(int)); 460#endif 461 462 if (UVM_AMAP_SMALL(amap)) 463 pool_put(&uvm_small_amap_pool[amap->am_nslot - 1], amap); 464 else { 465 TAILQ_FOREACH_SAFE(chunk, &amap->am_chunks, ac_list, tmp) 466 pool_put(&uvm_amap_chunk_pool, chunk); 467 free(amap->am_buckets, M_UVMAMAP, 468 amap->am_nbuckets * sizeof(*amap->am_buckets)); 469 pool_put(&uvm_amap_pool, amap); 470 } 471} 472 473/* 474 * amap_wipeout: wipeout all anon's in an amap; then free the amap! 475 * 476 * => Called from amap_unref(), when reference count drops to zero. 477 * => amap must be locked. 478 */ 479void 480amap_wipeout(struct vm_amap *amap) 481{ 482 int slot; 483 struct vm_anon *anon; 484 struct vm_amap_chunk *chunk; 485 486 KASSERT(rw_write_held(amap->am_lock)); 487 KASSERT(amap->am_ref == 0); 488 489 if (__predict_false((amap->am_flags & AMAP_SWAPOFF) != 0)) { 490 /* 491 * Note: amap_swap_off() will call us again. 492 */ 493 amap_unlock(amap); 494 return; 495 } 496 497 amap_list_remove(amap); 498 499 AMAP_CHUNK_FOREACH(chunk, amap) { 500 int i, refs, map = chunk->ac_usedmap; 501 502 for (i = ffs(map); i != 0; i = ffs(map)) { 503 slot = i - 1; 504 map ^= 1 << slot; 505 anon = chunk->ac_anon[slot]; 506 507 if (anon == NULL || anon->an_ref == 0) 508 panic("amap_wipeout: corrupt amap"); 509 KASSERT(anon->an_lock == amap->am_lock); 510 511 /* 512 * Drop the reference. 513 */ 514 refs = --anon->an_ref; 515 if (refs == 0) { 516 uvm_anfree(anon); 517 } 518 } 519 } 520 521 /* 522 * Finally, destroy the amap. 523 */ 524 amap->am_ref = 0; /* ... was one */ 525 amap->am_nused = 0; 526 amap_unlock(amap); 527 amap_free(amap); 528} 529 530/* 531 * amap_copy: ensure that a map entry's "needs_copy" flag is false 532 * by copying the amap if necessary. 533 * 534 * => an entry with a null amap pointer will get a new (blank) one. 535 * => the map that the map entry belongs to must be locked by caller. 536 * => the amap currently attached to "entry" (if any) must be unlocked. 537 * => if canchunk is true, then we may clip the entry into a chunk 538 * => "startva" and "endva" are used only if canchunk is true. they are 539 * used to limit chunking (e.g. if you have a large space that you 540 * know you are going to need to allocate amaps for, there is no point 541 * in allowing that to be chunked) 542 */ 543 544void 545amap_copy(struct vm_map *map, struct vm_map_entry *entry, int waitf, 546 boolean_t canchunk, vaddr_t startva, vaddr_t endva) 547{ 548 struct vm_amap *amap, *srcamap; 549 int slots, lcv, lazyalloc = 0; 550 vaddr_t chunksize; 551 int i, j, k, n, srcslot; 552 struct vm_amap_chunk *chunk = NULL, *srcchunk = NULL; 553 struct vm_anon *anon; 554 555 KASSERT(map != kernel_map); /* we use sleeping locks */ 556 557 /* 558 * Is there an amap to copy? If not, create one. 559 */ 560 if (entry->aref.ar_amap == NULL) { 561 /* 562 * Check to see if we have a large amap that we can 563 * chunk. We align startva/endva to chunk-sized 564 * boundaries and then clip to them. 565 * 566 * If we cannot chunk the amap, allocate it in a way 567 * that makes it grow or shrink dynamically with 568 * the number of slots. 569 */ 570 if (atop(entry->end - entry->start) >= UVM_AMAP_LARGE) { 571 if (canchunk) { 572 /* convert slots to bytes */ 573 chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT; 574 startva = (startva / chunksize) * chunksize; 575 endva = roundup(endva, chunksize); 576 UVM_MAP_CLIP_START(map, entry, startva); 577 /* watch out for endva wrap-around! */ 578 if (endva >= startva) 579 UVM_MAP_CLIP_END(map, entry, endva); 580 } else 581 lazyalloc = 1; 582 } 583 584 entry->aref.ar_pageoff = 0; 585 entry->aref.ar_amap = amap_alloc(entry->end - entry->start, 586 waitf, lazyalloc); 587 if (entry->aref.ar_amap != NULL) 588 entry->etype &= ~UVM_ET_NEEDSCOPY; 589 return; 590 } 591 592 /* 593 * First check and see if we are the only map entry referencing 594 * he amap we currently have. If so, then just take it over instead 595 * of copying it. Note that we are reading am_ref without lock held 596 * as the value can only be one if we have the only reference 597 * to the amap (via our locked map). If the value is greater than 598 * one, then allocate amap and re-check the value. 599 */ 600 if (entry->aref.ar_amap->am_ref == 1) { 601 entry->etype &= ~UVM_ET_NEEDSCOPY; 602 return; 603 } 604 605 /* 606 * Allocate a new amap (note: not initialised, etc). 607 */ 608 AMAP_B2SLOT(slots, entry->end - entry->start); 609 if (!UVM_AMAP_SMALL(entry->aref.ar_amap) && 610 entry->aref.ar_amap->am_hashshift != 0) 611 lazyalloc = 1; 612 amap = amap_alloc1(slots, waitf, lazyalloc); 613 if (amap == NULL) 614 return; 615 srcamap = entry->aref.ar_amap; 616 617 /* 618 * Make the new amap share the source amap's lock, and then lock 619 * both. 620 */ 621 amap->am_lock = srcamap->am_lock; 622 rw_obj_hold(amap->am_lock); 623 624 amap_lock(srcamap); 625 626 /* 627 * Re-check the reference count with the lock held. If it has 628 * dropped to one - we can take over the existing map. 629 */ 630 if (srcamap->am_ref == 1) { 631 /* Just take over the existing amap. */ 632 entry->etype &= ~UVM_ET_NEEDSCOPY; 633 amap_unlock(srcamap); 634 /* Destroy the new (unused) amap. */ 635 amap->am_ref--; 636 amap_free(amap); 637 return; 638 } 639 640 /* 641 * Copy the slots. 642 */ 643 for (lcv = 0; lcv < slots; lcv += n) { 644 srcslot = entry->aref.ar_pageoff + lcv; 645 i = UVM_AMAP_SLOTIDX(lcv); 646 j = UVM_AMAP_SLOTIDX(srcslot); 647 n = UVM_AMAP_CHUNK; 648 if (i > j) 649 n -= i; 650 else 651 n -= j; 652 if (lcv + n > slots) 653 n = slots - lcv; 654 655 srcchunk = amap_chunk_get(srcamap, srcslot, 0, PR_NOWAIT); 656 if (srcchunk == NULL) 657 continue; 658 659 chunk = amap_chunk_get(amap, lcv, 1, PR_NOWAIT); 660 if (chunk == NULL) { 661 amap_unlock(srcamap); 662 /* Destroy the new amap. */ 663 amap->am_ref--; 664 amap_free(amap); 665 return; 666 } 667 668 for (k = 0; k < n; i++, j++, k++) { 669 chunk->ac_anon[i] = anon = srcchunk->ac_anon[j]; 670 if (anon == NULL) 671 continue; 672 673 KASSERT(anon->an_lock == srcamap->am_lock); 674 KASSERT(anon->an_ref > 0); 675 chunk->ac_usedmap |= (1 << i); 676 anon->an_ref++; 677 amap->am_nused++; 678 } 679 } 680 681 /* 682 * Drop our reference to the old amap (srcamap) and unlock. 683 * Since the reference count on srcamap is greater than one, 684 * (we checked above), it cannot drop to zero while it is locked. 685 */ 686 srcamap->am_ref--; 687 KASSERT(srcamap->am_ref > 0); 688 689 if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) 690 srcamap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ 691#ifdef UVM_AMAP_PPREF 692 if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) { 693 amap_pp_adjref(srcamap, entry->aref.ar_pageoff, 694 (entry->end - entry->start) >> PAGE_SHIFT, -1); 695 } 696#endif 697 698 /* 699 * If we referenced any anons, then share the source amap's lock. 700 * Otherwise, we have nothing in common, so allocate a new one. 701 */ 702 KASSERT(amap->am_lock == srcamap->am_lock); 703 if (amap->am_nused == 0) { 704 rw_obj_free(amap->am_lock); 705 amap->am_lock = NULL; 706 } 707 amap_unlock(srcamap); 708 709 if (amap->am_lock == NULL) 710 amap_lock_alloc(amap); 711 712 /* 713 * Install new amap. 714 */ 715 entry->aref.ar_pageoff = 0; 716 entry->aref.ar_amap = amap; 717 entry->etype &= ~UVM_ET_NEEDSCOPY; 718 719 amap_list_insert(amap); 720} 721 722/* 723 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2) 724 * 725 * called during fork(2) when the parent process has a wired map 726 * entry. in that case we want to avoid write-protecting pages 727 * in the parent's map (e.g. like what you'd do for a COW page) 728 * so we resolve the COW here. 729 * 730 * => assume parent's entry was wired, thus all pages are resident. 731 * => the parent and child vm_map must both be locked. 732 * => caller passes child's map/entry in to us 733 * => XXXCDC: out of memory should cause fork to fail, but there is 734 * currently no easy way to do this (needs fix) 735 */ 736 737void 738amap_cow_now(struct vm_map *map, struct vm_map_entry *entry) 739{ 740 struct vm_amap *amap = entry->aref.ar_amap; 741 int slot; 742 struct vm_anon *anon, *nanon; 743 struct vm_page *pg, *npg; 744 struct vm_amap_chunk *chunk; 745 746 /* 747 * note that if we unlock the amap then we must ReStart the "lcv" for 748 * loop because some other process could reorder the anon's in the 749 * am_anon[] array on us while the lock is dropped. 750 */ 751ReStart: 752 amap_lock(amap); 753 AMAP_CHUNK_FOREACH(chunk, amap) { 754 int i, map = chunk->ac_usedmap; 755 756 for (i = ffs(map); i != 0; i = ffs(map)) { 757 slot = i - 1; 758 map ^= 1 << slot; 759 anon = chunk->ac_anon[slot]; 760 pg = anon->an_page; 761 KASSERT(anon->an_lock == amap->am_lock); 762 763 /* 764 * The old page must be resident since the parent is 765 * wired. 766 */ 767 KASSERT(pg != NULL); 768 769 /* 770 * if the anon ref count is one, we are safe (the child 771 * has exclusive access to the page). 772 */ 773 if (anon->an_ref <= 1) 774 continue; 775 776 /* 777 * If the page is busy, then we have to unlock, wait for 778 * it and then restart. 779 */ 780 if (pg->pg_flags & PG_BUSY) { 781 uvm_pagewait(pg, amap->am_lock, "cownow"); 782 goto ReStart; 783 } 784 785 /* 786 * Perform a copy-on-write. 787 * First - get a new anon and a page. 788 */ 789 nanon = uvm_analloc(); 790 if (nanon != NULL) { 791 /* the new anon will share the amap's lock */ 792 nanon->an_lock = amap->am_lock; 793 npg = uvm_pagealloc(NULL, 0, nanon, 0); 794 } else 795 npg = NULL; /* XXX: quiet gcc warning */ 796 797 if (nanon == NULL || npg == NULL) { 798 /* out of memory */ 799 amap_unlock(amap); 800 if (nanon != NULL) { 801 nanon->an_lock = NULL; 802 nanon->an_ref--; 803 KASSERT(nanon->an_ref == 0); 804 uvm_anfree(nanon); 805 } 806 uvm_wait("cownowpage"); 807 goto ReStart; 808 } 809 810 /* 811 * Copy the data and replace anon with the new one. 812 * Also, setup its lock (share the with amap's lock). 813 */ 814 uvm_pagecopy(pg, npg); 815 anon->an_ref--; 816 KASSERT(anon->an_ref > 0); 817 chunk->ac_anon[slot] = nanon; 818 819 /* 820 * Drop PG_BUSY on new page. Since its owner was write 821 * locked all this time - it cannot be PG_RELEASED or 822 * PG_WANTED. 823 */ 824 atomic_clearbits_int(&npg->pg_flags, PG_BUSY|PG_FAKE); 825 UVM_PAGE_OWN(npg, NULL); 826 uvm_lock_pageq(); 827 uvm_pageactivate(npg); 828 uvm_unlock_pageq(); 829 } 830 } 831 amap_unlock(amap); 832} 833 834/* 835 * amap_splitref: split a single reference into two separate references 836 * 837 * => called from uvm_map's clip routines 838 * => origref's map should be locked 839 * => origref->ar_amap should be unlocked (we will lock) 840 */ 841void 842amap_splitref(struct vm_aref *origref, struct vm_aref *splitref, vaddr_t offset) 843{ 844 struct vm_amap *amap = origref->ar_amap; 845 int leftslots; 846 847 KASSERT(splitref->ar_amap == amap); 848 AMAP_B2SLOT(leftslots, offset); 849 if (leftslots == 0) 850 panic("amap_splitref: split at zero offset"); 851 852 amap_lock(amap); 853 854 if (amap->am_nslot - origref->ar_pageoff - leftslots <= 0) 855 panic("amap_splitref: map size check failed"); 856 857#ifdef UVM_AMAP_PPREF 858 /* Establish ppref before we add a duplicate reference to the amap. */ 859 if (amap->am_ppref == NULL) 860 amap_pp_establish(amap); 861#endif 862 863 /* Note: not a share reference. */ 864 amap->am_ref++; 865 splitref->ar_amap = amap; 866 splitref->ar_pageoff = origref->ar_pageoff + leftslots; 867 amap_unlock(amap); 868} 869 870#ifdef UVM_AMAP_PPREF 871 872/* 873 * amap_pp_establish: add a ppref array to an amap, if possible. 874 * 875 * => amap should be locked by caller* => amap should be locked by caller 876 */ 877void 878amap_pp_establish(struct vm_amap *amap) 879{ 880 881 KASSERT(rw_write_held(amap->am_lock)); 882 amap->am_ppref = mallocarray(amap->am_nslot, sizeof(int), 883 M_UVMAMAP, M_NOWAIT|M_ZERO); 884 885 if (amap->am_ppref == NULL) { 886 /* Failure - just do not use ppref. */ 887 amap->am_ppref = PPREF_NONE; 888 return; 889 } 890 891 pp_setreflen(amap->am_ppref, 0, amap->am_ref, amap->am_nslot); 892} 893 894/* 895 * amap_pp_adjref: adjust reference count to a part of an amap using the 896 * per-page reference count array. 897 * 898 * => caller must check that ppref != PPREF_NONE before calling. 899 * => map and amap must be locked. 900 */ 901void 902amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval) 903{ 904 int stopslot, *ppref, lcv, prevlcv; 905 int ref, len, prevref, prevlen; 906 907 KASSERT(rw_write_held(amap->am_lock)); 908 909 stopslot = curslot + slotlen; 910 ppref = amap->am_ppref; 911 prevlcv = 0; 912 913 /* 914 * Advance to the correct place in the array, fragment if needed. 915 */ 916 for (lcv = 0 ; lcv < curslot ; lcv += len) { 917 pp_getreflen(ppref, lcv, &ref, &len); 918 if (lcv + len > curslot) { /* goes past start? */ 919 pp_setreflen(ppref, lcv, ref, curslot - lcv); 920 pp_setreflen(ppref, curslot, ref, len - (curslot -lcv)); 921 len = curslot - lcv; /* new length of entry @ lcv */ 922 } 923 prevlcv = lcv; 924 } 925 if (lcv != 0) 926 pp_getreflen(ppref, prevlcv, &prevref, &prevlen); 927 else { 928 /* 929 * Ensure that the "prevref == ref" test below always 930 * fails, since we are starting from the beginning of 931 * the ppref array; that is, there is no previous chunk. 932 */ 933 prevref = -1; 934 prevlen = 0; 935 } 936 937 /* 938 * Now adjust reference counts in range. Merge the first 939 * changed entry with the last unchanged entry if possible. 940 */ 941 if (lcv != curslot) 942 panic("amap_pp_adjref: overshot target"); 943 944 for (/* lcv already set */; lcv < stopslot ; lcv += len) { 945 pp_getreflen(ppref, lcv, &ref, &len); 946 if (lcv + len > stopslot) { /* goes past end? */ 947 pp_setreflen(ppref, lcv, ref, stopslot - lcv); 948 pp_setreflen(ppref, stopslot, ref, 949 len - (stopslot - lcv)); 950 len = stopslot - lcv; 951 } 952 ref += adjval; 953 if (ref < 0) 954 panic("amap_pp_adjref: negative reference count"); 955 if (lcv == prevlcv + prevlen && ref == prevref) { 956 pp_setreflen(ppref, prevlcv, ref, prevlen + len); 957 } else { 958 pp_setreflen(ppref, lcv, ref, len); 959 } 960 if (ref == 0) 961 amap_wiperange(amap, lcv, len); 962 } 963 964} 965 966void 967amap_wiperange_chunk(struct vm_amap *amap, struct vm_amap_chunk *chunk, 968 int slotoff, int slots) 969{ 970 int curslot, i, map; 971 int startbase, endbase; 972 struct vm_anon *anon; 973 974 startbase = AMAP_BASE_SLOT(slotoff); 975 endbase = AMAP_BASE_SLOT(slotoff + slots - 1); 976 977 map = chunk->ac_usedmap; 978 if (startbase == chunk->ac_baseslot) 979 map &= ~((1 << (slotoff - startbase)) - 1); 980 if (endbase == chunk->ac_baseslot) 981 map &= (1 << (slotoff + slots - endbase)) - 1; 982 983 for (i = ffs(map); i != 0; i = ffs(map)) { 984 int refs; 985 986 curslot = i - 1; 987 map ^= 1 << curslot; 988 chunk->ac_usedmap ^= 1 << curslot; 989 anon = chunk->ac_anon[curslot]; 990 KASSERT(anon->an_lock == amap->am_lock); 991 992 /* remove it from the amap */ 993 chunk->ac_anon[curslot] = NULL; 994 995 amap->am_nused--; 996 997 /* drop anon reference count */ 998 refs = --anon->an_ref; 999 if (refs == 0) { 1000 uvm_anfree(anon); 1001 } 1002 1003 /* 1004 * done with this anon, next ...! 1005 */ 1006 1007 } /* end of 'for' loop */ 1008} 1009 1010/* 1011 * amap_wiperange: wipe out a range of an amap. 1012 * Note: different from amap_wipeout because the amap is kept intact. 1013 * 1014 * => Both map and amap must be locked by caller. 1015 */ 1016void 1017amap_wiperange(struct vm_amap *amap, int slotoff, int slots) 1018{ 1019 int bucket, startbucket, endbucket; 1020 struct vm_amap_chunk *chunk, *nchunk; 1021 1022 KASSERT(rw_write_held(amap->am_lock)); 1023 1024 startbucket = UVM_AMAP_BUCKET(amap, slotoff); 1025 endbucket = UVM_AMAP_BUCKET(amap, slotoff + slots - 1); 1026 1027 /* 1028 * We can either traverse the amap by am_chunks or by am_buckets. 1029 * Determine which way is less expensive. 1030 */ 1031 if (UVM_AMAP_SMALL(amap)) 1032 amap_wiperange_chunk(amap, &amap->am_small, slotoff, slots); 1033 else if (endbucket + 1 - startbucket >= amap->am_ncused) { 1034 TAILQ_FOREACH_SAFE(chunk, &amap->am_chunks, ac_list, nchunk) { 1035 if (chunk->ac_baseslot + chunk->ac_nslot <= slotoff) 1036 continue; 1037 if (chunk->ac_baseslot >= slotoff + slots) 1038 continue; 1039 1040 amap_wiperange_chunk(amap, chunk, slotoff, slots); 1041 if (chunk->ac_usedmap == 0) 1042 amap_chunk_free(amap, chunk); 1043 } 1044 } else { 1045 for (bucket = startbucket; bucket <= endbucket; bucket++) { 1046 for (chunk = amap->am_buckets[bucket]; chunk != NULL; 1047 chunk = nchunk) { 1048 nchunk = TAILQ_NEXT(chunk, ac_list); 1049 1050 if (UVM_AMAP_BUCKET(amap, chunk->ac_baseslot) != 1051 bucket) 1052 break; 1053 if (chunk->ac_baseslot + chunk->ac_nslot <= 1054 slotoff) 1055 continue; 1056 if (chunk->ac_baseslot >= slotoff + slots) 1057 continue; 1058 1059 amap_wiperange_chunk(amap, chunk, slotoff, 1060 slots); 1061 if (chunk->ac_usedmap == 0) 1062 amap_chunk_free(amap, chunk); 1063 } 1064 } 1065 } 1066} 1067 1068#endif 1069 1070/* 1071 * amap_swap_off: pagein anonymous pages in amaps and drop swap slots. 1072 * 1073 * => note that we don't always traverse all anons. 1074 * eg. amaps being wiped out, released anons. 1075 * => return TRUE if failed. 1076 */ 1077 1078boolean_t 1079amap_swap_off(int startslot, int endslot) 1080{ 1081 struct vm_amap *am; 1082 struct vm_amap *am_next; 1083 struct vm_amap marker; 1084 boolean_t rv = FALSE; 1085 1086 amap_lock_list(); 1087 for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) { 1088 int i, map; 1089 struct vm_amap_chunk *chunk; 1090 1091 amap_lock(am); 1092 if (am->am_nused == 0) { 1093 amap_unlock(am); 1094 am_next = LIST_NEXT(am, am_list); 1095 continue; 1096 } 1097 1098 LIST_INSERT_AFTER(am, &marker, am_list); 1099 amap_unlock_list(); 1100 1101again: 1102 AMAP_CHUNK_FOREACH(chunk, am) { 1103 map = chunk->ac_usedmap; 1104 1105 for (i = ffs(map); i != 0; i = ffs(map)) { 1106 int swslot; 1107 int slot = i - 1; 1108 struct vm_anon *anon; 1109 1110 map ^= 1 << slot; 1111 anon = chunk->ac_anon[slot]; 1112 1113 swslot = anon->an_swslot; 1114 if (swslot < startslot || endslot <= swslot) { 1115 continue; 1116 } 1117 1118 am->am_flags |= AMAP_SWAPOFF; 1119 1120 rv = uvm_anon_pagein(am, anon); 1121 amap_lock(am); 1122 1123 am->am_flags &= ~AMAP_SWAPOFF; 1124 if (amap_refs(am) == 0) { 1125 amap_wipeout(am); 1126 am = NULL; 1127 goto nextamap; 1128 } 1129 if (rv) 1130 goto nextamap; 1131 goto again; 1132 } 1133 } 1134nextamap: 1135 if (am != NULL) 1136 amap_unlock(am); 1137 amap_lock_list(); 1138 am_next = LIST_NEXT(&marker, am_list); 1139 LIST_REMOVE(&marker, am_list); 1140 } 1141 amap_unlock_list(); 1142 1143 return rv; 1144} 1145 1146/* 1147 * amap_lookup: look up a page in an amap. 1148 * 1149 * => amap should be locked by caller. 1150 */ 1151struct vm_anon * 1152amap_lookup(struct vm_aref *aref, vaddr_t offset) 1153{ 1154 int slot; 1155 struct vm_amap *amap = aref->ar_amap; 1156 struct vm_amap_chunk *chunk; 1157 1158 AMAP_B2SLOT(slot, offset); 1159 slot += aref->ar_pageoff; 1160 KASSERT(slot < amap->am_nslot); 1161 1162 chunk = amap_chunk_get(amap, slot, 0, PR_NOWAIT); 1163 if (chunk == NULL) 1164 return NULL; 1165 1166 return chunk->ac_anon[UVM_AMAP_SLOTIDX(slot)]; 1167} 1168 1169/* 1170 * amap_lookups: look up a range of pages in an amap. 1171 * 1172 * => amap should be locked by caller. 1173 * => XXXCDC: this interface is biased toward array-based amaps. fix. 1174 */ 1175void 1176amap_lookups(struct vm_aref *aref, vaddr_t offset, 1177 struct vm_anon **anons, int npages) 1178{ 1179 int i, lcv, n, slot; 1180 struct vm_amap *amap = aref->ar_amap; 1181 struct vm_amap_chunk *chunk = NULL; 1182 1183 AMAP_B2SLOT(slot, offset); 1184 slot += aref->ar_pageoff; 1185 1186 KASSERT((slot + (npages - 1)) < amap->am_nslot); 1187 1188 for (i = 0, lcv = slot; lcv < slot + npages; i += n, lcv += n) { 1189 n = UVM_AMAP_CHUNK - UVM_AMAP_SLOTIDX(lcv); 1190 if (lcv + n > slot + npages) 1191 n = slot + npages - lcv; 1192 1193 chunk = amap_chunk_get(amap, lcv, 0, PR_NOWAIT); 1194 if (chunk == NULL) 1195 memset(&anons[i], 0, n * sizeof(*anons)); 1196 else 1197 memcpy(&anons[i], 1198 &chunk->ac_anon[UVM_AMAP_SLOTIDX(lcv)], 1199 n * sizeof(*anons)); 1200 } 1201} 1202 1203/* 1204 * amap_populate: ensure that the amap can store an anon for the page at 1205 * offset. This function can sleep until memory to store the anon is 1206 * available. 1207 */ 1208void 1209amap_populate(struct vm_aref *aref, vaddr_t offset) 1210{ 1211 int slot; 1212 struct vm_amap *amap = aref->ar_amap; 1213 struct vm_amap_chunk *chunk; 1214 1215 AMAP_B2SLOT(slot, offset); 1216 slot += aref->ar_pageoff; 1217 KASSERT(slot < amap->am_nslot); 1218 1219 chunk = amap_chunk_get(amap, slot, 1, PR_WAITOK); 1220 KASSERT(chunk != NULL); 1221} 1222 1223/* 1224 * amap_add: add (or replace) a page to an amap. 1225 * 1226 * => amap should be locked by caller. 1227 * => anon must have the lock associated with this amap. 1228 */ 1229int 1230amap_add(struct vm_aref *aref, vaddr_t offset, struct vm_anon *anon, 1231 boolean_t replace) 1232{ 1233 int slot; 1234 struct vm_amap *amap = aref->ar_amap; 1235 struct vm_amap_chunk *chunk; 1236 1237 AMAP_B2SLOT(slot, offset); 1238 slot += aref->ar_pageoff; 1239 KASSERT(slot < amap->am_nslot); 1240 1241 chunk = amap_chunk_get(amap, slot, 1, PR_NOWAIT); 1242 if (chunk == NULL) 1243 return 1; 1244 1245 slot = UVM_AMAP_SLOTIDX(slot); 1246 if (replace) { 1247 struct vm_anon *oanon = chunk->ac_anon[slot]; 1248 1249 KASSERT(oanon != NULL); 1250 if (oanon->an_page && (amap->am_flags & AMAP_SHARED) != 0) { 1251 pmap_page_protect(oanon->an_page, PROT_NONE); 1252 /* 1253 * XXX: suppose page is supposed to be wired somewhere? 1254 */ 1255 } 1256 } else { /* !replace */ 1257 if (chunk->ac_anon[slot] != NULL) 1258 panic("amap_add: slot in use"); 1259 1260 chunk->ac_usedmap |= 1 << slot; 1261 amap->am_nused++; 1262 } 1263 chunk->ac_anon[slot] = anon; 1264 1265 return 0; 1266} 1267 1268/* 1269 * amap_unadd: remove a page from an amap. 1270 * 1271 * => amap should be locked by caller. 1272 */ 1273void 1274amap_unadd(struct vm_aref *aref, vaddr_t offset) 1275{ 1276 struct vm_amap *amap = aref->ar_amap; 1277 struct vm_amap_chunk *chunk; 1278 int slot; 1279 1280 KASSERT(rw_write_held(amap->am_lock)); 1281 1282 AMAP_B2SLOT(slot, offset); 1283 slot += aref->ar_pageoff; 1284 KASSERT(slot < amap->am_nslot); 1285 chunk = amap_chunk_get(amap, slot, 0, PR_NOWAIT); 1286 KASSERT(chunk != NULL); 1287 1288 slot = UVM_AMAP_SLOTIDX(slot); 1289 KASSERT(chunk->ac_anon[slot] != NULL); 1290 1291 chunk->ac_anon[slot] = NULL; 1292 chunk->ac_usedmap &= ~(1 << slot); 1293 amap->am_nused--; 1294 1295 if (chunk->ac_usedmap == 0) 1296 amap_chunk_free(amap, chunk); 1297} 1298 1299/* 1300 * amap_adjref_anons: adjust the reference count(s) on amap and its anons. 1301 */ 1302static void 1303amap_adjref_anons(struct vm_amap *amap, vaddr_t offset, vsize_t len, 1304 int refv, boolean_t all) 1305{ 1306#ifdef UVM_AMAP_PPREF 1307 KASSERT(rw_write_held(amap->am_lock)); 1308 1309 /* 1310 * We must establish the ppref array before changing am_ref 1311 * so that the ppref values match the current amap refcount. 1312 */ 1313 if (amap->am_ppref == NULL && !all && len != amap->am_nslot) { 1314 amap_pp_establish(amap); 1315 } 1316#endif 1317 1318 amap->am_ref += refv; 1319 1320#ifdef UVM_AMAP_PPREF 1321 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 1322 if (all) { 1323 amap_pp_adjref(amap, 0, amap->am_nslot, refv); 1324 } else { 1325 amap_pp_adjref(amap, offset, len, refv); 1326 } 1327 } 1328#endif 1329 amap_unlock(amap); 1330} 1331 1332/* 1333 * amap_ref: gain a reference to an amap. 1334 * 1335 * => amap must not be locked (we will lock). 1336 * => "offset" and "len" are in units of pages. 1337 * => Called at fork time to gain the child's reference. 1338 */ 1339void 1340amap_ref(struct vm_amap *amap, vaddr_t offset, vsize_t len, int flags) 1341{ 1342 amap_lock(amap); 1343 if (flags & AMAP_SHARED) 1344 amap->am_flags |= AMAP_SHARED; 1345 amap_adjref_anons(amap, offset, len, 1, (flags & AMAP_REFALL) != 0); 1346} 1347 1348/* 1349 * amap_unref: remove a reference to an amap. 1350 * 1351 * => All pmap-level references to this amap must be already removed. 1352 * => Called from uvm_unmap_detach(); entry is already removed from the map. 1353 * => We will lock amap, so it must be unlocked. 1354 */ 1355void 1356amap_unref(struct vm_amap *amap, vaddr_t offset, vsize_t len, boolean_t all) 1357{ 1358 amap_lock(amap); 1359 1360 KASSERT(amap->am_ref > 0); 1361 1362 if (amap->am_ref == 1) { 1363 /* 1364 * If the last reference - wipeout and destroy the amap. 1365 */ 1366 amap->am_ref--; 1367 amap_wipeout(amap); 1368 return; 1369 } 1370 1371 /* 1372 * Otherwise, drop the reference count(s) on anons. 1373 */ 1374 if (amap->am_ref == 2 && (amap->am_flags & AMAP_SHARED) != 0) { 1375 amap->am_flags &= ~AMAP_SHARED; 1376 } 1377 amap_adjref_anons(amap, offset, len, -1, all); 1378} 1379