1234370Sjasone/******************************************************************************/ 2234370Sjasone#ifdef JEMALLOC_H_TYPES 3234370Sjasone 4234370Sjasone/* 5234370Sjasone * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized 6234370Sjasone * as small as possible such that this setting is still honored, without 7234370Sjasone * violating other constraints. The goal is to make runs as small as possible 8234370Sjasone * without exceeding a per run external fragmentation threshold. 9234370Sjasone * 10234370Sjasone * We use binary fixed point math for overhead computations, where the binary 11234370Sjasone * point is implicitly RUN_BFP bits to the left. 12234370Sjasone * 13234370Sjasone * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be 14234370Sjasone * honored for some/all object sizes, since when heap profiling is enabled 15234370Sjasone * there is one pointer of header overhead per object (plus a constant). This 16234370Sjasone * constraint is relaxed (ignored) for runs that are so small that the 17234370Sjasone * per-region overhead is greater than: 18234370Sjasone * 19234370Sjasone * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) 20234370Sjasone */ 21234370Sjasone#define RUN_BFP 12 22234370Sjasone/* \/ Implicit binary fixed point. */ 23234370Sjasone#define RUN_MAX_OVRHD 0x0000003dU 24234370Sjasone#define RUN_MAX_OVRHD_RELAX 0x00001800U 25234370Sjasone 26234370Sjasone/* Maximum number of regions in one run. */ 27234370Sjasone#define LG_RUN_MAXREGS 11 28234370Sjasone#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) 29234370Sjasone 30234370Sjasone/* 31234370Sjasone * Minimum redzone size. Redzones may be larger than this if necessary to 32234370Sjasone * preserve region alignment. 33234370Sjasone */ 34234370Sjasone#define REDZONE_MINSIZE 16 35234370Sjasone 36234370Sjasone/* 37234370Sjasone * The minimum ratio of active:dirty pages per arena is computed as: 38234370Sjasone * 39234370Sjasone * (nactive >> opt_lg_dirty_mult) >= ndirty 40234370Sjasone * 41242844Sjasone * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times 42242844Sjasone * as many active pages as dirty pages. 43234370Sjasone */ 44242844Sjasone#define LG_DIRTY_MULT_DEFAULT 3 45234370Sjasone 46234370Sjasonetypedef struct arena_chunk_map_s arena_chunk_map_t; 47234370Sjasonetypedef struct arena_chunk_s arena_chunk_t; 48234370Sjasonetypedef struct arena_run_s arena_run_t; 49234370Sjasonetypedef struct arena_bin_info_s arena_bin_info_t; 50234370Sjasonetypedef struct arena_bin_s arena_bin_t; 51234370Sjasonetypedef struct arena_s arena_t; 52234370Sjasone 53234370Sjasone#endif /* JEMALLOC_H_TYPES */ 54234370Sjasone/******************************************************************************/ 55234370Sjasone#ifdef JEMALLOC_H_STRUCTS 56234370Sjasone 57234370Sjasone/* Each element of the chunk map corresponds to one page within the chunk. */ 58234370Sjasonestruct arena_chunk_map_s { 59234370Sjasone#ifndef JEMALLOC_PROF 60234370Sjasone /* 61234370Sjasone * Overlay prof_ctx in order to allow it to be referenced by dead code. 62234370Sjasone * Such antics aren't warranted for per arena data structures, but 63234370Sjasone * chunk map overhead accounts for a percentage of memory, rather than 64234370Sjasone * being just a fixed cost. 65234370Sjasone */ 66234370Sjasone union { 67234370Sjasone#endif 68234370Sjasone union { 69234370Sjasone /* 70234370Sjasone * Linkage for run trees. There are two disjoint uses: 71234370Sjasone * 72242844Sjasone * 1) arena_t's runs_avail tree. 73234370Sjasone * 2) arena_run_t conceptually uses this linkage for in-use 74234370Sjasone * non-full runs, rather than directly embedding linkage. 75234370Sjasone */ 76234370Sjasone rb_node(arena_chunk_map_t) rb_link; 77234370Sjasone /* 78234370Sjasone * List of runs currently in purgatory. arena_chunk_purge() 79234370Sjasone * temporarily allocates runs that contain dirty pages while 80234370Sjasone * purging, so that other threads cannot use the runs while the 81234370Sjasone * purging thread is operating without the arena lock held. 82234370Sjasone */ 83234370Sjasone ql_elm(arena_chunk_map_t) ql_link; 84234370Sjasone } u; 85234370Sjasone 86234370Sjasone /* Profile counters, used for large object runs. */ 87234370Sjasone prof_ctx_t *prof_ctx; 88234370Sjasone#ifndef JEMALLOC_PROF 89234370Sjasone }; /* union { ... }; */ 90234370Sjasone#endif 91234370Sjasone 92234370Sjasone /* 93234370Sjasone * Run address (or size) and various flags are stored together. The bit 94234370Sjasone * layout looks like (assuming 32-bit system): 95234370Sjasone * 96235322Sjasone * ???????? ???????? ????nnnn nnnndula 97234370Sjasone * 98234370Sjasone * ? : Unallocated: Run address for first/last pages, unset for internal 99234370Sjasone * pages. 100234370Sjasone * Small: Run page offset. 101234370Sjasone * Large: Run size for first page, unset for trailing pages. 102235322Sjasone * n : binind for small size class, BININD_INVALID for large size class. 103234370Sjasone * d : dirty? 104234370Sjasone * u : unzeroed? 105234370Sjasone * l : large? 106234370Sjasone * a : allocated? 107234370Sjasone * 108234370Sjasone * Following are example bit patterns for the three types of runs. 109234370Sjasone * 110234370Sjasone * p : run page offset 111234370Sjasone * s : run size 112235238Sjasone * n : binind for size class; large objects set these to BININD_INVALID 113235238Sjasone * except for promoted allocations (see prof_promote) 114234370Sjasone * x : don't care 115234370Sjasone * - : 0 116234370Sjasone * + : 1 117234370Sjasone * [DULA] : bit set 118234370Sjasone * [dula] : bit unset 119234370Sjasone * 120234370Sjasone * Unallocated (clean): 121235322Sjasone * ssssssss ssssssss ssss++++ ++++du-a 122235238Sjasone * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx 123235322Sjasone * ssssssss ssssssss ssss++++ ++++dU-a 124234370Sjasone * 125234370Sjasone * Unallocated (dirty): 126235322Sjasone * ssssssss ssssssss ssss++++ ++++D--a 127235238Sjasone * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 128235322Sjasone * ssssssss ssssssss ssss++++ ++++D--a 129234370Sjasone * 130234370Sjasone * Small: 131235238Sjasone * pppppppp pppppppp ppppnnnn nnnnd--A 132235238Sjasone * pppppppp pppppppp ppppnnnn nnnn---A 133235238Sjasone * pppppppp pppppppp ppppnnnn nnnnd--A 134234370Sjasone * 135234370Sjasone * Large: 136235322Sjasone * ssssssss ssssssss ssss++++ ++++D-LA 137235238Sjasone * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx 138235322Sjasone * -------- -------- ----++++ ++++D-LA 139234370Sjasone * 140234370Sjasone * Large (sampled, size <= PAGE): 141235238Sjasone * ssssssss ssssssss ssssnnnn nnnnD-LA 142234370Sjasone * 143234370Sjasone * Large (not sampled, size == PAGE): 144235322Sjasone * ssssssss ssssssss ssss++++ ++++D-LA 145234370Sjasone */ 146234370Sjasone size_t bits; 147235238Sjasone#define CHUNK_MAP_BININD_SHIFT 4 148235238Sjasone#define BININD_INVALID ((size_t)0xffU) 149235238Sjasone/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ 150235238Sjasone#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) 151235238Sjasone#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK 152235238Sjasone#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) 153234370Sjasone#define CHUNK_MAP_DIRTY ((size_t)0x8U) 154234370Sjasone#define CHUNK_MAP_UNZEROED ((size_t)0x4U) 155234370Sjasone#define CHUNK_MAP_LARGE ((size_t)0x2U) 156234370Sjasone#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) 157234370Sjasone#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED 158234370Sjasone}; 159234370Sjasonetypedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; 160234370Sjasonetypedef rb_tree(arena_chunk_map_t) arena_run_tree_t; 161234370Sjasone 162234370Sjasone/* Arena chunk header. */ 163234370Sjasonestruct arena_chunk_s { 164234370Sjasone /* Arena that owns the chunk. */ 165242844Sjasone arena_t *arena; 166234370Sjasone 167242844Sjasone /* Linkage for tree of arena chunks that contain dirty runs. */ 168242844Sjasone rb_node(arena_chunk_t) dirty_link; 169234370Sjasone 170234370Sjasone /* Number of dirty pages. */ 171242844Sjasone size_t ndirty; 172234370Sjasone 173242844Sjasone /* Number of available runs. */ 174242844Sjasone size_t nruns_avail; 175242844Sjasone 176234370Sjasone /* 177242844Sjasone * Number of available run adjacencies. Clean and dirty available runs 178242844Sjasone * are not coalesced, which causes virtual memory fragmentation. The 179242844Sjasone * ratio of (nruns_avail-nruns_adjac):nruns_adjac is used for tracking 180242844Sjasone * this fragmentation. 181242844Sjasone * */ 182242844Sjasone size_t nruns_adjac; 183242844Sjasone 184242844Sjasone /* 185234370Sjasone * Map of pages within chunk that keeps track of free/large/small. The 186234370Sjasone * first map_bias entries are omitted, since the chunk header does not 187234370Sjasone * need to be tracked in the map. This omission saves a header page 188234370Sjasone * for common chunk sizes (e.g. 4 MiB). 189234370Sjasone */ 190242844Sjasone arena_chunk_map_t map[1]; /* Dynamically sized. */ 191234370Sjasone}; 192234370Sjasonetypedef rb_tree(arena_chunk_t) arena_chunk_tree_t; 193234370Sjasone 194234370Sjasonestruct arena_run_s { 195234370Sjasone /* Bin this run is associated with. */ 196234370Sjasone arena_bin_t *bin; 197234370Sjasone 198234370Sjasone /* Index of next region that has never been allocated, or nregs. */ 199234370Sjasone uint32_t nextind; 200234370Sjasone 201234370Sjasone /* Number of free regions in run. */ 202234370Sjasone unsigned nfree; 203234370Sjasone}; 204234370Sjasone 205234370Sjasone/* 206234370Sjasone * Read-only information associated with each element of arena_t's bins array 207234370Sjasone * is stored separately, partly to reduce memory usage (only one copy, rather 208234370Sjasone * than one per arena), but mainly to avoid false cacheline sharing. 209234370Sjasone * 210234370Sjasone * Each run has the following layout: 211234370Sjasone * 212234370Sjasone * /--------------------\ 213234370Sjasone * | arena_run_t header | 214234370Sjasone * | ... | 215234370Sjasone * bitmap_offset | bitmap | 216234370Sjasone * | ... | 217234370Sjasone * ctx0_offset | ctx map | 218234370Sjasone * | ... | 219234370Sjasone * |--------------------| 220234370Sjasone * | redzone | 221234370Sjasone * reg0_offset | region 0 | 222234370Sjasone * | redzone | 223234370Sjasone * |--------------------| \ 224234370Sjasone * | redzone | | 225234370Sjasone * | region 1 | > reg_interval 226234370Sjasone * | redzone | / 227234370Sjasone * |--------------------| 228234370Sjasone * | ... | 229234370Sjasone * | ... | 230234370Sjasone * | ... | 231234370Sjasone * |--------------------| 232234370Sjasone * | redzone | 233234370Sjasone * | region nregs-1 | 234234370Sjasone * | redzone | 235234370Sjasone * |--------------------| 236234370Sjasone * | alignment pad? | 237234370Sjasone * \--------------------/ 238234370Sjasone * 239234370Sjasone * reg_interval has at least the same minimum alignment as reg_size; this 240234370Sjasone * preserves the alignment constraint that sa2u() depends on. Alignment pad is 241234370Sjasone * either 0 or redzone_size; it is present only if needed to align reg0_offset. 242234370Sjasone */ 243234370Sjasonestruct arena_bin_info_s { 244234370Sjasone /* Size of regions in a run for this bin's size class. */ 245234370Sjasone size_t reg_size; 246234370Sjasone 247234370Sjasone /* Redzone size. */ 248234370Sjasone size_t redzone_size; 249234370Sjasone 250234370Sjasone /* Interval between regions (reg_size + (redzone_size << 1)). */ 251234370Sjasone size_t reg_interval; 252234370Sjasone 253234370Sjasone /* Total size of a run for this bin's size class. */ 254234370Sjasone size_t run_size; 255234370Sjasone 256234370Sjasone /* Total number of regions in a run for this bin's size class. */ 257234370Sjasone uint32_t nregs; 258234370Sjasone 259234370Sjasone /* 260234370Sjasone * Offset of first bitmap_t element in a run header for this bin's size 261234370Sjasone * class. 262234370Sjasone */ 263234370Sjasone uint32_t bitmap_offset; 264234370Sjasone 265234370Sjasone /* 266234370Sjasone * Metadata used to manipulate bitmaps for runs associated with this 267234370Sjasone * bin. 268234370Sjasone */ 269234370Sjasone bitmap_info_t bitmap_info; 270234370Sjasone 271234370Sjasone /* 272234370Sjasone * Offset of first (prof_ctx_t *) in a run header for this bin's size 273234370Sjasone * class, or 0 if (config_prof == false || opt_prof == false). 274234370Sjasone */ 275234370Sjasone uint32_t ctx0_offset; 276234370Sjasone 277234370Sjasone /* Offset of first region in a run for this bin's size class. */ 278234370Sjasone uint32_t reg0_offset; 279234370Sjasone}; 280234370Sjasone 281234370Sjasonestruct arena_bin_s { 282234370Sjasone /* 283234370Sjasone * All operations on runcur, runs, and stats require that lock be 284234370Sjasone * locked. Run allocation/deallocation are protected by the arena lock, 285234370Sjasone * which may be acquired while holding one or more bin locks, but not 286234370Sjasone * vise versa. 287234370Sjasone */ 288234370Sjasone malloc_mutex_t lock; 289234370Sjasone 290234370Sjasone /* 291234370Sjasone * Current run being used to service allocations of this bin's size 292234370Sjasone * class. 293234370Sjasone */ 294234370Sjasone arena_run_t *runcur; 295234370Sjasone 296234370Sjasone /* 297234370Sjasone * Tree of non-full runs. This tree is used when looking for an 298234370Sjasone * existing run when runcur is no longer usable. We choose the 299234370Sjasone * non-full run that is lowest in memory; this policy tends to keep 300234370Sjasone * objects packed well, and it can also help reduce the number of 301234370Sjasone * almost-empty chunks. 302234370Sjasone */ 303234370Sjasone arena_run_tree_t runs; 304234370Sjasone 305234370Sjasone /* Bin statistics. */ 306234370Sjasone malloc_bin_stats_t stats; 307234370Sjasone}; 308234370Sjasone 309234370Sjasonestruct arena_s { 310234370Sjasone /* This arena's index within the arenas array. */ 311234370Sjasone unsigned ind; 312234370Sjasone 313234370Sjasone /* 314234370Sjasone * Number of threads currently assigned to this arena. This field is 315234370Sjasone * protected by arenas_lock. 316234370Sjasone */ 317234370Sjasone unsigned nthreads; 318234370Sjasone 319234370Sjasone /* 320234370Sjasone * There are three classes of arena operations from a locking 321234370Sjasone * perspective: 322234370Sjasone * 1) Thread asssignment (modifies nthreads) is protected by 323234370Sjasone * arenas_lock. 324234370Sjasone * 2) Bin-related operations are protected by bin locks. 325234370Sjasone * 3) Chunk- and run-related operations are protected by this mutex. 326234370Sjasone */ 327234370Sjasone malloc_mutex_t lock; 328234370Sjasone 329234370Sjasone arena_stats_t stats; 330234370Sjasone /* 331234370Sjasone * List of tcaches for extant threads associated with this arena. 332234370Sjasone * Stats from these are merged incrementally, and at exit. 333234370Sjasone */ 334234370Sjasone ql_head(tcache_t) tcache_ql; 335234370Sjasone 336234370Sjasone uint64_t prof_accumbytes; 337234370Sjasone 338242844Sjasone dss_prec_t dss_prec; 339234370Sjasone 340242844Sjasone /* Tree of dirty-page-containing chunks this arena manages. */ 341242844Sjasone arena_chunk_tree_t chunks_dirty; 342242844Sjasone 343234370Sjasone /* 344234370Sjasone * In order to avoid rapid chunk allocation/deallocation when an arena 345234370Sjasone * oscillates right on the cusp of needing a new chunk, cache the most 346234370Sjasone * recently freed chunk. The spare is left in the arena's chunk trees 347234370Sjasone * until it is deleted. 348234370Sjasone * 349234370Sjasone * There is one spare chunk per arena, rather than one spare total, in 350234370Sjasone * order to avoid interactions between multiple threads that could make 351234370Sjasone * a single spare inadequate. 352234370Sjasone */ 353234370Sjasone arena_chunk_t *spare; 354234370Sjasone 355234370Sjasone /* Number of pages in active runs. */ 356234370Sjasone size_t nactive; 357234370Sjasone 358234370Sjasone /* 359234370Sjasone * Current count of pages within unused runs that are potentially 360234370Sjasone * dirty, and for which madvise(... MADV_DONTNEED) has not been called. 361234370Sjasone * By tracking this, we can institute a limit on how much dirty unused 362234370Sjasone * memory is mapped for each arena. 363234370Sjasone */ 364234370Sjasone size_t ndirty; 365234370Sjasone 366234370Sjasone /* 367234370Sjasone * Approximate number of pages being purged. It is possible for 368234370Sjasone * multiple threads to purge dirty pages concurrently, and they use 369234370Sjasone * npurgatory to indicate the total number of pages all threads are 370234370Sjasone * attempting to purge. 371234370Sjasone */ 372234370Sjasone size_t npurgatory; 373234370Sjasone 374234370Sjasone /* 375234370Sjasone * Size/address-ordered trees of this arena's available runs. The trees 376242844Sjasone * are used for first-best-fit run allocation. 377234370Sjasone */ 378242844Sjasone arena_avail_tree_t runs_avail; 379234370Sjasone 380234370Sjasone /* bins is used to store trees of free regions. */ 381234370Sjasone arena_bin_t bins[NBINS]; 382234370Sjasone}; 383234370Sjasone 384234370Sjasone#endif /* JEMALLOC_H_STRUCTS */ 385234370Sjasone/******************************************************************************/ 386234370Sjasone#ifdef JEMALLOC_H_EXTERNS 387234370Sjasone 388234370Sjasoneextern ssize_t opt_lg_dirty_mult; 389234370Sjasone/* 390234370Sjasone * small_size2bin is a compact lookup table that rounds request sizes up to 391234370Sjasone * size classes. In order to reduce cache footprint, the table is compressed, 392234370Sjasone * and all accesses are via the SMALL_SIZE2BIN macro. 393234370Sjasone */ 394234370Sjasoneextern uint8_t const small_size2bin[]; 395234370Sjasone#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) 396234370Sjasone 397234370Sjasoneextern arena_bin_info_t arena_bin_info[NBINS]; 398234370Sjasone 399234370Sjasone/* Number of large size classes. */ 400234370Sjasone#define nlclasses (chunk_npages - map_bias) 401234370Sjasone 402234370Sjasonevoid arena_purge_all(arena_t *arena); 403234370Sjasonevoid arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, 404234370Sjasone size_t binind, uint64_t prof_accumbytes); 405234370Sjasonevoid arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, 406234370Sjasone bool zero); 407234370Sjasonevoid arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); 408234370Sjasonevoid *arena_malloc_small(arena_t *arena, size_t size, bool zero); 409234370Sjasonevoid *arena_malloc_large(arena_t *arena, size_t size, bool zero); 410234370Sjasonevoid *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); 411234370Sjasonevoid arena_prof_promoted(const void *ptr, size_t size); 412235238Sjasonevoid arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, 413235238Sjasone arena_chunk_map_t *mapelm); 414234370Sjasonevoid arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, 415235238Sjasone size_t pageind, arena_chunk_map_t *mapelm); 416235238Sjasonevoid arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, 417235238Sjasone size_t pageind); 418235238Sjasonevoid arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, 419235238Sjasone void *ptr); 420234370Sjasonevoid arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); 421234370Sjasonevoid *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, 422234370Sjasone size_t extra, bool zero); 423242844Sjasonevoid *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, 424242844Sjasone size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, 425242844Sjasone bool try_tcache_dalloc); 426242844Sjasonedss_prec_t arena_dss_prec_get(arena_t *arena); 427242844Sjasonevoid arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); 428242844Sjasonevoid arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, 429242844Sjasone size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, 430242844Sjasone malloc_large_stats_t *lstats); 431234370Sjasonebool arena_new(arena_t *arena, unsigned ind); 432234370Sjasonevoid arena_boot(void); 433234370Sjasonevoid arena_prefork(arena_t *arena); 434234370Sjasonevoid arena_postfork_parent(arena_t *arena); 435234370Sjasonevoid arena_postfork_child(arena_t *arena); 436234370Sjasone 437234370Sjasone#endif /* JEMALLOC_H_EXTERNS */ 438234370Sjasone/******************************************************************************/ 439234370Sjasone#ifdef JEMALLOC_H_INLINES 440234370Sjasone 441234370Sjasone#ifndef JEMALLOC_ENABLE_INLINE 442235238Sjasonearena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); 443235238Sjasonesize_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); 444235238Sjasonesize_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); 445235238Sjasonesize_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, 446235238Sjasone size_t pageind); 447235238Sjasonesize_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); 448235238Sjasonesize_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); 449235238Sjasonesize_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); 450235238Sjasonesize_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); 451235238Sjasonesize_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); 452235238Sjasonesize_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); 453235238Sjasonesize_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); 454235238Sjasonevoid arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, 455235238Sjasone size_t size, size_t flags); 456235238Sjasonevoid arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 457235238Sjasone size_t size); 458235238Sjasonevoid arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, 459235238Sjasone size_t size, size_t flags); 460235238Sjasonevoid arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 461235238Sjasone size_t binind); 462235238Sjasonevoid arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, 463235238Sjasone size_t runind, size_t binind, size_t flags); 464235238Sjasonevoid arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 465235238Sjasone size_t unzeroed); 466251300Sjasonebool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); 467251300Sjasonebool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); 468251300Sjasonebool arena_prof_accum(arena_t *arena, uint64_t accumbytes); 469235238Sjasonesize_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); 470234370Sjasonesize_t arena_bin_index(arena_t *arena, arena_bin_t *bin); 471234370Sjasoneunsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, 472234370Sjasone const void *ptr); 473234370Sjasoneprof_ctx_t *arena_prof_ctx_get(const void *ptr); 474234370Sjasonevoid arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); 475234370Sjasonevoid *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); 476234543Sjasonesize_t arena_salloc(const void *ptr, bool demote); 477234370Sjasonevoid arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, 478234370Sjasone bool try_tcache); 479234370Sjasone#endif 480234370Sjasone 481234370Sjasone#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) 482235238Sjasone# ifdef JEMALLOC_ARENA_INLINE_A 483245868SjasoneJEMALLOC_ALWAYS_INLINE arena_chunk_map_t * 484235238Sjasonearena_mapp_get(arena_chunk_t *chunk, size_t pageind) 485235238Sjasone{ 486235238Sjasone 487235238Sjasone assert(pageind >= map_bias); 488235238Sjasone assert(pageind < chunk_npages); 489235238Sjasone 490235238Sjasone return (&chunk->map[pageind-map_bias]); 491235238Sjasone} 492235238Sjasone 493245868SjasoneJEMALLOC_ALWAYS_INLINE size_t * 494235238Sjasonearena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) 495235238Sjasone{ 496235238Sjasone 497235238Sjasone return (&arena_mapp_get(chunk, pageind)->bits); 498235238Sjasone} 499235238Sjasone 500245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 501235238Sjasonearena_mapbits_get(arena_chunk_t *chunk, size_t pageind) 502235238Sjasone{ 503235238Sjasone 504235238Sjasone return (*arena_mapbitsp_get(chunk, pageind)); 505235238Sjasone} 506235238Sjasone 507245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 508235238Sjasonearena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) 509235238Sjasone{ 510235238Sjasone size_t mapbits; 511235238Sjasone 512235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 513235238Sjasone assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 514235238Sjasone return (mapbits & ~PAGE_MASK); 515235238Sjasone} 516235238Sjasone 517245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 518235238Sjasonearena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) 519235238Sjasone{ 520235238Sjasone size_t mapbits; 521235238Sjasone 522235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 523235238Sjasone assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 524235238Sjasone (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); 525235238Sjasone return (mapbits & ~PAGE_MASK); 526235238Sjasone} 527235238Sjasone 528245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 529235238Sjasonearena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) 530235238Sjasone{ 531235238Sjasone size_t mapbits; 532235238Sjasone 533235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 534235238Sjasone assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 535235238Sjasone CHUNK_MAP_ALLOCATED); 536235238Sjasone return (mapbits >> LG_PAGE); 537235238Sjasone} 538235238Sjasone 539245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 540235238Sjasonearena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) 541235238Sjasone{ 542235238Sjasone size_t mapbits; 543235238Sjasone size_t binind; 544235238Sjasone 545235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 546235238Sjasone binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 547235238Sjasone assert(binind < NBINS || binind == BININD_INVALID); 548235238Sjasone return (binind); 549235238Sjasone} 550235238Sjasone 551245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 552235238Sjasonearena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) 553235238Sjasone{ 554235238Sjasone size_t mapbits; 555235238Sjasone 556235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 557235238Sjasone return (mapbits & CHUNK_MAP_DIRTY); 558235238Sjasone} 559235238Sjasone 560245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 561235238Sjasonearena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) 562235238Sjasone{ 563235238Sjasone size_t mapbits; 564235238Sjasone 565235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 566235238Sjasone return (mapbits & CHUNK_MAP_UNZEROED); 567235238Sjasone} 568235238Sjasone 569245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 570235238Sjasonearena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) 571235238Sjasone{ 572235238Sjasone size_t mapbits; 573235238Sjasone 574235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 575235238Sjasone return (mapbits & CHUNK_MAP_LARGE); 576235238Sjasone} 577235238Sjasone 578245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 579235238Sjasonearena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) 580235238Sjasone{ 581235238Sjasone size_t mapbits; 582235238Sjasone 583235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 584235238Sjasone return (mapbits & CHUNK_MAP_ALLOCATED); 585235238Sjasone} 586235238Sjasone 587245868SjasoneJEMALLOC_ALWAYS_INLINE void 588235238Sjasonearena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, 589235238Sjasone size_t flags) 590235238Sjasone{ 591235238Sjasone size_t *mapbitsp; 592235238Sjasone 593235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 594235238Sjasone assert((size & PAGE_MASK) == 0); 595235238Sjasone assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); 596235322Sjasone assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); 597235238Sjasone *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags; 598235238Sjasone} 599235238Sjasone 600245868SjasoneJEMALLOC_ALWAYS_INLINE void 601235238Sjasonearena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, 602235238Sjasone size_t size) 603235238Sjasone{ 604235238Sjasone size_t *mapbitsp; 605235238Sjasone 606235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 607235238Sjasone assert((size & PAGE_MASK) == 0); 608235238Sjasone assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); 609235238Sjasone *mapbitsp = size | (*mapbitsp & PAGE_MASK); 610235238Sjasone} 611235238Sjasone 612245868SjasoneJEMALLOC_ALWAYS_INLINE void 613235238Sjasonearena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, 614235238Sjasone size_t flags) 615235238Sjasone{ 616235238Sjasone size_t *mapbitsp; 617235322Sjasone size_t unzeroed; 618235238Sjasone 619235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 620235238Sjasone assert((size & PAGE_MASK) == 0); 621235322Sjasone assert((flags & CHUNK_MAP_DIRTY) == flags); 622235322Sjasone unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 623235322Sjasone *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | unzeroed | 624235322Sjasone CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; 625235238Sjasone} 626235238Sjasone 627245868SjasoneJEMALLOC_ALWAYS_INLINE void 628235238Sjasonearena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, 629235238Sjasone size_t binind) 630235238Sjasone{ 631235238Sjasone size_t *mapbitsp; 632235238Sjasone 633235238Sjasone assert(binind <= BININD_INVALID); 634235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 635235238Sjasone assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); 636235238Sjasone *mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind << 637235238Sjasone CHUNK_MAP_BININD_SHIFT); 638235238Sjasone} 639235238Sjasone 640245868SjasoneJEMALLOC_ALWAYS_INLINE void 641235238Sjasonearena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, 642235238Sjasone size_t binind, size_t flags) 643235238Sjasone{ 644235238Sjasone size_t *mapbitsp; 645235322Sjasone size_t unzeroed; 646235238Sjasone 647235238Sjasone assert(binind < BININD_INVALID); 648235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 649235238Sjasone assert(pageind - runind >= map_bias); 650235322Sjasone assert((flags & CHUNK_MAP_DIRTY) == flags); 651235322Sjasone unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ 652235238Sjasone *mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) | 653235322Sjasone flags | unzeroed | CHUNK_MAP_ALLOCATED; 654235238Sjasone} 655235238Sjasone 656245868SjasoneJEMALLOC_ALWAYS_INLINE void 657235238Sjasonearena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, 658235238Sjasone size_t unzeroed) 659235238Sjasone{ 660235238Sjasone size_t *mapbitsp; 661235238Sjasone 662235238Sjasone mapbitsp = arena_mapbitsp_get(chunk, pageind); 663235238Sjasone *mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed; 664235238Sjasone} 665235238Sjasone 666251300SjasoneJEMALLOC_INLINE bool 667245868Sjasonearena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) 668245868Sjasone{ 669245868Sjasone 670245868Sjasone cassert(config_prof); 671245868Sjasone assert(prof_interval != 0); 672245868Sjasone 673245868Sjasone arena->prof_accumbytes += accumbytes; 674245868Sjasone if (arena->prof_accumbytes >= prof_interval) { 675245868Sjasone arena->prof_accumbytes -= prof_interval; 676251300Sjasone return (true); 677245868Sjasone } 678251300Sjasone return (false); 679245868Sjasone} 680245868Sjasone 681251300SjasoneJEMALLOC_INLINE bool 682245868Sjasonearena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) 683245868Sjasone{ 684245868Sjasone 685245868Sjasone cassert(config_prof); 686245868Sjasone 687245868Sjasone if (prof_interval == 0) 688251300Sjasone return (false); 689251300Sjasone return (arena_prof_accum_impl(arena, accumbytes)); 690245868Sjasone} 691245868Sjasone 692251300SjasoneJEMALLOC_INLINE bool 693245868Sjasonearena_prof_accum(arena_t *arena, uint64_t accumbytes) 694245868Sjasone{ 695245868Sjasone 696245868Sjasone cassert(config_prof); 697245868Sjasone 698245868Sjasone if (prof_interval == 0) 699251300Sjasone return (false); 700251300Sjasone 701251300Sjasone { 702251300Sjasone bool ret; 703251300Sjasone 704251300Sjasone malloc_mutex_lock(&arena->lock); 705251300Sjasone ret = arena_prof_accum_impl(arena, accumbytes); 706251300Sjasone malloc_mutex_unlock(&arena->lock); 707251300Sjasone return (ret); 708251300Sjasone } 709245868Sjasone} 710245868Sjasone 711245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 712235238Sjasonearena_ptr_small_binind_get(const void *ptr, size_t mapbits) 713235238Sjasone{ 714235238Sjasone size_t binind; 715235238Sjasone 716235238Sjasone binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; 717235238Sjasone 718235238Sjasone if (config_debug) { 719235238Sjasone arena_chunk_t *chunk; 720235238Sjasone arena_t *arena; 721235238Sjasone size_t pageind; 722235238Sjasone size_t actual_mapbits; 723235238Sjasone arena_run_t *run; 724235238Sjasone arena_bin_t *bin; 725235238Sjasone size_t actual_binind; 726235238Sjasone arena_bin_info_t *bin_info; 727235238Sjasone 728235238Sjasone assert(binind != BININD_INVALID); 729235238Sjasone assert(binind < NBINS); 730235238Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 731235238Sjasone arena = chunk->arena; 732235238Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 733235238Sjasone actual_mapbits = arena_mapbits_get(chunk, pageind); 734235238Sjasone assert(mapbits == actual_mapbits); 735235238Sjasone assert(arena_mapbits_large_get(chunk, pageind) == 0); 736235238Sjasone assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 737235238Sjasone run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - 738235238Sjasone (actual_mapbits >> LG_PAGE)) << LG_PAGE)); 739235238Sjasone bin = run->bin; 740235238Sjasone actual_binind = bin - arena->bins; 741235238Sjasone assert(binind == actual_binind); 742235238Sjasone bin_info = &arena_bin_info[actual_binind]; 743235238Sjasone assert(((uintptr_t)ptr - ((uintptr_t)run + 744235238Sjasone (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval 745235238Sjasone == 0); 746235238Sjasone } 747235238Sjasone 748235238Sjasone return (binind); 749235238Sjasone} 750235238Sjasone# endif /* JEMALLOC_ARENA_INLINE_A */ 751235238Sjasone 752235238Sjasone# ifdef JEMALLOC_ARENA_INLINE_B 753235238SjasoneJEMALLOC_INLINE size_t 754234370Sjasonearena_bin_index(arena_t *arena, arena_bin_t *bin) 755234370Sjasone{ 756234370Sjasone size_t binind = bin - arena->bins; 757234370Sjasone assert(binind < NBINS); 758234370Sjasone return (binind); 759234370Sjasone} 760234370Sjasone 761234370SjasoneJEMALLOC_INLINE unsigned 762234370Sjasonearena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) 763234370Sjasone{ 764234370Sjasone unsigned shift, diff, regind; 765234370Sjasone size_t interval; 766234370Sjasone 767234370Sjasone /* 768234370Sjasone * Freeing a pointer lower than region zero can cause assertion 769234370Sjasone * failure. 770234370Sjasone */ 771234370Sjasone assert((uintptr_t)ptr >= (uintptr_t)run + 772234370Sjasone (uintptr_t)bin_info->reg0_offset); 773234370Sjasone 774234370Sjasone /* 775234370Sjasone * Avoid doing division with a variable divisor if possible. Using 776234370Sjasone * actual division here can reduce allocator throughput by over 20%! 777234370Sjasone */ 778234370Sjasone diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - 779234370Sjasone bin_info->reg0_offset); 780234370Sjasone 781234370Sjasone /* Rescale (factor powers of 2 out of the numerator and denominator). */ 782234370Sjasone interval = bin_info->reg_interval; 783234370Sjasone shift = ffs(interval) - 1; 784234370Sjasone diff >>= shift; 785234370Sjasone interval >>= shift; 786234370Sjasone 787234370Sjasone if (interval == 1) { 788234370Sjasone /* The divisor was a power of 2. */ 789234370Sjasone regind = diff; 790234370Sjasone } else { 791234370Sjasone /* 792234370Sjasone * To divide by a number D that is not a power of two we 793234370Sjasone * multiply by (2^21 / D) and then right shift by 21 positions. 794234370Sjasone * 795234370Sjasone * X / D 796234370Sjasone * 797234370Sjasone * becomes 798234370Sjasone * 799234370Sjasone * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT 800234370Sjasone * 801234370Sjasone * We can omit the first three elements, because we never 802234370Sjasone * divide by 0, and 1 and 2 are both powers of two, which are 803234370Sjasone * handled above. 804234370Sjasone */ 805234370Sjasone#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) 806234370Sjasone#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) 807234370Sjasone static const unsigned interval_invs[] = { 808234370Sjasone SIZE_INV(3), 809234370Sjasone SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), 810234370Sjasone SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), 811234370Sjasone SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), 812234370Sjasone SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), 813234370Sjasone SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), 814234370Sjasone SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), 815234370Sjasone SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) 816234370Sjasone }; 817234370Sjasone 818234370Sjasone if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + 819234370Sjasone 2)) { 820234370Sjasone regind = (diff * interval_invs[interval - 3]) >> 821234370Sjasone SIZE_INV_SHIFT; 822234370Sjasone } else 823234370Sjasone regind = diff / interval; 824234370Sjasone#undef SIZE_INV 825234370Sjasone#undef SIZE_INV_SHIFT 826234370Sjasone } 827234370Sjasone assert(diff == regind * interval); 828234370Sjasone assert(regind < bin_info->nregs); 829234370Sjasone 830234370Sjasone return (regind); 831234370Sjasone} 832234370Sjasone 833234370SjasoneJEMALLOC_INLINE prof_ctx_t * 834234370Sjasonearena_prof_ctx_get(const void *ptr) 835234370Sjasone{ 836234370Sjasone prof_ctx_t *ret; 837234370Sjasone arena_chunk_t *chunk; 838234370Sjasone size_t pageind, mapbits; 839234370Sjasone 840234370Sjasone cassert(config_prof); 841234370Sjasone assert(ptr != NULL); 842234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 843234370Sjasone 844234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 845234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 846235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 847234370Sjasone assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 848234370Sjasone if ((mapbits & CHUNK_MAP_LARGE) == 0) { 849234370Sjasone if (prof_promote) 850234370Sjasone ret = (prof_ctx_t *)(uintptr_t)1U; 851234370Sjasone else { 852234370Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 853234370Sjasone (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << 854234370Sjasone LG_PAGE)); 855235238Sjasone size_t binind = arena_ptr_small_binind_get(ptr, 856235238Sjasone mapbits); 857234370Sjasone arena_bin_info_t *bin_info = &arena_bin_info[binind]; 858234370Sjasone unsigned regind; 859234370Sjasone 860234370Sjasone regind = arena_run_regind(run, bin_info, ptr); 861234370Sjasone ret = *(prof_ctx_t **)((uintptr_t)run + 862234370Sjasone bin_info->ctx0_offset + (regind * 863234370Sjasone sizeof(prof_ctx_t *))); 864234370Sjasone } 865234370Sjasone } else 866235238Sjasone ret = arena_mapp_get(chunk, pageind)->prof_ctx; 867234370Sjasone 868234370Sjasone return (ret); 869234370Sjasone} 870234370Sjasone 871234370SjasoneJEMALLOC_INLINE void 872234370Sjasonearena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) 873234370Sjasone{ 874234370Sjasone arena_chunk_t *chunk; 875234370Sjasone size_t pageind, mapbits; 876234370Sjasone 877234370Sjasone cassert(config_prof); 878234370Sjasone assert(ptr != NULL); 879234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 880234370Sjasone 881234370Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 882234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 883235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 884234370Sjasone assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); 885234370Sjasone if ((mapbits & CHUNK_MAP_LARGE) == 0) { 886234370Sjasone if (prof_promote == false) { 887234370Sjasone arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + 888234370Sjasone (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << 889234370Sjasone LG_PAGE)); 890234370Sjasone size_t binind; 891234370Sjasone arena_bin_info_t *bin_info; 892234370Sjasone unsigned regind; 893234370Sjasone 894235238Sjasone binind = arena_ptr_small_binind_get(ptr, mapbits); 895234370Sjasone bin_info = &arena_bin_info[binind]; 896234370Sjasone regind = arena_run_regind(run, bin_info, ptr); 897234370Sjasone 898234370Sjasone *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset 899234370Sjasone + (regind * sizeof(prof_ctx_t *)))) = ctx; 900234370Sjasone } else 901234370Sjasone assert((uintptr_t)ctx == (uintptr_t)1U); 902234370Sjasone } else 903235238Sjasone arena_mapp_get(chunk, pageind)->prof_ctx = ctx; 904234370Sjasone} 905234370Sjasone 906245868SjasoneJEMALLOC_ALWAYS_INLINE void * 907234370Sjasonearena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) 908234370Sjasone{ 909234370Sjasone tcache_t *tcache; 910234370Sjasone 911234370Sjasone assert(size != 0); 912234370Sjasone assert(size <= arena_maxclass); 913234370Sjasone 914234370Sjasone if (size <= SMALL_MAXCLASS) { 915234370Sjasone if (try_tcache && (tcache = tcache_get(true)) != NULL) 916234370Sjasone return (tcache_alloc_small(tcache, size, zero)); 917234370Sjasone else { 918234370Sjasone return (arena_malloc_small(choose_arena(arena), size, 919234370Sjasone zero)); 920234370Sjasone } 921234370Sjasone } else { 922234370Sjasone /* 923234370Sjasone * Initialize tcache after checking size in order to avoid 924234370Sjasone * infinite recursion during tcache initialization. 925234370Sjasone */ 926234370Sjasone if (try_tcache && size <= tcache_maxclass && (tcache = 927234370Sjasone tcache_get(true)) != NULL) 928234370Sjasone return (tcache_alloc_large(tcache, size, zero)); 929234370Sjasone else { 930234370Sjasone return (arena_malloc_large(choose_arena(arena), size, 931234370Sjasone zero)); 932234370Sjasone } 933234370Sjasone } 934234370Sjasone} 935234370Sjasone 936234543Sjasone/* Return the size of the allocation pointed to by ptr. */ 937245868SjasoneJEMALLOC_ALWAYS_INLINE size_t 938234543Sjasonearena_salloc(const void *ptr, bool demote) 939234543Sjasone{ 940234543Sjasone size_t ret; 941234543Sjasone arena_chunk_t *chunk; 942235238Sjasone size_t pageind, binind; 943234543Sjasone 944234543Sjasone assert(ptr != NULL); 945234543Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 946234543Sjasone 947234543Sjasone chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); 948234543Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 949235238Sjasone assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 950235238Sjasone binind = arena_mapbits_binind_get(chunk, pageind); 951235238Sjasone if (binind == BININD_INVALID || (config_prof && demote == false && 952235238Sjasone prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) { 953235238Sjasone /* 954235238Sjasone * Large allocation. In the common case (demote == true), and 955235238Sjasone * as this is an inline function, most callers will only end up 956235238Sjasone * looking at binind to determine that ptr is a small 957235238Sjasone * allocation. 958235238Sjasone */ 959234543Sjasone assert(((uintptr_t)ptr & PAGE_MASK) == 0); 960235238Sjasone ret = arena_mapbits_large_size_get(chunk, pageind); 961234543Sjasone assert(ret != 0); 962235238Sjasone assert(pageind + (ret>>LG_PAGE) <= chunk_npages); 963235238Sjasone assert(ret == PAGE || arena_mapbits_large_size_get(chunk, 964235238Sjasone pageind+(ret>>LG_PAGE)-1) == 0); 965235238Sjasone assert(binind == arena_mapbits_binind_get(chunk, 966235238Sjasone pageind+(ret>>LG_PAGE)-1)); 967235238Sjasone assert(arena_mapbits_dirty_get(chunk, pageind) == 968235238Sjasone arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); 969235238Sjasone } else { 970235238Sjasone /* 971235238Sjasone * Small allocation (possibly promoted to a large object due to 972235238Sjasone * prof_promote). 973235238Sjasone */ 974235238Sjasone assert(arena_mapbits_large_get(chunk, pageind) != 0 || 975235238Sjasone arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, 976235238Sjasone pageind)) == binind); 977235238Sjasone ret = arena_bin_info[binind].reg_size; 978234543Sjasone } 979234543Sjasone 980234543Sjasone return (ret); 981234543Sjasone} 982234543Sjasone 983245868SjasoneJEMALLOC_ALWAYS_INLINE void 984234370Sjasonearena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) 985234370Sjasone{ 986235238Sjasone size_t pageind, mapbits; 987234370Sjasone tcache_t *tcache; 988234370Sjasone 989234370Sjasone assert(arena != NULL); 990234370Sjasone assert(chunk->arena == arena); 991234370Sjasone assert(ptr != NULL); 992234370Sjasone assert(CHUNK_ADDR2BASE(ptr) != ptr); 993234370Sjasone 994234370Sjasone pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; 995235238Sjasone mapbits = arena_mapbits_get(chunk, pageind); 996235238Sjasone assert(arena_mapbits_allocated_get(chunk, pageind) != 0); 997235238Sjasone if ((mapbits & CHUNK_MAP_LARGE) == 0) { 998234370Sjasone /* Small allocation. */ 999235238Sjasone if (try_tcache && (tcache = tcache_get(false)) != NULL) { 1000235238Sjasone size_t binind; 1001234370Sjasone 1002235238Sjasone binind = arena_ptr_small_binind_get(ptr, mapbits); 1003235238Sjasone tcache_dalloc_small(tcache, ptr, binind); 1004235238Sjasone } else 1005235238Sjasone arena_dalloc_small(arena, chunk, ptr, pageind); 1006234370Sjasone } else { 1007235238Sjasone size_t size = arena_mapbits_large_size_get(chunk, pageind); 1008234370Sjasone 1009234370Sjasone assert(((uintptr_t)ptr & PAGE_MASK) == 0); 1010234370Sjasone 1011234370Sjasone if (try_tcache && size <= tcache_maxclass && (tcache = 1012234370Sjasone tcache_get(false)) != NULL) { 1013234370Sjasone tcache_dalloc_large(tcache, ptr, size); 1014235238Sjasone } else 1015234370Sjasone arena_dalloc_large(arena, chunk, ptr); 1016234370Sjasone } 1017234370Sjasone} 1018235238Sjasone# endif /* JEMALLOC_ARENA_INLINE_B */ 1019234370Sjasone#endif 1020234370Sjasone 1021234370Sjasone#endif /* JEMALLOC_H_INLINES */ 1022234370Sjasone/******************************************************************************/ 1023