jemalloc.c revision 245868 
1#define	JEMALLOC_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7malloc_tsd_data(, arenas, arena_t *, NULL)
8malloc_tsd_data(, thread_allocated, thread_allocated_t,
9    THREAD_ALLOCATED_INITIALIZER)
10
11/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
12const char	*__malloc_options_1_0 = NULL;
13__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
14
15/* Runtime configuration options. */
16const char	*je_malloc_conf;
17bool	opt_abort =
18#ifdef JEMALLOC_DEBUG
19    true
20#else
21    false
22#endif
23    ;
24bool	opt_junk =
25#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
26    true
27#else
28    false
29#endif
30    ;
31size_t	opt_quarantine = ZU(0);
32bool	opt_redzone = false;
33bool	opt_utrace = false;
34bool	opt_valgrind = false;
35bool	opt_xmalloc = false;
36bool	opt_zero = false;
37size_t	opt_narenas = 0;
38
39unsigned	ncpus;
40
41malloc_mutex_t		arenas_lock;
42arena_t			**arenas;
43unsigned		narenas_total;
44unsigned		narenas_auto;
45
46/* Set to true once the allocator has been initialized. */
47static bool		malloc_initialized = false;
48
49#ifdef JEMALLOC_THREADED_INIT
50/* Used to let the initializing thread recursively allocate. */
51#  define NO_INITIALIZER	((unsigned long)0)
52#  define INITIALIZER		pthread_self()
53#  define IS_INITIALIZER	(malloc_initializer == pthread_self())
54static pthread_t		malloc_initializer = NO_INITIALIZER;
55#else
56#  define NO_INITIALIZER	false
57#  define INITIALIZER		true
58#  define IS_INITIALIZER	malloc_initializer
59static bool			malloc_initializer = NO_INITIALIZER;
60#endif
61
62/* Used to avoid initialization races. */
63#ifdef _WIN32
64static malloc_mutex_t	init_lock;
65
66JEMALLOC_ATTR(constructor)
67static void WINAPI
68_init_init_lock(void)
69{
70
71	malloc_mutex_init(&init_lock);
72}
73
74#ifdef _MSC_VER
75#  pragma section(".CRT$XCU", read)
76JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
77static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
78#endif
79
80#else
81static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
82#endif
83
84typedef struct {
85	void	*p;	/* Input pointer (as in realloc(p, s)). */
86	size_t	s;	/* Request size. */
87	void	*r;	/* Result pointer. */
88} malloc_utrace_t;
89
90#ifdef JEMALLOC_UTRACE
91#  define UTRACE(a, b, c) do {						\
92	if (opt_utrace) {						\
93		int utrace_serrno = errno;				\
94		malloc_utrace_t ut;					\
95		ut.p = (a);						\
96		ut.s = (b);						\
97		ut.r = (c);						\
98		utrace(&ut, sizeof(ut));				\
99		errno = utrace_serrno;					\
100	}								\
101} while (0)
102#else
103#  define UTRACE(a, b, c)
104#endif
105
106/******************************************************************************/
107/* Function prototypes for non-inline static functions. */
108
109static void	stats_print_atexit(void);
110static unsigned	malloc_ncpus(void);
111static bool	malloc_conf_next(char const **opts_p, char const **k_p,
112    size_t *klen_p, char const **v_p, size_t *vlen_p);
113static void	malloc_conf_error(const char *msg, const char *k, size_t klen,
114    const char *v, size_t vlen);
115static void	malloc_conf_init(void);
116static bool	malloc_init_hard(void);
117static int	imemalign(void **memptr, size_t alignment, size_t size,
118    size_t min_alignment);
119
120/******************************************************************************/
121/*
122 * Begin miscellaneous support functions.
123 */
124
125/* Create a new arena and insert it into the arenas array at index ind. */
126arena_t *
127arenas_extend(unsigned ind)
128{
129	arena_t *ret;
130
131	ret = (arena_t *)base_alloc(sizeof(arena_t));
132	if (ret != NULL && arena_new(ret, ind) == false) {
133		arenas[ind] = ret;
134		return (ret);
135	}
136	/* Only reached if there is an OOM error. */
137
138	/*
139	 * OOM here is quite inconvenient to propagate, since dealing with it
140	 * would require a check for failure in the fast path.  Instead, punt
141	 * by using arenas[0].  In practice, this is an extremely unlikely
142	 * failure.
143	 */
144	malloc_write("<jemalloc>: Error initializing arena\n");
145	if (opt_abort)
146		abort();
147
148	return (arenas[0]);
149}
150
151/* Slow path, called only by choose_arena(). */
152arena_t *
153choose_arena_hard(void)
154{
155	arena_t *ret;
156
157	if (narenas_auto > 1) {
158		unsigned i, choose, first_null;
159
160		choose = 0;
161		first_null = narenas_auto;
162		malloc_mutex_lock(&arenas_lock);
163		assert(arenas[0] != NULL);
164		for (i = 1; i < narenas_auto; i++) {
165			if (arenas[i] != NULL) {
166				/*
167				 * Choose the first arena that has the lowest
168				 * number of threads assigned to it.
169				 */
170				if (arenas[i]->nthreads <
171				    arenas[choose]->nthreads)
172					choose = i;
173			} else if (first_null == narenas_auto) {
174				/*
175				 * Record the index of the first uninitialized
176				 * arena, in case all extant arenas are in use.
177				 *
178				 * NB: It is possible for there to be
179				 * discontinuities in terms of initialized
180				 * versus uninitialized arenas, due to the
181				 * "thread.arena" mallctl.
182				 */
183				first_null = i;
184			}
185		}
186
187		if (arenas[choose]->nthreads == 0
188		    || first_null == narenas_auto) {
189			/*
190			 * Use an unloaded arena, or the least loaded arena if
191			 * all arenas are already initialized.
192			 */
193			ret = arenas[choose];
194		} else {
195			/* Initialize a new arena. */
196			ret = arenas_extend(first_null);
197		}
198		ret->nthreads++;
199		malloc_mutex_unlock(&arenas_lock);
200	} else {
201		ret = arenas[0];
202		malloc_mutex_lock(&arenas_lock);
203		ret->nthreads++;
204		malloc_mutex_unlock(&arenas_lock);
205	}
206
207	arenas_tsd_set(&ret);
208
209	return (ret);
210}
211
212static void
213stats_print_atexit(void)
214{
215
216	if (config_tcache && config_stats) {
217		unsigned narenas, i;
218
219		/*
220		 * Merge stats from extant threads.  This is racy, since
221		 * individual threads do not lock when recording tcache stats
222		 * events.  As a consequence, the final stats may be slightly
223		 * out of date by the time they are reported, if other threads
224		 * continue to allocate.
225		 */
226		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
227			arena_t *arena = arenas[i];
228			if (arena != NULL) {
229				tcache_t *tcache;
230
231				/*
232				 * tcache_stats_merge() locks bins, so if any
233				 * code is introduced that acquires both arena
234				 * and bin locks in the opposite order,
235				 * deadlocks may result.
236				 */
237				malloc_mutex_lock(&arena->lock);
238				ql_foreach(tcache, &arena->tcache_ql, link) {
239					tcache_stats_merge(tcache, arena);
240				}
241				malloc_mutex_unlock(&arena->lock);
242			}
243		}
244	}
245	je_malloc_stats_print(NULL, NULL, NULL);
246}
247
248/*
249 * End miscellaneous support functions.
250 */
251/******************************************************************************/
252/*
253 * Begin initialization functions.
254 */
255
256static unsigned
257malloc_ncpus(void)
258{
259	unsigned ret;
260	long result;
261
262#ifdef _WIN32
263	SYSTEM_INFO si;
264	GetSystemInfo(&si);
265	result = si.dwNumberOfProcessors;
266#else
267	result = sysconf(_SC_NPROCESSORS_ONLN);
268#endif
269	if (result == -1) {
270		/* Error. */
271		ret = 1;
272	}  else {
273    ret = (unsigned)result;
274  }
275
276	return (ret);
277}
278
279void
280arenas_cleanup(void *arg)
281{
282	arena_t *arena = *(arena_t **)arg;
283
284	malloc_mutex_lock(&arenas_lock);
285	arena->nthreads--;
286	malloc_mutex_unlock(&arenas_lock);
287}
288
289static JEMALLOC_ATTR(always_inline) bool
290malloc_init(void)
291{
292
293	if (malloc_initialized == false)
294		return (malloc_init_hard());
295
296	return (false);
297}
298
299static bool
300malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
301    char const **v_p, size_t *vlen_p)
302{
303	bool accept;
304	const char *opts = *opts_p;
305
306	*k_p = opts;
307
308	for (accept = false; accept == false;) {
309		switch (*opts) {
310		case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
311		case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
312		case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
313		case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
314		case 'Y': case 'Z':
315		case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
316		case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
317		case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
318		case 's': case 't': case 'u': case 'v': case 'w': case 'x':
319		case 'y': case 'z':
320		case '0': case '1': case '2': case '3': case '4': case '5':
321		case '6': case '7': case '8': case '9':
322		case '_':
323			opts++;
324			break;
325		case ':':
326			opts++;
327			*klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p;
328			*v_p = opts;
329			accept = true;
330			break;
331		case '\0':
332			if (opts != *opts_p) {
333				malloc_write("<jemalloc>: Conf string ends "
334				    "with key\n");
335			}
336			return (true);
337		default:
338			malloc_write("<jemalloc>: Malformed conf string\n");
339			return (true);
340		}
341	}
342
343	for (accept = false; accept == false;) {
344		switch (*opts) {
345		case ',':
346			opts++;
347			/*
348			 * Look ahead one character here, because the next time
349			 * this function is called, it will assume that end of
350			 * input has been cleanly reached if no input remains,
351			 * but we have optimistically already consumed the
352			 * comma if one exists.
353			 */
354			if (*opts == '\0') {
355				malloc_write("<jemalloc>: Conf string ends "
356				    "with comma\n");
357			}
358			*vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p;
359			accept = true;
360			break;
361		case '\0':
362			*vlen_p = (uintptr_t)opts - (uintptr_t)*v_p;
363			accept = true;
364			break;
365		default:
366			opts++;
367			break;
368		}
369	}
370
371	*opts_p = opts;
372	return (false);
373}
374
375static void
376malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v,
377    size_t vlen)
378{
379
380	malloc_printf("<jemalloc>: %s: %.*s:%.*s\n", msg, (int)klen, k,
381	    (int)vlen, v);
382}
383
384static void
385malloc_conf_init(void)
386{
387	unsigned i;
388	char buf[PATH_MAX + 1];
389	const char *opts, *k, *v;
390	size_t klen, vlen;
391
392	/*
393	 * Automatically configure valgrind before processing options.  The
394	 * valgrind option remains in jemalloc 3.x for compatibility reasons.
395	 */
396	if (config_valgrind) {
397		opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
398		if (config_fill && opt_valgrind) {
399			opt_junk = false;
400			assert(opt_zero == false);
401			opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
402			opt_redzone = true;
403		}
404		if (config_tcache && opt_valgrind)
405			opt_tcache = false;
406	}
407
408	for (i = 0; i < 3; i++) {
409		/* Get runtime configuration. */
410		switch (i) {
411		case 0:
412			if (je_malloc_conf != NULL) {
413				/*
414				 * Use options that were compiled into the
415				 * program.
416				 */
417				opts = je_malloc_conf;
418			} else {
419				/* No configuration specified. */
420				buf[0] = '\0';
421				opts = buf;
422			}
423			break;
424		case 1: {
425#ifndef _WIN32
426			int linklen;
427			const char *linkname =
428#  ifdef JEMALLOC_PREFIX
429			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
430#  else
431			    "/etc/malloc.conf"
432#  endif
433			    ;
434
435			if ((linklen = readlink(linkname, buf,
436			    sizeof(buf) - 1)) != -1) {
437				/*
438				 * Use the contents of the "/etc/malloc.conf"
439				 * symbolic link's name.
440				 */
441				buf[linklen] = '\0';
442				opts = buf;
443			} else
444#endif
445			{
446				/* No configuration specified. */
447				buf[0] = '\0';
448				opts = buf;
449			}
450			break;
451		} case 2: {
452			const char *envname =
453#ifdef JEMALLOC_PREFIX
454			    JEMALLOC_CPREFIX"MALLOC_CONF"
455#else
456			    "MALLOC_CONF"
457#endif
458			    ;
459
460			if (issetugid() == 0 && (opts = getenv(envname)) !=
461			    NULL) {
462				/*
463				 * Do nothing; opts is already initialized to
464				 * the value of the MALLOC_CONF environment
465				 * variable.
466				 */
467			} else {
468				/* No configuration specified. */
469				buf[0] = '\0';
470				opts = buf;
471			}
472			break;
473		} default:
474			/* NOTREACHED */
475			assert(false);
476			buf[0] = '\0';
477			opts = buf;
478		}
479
480		while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
481		    &vlen) == false) {
482#define	CONF_HANDLE_BOOL(o, n)						\
483			if (sizeof(n)-1 == klen && strncmp(n, k,	\
484			    klen) == 0) {				\
485				if (strncmp("true", v, vlen) == 0 &&	\
486				    vlen == sizeof("true")-1)		\
487					o = true;			\
488				else if (strncmp("false", v, vlen) ==	\
489				    0 && vlen == sizeof("false")-1)	\
490					o = false;			\
491				else {					\
492					malloc_conf_error(		\
493					    "Invalid conf value",	\
494					    k, klen, v, vlen);		\
495				}					\
496				continue;				\
497			}
498#define	CONF_HANDLE_SIZE_T(o, n, min, max, clip)			\
499			if (sizeof(n)-1 == klen && strncmp(n, k,	\
500			    klen) == 0) {				\
501				uintmax_t um;				\
502				char *end;				\
503									\
504				set_errno(0);				\
505				um = malloc_strtoumax(v, &end, 0);	\
506				if (get_errno() != 0 || (uintptr_t)end -\
507				    (uintptr_t)v != vlen) {		\
508					malloc_conf_error(		\
509					    "Invalid conf value",	\
510					    k, klen, v, vlen);		\
511				} else if (clip) {			\
512					if (um < min)			\
513						o = min;		\
514					else if (um > max)		\
515						o = max;		\
516					else				\
517						o = um;			\
518				} else {				\
519					if (um < min || um > max) {	\
520						malloc_conf_error(	\
521						    "Out-of-range "	\
522						    "conf value",	\
523						    k, klen, v, vlen);	\
524					} else				\
525						o = um;			\
526				}					\
527				continue;				\
528			}
529#define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
530			if (sizeof(n)-1 == klen && strncmp(n, k,	\
531			    klen) == 0) {				\
532				long l;					\
533				char *end;				\
534									\
535				set_errno(0);				\
536				l = strtol(v, &end, 0);			\
537				if (get_errno() != 0 || (uintptr_t)end -\
538				    (uintptr_t)v != vlen) {		\
539					malloc_conf_error(		\
540					    "Invalid conf value",	\
541					    k, klen, v, vlen);		\
542				} else if (l < (ssize_t)min || l >	\
543				    (ssize_t)max) {			\
544					malloc_conf_error(		\
545					    "Out-of-range conf value",	\
546					    k, klen, v, vlen);		\
547				} else					\
548					o = l;				\
549				continue;				\
550			}
551#define	CONF_HANDLE_CHAR_P(o, n, d)					\
552			if (sizeof(n)-1 == klen && strncmp(n, k,	\
553			    klen) == 0) {				\
554				size_t cpylen = (vlen <=		\
555				    sizeof(o)-1) ? vlen :		\
556				    sizeof(o)-1;			\
557				strncpy(o, v, cpylen);			\
558				o[cpylen] = '\0';			\
559				continue;				\
560			}
561
562			CONF_HANDLE_BOOL(opt_abort, "abort")
563			/*
564			 * Chunks always require at least one header page, plus
565			 * one data page in the absence of redzones, or three
566			 * pages in the presence of redzones.  In order to
567			 * simplify options processing, fix the limit based on
568			 * config_fill.
569			 */
570			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
571			    (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
572			    true)
573			if (strncmp("dss", k, klen) == 0) {
574				int i;
575				bool match = false;
576				for (i = 0; i < dss_prec_limit; i++) {
577					if (strncmp(dss_prec_names[i], v, vlen)
578					    == 0) {
579						if (chunk_dss_prec_set(i)) {
580							malloc_conf_error(
581							    "Error setting dss",
582							    k, klen, v, vlen);
583						} else {
584							opt_dss =
585							    dss_prec_names[i];
586							match = true;
587							break;
588						}
589					}
590				}
591				if (match == false) {
592					malloc_conf_error("Invalid conf value",
593					    k, klen, v, vlen);
594				}
595				continue;
596			}
597			CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
598			    SIZE_T_MAX, false)
599			CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
600			    -1, (sizeof(size_t) << 3) - 1)
601			CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
602			if (config_fill) {
603				CONF_HANDLE_BOOL(opt_junk, "junk")
604				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
605				    0, SIZE_T_MAX, false)
606				CONF_HANDLE_BOOL(opt_redzone, "redzone")
607				CONF_HANDLE_BOOL(opt_zero, "zero")
608			}
609			if (config_utrace) {
610				CONF_HANDLE_BOOL(opt_utrace, "utrace")
611			}
612			if (config_valgrind) {
613				CONF_HANDLE_BOOL(opt_valgrind, "valgrind")
614			}
615			if (config_xmalloc) {
616				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
617			}
618			if (config_tcache) {
619				CONF_HANDLE_BOOL(opt_tcache, "tcache")
620				CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
621				    "lg_tcache_max", -1,
622				    (sizeof(size_t) << 3) - 1)
623			}
624			if (config_prof) {
625				CONF_HANDLE_BOOL(opt_prof, "prof")
626				CONF_HANDLE_CHAR_P(opt_prof_prefix,
627				    "prof_prefix", "jeprof")
628				CONF_HANDLE_BOOL(opt_prof_active, "prof_active")
629				CONF_HANDLE_SSIZE_T(opt_lg_prof_sample,
630				    "lg_prof_sample", 0,
631				    (sizeof(uint64_t) << 3) - 1)
632				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
633				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
634				    "lg_prof_interval", -1,
635				    (sizeof(uint64_t) << 3) - 1)
636				CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump")
637				CONF_HANDLE_BOOL(opt_prof_final, "prof_final")
638				CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak")
639			}
640			malloc_conf_error("Invalid conf pair", k, klen, v,
641			    vlen);
642#undef CONF_HANDLE_BOOL
643#undef CONF_HANDLE_SIZE_T
644#undef CONF_HANDLE_SSIZE_T
645#undef CONF_HANDLE_CHAR_P
646		}
647	}
648}
649
650static bool
651malloc_init_hard(void)
652{
653	arena_t *init_arenas[1];
654
655	malloc_mutex_lock(&init_lock);
656	if (malloc_initialized || IS_INITIALIZER) {
657		/*
658		 * Another thread initialized the allocator before this one
659		 * acquired init_lock, or this thread is the initializing
660		 * thread, and it is recursively allocating.
661		 */
662		malloc_mutex_unlock(&init_lock);
663		return (false);
664	}
665#ifdef JEMALLOC_THREADED_INIT
666	if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) {
667		/* Busy-wait until the initializing thread completes. */
668		do {
669			malloc_mutex_unlock(&init_lock);
670			CPU_SPINWAIT;
671			malloc_mutex_lock(&init_lock);
672		} while (malloc_initialized == false);
673		malloc_mutex_unlock(&init_lock);
674		return (false);
675	}
676#endif
677	malloc_initializer = INITIALIZER;
678
679	malloc_tsd_boot();
680	if (config_prof)
681		prof_boot0();
682
683	malloc_conf_init();
684
685#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
686    && !defined(_WIN32))
687	/* Register fork handlers. */
688	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
689	    jemalloc_postfork_child) != 0) {
690		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
691		if (opt_abort)
692			abort();
693	}
694#endif
695
696	if (opt_stats_print) {
697		/* Print statistics at exit. */
698		if (atexit(stats_print_atexit) != 0) {
699			malloc_write("<jemalloc>: Error in atexit()\n");
700			if (opt_abort)
701				abort();
702		}
703	}
704
705	if (base_boot()) {
706		malloc_mutex_unlock(&init_lock);
707		return (true);
708	}
709
710	if (chunk_boot()) {
711		malloc_mutex_unlock(&init_lock);
712		return (true);
713	}
714
715	if (ctl_boot()) {
716		malloc_mutex_unlock(&init_lock);
717		return (true);
718	}
719
720	if (config_prof)
721		prof_boot1();
722
723	arena_boot();
724
725	if (config_tcache && tcache_boot0()) {
726		malloc_mutex_unlock(&init_lock);
727		return (true);
728	}
729
730	if (huge_boot()) {
731		malloc_mutex_unlock(&init_lock);
732		return (true);
733	}
734
735	if (malloc_mutex_init(&arenas_lock))
736		return (true);
737
738	/*
739	 * Create enough scaffolding to allow recursive allocation in
740	 * malloc_ncpus().
741	 */
742	narenas_total = narenas_auto = 1;
743	arenas = init_arenas;
744	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
745
746	/*
747	 * Initialize one arena here.  The rest are lazily created in
748	 * choose_arena_hard().
749	 */
750	arenas_extend(0);
751	if (arenas[0] == NULL) {
752		malloc_mutex_unlock(&init_lock);
753		return (true);
754	}
755
756	/* Initialize allocation counters before any allocations can occur. */
757	if (config_stats && thread_allocated_tsd_boot()) {
758		malloc_mutex_unlock(&init_lock);
759		return (true);
760	}
761
762	if (arenas_tsd_boot()) {
763		malloc_mutex_unlock(&init_lock);
764		return (true);
765	}
766
767	if (config_tcache && tcache_boot1()) {
768		malloc_mutex_unlock(&init_lock);
769		return (true);
770	}
771
772	if (config_fill && quarantine_boot()) {
773		malloc_mutex_unlock(&init_lock);
774		return (true);
775	}
776
777	if (config_prof && prof_boot2()) {
778		malloc_mutex_unlock(&init_lock);
779		return (true);
780	}
781
782	/* Get number of CPUs. */
783	malloc_mutex_unlock(&init_lock);
784	ncpus = malloc_ncpus();
785	malloc_mutex_lock(&init_lock);
786
787	if (mutex_boot()) {
788		malloc_mutex_unlock(&init_lock);
789		return (true);
790	}
791
792	if (opt_narenas == 0) {
793		/*
794		 * For SMP systems, create more than one arena per CPU by
795		 * default.
796		 */
797		if (ncpus > 1)
798			opt_narenas = ncpus << 2;
799		else
800			opt_narenas = 1;
801	}
802	narenas_auto = opt_narenas;
803	/*
804	 * Make sure that the arenas array can be allocated.  In practice, this
805	 * limit is enough to allow the allocator to function, but the ctl
806	 * machinery will fail to allocate memory at far lower limits.
807	 */
808	if (narenas_auto > chunksize / sizeof(arena_t *)) {
809		narenas_auto = chunksize / sizeof(arena_t *);
810		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
811		    narenas_auto);
812	}
813	narenas_total = narenas_auto;
814
815	/* Allocate and initialize arenas. */
816	arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total);
817	if (arenas == NULL) {
818		malloc_mutex_unlock(&init_lock);
819		return (true);
820	}
821	/*
822	 * Zero the array.  In practice, this should always be pre-zeroed,
823	 * since it was just mmap()ed, but let's be sure.
824	 */
825	memset(arenas, 0, sizeof(arena_t *) * narenas_total);
826	/* Copy the pointer to the one arena that was already initialized. */
827	arenas[0] = init_arenas[0];
828
829	malloc_initialized = true;
830	malloc_mutex_unlock(&init_lock);
831	return (false);
832}
833
834/*
835 * End initialization functions.
836 */
837/******************************************************************************/
838/*
839 * Begin malloc(3)-compatible functions.
840 */
841
842void *
843je_malloc(size_t size)
844{
845	void *ret;
846	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
847	prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
848
849	if (malloc_init()) {
850		ret = NULL;
851		goto label_oom;
852	}
853
854	if (size == 0)
855		size = 1;
856
857	if (config_prof && opt_prof) {
858		usize = s2u(size);
859		PROF_ALLOC_PREP(1, usize, cnt);
860		if (cnt == NULL) {
861			ret = NULL;
862			goto label_oom;
863		}
864		if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <=
865		    SMALL_MAXCLASS) {
866			ret = imalloc(SMALL_MAXCLASS+1);
867			if (ret != NULL)
868				arena_prof_promoted(ret, usize);
869		} else
870			ret = imalloc(size);
871	} else {
872		if (config_stats || (config_valgrind && opt_valgrind))
873			usize = s2u(size);
874		ret = imalloc(size);
875	}
876
877label_oom:
878	if (ret == NULL) {
879		if (config_xmalloc && opt_xmalloc) {
880			malloc_write("<jemalloc>: Error in malloc(): "
881			    "out of memory\n");
882			abort();
883		}
884		set_errno(ENOMEM);
885	}
886	if (config_prof && opt_prof && ret != NULL)
887		prof_malloc(ret, usize, cnt);
888	if (config_stats && ret != NULL) {
889		assert(usize == isalloc(ret, config_prof));
890		thread_allocated_tsd_get()->allocated += usize;
891	}
892	UTRACE(0, size, ret);
893	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
894	return (ret);
895}
896
897JEMALLOC_ATTR(nonnull(1))
898#ifdef JEMALLOC_PROF
899/*
900 * Avoid any uncertainty as to how many backtrace frames to ignore in
901 * PROF_ALLOC_PREP().
902 */
903JEMALLOC_NOINLINE
904#endif
905static int
906imemalign(void **memptr, size_t alignment, size_t size,
907    size_t min_alignment)
908{
909	int ret;
910	size_t usize;
911	void *result;
912	prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
913
914	assert(min_alignment != 0);
915
916	if (malloc_init())
917		result = NULL;
918	else {
919		if (size == 0)
920			size = 1;
921
922		/* Make sure that alignment is a large enough power of 2. */
923		if (((alignment - 1) & alignment) != 0
924		    || (alignment < min_alignment)) {
925			if (config_xmalloc && opt_xmalloc) {
926				malloc_write("<jemalloc>: Error allocating "
927				    "aligned memory: invalid alignment\n");
928				abort();
929			}
930			result = NULL;
931			ret = EINVAL;
932			goto label_return;
933		}
934
935		usize = sa2u(size, alignment);
936		if (usize == 0) {
937			result = NULL;
938			ret = ENOMEM;
939			goto label_return;
940		}
941
942		if (config_prof && opt_prof) {
943			PROF_ALLOC_PREP(2, usize, cnt);
944			if (cnt == NULL) {
945				result = NULL;
946				ret = EINVAL;
947			} else {
948				if (prof_promote && (uintptr_t)cnt !=
949				    (uintptr_t)1U && usize <= SMALL_MAXCLASS) {
950					assert(sa2u(SMALL_MAXCLASS+1,
951					    alignment) != 0);
952					result = ipalloc(sa2u(SMALL_MAXCLASS+1,
953					    alignment), alignment, false);
954					if (result != NULL) {
955						arena_prof_promoted(result,
956						    usize);
957					}
958				} else {
959					result = ipalloc(usize, alignment,
960					    false);
961				}
962			}
963		} else
964			result = ipalloc(usize, alignment, false);
965	}
966
967	if (result == NULL) {
968		if (config_xmalloc && opt_xmalloc) {
969			malloc_write("<jemalloc>: Error allocating aligned "
970			    "memory: out of memory\n");
971			abort();
972		}
973		ret = ENOMEM;
974		goto label_return;
975	}
976
977	*memptr = result;
978	ret = 0;
979
980label_return:
981	if (config_stats && result != NULL) {
982		assert(usize == isalloc(result, config_prof));
983		thread_allocated_tsd_get()->allocated += usize;
984	}
985	if (config_prof && opt_prof && result != NULL)
986		prof_malloc(result, usize, cnt);
987	UTRACE(0, size, result);
988	return (ret);
989}
990
991int
992je_posix_memalign(void **memptr, size_t alignment, size_t size)
993{
994	int ret = imemalign(memptr, alignment, size, sizeof(void *));
995	JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
996	    config_prof), false);
997	return (ret);
998}
999
1000void *
1001je_aligned_alloc(size_t alignment, size_t size)
1002{
1003	void *ret;
1004	int err;
1005
1006	if ((err = imemalign(&ret, alignment, size, 1)) != 0) {
1007		ret = NULL;
1008		set_errno(err);
1009	}
1010	JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
1011	    false);
1012	return (ret);
1013}
1014
1015void *
1016je_calloc(size_t num, size_t size)
1017{
1018	void *ret;
1019	size_t num_size;
1020	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1021	prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
1022
1023	if (malloc_init()) {
1024		num_size = 0;
1025		ret = NULL;
1026		goto label_return;
1027	}
1028
1029	num_size = num * size;
1030	if (num_size == 0) {
1031		if (num == 0 || size == 0)
1032			num_size = 1;
1033		else {
1034			ret = NULL;
1035			goto label_return;
1036		}
1037	/*
1038	 * Try to avoid division here.  We know that it isn't possible to
1039	 * overflow during multiplication if neither operand uses any of the
1040	 * most significant half of the bits in a size_t.
1041	 */
1042	} else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
1043	    && (num_size / size != num)) {
1044		/* size_t overflow. */
1045		ret = NULL;
1046		goto label_return;
1047	}
1048
1049	if (config_prof && opt_prof) {
1050		usize = s2u(num_size);
1051		PROF_ALLOC_PREP(1, usize, cnt);
1052		if (cnt == NULL) {
1053			ret = NULL;
1054			goto label_return;
1055		}
1056		if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize
1057		    <= SMALL_MAXCLASS) {
1058			ret = icalloc(SMALL_MAXCLASS+1);
1059			if (ret != NULL)
1060				arena_prof_promoted(ret, usize);
1061		} else
1062			ret = icalloc(num_size);
1063	} else {
1064		if (config_stats || (config_valgrind && opt_valgrind))
1065			usize = s2u(num_size);
1066		ret = icalloc(num_size);
1067	}
1068
1069label_return:
1070	if (ret == NULL) {
1071		if (config_xmalloc && opt_xmalloc) {
1072			malloc_write("<jemalloc>: Error in calloc(): out of "
1073			    "memory\n");
1074			abort();
1075		}
1076		set_errno(ENOMEM);
1077	}
1078
1079	if (config_prof && opt_prof && ret != NULL)
1080		prof_malloc(ret, usize, cnt);
1081	if (config_stats && ret != NULL) {
1082		assert(usize == isalloc(ret, config_prof));
1083		thread_allocated_tsd_get()->allocated += usize;
1084	}
1085	UTRACE(0, num_size, ret);
1086	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
1087	return (ret);
1088}
1089
1090void *
1091je_realloc(void *ptr, size_t size)
1092{
1093	void *ret;
1094	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1095	size_t old_size = 0;
1096	size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1097	prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
1098	prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL);
1099
1100	if (size == 0) {
1101		if (ptr != NULL) {
1102			/* realloc(ptr, 0) is equivalent to free(p). */
1103			if (config_prof) {
1104				old_size = isalloc(ptr, true);
1105				if (config_valgrind && opt_valgrind)
1106					old_rzsize = p2rz(ptr);
1107			} else if (config_stats) {
1108				old_size = isalloc(ptr, false);
1109				if (config_valgrind && opt_valgrind)
1110					old_rzsize = u2rz(old_size);
1111			} else if (config_valgrind && opt_valgrind) {
1112				old_size = isalloc(ptr, false);
1113				old_rzsize = u2rz(old_size);
1114			}
1115			if (config_prof && opt_prof) {
1116				old_ctx = prof_ctx_get(ptr);
1117				cnt = NULL;
1118			}
1119			iqalloc(ptr);
1120			ret = NULL;
1121			goto label_return;
1122		} else
1123			size = 1;
1124	}
1125
1126	if (ptr != NULL) {
1127		assert(malloc_initialized || IS_INITIALIZER);
1128
1129		if (config_prof) {
1130			old_size = isalloc(ptr, true);
1131			if (config_valgrind && opt_valgrind)
1132				old_rzsize = p2rz(ptr);
1133		} else if (config_stats) {
1134			old_size = isalloc(ptr, false);
1135			if (config_valgrind && opt_valgrind)
1136				old_rzsize = u2rz(old_size);
1137		} else if (config_valgrind && opt_valgrind) {
1138			old_size = isalloc(ptr, false);
1139			old_rzsize = u2rz(old_size);
1140		}
1141		if (config_prof && opt_prof) {
1142			usize = s2u(size);
1143			old_ctx = prof_ctx_get(ptr);
1144			PROF_ALLOC_PREP(1, usize, cnt);
1145			if (cnt == NULL) {
1146				old_ctx = NULL;
1147				ret = NULL;
1148				goto label_oom;
1149			}
1150			if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U &&
1151			    usize <= SMALL_MAXCLASS) {
1152				ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0,
1153				    false, false);
1154				if (ret != NULL)
1155					arena_prof_promoted(ret, usize);
1156				else
1157					old_ctx = NULL;
1158			} else {
1159				ret = iralloc(ptr, size, 0, 0, false, false);
1160				if (ret == NULL)
1161					old_ctx = NULL;
1162			}
1163		} else {
1164			if (config_stats || (config_valgrind && opt_valgrind))
1165				usize = s2u(size);
1166			ret = iralloc(ptr, size, 0, 0, false, false);
1167		}
1168
1169label_oom:
1170		if (ret == NULL) {
1171			if (config_xmalloc && opt_xmalloc) {
1172				malloc_write("<jemalloc>: Error in realloc(): "
1173				    "out of memory\n");
1174				abort();
1175			}
1176			set_errno(ENOMEM);
1177		}
1178	} else {
1179		/* realloc(NULL, size) is equivalent to malloc(size). */
1180		if (config_prof && opt_prof)
1181			old_ctx = NULL;
1182		if (malloc_init()) {
1183			if (config_prof && opt_prof)
1184				cnt = NULL;
1185			ret = NULL;
1186		} else {
1187			if (config_prof && opt_prof) {
1188				usize = s2u(size);
1189				PROF_ALLOC_PREP(1, usize, cnt);
1190				if (cnt == NULL)
1191					ret = NULL;
1192				else {
1193					if (prof_promote && (uintptr_t)cnt !=
1194					    (uintptr_t)1U && usize <=
1195					    SMALL_MAXCLASS) {
1196						ret = imalloc(SMALL_MAXCLASS+1);
1197						if (ret != NULL) {
1198							arena_prof_promoted(ret,
1199							    usize);
1200						}
1201					} else
1202						ret = imalloc(size);
1203				}
1204			} else {
1205				if (config_stats || (config_valgrind &&
1206				    opt_valgrind))
1207					usize = s2u(size);
1208				ret = imalloc(size);
1209			}
1210		}
1211
1212		if (ret == NULL) {
1213			if (config_xmalloc && opt_xmalloc) {
1214				malloc_write("<jemalloc>: Error in realloc(): "
1215				    "out of memory\n");
1216				abort();
1217			}
1218			set_errno(ENOMEM);
1219		}
1220	}
1221
1222label_return:
1223	if (config_prof && opt_prof)
1224		prof_realloc(ret, usize, cnt, old_size, old_ctx);
1225	if (config_stats && ret != NULL) {
1226		thread_allocated_t *ta;
1227		assert(usize == isalloc(ret, config_prof));
1228		ta = thread_allocated_tsd_get();
1229		ta->allocated += usize;
1230		ta->deallocated += old_size;
1231	}
1232	UTRACE(ptr, size, ret);
1233	JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false);
1234	return (ret);
1235}
1236
1237void
1238je_free(void *ptr)
1239{
1240
1241	UTRACE(ptr, 0, 0);
1242	if (ptr != NULL) {
1243		size_t usize;
1244		size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1245
1246		assert(malloc_initialized || IS_INITIALIZER);
1247
1248		if (config_prof && opt_prof) {
1249			usize = isalloc(ptr, config_prof);
1250			prof_free(ptr, usize);
1251		} else if (config_stats || config_valgrind)
1252			usize = isalloc(ptr, config_prof);
1253		if (config_stats)
1254			thread_allocated_tsd_get()->deallocated += usize;
1255		if (config_valgrind && opt_valgrind)
1256			rzsize = p2rz(ptr);
1257		iqalloc(ptr);
1258		JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1259	}
1260}
1261
1262/*
1263 * End malloc(3)-compatible functions.
1264 */
1265/******************************************************************************/
1266/*
1267 * Begin non-standard override functions.
1268 */
1269
1270#ifdef JEMALLOC_OVERRIDE_MEMALIGN
1271void *
1272je_memalign(size_t alignment, size_t size)
1273{
1274	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1275	imemalign(&ret, alignment, size, 1);
1276	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1277	return (ret);
1278}
1279#endif
1280
1281#ifdef JEMALLOC_OVERRIDE_VALLOC
1282void *
1283je_valloc(size_t size)
1284{
1285	void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
1286	imemalign(&ret, PAGE, size, 1);
1287	JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
1288	return (ret);
1289}
1290#endif
1291
1292/*
1293 * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has
1294 * #define je_malloc malloc
1295 */
1296#define	malloc_is_malloc 1
1297#define	is_malloc_(a) malloc_is_ ## a
1298#define	is_malloc(a) is_malloc_(a)
1299
1300#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__))
1301/*
1302 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
1303 * to inconsistently reference libc's malloc(3)-compatible functions
1304 * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541).
1305 *
1306 * These definitions interpose hooks in glibc.  The functions are actually
1307 * passed an extra argument for the caller return address, which will be
1308 * ignored.
1309 */
1310JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free;
1311JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc;
1312JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc;
1313JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) =
1314    je_memalign;
1315#endif
1316
1317/*
1318 * End non-standard override functions.
1319 */
1320/******************************************************************************/
1321/*
1322 * Begin non-standard functions.
1323 */
1324
1325size_t
1326je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
1327{
1328	size_t ret;
1329
1330	assert(malloc_initialized || IS_INITIALIZER);
1331
1332	if (config_ivsalloc)
1333		ret = ivsalloc(ptr, config_prof);
1334	else
1335		ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
1336
1337	return (ret);
1338}
1339
1340void
1341je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
1342    const char *opts)
1343{
1344
1345	stats_print(write_cb, cbopaque, opts);
1346}
1347
1348int
1349je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
1350    size_t newlen)
1351{
1352
1353	if (malloc_init())
1354		return (EAGAIN);
1355
1356	return (ctl_byname(name, oldp, oldlenp, newp, newlen));
1357}
1358
1359int
1360je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
1361{
1362
1363	if (malloc_init())
1364		return (EAGAIN);
1365
1366	return (ctl_nametomib(name, mibp, miblenp));
1367}
1368
1369int
1370je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
1371  void *newp, size_t newlen)
1372{
1373
1374	if (malloc_init())
1375		return (EAGAIN);
1376
1377	return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
1378}
1379
1380/*
1381 * End non-standard functions.
1382 */
1383/******************************************************************************/
1384/*
1385 * Begin experimental functions.
1386 */
1387#ifdef JEMALLOC_EXPERIMENTAL
1388
1389static JEMALLOC_ATTR(always_inline) void *
1390iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache,
1391    arena_t *arena)
1392{
1393
1394	assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize,
1395	    alignment)));
1396
1397	if (alignment != 0)
1398		return (ipallocx(usize, alignment, zero, try_tcache, arena));
1399	else if (zero)
1400		return (icallocx(usize, try_tcache, arena));
1401	else
1402		return (imallocx(usize, try_tcache, arena));
1403}
1404
1405int
1406je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
1407{
1408	void *p;
1409	size_t usize;
1410	size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
1411	    & (SIZE_T_MAX-1));
1412	bool zero = flags & ALLOCM_ZERO;
1413	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1414	arena_t *arena;
1415	bool try_tcache;
1416
1417	assert(ptr != NULL);
1418	assert(size != 0);
1419
1420	if (malloc_init())
1421		goto label_oom;
1422
1423	if (arena_ind != UINT_MAX) {
1424		arena = arenas[arena_ind];
1425		try_tcache = false;
1426	} else {
1427		arena = NULL;
1428		try_tcache = true;
1429	}
1430
1431	usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1432	if (usize == 0)
1433		goto label_oom;
1434
1435	if (config_prof && opt_prof) {
1436		prof_thr_cnt_t *cnt;
1437
1438		PROF_ALLOC_PREP(1, usize, cnt);
1439		if (cnt == NULL)
1440			goto label_oom;
1441		if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <=
1442		    SMALL_MAXCLASS) {
1443			size_t usize_promoted = (alignment == 0) ?
1444			    s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1,
1445			    alignment);
1446			assert(usize_promoted != 0);
1447			p = iallocm(usize_promoted, alignment, zero,
1448			    try_tcache, arena);
1449			if (p == NULL)
1450				goto label_oom;
1451			arena_prof_promoted(p, usize);
1452		} else {
1453			p = iallocm(usize, alignment, zero, try_tcache, arena);
1454			if (p == NULL)
1455				goto label_oom;
1456		}
1457		prof_malloc(p, usize, cnt);
1458	} else {
1459		p = iallocm(usize, alignment, zero, try_tcache, arena);
1460		if (p == NULL)
1461			goto label_oom;
1462	}
1463	if (rsize != NULL)
1464		*rsize = usize;
1465
1466	*ptr = p;
1467	if (config_stats) {
1468		assert(usize == isalloc(p, config_prof));
1469		thread_allocated_tsd_get()->allocated += usize;
1470	}
1471	UTRACE(0, size, p);
1472	JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
1473	return (ALLOCM_SUCCESS);
1474label_oom:
1475	if (config_xmalloc && opt_xmalloc) {
1476		malloc_write("<jemalloc>: Error in allocm(): "
1477		    "out of memory\n");
1478		abort();
1479	}
1480	*ptr = NULL;
1481	UTRACE(0, size, 0);
1482	return (ALLOCM_ERR_OOM);
1483}
1484
1485int
1486je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
1487{
1488	void *p, *q;
1489	size_t usize;
1490	size_t old_size;
1491	size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
1492	size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
1493	    & (SIZE_T_MAX-1));
1494	bool zero = flags & ALLOCM_ZERO;
1495	bool no_move = flags & ALLOCM_NO_MOVE;
1496	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1497	bool try_tcache_alloc, try_tcache_dalloc;
1498	arena_t *arena;
1499
1500	assert(ptr != NULL);
1501	assert(*ptr != NULL);
1502	assert(size != 0);
1503	assert(SIZE_T_MAX - size >= extra);
1504	assert(malloc_initialized || IS_INITIALIZER);
1505
1506	if (arena_ind != UINT_MAX) {
1507		arena_chunk_t *chunk;
1508		try_tcache_alloc = true;
1509		chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(*ptr);
1510		try_tcache_dalloc = (chunk == *ptr || chunk->arena !=
1511		    arenas[arena_ind]);
1512		arena = arenas[arena_ind];
1513	} else {
1514		try_tcache_alloc = true;
1515		try_tcache_dalloc = true;
1516		arena = NULL;
1517	}
1518
1519	p = *ptr;
1520	if (config_prof && opt_prof) {
1521		prof_thr_cnt_t *cnt;
1522
1523		/*
1524		 * usize isn't knowable before iralloc() returns when extra is
1525		 * non-zero.  Therefore, compute its maximum possible value and
1526		 * use that in PROF_ALLOC_PREP() to decide whether to capture a
1527		 * backtrace.  prof_realloc() will use the actual usize to
1528		 * decide whether to sample.
1529		 */
1530		size_t max_usize = (alignment == 0) ? s2u(size+extra) :
1531		    sa2u(size+extra, alignment);
1532		prof_ctx_t *old_ctx = prof_ctx_get(p);
1533		old_size = isalloc(p, true);
1534		if (config_valgrind && opt_valgrind)
1535			old_rzsize = p2rz(p);
1536		PROF_ALLOC_PREP(1, max_usize, cnt);
1537		if (cnt == NULL)
1538			goto label_oom;
1539		/*
1540		 * Use minimum usize to determine whether promotion may happen.
1541		 */
1542		if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U
1543		    && ((alignment == 0) ? s2u(size) : sa2u(size, alignment))
1544		    <= SMALL_MAXCLASS) {
1545			q = irallocx(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
1546			    size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1),
1547			    alignment, zero, no_move, try_tcache_alloc,
1548			    try_tcache_dalloc, arena);
1549			if (q == NULL)
1550				goto label_err;
1551			if (max_usize < PAGE) {
1552				usize = max_usize;
1553				arena_prof_promoted(q, usize);
1554			} else
1555				usize = isalloc(q, config_prof);
1556		} else {
1557			q = irallocx(p, size, extra, alignment, zero, no_move,
1558			    try_tcache_alloc, try_tcache_dalloc, arena);
1559			if (q == NULL)
1560				goto label_err;
1561			usize = isalloc(q, config_prof);
1562		}
1563		prof_realloc(q, usize, cnt, old_size, old_ctx);
1564		if (rsize != NULL)
1565			*rsize = usize;
1566	} else {
1567		if (config_stats) {
1568			old_size = isalloc(p, false);
1569			if (config_valgrind && opt_valgrind)
1570				old_rzsize = u2rz(old_size);
1571		} else if (config_valgrind && opt_valgrind) {
1572			old_size = isalloc(p, false);
1573			old_rzsize = u2rz(old_size);
1574		}
1575		q = irallocx(p, size, extra, alignment, zero, no_move,
1576		    try_tcache_alloc, try_tcache_dalloc, arena);
1577		if (q == NULL)
1578			goto label_err;
1579		if (config_stats)
1580			usize = isalloc(q, config_prof);
1581		if (rsize != NULL) {
1582			if (config_stats == false)
1583				usize = isalloc(q, config_prof);
1584			*rsize = usize;
1585		}
1586	}
1587
1588	*ptr = q;
1589	if (config_stats) {
1590		thread_allocated_t *ta;
1591		ta = thread_allocated_tsd_get();
1592		ta->allocated += usize;
1593		ta->deallocated += old_size;
1594	}
1595	UTRACE(p, size, q);
1596	JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero);
1597	return (ALLOCM_SUCCESS);
1598label_err:
1599	if (no_move) {
1600		UTRACE(p, size, q);
1601		return (ALLOCM_ERR_NOT_MOVED);
1602	}
1603label_oom:
1604	if (config_xmalloc && opt_xmalloc) {
1605		malloc_write("<jemalloc>: Error in rallocm(): "
1606		    "out of memory\n");
1607		abort();
1608	}
1609	UTRACE(p, size, 0);
1610	return (ALLOCM_ERR_OOM);
1611}
1612
1613int
1614je_sallocm(const void *ptr, size_t *rsize, int flags)
1615{
1616	size_t sz;
1617
1618	assert(malloc_initialized || IS_INITIALIZER);
1619
1620	if (config_ivsalloc)
1621		sz = ivsalloc(ptr, config_prof);
1622	else {
1623		assert(ptr != NULL);
1624		sz = isalloc(ptr, config_prof);
1625	}
1626	assert(rsize != NULL);
1627	*rsize = sz;
1628
1629	return (ALLOCM_SUCCESS);
1630}
1631
1632int
1633je_dallocm(void *ptr, int flags)
1634{
1635	size_t usize;
1636	size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
1637	unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
1638	bool try_tcache;
1639
1640	assert(ptr != NULL);
1641	assert(malloc_initialized || IS_INITIALIZER);
1642
1643	if (arena_ind != UINT_MAX) {
1644		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1645		try_tcache = (chunk == ptr || chunk->arena !=
1646		    arenas[arena_ind]);
1647	} else
1648		try_tcache = true;
1649
1650	UTRACE(ptr, 0, 0);
1651	if (config_stats || config_valgrind)
1652		usize = isalloc(ptr, config_prof);
1653	if (config_prof && opt_prof) {
1654		if (config_stats == false && config_valgrind == false)
1655			usize = isalloc(ptr, config_prof);
1656		prof_free(ptr, usize);
1657	}
1658	if (config_stats)
1659		thread_allocated_tsd_get()->deallocated += usize;
1660	if (config_valgrind && opt_valgrind)
1661		rzsize = p2rz(ptr);
1662	iqallocx(ptr, try_tcache);
1663	JEMALLOC_VALGRIND_FREE(ptr, rzsize);
1664
1665	return (ALLOCM_SUCCESS);
1666}
1667
1668int
1669je_nallocm(size_t *rsize, size_t size, int flags)
1670{
1671	size_t usize;
1672	size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
1673	    & (SIZE_T_MAX-1));
1674
1675	assert(size != 0);
1676
1677	if (malloc_init())
1678		return (ALLOCM_ERR_OOM);
1679
1680	usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
1681	if (usize == 0)
1682		return (ALLOCM_ERR_OOM);
1683
1684	if (rsize != NULL)
1685		*rsize = usize;
1686	return (ALLOCM_SUCCESS);
1687}
1688
1689#endif
1690/*
1691 * End experimental functions.
1692 */
1693/******************************************************************************/
1694/*
1695 * The following functions are used by threading libraries for protection of
1696 * malloc during fork().
1697 */
1698
1699/*
1700 * If an application creates a thread before doing any allocation in the main
1701 * thread, then calls fork(2) in the main thread followed by memory allocation
1702 * in the child process, a race can occur that results in deadlock within the
1703 * child: the main thread may have forked while the created thread had
1704 * partially initialized the allocator.  Ordinarily jemalloc prevents
1705 * fork/malloc races via the following functions it registers during
1706 * initialization using pthread_atfork(), but of course that does no good if
1707 * the allocator isn't fully initialized at fork time.  The following library
1708 * constructor is a partial solution to this problem.  It may still possible to
1709 * trigger the deadlock described above, but doing so would involve forking via
1710 * a library constructor that runs before jemalloc's runs.
1711 */
1712JEMALLOC_ATTR(constructor)
1713static void
1714jemalloc_constructor(void)
1715{
1716
1717	malloc_init();
1718}
1719
1720#ifndef JEMALLOC_MUTEX_INIT_CB
1721void
1722jemalloc_prefork(void)
1723#else
1724JEMALLOC_EXPORT void
1725_malloc_prefork(void)
1726#endif
1727{
1728	unsigned i;
1729
1730#ifdef JEMALLOC_MUTEX_INIT_CB
1731	if (malloc_initialized == false)
1732		return;
1733#endif
1734	assert(malloc_initialized);
1735
1736	/* Acquire all mutexes in a safe order. */
1737	ctl_prefork();
1738	malloc_mutex_prefork(&arenas_lock);
1739	for (i = 0; i < narenas_total; i++) {
1740		if (arenas[i] != NULL)
1741			arena_prefork(arenas[i]);
1742	}
1743	prof_prefork();
1744	chunk_prefork();
1745	base_prefork();
1746	huge_prefork();
1747}
1748
1749#ifndef JEMALLOC_MUTEX_INIT_CB
1750void
1751jemalloc_postfork_parent(void)
1752#else
1753JEMALLOC_EXPORT void
1754_malloc_postfork(void)
1755#endif
1756{
1757	unsigned i;
1758
1759#ifdef JEMALLOC_MUTEX_INIT_CB
1760	if (malloc_initialized == false)
1761		return;
1762#endif
1763	assert(malloc_initialized);
1764
1765	/* Release all mutexes, now that fork() has completed. */
1766	huge_postfork_parent();
1767	base_postfork_parent();
1768	chunk_postfork_parent();
1769	prof_postfork_parent();
1770	for (i = 0; i < narenas_total; i++) {
1771		if (arenas[i] != NULL)
1772			arena_postfork_parent(arenas[i]);
1773	}
1774	malloc_mutex_postfork_parent(&arenas_lock);
1775	ctl_postfork_parent();
1776}
1777
1778void
1779jemalloc_postfork_child(void)
1780{
1781	unsigned i;
1782
1783	assert(malloc_initialized);
1784
1785	/* Release all mutexes, now that fork() has completed. */
1786	huge_postfork_child();
1787	base_postfork_child();
1788	chunk_postfork_child();
1789	prof_postfork_child();
1790	for (i = 0; i < narenas_total; i++) {
1791		if (arenas[i] != NULL)
1792			arena_postfork_child(arenas[i]);
1793	}
1794	malloc_mutex_postfork_child(&arenas_lock);
1795	ctl_postfork_child();
1796}
1797
1798/******************************************************************************/
1799/*
1800 * The following functions are used for TLS allocation/deallocation in static
1801 * binaries on FreeBSD.  The primary difference between these and i[mcd]alloc()
1802 * is that these avoid accessing TLS variables.
1803 */
1804
1805static void *
1806a0alloc(size_t size, bool zero)
1807{
1808
1809	if (malloc_init())
1810		return (NULL);
1811
1812	if (size == 0)
1813		size = 1;
1814
1815	if (size <= arena_maxclass)
1816		return (arena_malloc(arenas[0], size, zero, false));
1817	else
1818		return (huge_malloc(size, zero));
1819}
1820
1821void *
1822a0malloc(size_t size)
1823{
1824
1825	return (a0alloc(size, false));
1826}
1827
1828void *
1829a0calloc(size_t num, size_t size)
1830{
1831
1832	return (a0alloc(num * size, true));
1833}
1834
1835void
1836a0free(void *ptr)
1837{
1838	arena_chunk_t *chunk;
1839
1840	if (ptr == NULL)
1841		return;
1842
1843	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
1844	if (chunk != ptr)
1845		arena_dalloc(chunk->arena, chunk, ptr, false);
1846	else
1847		huge_dalloc(ptr, true);
1848}
1849
1850/******************************************************************************/
1851