1diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in 2index 7fecda7c..d5ca5e86 100644 3--- a/doc/jemalloc.xml.in 4+++ b/doc/jemalloc.xml.in 5@@ -53,11 +53,22 @@ 6 <para>This manual describes jemalloc @jemalloc_version@. More information 7 can be found at the <ulink 8 url="http://jemalloc.net/">jemalloc website</ulink>.</para> 9+ 10+ <para>The following configuration options are enabled in libc's built-in 11+ jemalloc: <option>--enable-fill</option>, 12+ <option>--enable-lazy-lock</option>, <option>--enable-stats</option>, 13+ <option>--enable-utrace</option>, <option>--enable-xmalloc</option>, and 14+ <option>--with-malloc-conf=abort_conf:false</option>. 15+ Additionally, <option>--enable-debug</option> is enabled in development 16+ versions of FreeBSD (controlled by the 17+ <constant>MK_MALLOC_PRODUCTION</constant> make variable).</para> 18+ 19 </refsect1> 20 <refsynopsisdiv> 21 <title>SYNOPSIS</title> 22 <funcsynopsis> 23- <funcsynopsisinfo>#include <<filename class="headerfile">jemalloc/jemalloc.h</filename>></funcsynopsisinfo> 24+ <funcsynopsisinfo>#include <<filename class="headerfile">stdlib.h</filename>> 25+#include <<filename class="headerfile">malloc_np.h</filename>></funcsynopsisinfo> 26 <refsect2> 27 <title>Standard API</title> 28 <funcprototype> 29@@ -3510,4 +3521,18 @@ malloc_conf = "narenas:1";]]></programlisting></para> 30 <para>The <function>posix_memalign()</function> function conforms 31 to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para> 32 </refsect1> 33+ <refsect1 id="history"> 34+ <title>HISTORY</title> 35+ <para>The <function>malloc_usable_size()</function> and 36+ <function>posix_memalign()</function> functions first appeared in FreeBSD 37+ 7.0.</para> 38+ 39+ <para>The <function>aligned_alloc()</function>, 40+ <function>malloc_stats_print()</function>, and 41+ <function>mallctl*()</function> functions first appeared in FreeBSD 42+ 10.0.</para> 43+ 44+ <para>The <function>*allocx()</function> functions first appeared in FreeBSD 45+ 11.0.</para> 46+ </refsect1> 47 </refentry> 48diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h 49index 7d6053e2..a0e4f5af 100644 50--- a/include/jemalloc/internal/jemalloc_internal_decls.h 51+++ b/include/jemalloc/internal/jemalloc_internal_decls.h 52@@ -1,6 +1,9 @@ 53 #ifndef JEMALLOC_INTERNAL_DECLS_H 54 #define JEMALLOC_INTERNAL_DECLS_H 55 56+#include "libc_private.h" 57+#include "namespace.h" 58+ 59 #include <math.h> 60 #ifdef _WIN32 61 # include <windows.h> 62diff --git a/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h 63new file mode 100644 64index 00000000..0dab1296 65--- /dev/null 66+++ b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h 67@@ -0,0 +1,9 @@ 68+#ifndef __clang__ 69+# undef JEMALLOC_INTERNAL_UNREACHABLE 70+# define JEMALLOC_INTERNAL_UNREACHABLE abort 71+ 72+# undef JEMALLOC_C11_ATOMICS 73+# undef JEMALLOC_GCC_ATOMIC_ATOMICS 74+# undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS 75+# undef JEMALLOC_GCC_U8_SYNC_ATOMICS 76+#endif 77diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in 78index 3418cbfa..53e30dc4 100644 79--- a/include/jemalloc/internal/jemalloc_preamble.h.in 80+++ b/include/jemalloc/internal/jemalloc_preamble.h.in 81@@ -8,6 +8,9 @@ 82 #include <sys/ktrace.h> 83 #endif 84 85+#include "un-namespace.h" 86+#include "libc_private.h" 87+ 88 #define JEMALLOC_NO_DEMANGLE 89 #ifdef JEMALLOC_JET 90 # undef JEMALLOC_IS_MALLOC 91@@ -79,13 +82,7 @@ static const bool config_fill = 92 false 93 #endif 94 ; 95-static const bool config_lazy_lock = 96-#ifdef JEMALLOC_LAZY_LOCK 97- true 98-#else 99- false 100-#endif 101- ; 102+static const bool config_lazy_lock = true; 103 static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; 104 static const bool config_prof = 105 #ifdef JEMALLOC_PROF 106diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h 107index 7c24f072..94af1618 100644 108--- a/include/jemalloc/internal/mutex.h 109+++ b/include/jemalloc/internal/mutex.h 110@@ -135,9 +135,6 @@ struct malloc_mutex_s { 111 112 #ifdef JEMALLOC_LAZY_LOCK 113 extern bool isthreaded; 114-#else 115-# undef isthreaded /* Undo private_namespace.h definition. */ 116-# define isthreaded true 117 #endif 118 119 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 120@@ -145,6 +142,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 121 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); 122 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); 123 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); 124+bool malloc_mutex_first_thread(void); 125 bool malloc_mutex_boot(void); 126 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); 127 128diff --git a/include/jemalloc/internal/test_hooks.h b/include/jemalloc/internal/test_hooks.h 129index a6351e59..0780c52f 100644 130--- a/include/jemalloc/internal/test_hooks.h 131+++ b/include/jemalloc/internal/test_hooks.h 132@@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); 133 134 #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) 135 136-#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) 137-#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) 138-#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) 139-#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) 140-#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) 141-#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) 142-#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) 143 /* Note that this is undef'd and re-define'd in src/prof.c. */ 144 #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) 145 146diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h 147index 9ba26004..ecfda5d6 100644 148--- a/include/jemalloc/internal/tsd.h 149+++ b/include/jemalloc/internal/tsd.h 150@@ -198,7 +198,8 @@ struct tsd_s { 151 t TSD_MANGLE(n); 152 MALLOC_TSD 153 #undef O 154-}; 155+/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */ 156+} JEMALLOC_ALIGNED(16); 157 158 JEMALLOC_ALWAYS_INLINE uint8_t 159 tsd_state_get(tsd_t *tsd) { 160diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h 161new file mode 100644 162index 00000000..b752b0e7 163--- /dev/null 164+++ b/include/jemalloc/jemalloc_FreeBSD.h 165@@ -0,0 +1,185 @@ 166+/* 167+ * Override settings that were generated in jemalloc_defs.h as necessary. 168+ */ 169+ 170+#undef JEMALLOC_OVERRIDE_VALLOC 171+ 172+#ifndef MALLOC_PRODUCTION 173+#define JEMALLOC_DEBUG 174+#endif 175+ 176+#undef JEMALLOC_DSS 177+ 178+#undef JEMALLOC_BACKGROUND_THREAD 179+ 180+/* 181+ * The following are architecture-dependent, so conditionally define them for 182+ * each supported architecture. 183+ */ 184+#undef JEMALLOC_TLS_MODEL 185+#undef LG_PAGE 186+#undef LG_VADDR 187+#undef LG_SIZEOF_PTR 188+#undef LG_SIZEOF_INT 189+#undef LG_SIZEOF_LONG 190+#undef LG_SIZEOF_INTMAX_T 191+ 192+#ifdef __i386__ 193+# define LG_VADDR 32 194+# define LG_SIZEOF_PTR 2 195+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 196+#endif 197+#ifdef __ia64__ 198+# define LG_VADDR 64 199+# define LG_SIZEOF_PTR 3 200+#endif 201+#ifdef __sparc64__ 202+# define LG_VADDR 64 203+# define LG_SIZEOF_PTR 3 204+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 205+#endif 206+#ifdef __amd64__ 207+# define LG_VADDR 48 208+# define LG_SIZEOF_PTR 3 209+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) 210+#endif 211+#ifdef __arm__ 212+# define LG_VADDR 32 213+# define LG_SIZEOF_PTR 2 214+#endif 215+#ifdef __aarch64__ 216+# define LG_VADDR 48 217+# define LG_SIZEOF_PTR 3 218+#endif 219+#ifdef __mips__ 220+#ifdef __mips_n64 221+# define LG_VADDR 64 222+# define LG_SIZEOF_PTR 3 223+#else 224+# define LG_VADDR 32 225+# define LG_SIZEOF_PTR 2 226+#endif 227+#endif 228+#ifdef __powerpc64__ 229+# define LG_VADDR 64 230+# define LG_SIZEOF_PTR 3 231+#elif defined(__powerpc__) 232+# define LG_VADDR 32 233+# define LG_SIZEOF_PTR 2 234+#endif 235+#ifdef __riscv 236+# define LG_VADDR 48 237+# define LG_SIZEOF_PTR 3 238+#endif 239+ 240+#ifndef JEMALLOC_TLS_MODEL 241+# define JEMALLOC_TLS_MODEL /* Default. */ 242+#endif 243+ 244+#define LG_PAGE PAGE_SHIFT 245+#define LG_SIZEOF_INT 2 246+#define LG_SIZEOF_LONG LG_SIZEOF_PTR 247+#define LG_SIZEOF_INTMAX_T 3 248+ 249+#undef CPU_SPINWAIT 250+#include <machine/cpu.h> 251+#include <machine/cpufunc.h> 252+#define CPU_SPINWAIT cpu_spinwait() 253+ 254+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */ 255+#undef JEMALLOC_LAZY_LOCK 256+extern int __isthreaded; 257+#define isthreaded ((bool)__isthreaded) 258+ 259+/* Mangle. */ 260+#undef je_malloc 261+#undef je_calloc 262+#undef je_posix_memalign 263+#undef je_aligned_alloc 264+#undef je_realloc 265+#undef je_free 266+#undef je_malloc_usable_size 267+#undef je_mallocx 268+#undef je_rallocx 269+#undef je_xallocx 270+#undef je_sallocx 271+#undef je_dallocx 272+#undef je_sdallocx 273+#undef je_nallocx 274+#undef je_mallctl 275+#undef je_mallctlnametomib 276+#undef je_mallctlbymib 277+#undef je_malloc_stats_print 278+#undef je_allocm 279+#undef je_rallocm 280+#undef je_sallocm 281+#undef je_dallocm 282+#undef je_nallocm 283+#define je_malloc __malloc 284+#define je_calloc __calloc 285+#define je_posix_memalign __posix_memalign 286+#define je_aligned_alloc __aligned_alloc 287+#define je_realloc __realloc 288+#define je_free __free 289+#define je_malloc_usable_size __malloc_usable_size 290+#define je_mallocx __mallocx 291+#define je_rallocx __rallocx 292+#define je_xallocx __xallocx 293+#define je_sallocx __sallocx 294+#define je_dallocx __dallocx 295+#define je_sdallocx __sdallocx 296+#define je_nallocx __nallocx 297+#define je_mallctl __mallctl 298+#define je_mallctlnametomib __mallctlnametomib 299+#define je_mallctlbymib __mallctlbymib 300+#define je_malloc_stats_print __malloc_stats_print 301+#define je_allocm __allocm 302+#define je_rallocm __rallocm 303+#define je_sallocm __sallocm 304+#define je_dallocm __dallocm 305+#define je_nallocm __nallocm 306+#define open _open 307+#define read _read 308+#define write _write 309+#define close _close 310+#define pthread_join _pthread_join 311+#define pthread_once _pthread_once 312+#define pthread_self _pthread_self 313+#define pthread_equal _pthread_equal 314+#define pthread_mutex_lock _pthread_mutex_lock 315+#define pthread_mutex_trylock _pthread_mutex_trylock 316+#define pthread_mutex_unlock _pthread_mutex_unlock 317+#define pthread_cond_init _pthread_cond_init 318+#define pthread_cond_wait _pthread_cond_wait 319+#define pthread_cond_timedwait _pthread_cond_timedwait 320+#define pthread_cond_signal _pthread_cond_signal 321+ 322+#ifdef JEMALLOC_C_ 323+/* 324+ * Define 'weak' symbols so that an application can have its own versions 325+ * of malloc, calloc, realloc, free, et al. 326+ */ 327+__weak_reference(__malloc, malloc); 328+__weak_reference(__calloc, calloc); 329+__weak_reference(__posix_memalign, posix_memalign); 330+__weak_reference(__aligned_alloc, aligned_alloc); 331+__weak_reference(__realloc, realloc); 332+__weak_reference(__free, free); 333+__weak_reference(__malloc_usable_size, malloc_usable_size); 334+__weak_reference(__mallocx, mallocx); 335+__weak_reference(__rallocx, rallocx); 336+__weak_reference(__xallocx, xallocx); 337+__weak_reference(__sallocx, sallocx); 338+__weak_reference(__dallocx, dallocx); 339+__weak_reference(__sdallocx, sdallocx); 340+__weak_reference(__nallocx, nallocx); 341+__weak_reference(__mallctl, mallctl); 342+__weak_reference(__mallctlnametomib, mallctlnametomib); 343+__weak_reference(__mallctlbymib, mallctlbymib); 344+__weak_reference(__malloc_stats_print, malloc_stats_print); 345+__weak_reference(__allocm, allocm); 346+__weak_reference(__rallocm, rallocm); 347+__weak_reference(__sallocm, sallocm); 348+__weak_reference(__dallocm, dallocm); 349+__weak_reference(__nallocm, nallocm); 350+#endif 351diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh 352index f9438912..47d032c1 100755 353--- a/include/jemalloc/jemalloc_rename.sh 354+++ b/include/jemalloc/jemalloc_rename.sh 355@@ -19,4 +19,6 @@ done 356 357 cat <<EOF 358 #endif 359+ 360+#include "jemalloc_FreeBSD.h" 361 EOF 362diff --git a/src/jemalloc.c b/src/jemalloc.c 363index ed13718d..fefb719a 100644 364--- a/src/jemalloc.c 365+++ b/src/jemalloc.c 366@@ -23,6 +23,10 @@ 367 /******************************************************************************/ 368 /* Data. */ 369 370+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */ 371+const char *__malloc_options_1_0 = NULL; 372+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); 373+ 374 /* Runtime configuration options. */ 375 const char *je_malloc_conf 376 #ifndef _WIN32 377@@ -2660,25 +2664,6 @@ je_realloc(void *ptr, size_t arg_size) { 378 LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); 379 380 if (unlikely(size == 0)) { 381- if (ptr != NULL) { 382- /* realloc(ptr, 0) is equivalent to free(ptr). */ 383- UTRACE(ptr, 0, 0); 384- tcache_t *tcache; 385- tsd_t *tsd = tsd_fetch(); 386- if (tsd_reentrancy_level_get(tsd) == 0) { 387- tcache = tcache_get(tsd); 388- } else { 389- tcache = NULL; 390- } 391- 392- uintptr_t args[3] = {(uintptr_t)ptr, size}; 393- hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); 394- 395- ifree(tsd, ptr, tcache, true); 396- 397- LOG("core.realloc.exit", "result: %p", NULL); 398- return NULL; 399- } 400 size = 1; 401 } 402 403@@ -3750,6 +3735,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { 404 * End non-standard functions. 405 */ 406 /******************************************************************************/ 407+/* 408+ * Begin compatibility functions. 409+ */ 410+ 411+#define ALLOCM_LG_ALIGN(la) (la) 412+#define ALLOCM_ALIGN(a) (ffsl(a)-1) 413+#define ALLOCM_ZERO ((int)0x40) 414+#define ALLOCM_NO_MOVE ((int)0x80) 415+ 416+#define ALLOCM_SUCCESS 0 417+#define ALLOCM_ERR_OOM 1 418+#define ALLOCM_ERR_NOT_MOVED 2 419+ 420+int 421+je_allocm(void **ptr, size_t *rsize, size_t size, int flags) { 422+ assert(ptr != NULL); 423+ 424+ void *p = je_mallocx(size, flags); 425+ if (p == NULL) { 426+ return (ALLOCM_ERR_OOM); 427+ } 428+ if (rsize != NULL) { 429+ *rsize = isalloc(tsdn_fetch(), p); 430+ } 431+ *ptr = p; 432+ return ALLOCM_SUCCESS; 433+} 434+ 435+int 436+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) { 437+ assert(ptr != NULL); 438+ assert(*ptr != NULL); 439+ assert(size != 0); 440+ assert(SIZE_T_MAX - size >= extra); 441+ 442+ int ret; 443+ bool no_move = flags & ALLOCM_NO_MOVE; 444+ 445+ if (no_move) { 446+ size_t usize = je_xallocx(*ptr, size, extra, flags); 447+ ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; 448+ if (rsize != NULL) { 449+ *rsize = usize; 450+ } 451+ } else { 452+ void *p = je_rallocx(*ptr, size+extra, flags); 453+ if (p != NULL) { 454+ *ptr = p; 455+ ret = ALLOCM_SUCCESS; 456+ } else { 457+ ret = ALLOCM_ERR_OOM; 458+ } 459+ if (rsize != NULL) { 460+ *rsize = isalloc(tsdn_fetch(), *ptr); 461+ } 462+ } 463+ return ret; 464+} 465+ 466+int 467+je_sallocm(const void *ptr, size_t *rsize, int flags) { 468+ assert(rsize != NULL); 469+ *rsize = je_sallocx(ptr, flags); 470+ return ALLOCM_SUCCESS; 471+} 472+ 473+int 474+je_dallocm(void *ptr, int flags) { 475+ je_dallocx(ptr, flags); 476+ return ALLOCM_SUCCESS; 477+} 478+ 479+int 480+je_nallocm(size_t *rsize, size_t size, int flags) { 481+ size_t usize = je_nallocx(size, flags); 482+ if (usize == 0) { 483+ return ALLOCM_ERR_OOM; 484+ } 485+ if (rsize != NULL) { 486+ *rsize = usize; 487+ } 488+ return ALLOCM_SUCCESS; 489+} 490+ 491+#undef ALLOCM_LG_ALIGN 492+#undef ALLOCM_ALIGN 493+#undef ALLOCM_ZERO 494+#undef ALLOCM_NO_MOVE 495+ 496+#undef ALLOCM_SUCCESS 497+#undef ALLOCM_ERR_OOM 498+#undef ALLOCM_ERR_NOT_MOVED 499+ 500+/* 501+ * End compatibility functions. 502+ */ 503+/******************************************************************************/ 504 /* 505 * The following functions are used by threading libraries for protection of 506 * malloc during fork(). 507@@ -3919,4 +4001,11 @@ jemalloc_postfork_child(void) { 508 ctl_postfork_child(tsd_tsdn(tsd)); 509 } 510 511+void 512+_malloc_first_thread(void) 513+{ 514+ 515+ (void)malloc_mutex_first_thread(); 516+} 517+ 518 /******************************************************************************/ 519diff --git a/src/malloc_io.c b/src/malloc_io.c 520index d7cb0f52..cda589c4 100644 521--- a/src/malloc_io.c 522+++ b/src/malloc_io.c 523@@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) { 524 525 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 526 527+JEMALLOC_ATTR(visibility("hidden")) 528+void 529+wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) { 530+ 531+ wrtmessage(NULL, s1); 532+ wrtmessage(NULL, s2); 533+ wrtmessage(NULL, s3); 534+ wrtmessage(NULL, s4); 535+} 536+ 537+void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, 538+ const char *s4) = wrtmessage_1_0; 539+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); 540+ 541 /* 542 * Wrapper around malloc_message() that avoids the need for 543 * je_malloc_message(...) throughout the code. 544diff --git a/src/mutex.c b/src/mutex.c 545index 3f920f5b..88a7730c 100644 546--- a/src/mutex.c 547+++ b/src/mutex.c 548@@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread, 549 #ifdef JEMALLOC_MUTEX_INIT_CB 550 JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 551 void *(calloc_cb)(size_t, size_t)); 552+ 553+#pragma weak _pthread_mutex_init_calloc_cb 554+int 555+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, 556+ void *(calloc_cb)(size_t, size_t)) 557+{ 558+ 559+ return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t))) 560+ __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex, 561+ calloc_cb)); 562+} 563 #endif 564 565 void 566@@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1, 567 } 568 } 569 570+bool 571+malloc_mutex_first_thread(void) { 572+ 573+#ifndef JEMALLOC_MUTEX_INIT_CB 574+ return (malloc_mutex_first_thread()); 575+#else 576+ return (false); 577+#endif 578+} 579+ 580 bool 581 malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 582 witness_rank_t rank, malloc_mutex_lock_order_t lock_order) { 583