thr_stack.c revision 296732
1203134Sthompsa/* 2203134Sthompsa * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org> 3203134Sthompsa * Copyright (c) 2000-2001 Jason Evans <jasone@freebsd.org> 4203134Sthompsa * All rights reserved. 5203134Sthompsa * 6203134Sthompsa * Redistribution and use in source and binary forms, with or without 7203134Sthompsa * modification, are permitted provided that the following conditions 8203134Sthompsa * are met: 9203134Sthompsa * 1. Redistributions of source code must retain the above copyright 10203134Sthompsa * notice, this list of conditions and the following disclaimer. 11203134Sthompsa * 2. Redistributions in binary form must reproduce the above copyright 12203134Sthompsa * notice, this list of conditions and the following disclaimer in the 13203134Sthompsa * documentation and/or other materials provided with the distribution. 14203134Sthompsa * 15203134Sthompsa * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 16203134Sthompsa * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17203134Sthompsa * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18203134Sthompsa * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 19203134Sthompsa * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20203134Sthompsa * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21203134Sthompsa * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22203134Sthompsa * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23203134Sthompsa * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24203134Sthompsa * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25261868Skevlo * SUCH DAMAGE. 26261868Skevlo * 27203134Sthompsa * $FreeBSD: stable/10/lib/libthr/thread/thr_stack.c 296732 2016-03-12 17:33:40Z kib $ 28261868Skevlo */ 29203134Sthompsa 30203134Sthompsa#include <sys/types.h> 31261868Skevlo#include <sys/mman.h> 32261868Skevlo#include <sys/queue.h> 33261868Skevlo#include <sys/resource.h> 34261868Skevlo#include <sys/sysctl.h> 35261868Skevlo#include <stdlib.h> 36261868Skevlo#include <pthread.h> 37261868Skevlo#include <link.h> 38261868Skevlo 39261868Skevlo#include "thr_private.h" 40261868Skevlo 41261868Skevlo/* Spare thread stack. */ 42261868Skevlostruct stack { 43261868Skevlo LIST_ENTRY(stack) qe; /* Stack queue linkage. */ 44261868Skevlo size_t stacksize; /* Stack size (rounded up). */ 45261868Skevlo size_t guardsize; /* Guard size. */ 46261868Skevlo void *stackaddr; /* Stack address. */ 47261868Skevlo}; 48261868Skevlo 49261868Skevlo/* 50261868Skevlo * Default sized (stack and guard) spare stack queue. Stacks are cached 51261868Skevlo * to avoid additional complexity managing mmap()ed stack regions. Spare 52261868Skevlo * stacks are used in LIFO order to increase cache locality. 53203134Sthompsa */ 54203134Sthompsastatic LIST_HEAD(, stack) dstackq = LIST_HEAD_INITIALIZER(dstackq); 55261868Skevlo 56261868Skevlo/* 57261868Skevlo * Miscellaneous sized (non-default stack and/or guard) spare stack queue. 58261868Skevlo * Stacks are cached to avoid additional complexity managing mmap()ed 59261868Skevlo * stack regions. This list is unordered, since ordering on both stack 60261868Skevlo * size and guard size would be more trouble than it's worth. Stacks are 61261868Skevlo * allocated from this cache on a first size match basis. 62261868Skevlo */ 63261868Skevlostatic LIST_HEAD(, stack) mstackq = LIST_HEAD_INITIALIZER(mstackq); 64261868Skevlo 65261868Skevlo/** 66261868Skevlo * Base address of the last stack allocated (including its red zone, if 67261868Skevlo * there is one). Stacks are allocated contiguously, starting beyond the 68261868Skevlo * top of the main stack. When a new stack is created, a red zone is 69261868Skevlo * typically created (actually, the red zone is mapped with PROT_NONE) above 70203134Sthompsa * the top of the stack, such that the stack will not be able to grow all 71203134Sthompsa * the way to the bottom of the next stack. This isn't fool-proof. It is 72261868Skevlo * possible for a stack to grow by a large amount, such that it grows into 73261868Skevlo * the next stack, and as long as the memory within the red zone is never 74261868Skevlo * accessed, nothing will prevent one thread stack from trouncing all over 75261868Skevlo * the next. 76261868Skevlo * 77261868Skevlo * low memory 78261868Skevlo * . . . . . . . . . . . . . . . . . . 79261868Skevlo * | | 80203134Sthompsa * | stack 3 | start of 3rd thread stack 81259453Shselasky * +-----------------------------------+ 82261868Skevlo * | | 83259453Shselasky * | Red Zone (guard page) | red zone for 2nd thread 84203134Sthompsa * | | 85261868Skevlo * +-----------------------------------+ 86261868Skevlo * | stack 2 - _thr_stack_default | top of 2nd thread stack 87261868Skevlo * | | 88261868Skevlo * | | 89261868Skevlo * | | 90261868Skevlo * | | 91261868Skevlo * | stack 2 | 92261868Skevlo * +-----------------------------------+ <-- start of 2nd thread stack 93261868Skevlo * | | 94261868Skevlo * | Red Zone | red zone for 1st thread 95261868Skevlo * | | 96261868Skevlo * +-----------------------------------+ 97203134Sthompsa * | stack 1 - _thr_stack_default | top of 1st thread stack 98203134Sthompsa * | | 99261868Skevlo * | | 100203134Sthompsa * | | 101203134Sthompsa * | | 102261868Skevlo * | stack 1 | 103261868Skevlo * +-----------------------------------+ <-- start of 1st thread stack 104261868Skevlo * | | (initial value of last_stack) 105261868Skevlo * | Red Zone | 106261868Skevlo * | | red zone for main thread 107261868Skevlo * +-----------------------------------+ 108261868Skevlo * | USRSTACK - _thr_stack_initial | top of main thread stack 109261868Skevlo * | | ^ 110261868Skevlo * | | | 111261868Skevlo * | | | 112261868Skevlo * | | | stack growth 113261868Skevlo * | | 114261868Skevlo * +-----------------------------------+ <-- start of main thread stack 115203134Sthompsa * (USRSTACK) 116203134Sthompsa * high memory 117261868Skevlo * 118261868Skevlo */ 119261868Skevlostatic char *last_stack = NULL; 120203134Sthompsa 121203134Sthompsa/* 122261868Skevlo * Round size up to the nearest multiple of 123261868Skevlo * _thr_page_size. 124261868Skevlo */ 125261868Skevlostatic inline size_t 126261868Skevloround_up(size_t size) 127261868Skevlo{ 128261868Skevlo if (size % _thr_page_size != 0) 129261868Skevlo size = ((size / _thr_page_size) + 1) * 130261868Skevlo _thr_page_size; 131261868Skevlo return size; 132261868Skevlo} 133261868Skevlo 134261868Skevlovoid 135261868Skevlo_thr_stack_fix_protection(struct pthread *thrd) 136261868Skevlo{ 137261868Skevlo 138261868Skevlo mprotect((char *)thrd->attr.stackaddr_attr + 139261868Skevlo round_up(thrd->attr.guardsize_attr), 140261868Skevlo round_up(thrd->attr.stacksize_attr), 141261868Skevlo _rtld_get_stack_prot()); 142261868Skevlo} 143261868Skevlo 144261868Skevlostatic void 145261868Skevlosinglethread_map_stacks_exec(void) 146261868Skevlo{ 147261868Skevlo int mib[2]; 148203134Sthompsa struct rlimit rlim; 149203134Sthompsa u_long usrstack; 150261868Skevlo size_t len; 151261868Skevlo 152261868Skevlo mib[0] = CTL_KERN; 153261868Skevlo mib[1] = KERN_USRSTACK; 154261868Skevlo len = sizeof(usrstack); 155261868Skevlo if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), &usrstack, &len, NULL, 0) 156261868Skevlo == -1) 157203134Sthompsa return; 158203134Sthompsa if (getrlimit(RLIMIT_STACK, &rlim) == -1) 159261868Skevlo return; 160261868Skevlo mprotect((void *)(uintptr_t)(usrstack - rlim.rlim_cur), 161261868Skevlo rlim.rlim_cur, _rtld_get_stack_prot()); 162203134Sthompsa} 163203134Sthompsa 164261868Skevlovoid 165261868Skevlo__thr_map_stacks_exec(void) 166261868Skevlo{ 167261868Skevlo struct pthread *curthread, *thrd; 168261868Skevlo struct stack *st; 169261868Skevlo 170203134Sthompsa if (!_thr_is_inited()) { 171203134Sthompsa singlethread_map_stacks_exec(); 172261868Skevlo return; 173261868Skevlo } 174261868Skevlo curthread = _get_curthread(); 175261868Skevlo THREAD_LIST_RDLOCK(curthread); 176261868Skevlo LIST_FOREACH(st, &mstackq, qe) 177261868Skevlo mprotect((char *)st->stackaddr + st->guardsize, st->stacksize, 178261868Skevlo _rtld_get_stack_prot()); 179203134Sthompsa LIST_FOREACH(st, &dstackq, qe) 180203134Sthompsa mprotect((char *)st->stackaddr + st->guardsize, st->stacksize, 181261868Skevlo _rtld_get_stack_prot()); 182203134Sthompsa TAILQ_FOREACH(thrd, &_thread_gc_list, gcle) 183261868Skevlo _thr_stack_fix_protection(thrd); 184261868Skevlo TAILQ_FOREACH(thrd, &_thread_list, tle) 185203134Sthompsa _thr_stack_fix_protection(thrd); 186203134Sthompsa THREAD_LIST_UNLOCK(curthread); 187261868Skevlo} 188203134Sthompsa 189203134Sthompsaint 190261868Skevlo_thr_stack_alloc(struct pthread_attr *attr) 191203134Sthompsa{ 192203134Sthompsa struct pthread *curthread = _get_curthread(); 193261868Skevlo struct stack *spare_stack; 194203134Sthompsa size_t stacksize; 195203134Sthompsa size_t guardsize; 196261868Skevlo char *stackaddr; 197203134Sthompsa 198203134Sthompsa /* 199261868Skevlo * Round up stack size to nearest multiple of _thr_page_size so 200261868Skevlo * that mmap() * will work. If the stack size is not an even 201261868Skevlo * multiple, we end up initializing things such that there is 202261868Skevlo * unused space above the beginning of the stack, so the stack 203203134Sthompsa * sits snugly against its guard. 204203134Sthompsa */ 205261868Skevlo stacksize = round_up(attr->stacksize_attr); 206261868Skevlo guardsize = round_up(attr->guardsize_attr); 207261868Skevlo 208261868Skevlo attr->stackaddr_attr = NULL; 209261868Skevlo attr->flags &= ~THR_STACK_USER; 210261868Skevlo 211203134Sthompsa /* 212203134Sthompsa * Use the garbage collector lock for synchronization of the 213203134Sthompsa * spare stack lists and allocations from usrstack. 214261868Skevlo */ 215261868Skevlo THREAD_LIST_WRLOCK(curthread); 216261868Skevlo /* 217261868Skevlo * If the stack and guard sizes are default, try to allocate a stack 218261868Skevlo * from the default-size stack cache: 219261868Skevlo */ 220203134Sthompsa if ((stacksize == THR_STACK_DEFAULT) && 221203134Sthompsa (guardsize == _thr_guard_default)) { 222261868Skevlo if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) { 223261868Skevlo /* Use the spare stack. */ 224261868Skevlo LIST_REMOVE(spare_stack, qe); 225261868Skevlo attr->stackaddr_attr = spare_stack->stackaddr; 226261868Skevlo } 227261868Skevlo } 228261868Skevlo /* 229261868Skevlo * The user specified a non-default stack and/or guard size, so try to 230261868Skevlo * allocate a stack from the non-default size stack cache, using the 231261868Skevlo * rounded up stack size (stack_size) in the search: 232261868Skevlo */ 233261868Skevlo else { 234261868Skevlo LIST_FOREACH(spare_stack, &mstackq, qe) { 235261868Skevlo if (spare_stack->stacksize == stacksize && 236261868Skevlo spare_stack->guardsize == guardsize) { 237261868Skevlo LIST_REMOVE(spare_stack, qe); 238261868Skevlo attr->stackaddr_attr = spare_stack->stackaddr; 239261868Skevlo break; 240203134Sthompsa } 241203134Sthompsa } 242261868Skevlo } 243261868Skevlo if (attr->stackaddr_attr != NULL) { 244261868Skevlo /* A cached stack was found. Release the lock. */ 245261868Skevlo THREAD_LIST_UNLOCK(curthread); 246261868Skevlo } 247261868Skevlo else { 248261868Skevlo /* 249261868Skevlo * Allocate a stack from or below usrstack, depending 250261868Skevlo * on the LIBPTHREAD_BIGSTACK_MAIN env variable. 251261868Skevlo */ 252261868Skevlo if (last_stack == NULL) 253261868Skevlo last_stack = _usrstack - _thr_stack_initial - 254203134Sthompsa _thr_guard_default; 255203134Sthompsa 256261868Skevlo /* Allocate a new stack. */ 257261868Skevlo stackaddr = last_stack - stacksize - guardsize; 258261868Skevlo 259261868Skevlo /* 260261868Skevlo * Even if stack allocation fails, we don't want to try to 261261868Skevlo * use this location again, so unconditionally decrement 262203134Sthompsa * last_stack. Under normal operating conditions, the most 263203134Sthompsa * likely reason for an mmap() error is a stack overflow of 264261868Skevlo * the adjacent thread stack. 265261868Skevlo */ 266203134Sthompsa last_stack -= (stacksize + guardsize); 267203134Sthompsa 268261868Skevlo /* Release the lock before mmap'ing it. */ 269261868Skevlo THREAD_LIST_UNLOCK(curthread); 270261868Skevlo 271261868Skevlo /* Map the stack and guard page together, and split guard 272261868Skevlo page from allocated space: */ 273261868Skevlo if ((stackaddr = mmap(stackaddr, stacksize + guardsize, 274261868Skevlo _rtld_get_stack_prot(), MAP_STACK, 275261868Skevlo -1, 0)) != MAP_FAILED && 276261868Skevlo (guardsize == 0 || 277261868Skevlo mprotect(stackaddr, guardsize, PROT_NONE) == 0)) { 278261868Skevlo stackaddr += guardsize; 279261868Skevlo } else { 280203134Sthompsa if (stackaddr != MAP_FAILED) 281203134Sthompsa munmap(stackaddr, stacksize + guardsize); 282261868Skevlo stackaddr = NULL; 283261868Skevlo } 284261868Skevlo attr->stackaddr_attr = stackaddr; 285261868Skevlo } 286203134Sthompsa if (attr->stackaddr_attr != NULL) 287203134Sthompsa return (0); 288261868Skevlo else 289261868Skevlo return (-1); 290261868Skevlo} 291261868Skevlo 292261868Skevlo/* This function must be called with _thread_list_lock held. */ 293261868Skevlovoid 294261868Skevlo_thr_stack_free(struct pthread_attr *attr) 295261868Skevlo{ 296261868Skevlo struct stack *spare_stack; 297261868Skevlo 298261868Skevlo if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0) 299261868Skevlo && (attr->stackaddr_attr != NULL)) { 300261868Skevlo spare_stack = (struct stack *) 301203134Sthompsa ((char *)attr->stackaddr_attr + 302203134Sthompsa attr->stacksize_attr - sizeof(struct stack)); 303261868Skevlo spare_stack->stacksize = round_up(attr->stacksize_attr); 304261868Skevlo spare_stack->guardsize = round_up(attr->guardsize_attr); 305261868Skevlo spare_stack->stackaddr = attr->stackaddr_attr; 306261868Skevlo 307261868Skevlo if (spare_stack->stacksize == THR_STACK_DEFAULT && 308261868Skevlo spare_stack->guardsize == _thr_guard_default) { 309261868Skevlo /* Default stack/guard size. */ 310261868Skevlo LIST_INSERT_HEAD(&dstackq, spare_stack, qe); 311261868Skevlo } else { 312261868Skevlo /* Non-default stack/guard size. */ 313261868Skevlo LIST_INSERT_HEAD(&mstackq, spare_stack, qe); 314203134Sthompsa } 315203134Sthompsa attr->stackaddr_attr = NULL; 316261868Skevlo } 317261868Skevlo} 318261868Skevlo