1133064Sdfr/*- 2133064Sdfr * Copyright (c) 2004 Doug Rabson 3133064Sdfr * All rights reserved. 4133064Sdfr * 5133064Sdfr * Redistribution and use in source and binary forms, with or without 6133064Sdfr * modification, are permitted provided that the following conditions 7133064Sdfr * are met: 8133064Sdfr * 1. Redistributions of source code must retain the above copyright 9133064Sdfr * notice, this list of conditions and the following disclaimer. 10133064Sdfr * 2. Redistributions in binary form must reproduce the above copyright 11133064Sdfr * notice, this list of conditions and the following disclaimer in the 12133064Sdfr * documentation and/or other materials provided with the distribution. 13133064Sdfr * 14133064Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15133064Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16133064Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17133064Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18133064Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19133064Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20133064Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21133064Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22133064Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23133064Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24133064Sdfr * SUCH DAMAGE. 25133064Sdfr * 26133064Sdfr * $FreeBSD$ 27133064Sdfr */ 28133064Sdfr 29133064Sdfr/* 30133064Sdfr * Define stubs for TLS internals so that programs and libraries can 31133064Sdfr * link. These functions will be replaced by functional versions at 32133064Sdfr * runtime from ld-elf.so.1. 33133064Sdfr */ 34133064Sdfr 35143921Sdavidxu#include <sys/cdefs.h> 36133754Sdfr#include <stdlib.h> 37133754Sdfr#include <string.h> 38133754Sdfr#include <elf.h> 39143921Sdavidxu 40133754Sdfr#include "libc_private.h" 41133064Sdfr 42234370Sjasone/* Provided by jemalloc to avoid bootstrapping issues. */ 43234569Sjasonevoid *__jemalloc_a0malloc(size_t size); 44234569Sjasonevoid *__jemalloc_a0calloc(size_t num, size_t size); 45234569Sjasonevoid __jemalloc_a0free(void *ptr); 46234370Sjasone 47143921Sdavidxu__weak_reference(__libc_allocate_tls, _rtld_allocate_tls); 48143921Sdavidxu__weak_reference(__libc_free_tls, _rtld_free_tls); 49143921Sdavidxu 50143921Sdavidxu#ifdef __i386__ 51143921Sdavidxu 52143921Sdavidxu__weak_reference(___libc_tls_get_addr, ___tls_get_addr); 53143921Sdavidxu__attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *); 54143921Sdavidxu 55143921Sdavidxu#endif 56143921Sdavidxu 57143921Sdavidxuvoid * __libc_tls_get_addr(void *); 58143921Sdavidxu__weak_reference(__libc_tls_get_addr, __tls_get_addr); 59143921Sdavidxu 60143921Sdavidxuvoid *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 61143921Sdavidxuvoid _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 62143921Sdavidxuvoid *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign); 63143921Sdavidxuvoid __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign); 64143921Sdavidxu 65163118Skmacy#if defined(__ia64__) || defined(__amd64__) 66163118Skmacy#define TLS_TCB_ALIGN 16 67163118Skmacy#elif defined(__powerpc__) || defined(__i386__) || defined(__arm__) || \ 68178684Sgonzo defined(__sparc64__) || defined(__mips__) 69163118Skmacy#define TLS_TCB_ALIGN sizeof(void *) 70163118Skmacy#else 71163118Skmacy#error TLS_TCB_ALIGN undefined for target architecture 72163118Skmacy#endif 73163118Skmacy 74232582Sgonzo#if defined(__arm__) || defined(__ia64__) || defined(__mips__) || \ 75232582Sgonzo defined(__powerpc__) 76133754Sdfr#define TLS_VARIANT_I 77133754Sdfr#endif 78232582Sgonzo#if defined(__i386__) || defined(__amd64__) || defined(__sparc64__) 79133754Sdfr#define TLS_VARIANT_II 80133754Sdfr#endif 81133754Sdfr 82133754Sdfr#ifndef PIC 83133754Sdfr 84133754Sdfr#define round(size, align) \ 85133754Sdfr (((size) + (align) - 1) & ~((align) - 1)) 86133754Sdfr 87133754Sdfrstatic size_t tls_static_space; 88133754Sdfrstatic size_t tls_init_size; 89133754Sdfrstatic void *tls_init; 90133754Sdfr#endif 91133754Sdfr 92133064Sdfr#ifdef __i386__ 93133064Sdfr 94143921Sdavidxu/* GNU ABI */ 95133064Sdfr 96133064Sdfr__attribute__((__regparm__(1))) 97133064Sdfrvoid * 98143921Sdavidxu___libc_tls_get_addr(void *ti __unused) 99133064Sdfr{ 100133064Sdfr return (0); 101133064Sdfr} 102133064Sdfr 103133064Sdfr#endif 104133064Sdfr 105133064Sdfrvoid * 106143921Sdavidxu__libc_tls_get_addr(void *ti __unused) 107133064Sdfr{ 108133064Sdfr return (0); 109133064Sdfr} 110133064Sdfr 111143921Sdavidxu#ifndef PIC 112143921Sdavidxu 113133754Sdfr#ifdef TLS_VARIANT_I 114133754Sdfr 115161800Smarcel#define TLS_TCB_SIZE (2 * sizeof(void *)) 116161800Smarcel 117142560Sdavidxu/* 118142959Sdavidxu * Free Static TLS using the Variant I method. 119142560Sdavidxu */ 120133754Sdfrvoid 121161800Smarcel__libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused) 122133754Sdfr{ 123161800Smarcel Elf_Addr *dtv; 124161800Smarcel Elf_Addr **tls; 125133754Sdfr 126161800Smarcel tls = (Elf_Addr **)((Elf_Addr)tcb + tcbsize - TLS_TCB_SIZE); 127161800Smarcel dtv = tls[0]; 128234569Sjasone __jemalloc_a0free(dtv); 129234569Sjasone __jemalloc_a0free(tcb); 130133754Sdfr} 131133754Sdfr 132133754Sdfr/* 133133754Sdfr * Allocate Static TLS using the Variant I method. 134133754Sdfr */ 135133064Sdfrvoid * 136161800Smarcel__libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign __unused) 137133064Sdfr{ 138133754Sdfr Elf_Addr *dtv; 139161800Smarcel Elf_Addr **tls; 140161800Smarcel char *tcb; 141133754Sdfr 142161800Smarcel if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE) 143161800Smarcel return (oldtcb); 144133754Sdfr 145234569Sjasone tcb = __jemalloc_a0calloc(1, tls_static_space + tcbsize - TLS_TCB_SIZE); 146161800Smarcel tls = (Elf_Addr **)(tcb + tcbsize - TLS_TCB_SIZE); 147133754Sdfr 148161800Smarcel if (oldtcb != NULL) { 149203946Smarcel memcpy(tls, oldtcb, tls_static_space); 150234569Sjasone __jemalloc_a0free(oldtcb); 151133754Sdfr 152161800Smarcel /* Adjust the DTV. */ 153161800Smarcel dtv = tls[0]; 154161800Smarcel dtv[2] = (Elf_Addr)tls + TLS_TCB_SIZE; 155161800Smarcel } else { 156234569Sjasone dtv = __jemalloc_a0malloc(3 * sizeof(Elf_Addr)); 157161800Smarcel tls[0] = dtv; 158161800Smarcel dtv[0] = 1; 159161800Smarcel dtv[1] = 1; 160161800Smarcel dtv[2] = (Elf_Addr)tls + TLS_TCB_SIZE; 161133754Sdfr 162161800Smarcel if (tls_init_size > 0) 163161827Smarcel memcpy((void*)dtv[2], tls_init, tls_init_size); 164161800Smarcel if (tls_static_space > tls_init_size) 165161827Smarcel memset((void*)(dtv[2] + tls_init_size), 0, 166161827Smarcel tls_static_space - tls_init_size); 167133754Sdfr } 168133754Sdfr 169161800Smarcel return(tcb); 170133064Sdfr} 171133064Sdfr 172133754Sdfr#endif 173133754Sdfr 174133754Sdfr#ifdef TLS_VARIANT_II 175133754Sdfr 176161800Smarcel#define TLS_TCB_SIZE (3 * sizeof(Elf_Addr)) 177161800Smarcel 178133754Sdfr/* 179133754Sdfr * Free Static TLS using the Variant II method. 180133754Sdfr */ 181133064Sdfrvoid 182143921Sdavidxu__libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign) 183133064Sdfr{ 184133754Sdfr size_t size; 185133754Sdfr Elf_Addr* dtv; 186133754Sdfr Elf_Addr tlsstart, tlsend; 187133754Sdfr 188133754Sdfr /* 189133754Sdfr * Figure out the size of the initial TLS block so that we can 190133754Sdfr * find stuff which ___tls_get_addr() allocated dynamically. 191133754Sdfr */ 192133754Sdfr size = round(tls_static_space, tcbalign); 193133754Sdfr 194133754Sdfr dtv = ((Elf_Addr**)tcb)[1]; 195133754Sdfr tlsend = (Elf_Addr) tcb; 196133754Sdfr tlsstart = tlsend - size; 197234569Sjasone __jemalloc_a0free((void*) tlsstart); 198234569Sjasone __jemalloc_a0free(dtv); 199133064Sdfr} 200133754Sdfr 201133754Sdfr/* 202133754Sdfr * Allocate Static TLS using the Variant II method. 203133754Sdfr */ 204133754Sdfrvoid * 205143921Sdavidxu__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign) 206133754Sdfr{ 207133754Sdfr size_t size; 208133754Sdfr char *tls; 209133754Sdfr Elf_Addr *dtv; 210133754Sdfr Elf_Addr segbase, oldsegbase; 211133754Sdfr 212133754Sdfr size = round(tls_static_space, tcbalign); 213133754Sdfr 214166995Skientzle if (tcbsize < 2 * sizeof(Elf_Addr)) 215166995Skientzle tcbsize = 2 * sizeof(Elf_Addr); 216234569Sjasone tls = __jemalloc_a0calloc(1, size + tcbsize); 217234569Sjasone dtv = __jemalloc_a0malloc(3 * sizeof(Elf_Addr)); 218133754Sdfr 219133754Sdfr segbase = (Elf_Addr)(tls + size); 220133754Sdfr ((Elf_Addr*)segbase)[0] = segbase; 221133754Sdfr ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv; 222133754Sdfr 223133754Sdfr dtv[0] = 1; 224133754Sdfr dtv[1] = 1; 225133754Sdfr dtv[2] = segbase - tls_static_space; 226133754Sdfr 227133754Sdfr if (oldtls) { 228133754Sdfr /* 229133754Sdfr * Copy the static TLS block over whole. 230133754Sdfr */ 231133754Sdfr oldsegbase = (Elf_Addr) oldtls; 232133754Sdfr memcpy((void *)(segbase - tls_static_space), 233133754Sdfr (const void *)(oldsegbase - tls_static_space), 234133754Sdfr tls_static_space); 235133754Sdfr 236133754Sdfr /* 237133754Sdfr * We assume that this block was the one we created with 238133754Sdfr * allocate_initial_tls(). 239133754Sdfr */ 240133754Sdfr _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr)); 241133754Sdfr } else { 242133754Sdfr memcpy((void *)(segbase - tls_static_space), 243133754Sdfr tls_init, tls_init_size); 244133754Sdfr memset((void *)(segbase - tls_static_space + tls_init_size), 245133754Sdfr 0, tls_static_space - tls_init_size); 246133754Sdfr } 247133754Sdfr 248133754Sdfr return (void*) segbase; 249143921Sdavidxu} 250143921Sdavidxu 251143921Sdavidxu#endif /* TLS_VARIANT_II */ 252143921Sdavidxu 253133754Sdfr#else 254143921Sdavidxu 255143921Sdavidxuvoid * 256143921Sdavidxu__libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused, 257143921Sdavidxu size_t tcbalign __unused) 258143921Sdavidxu{ 259133754Sdfr return (0); 260133754Sdfr} 261133754Sdfr 262143921Sdavidxuvoid 263143921Sdavidxu__libc_free_tls(void *tcb __unused, size_t tcbsize __unused, 264143921Sdavidxu size_t tcbalign __unused) 265143921Sdavidxu{ 266143921Sdavidxu} 267133754Sdfr 268143921Sdavidxu#endif /* PIC */ 269143921Sdavidxu 270143921Sdavidxuextern char **environ; 271143921Sdavidxu 272133754Sdfrvoid 273133754Sdfr_init_tls() 274133754Sdfr{ 275133754Sdfr#ifndef PIC 276133754Sdfr Elf_Addr *sp; 277133754Sdfr Elf_Auxinfo *aux, *auxp; 278133754Sdfr Elf_Phdr *phdr; 279133754Sdfr size_t phent, phnum; 280133754Sdfr int i; 281133949Sdfr void *tls; 282133754Sdfr 283133754Sdfr sp = (Elf_Addr *) environ; 284133754Sdfr while (*sp++ != 0) 285133754Sdfr ; 286133754Sdfr aux = (Elf_Auxinfo *) sp; 287133754Sdfr phdr = 0; 288133754Sdfr phent = phnum = 0; 289133754Sdfr for (auxp = aux; auxp->a_type != AT_NULL; auxp++) { 290133754Sdfr switch (auxp->a_type) { 291133754Sdfr case AT_PHDR: 292133754Sdfr phdr = auxp->a_un.a_ptr; 293133754Sdfr break; 294133754Sdfr 295133754Sdfr case AT_PHENT: 296133754Sdfr phent = auxp->a_un.a_val; 297133754Sdfr break; 298133754Sdfr 299133754Sdfr case AT_PHNUM: 300133754Sdfr phnum = auxp->a_un.a_val; 301133754Sdfr break; 302133754Sdfr } 303133754Sdfr } 304133754Sdfr if (phdr == 0 || phent != sizeof(Elf_Phdr) || phnum == 0) 305133754Sdfr return; 306133754Sdfr 307143921Sdavidxu for (i = 0; (unsigned) i < phnum; i++) { 308133754Sdfr if (phdr[i].p_type == PT_TLS) { 309133754Sdfr tls_static_space = round(phdr[i].p_memsz, 310133754Sdfr phdr[i].p_align); 311133754Sdfr tls_init_size = phdr[i].p_filesz; 312133754Sdfr tls_init = (void*) phdr[i].p_vaddr; 313133754Sdfr } 314133754Sdfr } 315133754Sdfr 316232582Sgonzo#ifdef TLS_VARIANT_I 317232582Sgonzo /* 318232582Sgonzo * tls_static_space should include space for TLS structure 319232582Sgonzo */ 320232582Sgonzo tls_static_space += TLS_TCB_SIZE; 321232582Sgonzo#endif 322232582Sgonzo 323163118Skmacy tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN); 324133754Sdfr 325133949Sdfr _set_tp(tls); 326133754Sdfr#endif 327133754Sdfr} 328