2N/A/*
2N/A * CDDL HEADER START
2N/A *
2N/A * The contents of this file are subject to the terms of the
2N/A * Common Development and Distribution License (the "License").
2N/A * You may not use this file except in compliance with the License.
2N/A *
2N/A * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
2N/A * or http://www.opensolaris.org/os/licensing.
2N/A * See the License for the specific language governing permissions
2N/A * and limitations under the License.
2N/A *
2N/A * When distributing Covered Code, include this CDDL HEADER in each
2N/A * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
2N/A * If applicable, add the following below this CDDL HEADER, with the
2N/A * fields enclosed by brackets "[]" replaced with your own identifying
2N/A * information: Portions Copyright [yyyy] [name of copyright owner]
2N/A *
2N/A * CDDL HEADER END
2N/A */
2N/A
2N/A/*
2N/A * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
2N/A * Use is subject to license terms.
2N/A */
2N/A
2N/A#include "lint.h"
2N/A#include "thr_uberdata.h"
2N/A
2N/A#define MIN_MOD_SLOTS 8
2N/A
2N/A/*
2N/A * Used to inform libc_init() that we are on the primary link map,
2N/A * and to cause certain functions (like malloc() and sbrk()) to fail
2N/A * (with ENOTSUP) when they are called on an alternate link map.
2N/A */
2N/Aint primary_link_map = 0;
2N/A
2N/A#if defined(_LP64)
2N/A#define ALIGN 16
2N/A#else
2N/A#define ALIGN 8
2N/A#endif
2N/A
2N/A/*
2N/A * Grow the TLS module information array as necessary to include the
2N/A * specified module-id. tls_modinfo->tls_size must be a power of two.
2N/A * Return a pointer to the (possibly reallocated) module information array.
2N/A */
2N/Astatic TLS_modinfo *
2N/Atls_modinfo_alloc(tls_metadata_t *tlsm, ulong_t moduleid)
2N/A{
2N/A tls_t *tls_modinfo = &tlsm->tls_modinfo;
2N/A TLS_modinfo *modinfo;
2N/A size_t mod_slots;
2N/A
2N/A if ((modinfo = tls_modinfo->tls_data) == NULL ||
2N/A tls_modinfo->tls_size <= moduleid) {
2N/A if ((mod_slots = tls_modinfo->tls_size) == 0)
2N/A mod_slots = MIN_MOD_SLOTS;
2N/A while (mod_slots <= moduleid)
2N/A mod_slots *= 2;
2N/A modinfo = lmalloc(mod_slots * sizeof (TLS_modinfo));
2N/A if (tls_modinfo->tls_data != NULL) {
2N/A (void) memcpy(modinfo, tls_modinfo->tls_data,
2N/A tls_modinfo->tls_size * sizeof (TLS_modinfo));
2N/A lfree(tls_modinfo->tls_data,
2N/A tls_modinfo->tls_size * sizeof (TLS_modinfo));
2N/A }
2N/A tls_modinfo->tls_data = modinfo;
2N/A tls_modinfo->tls_size = mod_slots;
2N/A }
2N/A return (modinfo);
2N/A}
2N/A
2N/A/*
2N/A * This is called from the dynamic linker, before libc_init() is called,
2N/A * to setup all of the TLS blocks that are available at process startup
2N/A * and hence must be included as part of the static TLS block.
2N/A * No locks are needed because we are single-threaded at this point.
2N/A * We must be careful not to call any function that could possibly
2N/A * invoke the dynamic linker. That is, we must only call functions
2N/A * that are wholly private to libc.
2N/A */
2N/Avoid
2N/A__tls_static_mods(TLS_modinfo **tlslist, unsigned long statictlssize)
2N/A{
2N/A ulwp_t *oldself = __curthread();
2N/A tls_metadata_t *tlsm;
2N/A TLS_modinfo **tlspp;
2N/A TLS_modinfo *tlsp;
2N/A TLS_modinfo *modinfo;
2N/A caddr_t data;
2N/A caddr_t data_end;
2N/A int max_modid;
2N/A
2N/A primary_link_map = 1; /* inform libc_init */
2N/A if (statictlssize == 0)
2N/A return;
2N/A
2N/A /*
2N/A * Retrieve whatever dynamic TLS metadata was generated by code
2N/A * running on alternate link maps prior to now (we must be running
2N/A * on the primary link map now since __tls_static_mods() is only
2N/A * called on the primary link map).
2N/A */
2N/A tlsm = &__uberdata.tls_metadata;
2N/A if (oldself != NULL) {
2N/A (void) memcpy(tlsm,
2N/A &oldself->ul_uberdata->tls_metadata, sizeof (*tlsm));
2N/A ASSERT(tlsm->static_tls.tls_data == NULL);
2N/A }
2N/A
2N/A /*
2N/A * We call lmalloc() to allocate the template even though libc_init()
2N/A * has not yet been called. lmalloc() must and does deal with this.
2N/A */
2N/A ASSERT((statictlssize & (ALIGN - 1)) == 0);
2N/A tlsm->static_tls.tls_data = data = lmalloc(statictlssize);
2N/A data_end = data + statictlssize;
2N/A tlsm->static_tls.tls_size = statictlssize;
2N/A /*
2N/A * Initialize the static TLS template.
2N/A * We make no assumptions about the order in memory of the TLS
2N/A * modules we are processing, only that they fit within the
2N/A * total size we are given and that they are self-consistent.
2N/A * We do not assume any order for the moduleid's; we only assume
2N/A * that they are reasonably small integers.
2N/A */
2N/A for (max_modid = 0, tlspp = tlslist; (tlsp = *tlspp) != NULL; tlspp++) {
2N/A ASSERT(tlsp->tm_flags & TM_FLG_STATICTLS);
2N/A ASSERT(tlsp->tm_stattlsoffset > 0);
2N/A ASSERT(tlsp->tm_stattlsoffset <= statictlssize);
2N/A ASSERT((tlsp->tm_stattlsoffset & (ALIGN - 1)) == 0);
2N/A ASSERT(tlsp->tm_filesz <= tlsp->tm_memsz);
2N/A ASSERT(tlsp->tm_memsz <= tlsp->tm_stattlsoffset);
2N/A if (tlsp->tm_filesz)
2N/A (void) memcpy(data_end-tlsp->tm_stattlsoffset,
2N/A tlsp->tm_tlsblock, tlsp->tm_filesz);
2N/A if (max_modid < tlsp->tm_modid)
2N/A max_modid = tlsp->tm_modid;
2N/A }
2N/A /*
2N/A * Record the static TLS_modinfo information.
2N/A */
2N/A modinfo = tls_modinfo_alloc(tlsm, max_modid);
2N/A for (tlspp = tlslist; (tlsp = *tlspp) != NULL; tlspp++)
2N/A (void) memcpy(&modinfo[tlsp->tm_modid],
2N/A tlsp, sizeof (*tlsp));
2N/A
2N/A /*
2N/A * Copy the new tls_metadata back to the old, if any,
2N/A * since it will be copied up again in libc_init().
2N/A */
2N/A if (oldself != NULL)
2N/A (void) memcpy(&oldself->ul_uberdata->tls_metadata,
2N/A tlsm, sizeof (*tlsm));
2N/A}
2N/A
2N/A/*
2N/A * This is called from the dynamic linker for each module not included
2N/A * in the static TLS mod list, after the module has been loaded but
2N/A * before any of the module's init code has been executed.
2N/A */
2N/Avoid
2N/A__tls_mod_add(TLS_modinfo *tlsp)
2N/A{
2N/A tls_metadata_t *tlsm = &curthread->ul_uberdata->tls_metadata;
2N/A ulong_t moduleid = tlsp->tm_modid;
2N/A TLS_modinfo *modinfo;
2N/A
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A ASSERT(!(tlsp->tm_flags & TM_FLG_STATICTLS));
2N/A ASSERT(tlsp->tm_filesz <= tlsp->tm_memsz);
2N/A modinfo = tls_modinfo_alloc(tlsm, moduleid);
2N/A (void) memcpy(&modinfo[moduleid], tlsp, sizeof (*tlsp));
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A}
2N/A
2N/A/*
2N/A * Called for each module as it is unloaded from memory by dlclose().
2N/A */
2N/Avoid
2N/A__tls_mod_remove(TLS_modinfo *tlsp)
2N/A{
2N/A tls_metadata_t *tlsm = &curthread->ul_uberdata->tls_metadata;
2N/A ulong_t moduleid = tlsp->tm_modid;
2N/A TLS_modinfo *modinfo;
2N/A
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A ASSERT(tlsm->tls_modinfo.tls_data != NULL &&
2N/A moduleid < tlsm->tls_modinfo.tls_size);
2N/A modinfo = tlsm->tls_modinfo.tls_data;
2N/A (void) memset(&modinfo[moduleid], 0, sizeof (TLS_modinfo));
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A}
2N/A
2N/Aextern int _preexec_exit_handlers();
2N/Aextern void libc_init();
2N/A
2N/Aconst Lc_interface tls_rtldinfo[] = {
2N/A {CI_VERSION, (int(*)())CI_V_CURRENT},
2N/A {CI_ATEXIT, (int(*)())_preexec_exit_handlers},
2N/A {CI_TLS_MODADD, (int(*)())__tls_mod_add},
2N/A {CI_TLS_MODREM, (int(*)())__tls_mod_remove},
2N/A {CI_TLS_STATMOD, (int(*)())__tls_static_mods},
2N/A {CI_THRINIT, (int(*)())libc_init},
2N/A {CI_NULL, (int(*)())NULL}
2N/A};
2N/A
2N/A/*
2N/A * Return the address of a TLS variable for the current thread.
2N/A * Run the constructors for newly-allocated dynamic TLS.
2N/A */
2N/Avoid *
2N/Aslow_tls_get_addr(TLS_index *tls_index)
2N/A{
2N/A ulwp_t *self = curthread;
2N/A tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata;
2N/A TLS_modinfo *tlsp;
2N/A ulong_t moduleid;
2N/A tls_t *tlsent;
2N/A caddr_t base;
2N/A void (**initarray)(void);
2N/A ulong_t arraycnt = 0;
2N/A
2N/A /*
2N/A * Defer signals until we have finished calling
2N/A * all of the constructors.
2N/A */
2N/A sigoff(self);
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A if ((moduleid = tls_index->ti_moduleid) < self->ul_ntlsent)
2N/A tlsent = self->ul_tlsent;
2N/A else {
2N/A ASSERT(moduleid < tlsm->tls_modinfo.tls_size);
2N/A tlsent = lmalloc(tlsm->tls_modinfo.tls_size * sizeof (tls_t));
2N/A if (self->ul_tlsent != NULL) {
2N/A (void) memcpy(tlsent, self->ul_tlsent,
2N/A self->ul_ntlsent * sizeof (tls_t));
2N/A lfree(self->ul_tlsent,
2N/A self->ul_ntlsent * sizeof (tls_t));
2N/A }
2N/A self->ul_tlsent = tlsent;
2N/A self->ul_ntlsent = tlsm->tls_modinfo.tls_size;
2N/A }
2N/A tlsent += moduleid;
2N/A if ((base = tlsent->tls_data) == NULL) {
2N/A tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid;
2N/A if (tlsp->tm_memsz == 0) { /* dlclose()d module? */
2N/A base = NULL;
2N/A } else if (tlsp->tm_flags & TM_FLG_STATICTLS) {
2N/A /* static TLS is already allocated/initialized */
2N/A base = (caddr_t)self - tlsp->tm_stattlsoffset;
2N/A tlsent->tls_data = base;
2N/A tlsent->tls_size = 0; /* don't lfree() this space */
2N/A } else {
2N/A /* allocate/initialize the dynamic TLS */
2N/A base = lmalloc(tlsp->tm_memsz);
2N/A if (tlsp->tm_filesz != 0)
2N/A (void) memcpy(base, tlsp->tm_tlsblock,
2N/A tlsp->tm_filesz);
2N/A tlsent->tls_data = base;
2N/A tlsent->tls_size = tlsp->tm_memsz;
2N/A /* remember the constructors */
2N/A arraycnt = tlsp->tm_tlsinitarraycnt;
2N/A initarray = tlsp->tm_tlsinitarray;
2N/A }
2N/A }
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A
2N/A /*
2N/A * Call constructors, if any, in ascending order.
2N/A * We have to do this after dropping tls_lock because
2N/A * we have no idea what the constructors will do.
2N/A * At least we have signals deferred until they are done.
2N/A */
2N/A if (arraycnt) {
2N/A do {
2N/A (**initarray++)();
2N/A } while (--arraycnt != 0);
2N/A }
2N/A
2N/A if (base == NULL) /* kludge to get x86/x64 to boot */
2N/A base = (caddr_t)self - 512;
2N/A
2N/A sigon(self);
2N/A return (base + tls_index->ti_tlsoffset);
2N/A}
2N/A
2N/A#ifdef TLS_GET_ADDR_IS_WRITTEN_IN_ASSEMBLER
2N/A/*
2N/A * For speed, we do not make reference to any static data in this function.
2N/A * If necessary to do so, we do a tail call to slow_tls_get_addr().
2N/A */
2N/Avoid *
2N/A__tls_get_addr(TLS_index *tls_index)
2N/A{
2N/A ulwp_t *self = curthread;
2N/A tls_t *tlsent = self->ul_tlsent;
2N/A ulong_t moduleid;
2N/A caddr_t base;
2N/A
2N/A if ((moduleid = tls_index->ti_moduleid) < self->ul_ntlsent &&
2N/A (base = tlsent[moduleid].tls_data) != NULL)
2N/A return (base + tls_index->ti_tlsoffset);
2N/A
2N/A return (slow_tls_get_addr(tls_index));
2N/A}
2N/A#endif /* TLS_GET_ADDR_IS_WRITTEN_IN_ASSEMBLER */
2N/A
2N/A/*
2N/A * This is called by _thrp_setup() to initialize the thread's static TLS.
2N/A * Constructors for initially allocated static TLS are called here.
2N/A */
2N/Avoid
2N/Atls_setup()
2N/A{
2N/A ulwp_t *self = curthread;
2N/A tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata;
2N/A TLS_modinfo *tlsp;
2N/A long moduleid;
2N/A ulong_t nmods;
2N/A
2N/A if (tlsm->static_tls.tls_size == 0) /* no static TLS */
2N/A return;
2N/A
2N/A /* static TLS initialization */
2N/A (void) memcpy((caddr_t)self - tlsm->static_tls.tls_size,
2N/A tlsm->static_tls.tls_data, tlsm->static_tls.tls_size);
2N/A
2N/A /* call TLS constructors for the static TLS just initialized */
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A nmods = tlsm->tls_modinfo.tls_size;
2N/A for (moduleid = 0; moduleid < nmods; moduleid++) {
2N/A /*
2N/A * Resume where we left off in the module array.
2N/A * tls_modinfo.tls_data may have changed since we
2N/A * dropped and reacquired tls_lock, but TLS modules
2N/A * retain their positions in the new array.
2N/A */
2N/A tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid;
2N/A /*
2N/A * Call constructors for this module if there are any
2N/A * to be called and if it is part of the static TLS.
2N/A */
2N/A if (tlsp->tm_tlsinitarraycnt != 0 &&
2N/A (tlsp->tm_flags & TM_FLG_STATICTLS)) {
2N/A ulong_t arraycnt = tlsp->tm_tlsinitarraycnt;
2N/A void (**initarray)(void) = tlsp->tm_tlsinitarray;
2N/A
2N/A /*
2N/A * Call the constructors in ascending order.
2N/A * We must drop tls_lock while doing this because
2N/A * we have no idea what the constructors will do.
2N/A */
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A do {
2N/A (**initarray++)();
2N/A } while (--arraycnt != 0);
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A }
2N/A }
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A}
2N/A
2N/A/*
2N/A * This is called by _thrp_exit() to deallocate the thread's TLS.
2N/A * Destructors for all allocated TLS are called here.
2N/A */
2N/Avoid
2N/Atls_exit()
2N/A{
2N/A ulwp_t *self = curthread;
2N/A tls_metadata_t *tlsm = &self->ul_uberdata->tls_metadata;
2N/A tls_t *tlsent;
2N/A TLS_modinfo *tlsp;
2N/A long moduleid;
2N/A ulong_t nmods;
2N/A
2N/A if (tlsm->static_tls.tls_size == 0 && self->ul_ntlsent == 0)
2N/A return; /* no TLS */
2N/A
2N/A /*
2N/A * Call TLS destructors for all TLS allocated for this thread.
2N/A */
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A nmods = tlsm->tls_modinfo.tls_size;
2N/A for (moduleid = nmods - 1; moduleid >= 0; --moduleid) {
2N/A /*
2N/A * Resume where we left off in the module array.
2N/A * tls_modinfo.tls_data may have changed since we
2N/A * dropped and reacquired tls_lock, but TLS modules
2N/A * retain their positions in the new array.
2N/A */
2N/A tlsp = (TLS_modinfo *)tlsm->tls_modinfo.tls_data + moduleid;
2N/A /*
2N/A * Call destructors for this module if there are any
2N/A * to be called and if it is part of the static TLS or
2N/A * if the dynamic TLS for the module has been allocated.
2N/A */
2N/A if (tlsp->tm_tlsfiniarraycnt != 0 &&
2N/A ((tlsp->tm_flags & TM_FLG_STATICTLS) ||
2N/A (moduleid < self->ul_ntlsent &&
2N/A (tlsent = self->ul_tlsent) != NULL &&
2N/A tlsent[moduleid].tls_data != NULL))) {
2N/A ulong_t arraycnt = tlsp->tm_tlsfiniarraycnt;
2N/A void (**finiarray)(void) = tlsp->tm_tlsfiniarray;
2N/A
2N/A /*
2N/A * Call the destructors in descending order.
2N/A * We must drop tls_lock while doing this because
2N/A * we have no idea what the destructors will do.
2N/A */
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A finiarray += arraycnt;
2N/A do {
2N/A (**--finiarray)();
2N/A } while (--arraycnt != 0);
2N/A lmutex_lock(&tlsm->tls_lock);
2N/A }
2N/A }
2N/A lmutex_unlock(&tlsm->tls_lock);
2N/A
2N/A tls_free(self);
2N/A}
2N/A
2N/A/*
2N/A * We only free the dynamically allocated TLS; the statically
2N/A * allocated TLS is reused when the ulwp_t is reallocated.
2N/A */
2N/Avoid
2N/Atls_free(ulwp_t *ulwp)
2N/A{
2N/A ulong_t moduleid;
2N/A tls_t *tlsent;
2N/A size_t ntlsent;
2N/A void *base;
2N/A size_t size;
2N/A
2N/A if ((tlsent = ulwp->ul_tlsent) == NULL ||
2N/A (ntlsent = ulwp->ul_ntlsent) == 0)
2N/A return;
2N/A
2N/A for (moduleid = 0; moduleid < ntlsent; moduleid++, tlsent++) {
2N/A if ((base = tlsent->tls_data) != NULL &&
2N/A (size = tlsent->tls_size) != 0)
2N/A lfree(base, size);
2N/A tlsent->tls_data = NULL; /* paranoia */
2N/A tlsent->tls_size = 0;
2N/A }
2N/A lfree(ulwp->ul_tlsent, ntlsent * sizeof (tls_t));
2N/A ulwp->ul_tlsent = NULL;
2N/A ulwp->ul_ntlsent = 0;
2N/A}