lwp_sobj.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/sysmacros.h>
#include <sys/watchpoint.h>
#include <sys/machlock.h>
#include <sys/schedctl.h>
#include <sys/tnf_probe.h>
#include <sys/lwpchan_impl.h>
#include <sys/turnstile.h>
#include <sys/lwp_timer_impl.h>
#include <sys/lwp_upimutex_impl.h>
static void lwp_unsleep(kthread_t *t);
/*
* Maximum number of user prio inheritance locks that can be held by a thread.
* Used to limit kmem for each thread. This is a per-thread limit that
*
* Also, when a limit, say maxlwps is added for numbers of lwps within a
* process, the per-thread limit automatically becomes a process-wide limit
* of maximum number of held upi locks within a process:
* maxheldupimx = maxnestupimx * maxlwps;
*/
/*
* The sobj_ops vector exports a set of functions needed when a thread
* is asleep on a synchronization object of this type.
*/
static sobj_ops_t lwp_sobj_ops = {
};
static sobj_ops_t lwp_sobj_pi_ops = {
};
/*
* We know that both lc_wchan and lc_wchan0 are addresses that most
* likely are 8-byte aligned, so we shift off the low-order 3 bits.
* 'pool' is either 0 or 1.
*/
#define LWPCHAN_LOCK_HASH(X, pool) \
/*
* Is this a POSIX threads user-level lock requiring priority inheritance?
*/
static sleepq_head_t *
{
return (&lwpsleepq[SQHASHINDEX(x)]);
}
/*
* Lock an lwpchan.
* Keep this in sync with lwpchan_unlock(), below.
*/
static void
{
}
/*
* Unlock an lwpchan.
* Keep this in sync with lwpchan_lock(), above.
*/
static void
{
}
/*
* Delete mappings from the lwpchan cache for pages that are being
* unmapped by as_unmap(). Given a range of addresses, "start" to "end",
* all mappings within the range are deleted from the lwpchan cache.
*/
void
{
mutex_enter(&p->p_lcp_lock);
continue;
/* check entire chain */
} else {
}
}
}
mutex_exit(&p->p_lcp_lock);
}
/*
* Given an lwpchan cache pointer and a process virtual address,
* return a pointer to the corresponding lwpchan hash bucket.
*/
static lwpchan_hashbucket_t *
{
uint_t i;
/*
* All user-level sync object addresses are 8-byte aligned.
* Ignore the lowest 3 bits of the address and use the
* higher-order 2*lwpchan_bits bits for the hash index.
*/
addr >>= 3;
return (lcp->lwpchan_cache + i);
}
/*
* (Re)allocate the per-process lwpchan cache.
*/
static void
{
lcp->lwpchan_entries = 0;
sizeof (lwpchan_hashbucket_t), KM_SLEEP);
mutex_enter(&p->p_lcp_lock);
/* someone beat us to it */
mutex_exit(&p->p_lcp_lock);
sizeof (lwpchan_hashbucket_t));
return;
}
/*
* Acquire all of the old hash table locks.
*/
/*
* Move all of the old hash table entries to the
* new hash table. The new hash table has not yet
* been installed so we don't need any of its locks.
*/
count = 0;
count++;
}
}
}
/*
* Retire the old hash table. We can't actually kmem_free() it
* now because someone may still have a pointer to it. Instead,
* we link it onto the new hash table's list of retired hash tables.
* The new hash table is double the size of the previous one, so
* the total size of all retired hash tables is less than the size
* of the new one. exit() and exec() free the retired hash tables
* (see lwpchan_destroy_cache(), below).
*/
/*
* As soon as we store the new lcp, future locking operations will
* use it. Therefore, we must ensure that all the state we've just
* established reaches global visibility before the new lcp does.
*/
/*
* Release all of the old hash table locks.
*/
}
mutex_exit(&p->p_lcp_lock);
}
/*
* Deallocate the lwpchan cache, and any dynamically allocated mappings.
* Called when the process exits or execs. All lwps except one have
* exited so we need no locks here.
*/
void
{
}
}
sizeof (lwpchan_hashbucket_t));
}
}
/*
* Return zero when there is an entry in the lwpchan cache for the
* given process virtual address and non-zero when there is not.
* The returned non-zero value is the current length of the
* hash chain plus one. The caller holds the hash bucket lock.
*/
static uint_t
{
/*
* This shouldn't happen, but might if the
* process reuses its memory for different
* types of sync objects. We test first
* to avoid grabbing the memory cache line.
*/
}
return (0);
}
count++;
}
return (count);
}
/*
* Return the cached lwpchan mapping if cached, otherwise insert
* a virtual address to lwpchan mapping into the cache.
*/
static int
{
top:
/* initialize the lwpchan cache, if necesary */
goto top;
}
/* someone resized the lwpchan cache; start over */
goto top;
}
/* it's in the cache */
return (1);
}
return (0);
/* someone resized the lwpchan cache; start over */
goto top;
}
if (count == 0) {
/* someone else added this entry to the cache */
return (1);
}
/* hash chain too long; reallocate the hash table */
goto top;
}
return (1);
}
/*
* Return a unique pair of identifiers that corresponds to a
* synchronization object's virtual address. Process-shared
*/
static int
{
/*
* If the lwp synch object is defined to be process-private,
* we just make the first field of the lwpchan be 'as' and
* the second field be the synch object's virtual address.
* (segvn_getmemid() does the same for MAP_PRIVATE mappings.)
* The lwpchan cache is used only for process-shared objects.
*/
return (1);
}
/* check the lwpchan cache for mapping */
}
static void
{
thread_lock(t);
t->t_flag |= T_WAKEABLE;
t->t_sobj_ops = &lwp_sobj_ops;
t->t_release = 0;
CL_SLEEP(t);
thread_unlock(t);
lwp->lwp_sysabort = 0;
}
static kthread_t *
{
}
static struct upimutex *
{
break;
}
return (upip);
}
static void
{
/*
* Insert upimutex at front of list. Maybe a bit unfair
* but assume that not many lwpchans hash to the same
* upimutextab bucket, i.e. the list of upimutexes from
* upib_first is not too long.
*/
}
static void
{
}
}
/*
* Add upimutex to chain of upimutexes held by curthread.
* Returns number of upimutexes held by curthread.
*/
static uint32_t
{
/*
* Insert upimutex at front of list of upimutexes owned by t. This
* would match typical LIFO order in which nested locks are acquired
* and released.
*/
t->t_upimutex = upimutex;
t->t_nupinest++;
ASSERT(t->t_nupinest > 0);
return (t->t_nupinest);
}
/*
* Delete upimutex from list of upimutexes owned by curthread.
*/
static void
{
/*
* Since the order in which nested locks are acquired and released,
* is typically LIFO, and typical nesting levels are not too deep, the
* following should not be expensive in the general case.
*/
prev = &t->t_upimutex;
}
ASSERT(t->t_nupinest > 0);
t->t_nupinest--;
}
/*
* Returns true if upimutex is owned. Should be called only when upim points
* to kmem which cannot disappear from underneath.
*/
static int
{
}
/*
* Returns pointer to kernel object (upimutex_t *) if lp is owned.
*/
static struct upimutex *
{
return (NULL);
return (NULL);
}
return (upimutex);
}
/*
* Unlocks upimutex, waking up waiters if any. upimutex kmem is freed if
* no lock hand-off occurrs.
*/
static void
{
/* hand-off lock to highest prio waiter */
upimutex->upi_waiter = 0;
return;
/* LOCK_NOTRECOVERABLE: wakeup all */
} else {
/*
* Misleading w bit. Waiters might have been
* interrupted. No need to clear the w bit (upimutex
* will soon be freed). Re-calculate PI from existing
* waiters.
*/
}
}
/*
* no waiters, or LOCK_NOTRECOVERABLE.
* remove from the bucket chain of upi mutexes.
* de-allocate kernel memory (upimutex).
*/
}
static int
{
int error = 0;
volatile int upilocked = 0;
if (upilocked)
goto out;
}
/*
* The apparent assumption made in implementing other _lwp_* synch
* primitives, is that get_lwpchan() does not return a unique cookie
* for the case where 2 processes (one forked from the other) point
* at the same underlying object, which is typed USYNC_PROCESS, but
* mapped MAP_PRIVATE, since the object has not yet been written to,
* in the child process.
*
* Since get_lwpchan() has been fixed, it is not necessary to do the
* dummy writes to force a COW fault as in other places (which should
* be fixed).
*/
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
/* lock available since lwpchan has no upimutex */
upilocked = 1;
if (nupinest > maxnestupimx &&
secpolicy_resource(CRED()) != 0) {
goto out;
}
if (flag & LOCK_OWNERDEAD) {
/*
* Return with upimutex held.
*/
error = EOWNERDEAD;
} else if (flag & LOCK_NOTRECOVERABLE) {
/*
* Since the setting of LOCK_NOTRECOVERABLE
* was done under the high-level upi mutex,
* in lwp_upimutex_unlock(), this flag needs to
* be checked while holding the upi mutex.
* If set, this thread should return without
* the lock held, and with the right error
* code.
*/
upilocked = 0;
}
goto out;
}
/*
* If a upimutex object exists, it must have an owner.
* This is due to lock hand-off, and release of upimutex when no
* waiters are present at unlock time,
*/
/*
* The user wrapper can check if the mutex type is
* ERRORCHECK: if not, it should stall at user-level.
* If so, it should return the error code.
*/
goto out;
}
if (try == UPIMUTEX_TRY) {
goto out;
}
/*
* Block for the lock.
* Put the lwp in an orderly state for debugging.
* Calling prstop() has to be done here, and not in
* turnstile_block(), since the preceding call to
* turnstile_lookup() raises the PIL to a level
* at which calls to prstop() should not be made.
*/
/*
* The SUSV3 Posix spec is very clear that we
* should get no error from validating the
* timer until we would actually sleep.
*/
goto out;
}
prstop(PR_REQUESTED, 0);
/*
* If we successfully queue the timeout
* (lwp_timer_enqueue() returns zero),
* then don't drop t_delay_lock until we are
* on the sleep queue (in turnstile_block()).
* Otherwise we will get an immediate timeout
* when we attempt to sleep in turnstile_block().
*/
if (lwp_timer_enqueue(lwptp) != 0)
}
/*
* Now, set the waiter bit and block for the lock in turnstile_block().
* No need to preserve the previous wbit since a lock try is not
* attempted after setting the wait bit. Wait bit is set under
* the upib_lock, which is not released until the turnstile lock
* is acquired. Say, the upimutex is L:
*
* 1. upib_lock is held so the waiter does not have to retry L after
* setting the wait bit: since the owner has to grab the upib_lock
* to unlock L, it will certainly see the wait bit set.
* 2. upib_lock is not released until the turnstile lock is acquired.
* This is the key to preventing a missed wake-up. Otherwise, the
* owner could acquire the upib_lock, and the tc_lock, to call
* turnstile_wakeup(). All this, before the waiter gets tc_lock
* to sleep in turnstile_block(). turnstile_wakeup() will then not
* find this waiter, resulting in the missed wakeup.
* 3. The upib_lock, being a kernel mutex, cannot be released while
* holding the tc_lock (since mutex_exit() could need to acquire
* the same tc_lock)...and so is held when calling turnstile_block().
* The address of upib_lock is passed to turnstile_block() which
* releases it after releasing all turnstile locks, and before going
* to sleep in swtch().
* 4. The waiter value cannot be a count of waiters, because a waiter
* can be interrupted. The interrupt occurs under the tc_lock, at
* which point, the upib_lock cannot be locked, to decrement waiter
* count. So, just treat the waiter state as a bit, not a count.
*/
/*
* Hand-off implies that we wakeup holding the lock, except when:
* - deadlock is detected
* - lock is not recoverable
* - we got an interrupt or timeout
* If we wake up due to an interrupt or timeout, we may
* or may not be holding the lock due to mutex hand-off.
* Use lwp_upimutex_owned() to check if we do hold the lock.
*/
if (error != 0) {
/*
* Unlock and return - the re-startable syscall will
* try the lock again if we got EINTR.
*/
}
/*
* The only other possible error is EDEADLK. If so, upimutex
* is valid, since its owner is deadlocked with curthread.
*/
goto out;
}
upilocked = 1;
}
/*
* Now, need to read the user-level lp->mutex_flag to do the following:
*
* - if lock is held, check if EOWNERDEAD should be returned
* - if lock isn't held, check if ENOTRECOVERABLE should be returned
*
* Now, either lp->mutex_flag is readable or it's not. If not
* readable, the on_fault path will cause a return with EFAULT as
* it should. If it is readable, the state of the flag encodes the
* robustness state of the lock:
*
* If the upimutex is locked here, the flag's LOCK_OWNERDEAD setting
* will influence the return code appropriately. If the upimutex is
* not locked here, this could be due to a spurious wake-up or a
* NOTRECOVERABLE event. The flag's setting can be used to distinguish
* between these two events.
*/
if (upilocked) {
/*
* If the thread wakes up from turnstile_block with the lock
* held, the flag could not be set to LOCK_NOTRECOVERABLE,
* since it would not have been handed-off the lock.
* So, no need to check for this case.
*/
if (nupinest > maxnestupimx &&
secpolicy_resource(CRED()) != 0) {
upilocked = 0;
} else if (flag & LOCK_OWNERDEAD) {
error = EOWNERDEAD;
}
} else {
/*
* Wake-up without the upimutex held. Either this is a
* spurious wake-up (due to signals, forkall(), whatever), or
* it is a LOCK_NOTRECOVERABLE robustness event. The setting
* of the mutex flag can be used to distinguish between the
* two events.
*/
if (flag & LOCK_NOTRECOVERABLE) {
} else {
/*
* Here, the flag could be set to LOCK_OWNERDEAD or
* not. In both cases, this is a spurious wakeup,
* since the upi lock is not held, but the thread
* has returned from turnstile_block().
*
* The user flag could be LOCK_OWNERDEAD if, at the
* same time as curthread having been woken up
* spuriously, the owner (say Tdead) has died, marked
* the mutex flag accordingly, and handed off the lock
* to some other waiter (say Tnew). curthread just
* happened to read the flag while Tnew has yet to deal
* with the owner-dead event.
*
* In this event, curthread should retry the lock.
* If Tnew is able to cleanup the lock, curthread
* will eventually get the lock with a zero error code,
* If Tnew is unable to cleanup, its eventual call to
* unlock the lock will result in the mutex flag being
* set to LOCK_NOTRECOVERABLE, and the wake-up of
* all waiters, including curthread, which will then
* eventually return ENOTRECOVERABLE due to the above
* check.
*
* Of course, if the user-flag is not set with
* LOCK_OWNERDEAD, retrying is the thing to do, since
* this is definitely a spurious wakeup.
*/
goto retry;
}
}
out:
no_fault();
return (error);
}
static int
{
int error = 0;
volatile int upilocked = 0;
if (upilocked)
goto out;
}
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
/*
* If the lock is not held, or the owner is not curthread, return
* error. The user-level wrapper can return this error or stall,
* depending on whether mutex is of ERRORCHECK type or not.
*/
goto out;
}
upilocked = 1;
if (flag & LOCK_OWNERDEAD) {
/*
* transition mutex to the LOCK_NOTRECOVERABLE state.
*/
flag &= ~LOCK_OWNERDEAD;
}
upilocked = 0;
out:
no_fault();
return (error);
}
/*
* Mark user mutex state, corresponding to kernel upimutex, as LOCK_OWNERDEAD.
*/
static int
{
int error = 0;
goto out;
}
flag |= LOCK_OWNERDEAD;
out:
no_fault();
return (error);
}
/*
* Unlock all upimutexes held by curthread, since curthread is dying.
* For each upimutex, attempt to mark its corresponding user mutex object as
* dead.
*/
void
{
/*
* If the user object associated with this upimutex is
* unmapped, unlock upimutex with the
* LOCK_NOTRECOVERABLE flag, so that all waiters are
* woken up. Since user object is unmapped, it could
* not be marked as dead or notrecoverable.
* The waiters will now all wake up and return
* ENOTRECOVERABLE, since they would find that the lock
* has not been handed-off to them.
* See lwp_upimutex_lock().
*/
} else {
/*
* The user object has been updated as dead.
* Unlock the upimutex: if no waiters, upip kmem will
* be freed. If there is a waiter, the lock will be
* handed off. If exit() is in progress, each existing
* waiter will successively get the lock, as owners
* die, and each new owner will call this routine as
* it dies. The last owner will free kmem, since
* it will find the upimutex has no waiters. So,
* eventually, the kmem is guaranteed to be freed.
*/
upimutex_unlock(upip, 0);
}
/*
* Note that the call to upimutex_unlock() above will delete
* upimutex from the t_upimutexes chain. And so the
* while loop will eventually terminate.
*/
}
}
int
{
int error = 0;
int time_error;
volatile int locked = 0;
volatile int watched = 0;
static int iswanted();
int imm_timeout = 0;
imm_timeout = 1;
}
/*
* Although LMS_USER_LOCK implies "asleep waiting for user-mode lock",
* this micro state is really a run state. If the thread indeed blocks,
* this state becomes valid. If not, the state is converted back to
* LMS_SYSTEM. So, it is OK to set the mstate here, instead of just
* when blocking.
*/
(void) new_mstate(t, LMS_USER_LOCK);
if (locked)
goto out;
}
no_fault();
(type & USYNC_PROCESS))
if (error)
return (0);
}
/*
* Force Copy-on-write fault if lwp_mutex_t object is
* defined to be MAP_PRIVATE and it was initialized to
* USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
locked = 1;
if (type & USYNC_PROCESS_ROBUST) {
if (flag & LOCK_NOTRECOVERABLE) {
goto out;
}
}
/*
* If watchpoints are set, they need to be restored, since
* atomic accesses of memory such as the call to ulock_try()
* below cannot be watched.
*/
if (time_error) {
/*
* The SUSV3 Posix spec is very clear that we
* should get no error from validating the
* timer until we would actually sleep.
*/
error = time_error;
break;
}
if (watched) {
watched = 0;
}
/*
* Put the lwp in an orderly state for debugging.
*/
prstop(PR_REQUESTED, 0);
if (timedwait) {
/*
* If we successfully queue the timeout,
* then don't drop t_delay_lock until
* we are on the sleep queue (below).
*/
mutex_enter(&t->t_delay_lock);
if (lwp_timer_enqueue(&lwpt) != 0) {
mutex_exit(&t->t_delay_lock);
imm_timeout = 1;
}
}
/*
* Nothing should happen to cause the lwp to go to
* sleep again until after it returns from swtch().
*/
if (timedwait)
mutex_exit(&t->t_delay_lock);
locked = 0;
setrun(t);
swtch();
t->t_flag &= ~T_WAKEABLE;
if (timedwait)
setallwatch();
if (error) {
lwp->lwp_asleep = 0;
lwp->lwp_sysabort = 0;
S_WRITE);
/*
* Need to re-compute waiters bit. The waiters field in
* the lock is not reliable. Either of two things could
* have occurred: no lwp may have called lwp_release()
* for me but I have woken up due to a signal or
* timeout. In this case, the waiter bit is incorrect
* since it is still set to 1, set above.
* OR an lwp_release() did occur for some other lwp on
* the same lwpchan. In this case, the waiter bit is
* correct. But which event occurred, one can't tell.
* So, recompute.
*/
locked = 1;
break;
}
lwp->lwp_asleep = 0;
S_WRITE);
locked = 1;
if (type & USYNC_PROCESS_ROBUST) {
if (flag & LOCK_NOTRECOVERABLE) {
break;
}
}
}
if (t->t_mstate == LMS_USER_LOCK)
(void) new_mstate(t, LMS_SYSTEM);
if (type & USYNC_PROCESS_ROBUST) {
if (flag & LOCK_OWNERDEAD)
error = EOWNERDEAD;
else if (flag & LOCK_UNMAPPED)
}
}
locked = 0;
out:
no_fault();
if (watched)
if (error)
return (0);
}
/*
* Obsolete lwp_mutex_lock() interface, no longer called from libc.
* libc now calls lwp_mutex_timedlock(lp, NULL).
* This system call trap continues to exist solely for the benefit
* of old statically-linked binaries from Solaris 9 and before.
* It should be removed from the system when we no longer care
* about such applications.
*/
int
{
}
static int
{
/*
* The caller holds the dispatcher lock on the sleep queue.
*/
while (t != NULL) {
return (1);
t = t->t_link;
}
return (0);
}
/*
* Return the highest priority thread sleeping on this lwpchan.
*/
static kthread_t *
{
break;
}
return (tp);
}
static int
{
/*
* The following is typically false. It could be true
* only if lwp_release() is called from
* lwp_mutex_wakeup() after reading the waiters field
* from memory in which the lwp lock used to be, but has
* since been re-used to hold a lwp cv or lwp semaphore.
* The thread "tp" found to match the lwp lock's wchan
* is actually sleeping for the cv or semaphore which
* now has the same wchan. In this case, lwp_release()
* should return failure.
*/
/*
* assert that this can happen only for mutexes
* i.e. sync_type == 0, for correctly written
* user programs.
*/
return (0);
}
return (1);
}
}
*waiters = 0;
return (0);
}
static void
{
} else {
}
}
}
/*
* unblock a lwp that is trying to acquire this mutex. the blocked
* lwp resumes and retries to acquire the lock.
*/
int
{
volatile int locked = 0;
volatile int watched = 0;
int error = 0;
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_mutex_t object is
* defined to be MAP_PRIVATE, and type is USYNC_PROCESS
*/
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
locked = 1;
/*
* Always wake up an lwp (if any) waiting on lwpchan. The woken lwp will
* re-try the lock in lwp_mutex_timedlock(). The call to lwp_release()
* may fail. If it fails, do not write into the waiter bit.
* The call to lwp_release() might fail due to one of three reasons:
*
* 1. due to the thread which set the waiter bit not actually
* sleeping since it got the lock on the re-try. The waiter
* bit will then be correctly updated by that thread. This
* window may be closed by reading the wait bit again here
* and not calling lwp_release() at all if it is zero.
* 2. the thread which set the waiter bit and went to sleep
* was woken up by a signal. This time, the waiter recomputes
* the wait bit in the return with EINTR code.
* 3. the waiter bit read by lwp_mutex_wakeup() was in
* memory that has been re-used after the lock was dropped.
* In this case, writing into the waiter bit would cause data
* corruption.
*/
}
out:
no_fault();
if (watched)
if (error)
return (0);
}
/*
* lwp_cond_wait() has four arguments, a pointer to a condition variable,
* a pointer to a mutex, a pointer to a timespec for a timed wait and
* schedctl parking protocol (see schedctl_is_park() in schedctl.c).
* The kernel puts the lwp to sleep on a unique pair of caddr_t's called an
* lwpchan, returned by get_lwpchan(). If the timespec pointer is non-NULL,
* time until timeout. On exit, we copyout the residual time left to it.
*/
int
{
volatile int error;
volatile int locked = 0;
volatile int m_locked = 0;
volatile int cvwatched = 0;
volatile int mpwatched = 0;
volatile int no_lwpchan = 1;
int imm_timeout = 0;
int imm_unpark = 0;
if (lwpt.lwpt_imm_timeout) {
imm_timeout = 1;
}
(void) new_mstate(t, LMS_USER_LOCK);
if (no_lwpchan) {
goto out;
}
if (m_locked) {
m_locked = 0;
}
if (locked) {
locked = 0;
}
/*
* set up another on_fault() for a possible fault
* on the user lock accessed at "efault"
*/
if (m_locked) {
m_locked = 0;
}
goto out;
}
goto efault;
}
/*
* Force Copy-on-write fault if lwp_cond_t and lwp_mutex_t
* objects are defined to be MAP_PRIVATE, and are USYNC_PROCESS
*/
/* convert user level mutex, "mp", to a unique lwpchan */
/* check if mtype is ok to use below, instead of type from cv */
&m_lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
}
/* convert user level condition variable, "cv", to a unique lwpchan */
&cv_lwpchan, LWPCHAN_CVPOOL)) {
goto out;
}
no_lwpchan = 0;
S_WRITE);
/*
* lwpchan_lock ensures that the calling lwp is put to sleep atomically
* with respect to a possible wakeup which is a result of either
* an lwp_cond_signal() or an lwp_cond_broadcast().
*
* What's misleading, is that the lwp is put to sleep after the
* condition variable's mutex is released. This is OK as long as
* the release operation is also done while holding lwpchan_lock.
* The lwp is then put to sleep when the possibility of pagefaulting
* or sleeping is completely eliminated.
*/
locked = 1;
m_locked = 1;
/*
* unlock the condition variable's mutex. (pagefaults are
* possible here.)
*/
if (waiters != 0) {
/*
* Given the locking of lwpchan_lock around the release
* of the mutex and checking for waiters, the following
* call to lwp_release() can fail ONLY if the lock
* acquirer is interrupted after setting the waiter bit,
* calling lwp_block() and releasing lwpchan_lock.
* In this case, it could get pulled off the lwp sleep
* q (via setrun()) before the following call to
* lwp_release() occurs. In this case, the lock
* requestor will update the waiter bit correctly by
* re-evaluating it.
*/
}
m_locked = 0;
} else {
if (error) { /* if the upimutex unlock failed */
locked = 0;
goto out;
}
}
no_fault();
if (mpwatched) {
mpwatched = 0;
}
if (cvwatched) {
cvwatched = 0;
}
/*
* Put the lwp in an orderly state for debugging.
*/
prstop(PR_REQUESTED, 0);
/*
* We received a signal at user-level before calling here
* or another thread wants us to return immediately
* with EINTR. See lwp_unpark().
*/
imm_unpark = 1;
t->t_unpark = 0;
} else if (timedwait) {
/*
* If we successfully queue the timeout,
* then don't drop t_delay_lock until
* we are on the sleep queue (below).
*/
mutex_enter(&t->t_delay_lock);
if (lwp_timer_enqueue(&lwpt) != 0) {
mutex_exit(&t->t_delay_lock);
imm_timeout = 1;
}
}
t->t_flag |= T_WAITCVSEM;
/*
* Nothing should happen to cause the lwp to go to sleep
* until after it returns from swtch().
*/
if (timedwait)
mutex_exit(&t->t_delay_lock);
locked = 0;
(imm_timeout | imm_unpark))
setrun(t);
swtch();
if (timedwait)
MUSTRETURN(p, t) || imm_unpark)
lwp->lwp_asleep = 0;
lwp->lwp_sysabort = 0;
setallwatch();
if (t->t_mstate == LMS_USER_LOCK)
(void) new_mstate(t, LMS_SYSTEM);
/* the mutex is reacquired by the caller on return to user level */
if (error) {
/*
* If we were concurrently lwp_cond_signal()d and we
* received a UNIX signal or got a timeout, then perform
* another lwp_cond_signal() to avoid consuming the wakeup.
*/
if (t->t_release)
(void) lwp_cond_signal(cv);
}
return (0);
/*
* make sure that the user level lock is dropped before
* returning to caller, since the caller always re-acquires it.
*/
m_locked = 1;
if (waiters != 0) {
/*
* See comment above on lock clearing and lwp_release()
*/
}
m_locked = 0;
} else {
}
out:
no_fault();
if (mpwatched)
if (cvwatched)
if (t->t_mstate == LMS_USER_LOCK)
(void) new_mstate(t, LMS_SYSTEM);
}
/*
* wakeup one lwp that's blocked on this condition variable.
*/
int
{
volatile int locked = 0;
volatile int watched = 0;
int error = 0;
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_cond_t object is
* defined to be MAP_PRIVATE, and is USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_CVPOOL)) {
goto out;
}
locked = 1;
if (waiters != 0) {
/*
* The following call to lwp_release() might fail but it is
* OK to write into the waiters bit below, since the memory
* could not have been re-used or unmapped (for correctly
* written user programs) as in the case of lwp_mutex_wakeup().
* For an incorrect program, we should not care about data
* corruption since this is just one instance of other places
* where corruption can occur for such a program. Of course
* if the memory is unmapped, normal fault recovery occurs.
*/
}
out:
no_fault();
if (watched)
if (error)
return (0);
}
/*
* wakeup every lwp that's blocked on this condition variable.
*/
int
{
volatile int locked = 0;
volatile int watched = 0;
int error = 0;
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_cond_t object is
* defined to be MAP_PRIVATE, and is USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_CVPOOL)) {
goto out;
}
locked = 1;
if (waiters != 0) {
}
out:
no_fault();
if (watched)
if (error)
return (0);
}
int
{
volatile int locked = 0;
volatile int watched = 0;
int count;
int error = 0;
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_sema_t object is
* defined to be MAP_PRIVATE, and is USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_CVPOOL)) {
goto out;
}
locked = 1;
if (count == 0)
else
if (count != 0) {
if (waiters != 0) {
}
}
out:
no_fault();
if (watched)
if (error)
return (0);
}
/*
* See lwp_cond_wait(), above, for an explanation of the 'check_park' argument.
*/
int
{
volatile int locked = 0;
volatile int watched = 0;
int count;
int error = 0;
int time_error;
int imm_timeout = 0;
int imm_unpark = 0;
imm_timeout = 1;
}
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_sema_t object is
* defined to be MAP_PRIVATE, and is USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_CVPOOL)) {
goto out;
}
locked = 1;
if (time_error) {
/*
* The SUSV3 Posix spec is very clear that we
* should get no error from validating the
* timer until we would actually sleep.
*/
error = time_error;
break;
}
if (watched)
/*
* Put the lwp in an orderly state for debugging.
*/
prstop(PR_REQUESTED, 0);
/*
* We received a signal at user-level before calling
* here or another thread wants us to return
* immediately with EINTR. See lwp_unpark().
*/
imm_unpark = 1;
t->t_unpark = 0;
} else if (timedwait) {
/*
* If we successfully queue the timeout,
* then don't drop t_delay_lock until
* we are on the sleep queue (below).
*/
mutex_enter(&t->t_delay_lock);
if (lwp_timer_enqueue(&lwpt) != 0) {
mutex_exit(&t->t_delay_lock);
imm_timeout = 1;
}
}
t->t_flag |= T_WAITCVSEM;
/*
* Nothing should happen to cause the lwp to sleep
* again until after it returns from swtch().
*/
if (timedwait)
mutex_exit(&t->t_delay_lock);
locked = 0;
(imm_timeout | imm_unpark))
setrun(t);
swtch();
if (timedwait)
setallwatch();
MUSTRETURN(p, t) || imm_unpark)
lwp->lwp_asleep = 0;
lwp->lwp_sysabort = 0;
locked = 1;
}
if (error == 0)
if (count != 0) {
}
out:
no_fault();
if (watched)
if (error)
return (0);
}
/*
* Obsolete lwp_sema_wait() interface, no longer called from libc.
* libc now calls lwp_sema_timedwait().
* This system call trap exists solely for the benefit of old
* statically linked applications from Solaris 9 and before.
* It should be removed when we no longer care about such applications.
*/
int
{
}
int
{
volatile int locked = 0;
volatile int watched = 0;
int count;
int error = 0;
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_sema_t object is
* defined to be MAP_PRIVATE, and is USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_CVPOOL)) {
goto out;
}
locked = 1;
if (count == _SEM_VALUE_MAX)
else
if (count == 1) {
if (waiters) {
}
}
out:
no_fault();
if (watched)
if (error)
return (0);
}
#define TRW_WANT_WRITE 0x1
#define TRW_LOCK_GRANTED 0x2
#define READ_LOCK 0
#define WRITE_LOCK 1
#define TRY_FLAG 0x10
/*
* Release one writer or one or more readers. Compute the rwstate word to
* reflect the new state of the queue. For a safe hand-off we copy the new
* rwstate value back to userland before we wake any of the new lock holders.
*
* Note that sleepq_insert() implements a prioritized FIFO (with writers
* being given precedence over readers of the same priority).
*
* If the first thread is a reader we scan the queue releasing all readers
* until we hit a writer or the end of the queue. If the first thread is a
* writer we still need to check for another writer (i.e. URW_WRITE_WANTED).
*/
void
{
int wcount = 0;
int rcount = 0;
/* Just one writer to wake. */
/* tpp already set for next thread. */
continue;
} else {
rwstate |=
/* We need look no further. */
break;
}
} else {
rcount++;
if (wcount == 0) {
rwstate++;
/* Add reader to wake list. */
/* tpp already set for next thread. */
continue;
} else
}
}
}
/* Copy the new rwstate back to userland. */
/* Wake the new lock holder(s) up. */
}
}
/*
* We enter here holding the user-level mutex, which we must release before
* returning or blocking. Based on lwp_cond_wait().
*/
static int
{
volatile int error = 0;
int time_error;
volatile int locked = 0;
volatile int mlocked = 0;
volatile int watched = 0;
volatile int mwatched = 0;
volatile int no_lwpchan = 1;
int imm_timeout = 0;
int try_flag;
int acquired = 0;
/* We only check rw because the mutex is included in it. */
/* We must only report this error if we are about to sleep (later). */
imm_timeout = 1;
}
(void) new_mstate(t, LMS_USER_LOCK);
if (no_lwpchan) {
goto out_nodrop;
}
if (mlocked) {
mlocked = 0;
}
if (locked) {
locked = 0;
}
/*
* Set up another on_fault() for a possible fault
* on the user lock accessed at "out_drop".
*/
if (mlocked) {
mlocked = 0;
}
goto out_nodrop;
}
goto out_nodrop;
}
/* Process rd_wr (including sanity check). */
goto out_nodrop;
}
/* We can only continue for simple USYNC_PROCESS locks. */
goto out_nodrop;
}
/* Force Copy-on-write fault incase objects are MAP_PRIVATE. */
/* Convert user level mutex, "mp", to a unique lwpchan. */
&mlwpchan, LWPCHAN_MPPOOL)) {
goto out_nodrop;
}
/* Convert user level rwlock, "rw", to a unique lwpchan. */
&lwpchan, LWPCHAN_CVPOOL)) {
goto out_nodrop;
}
no_lwpchan = 0;
/*
* lwpchan_lock() ensures that the calling LWP is put to sleep
* atomically with respect to a possible wakeup which is a result
* of lwp_rwlock_unlock().
*
* What's misleading is that the LWP is put to sleep after the
* rwlock's mutex is released. This is OK as long as the release
* operation is also done while holding mlwpchan. The LWP is then
* put to sleep when the possibility of pagefaulting or sleeping
* has been completely eliminated.
*/
locked = 1;
mlocked = 1;
/*
* Fetch the current rwlock state.
*
* The possibility of spurious wake-ups or killed waiters means that
* rwstate's URW_HAS_WAITERS and URW_WRITE_WANTED bits may indicate
* false positives. We only fix these if they are important to us.
*
* Although various error states can be observed here (e.g. the lock
* is not held, but there are waiters) we assume these are applicaton
* errors and so we take no corrective action.
*/
/*
* If the lock is uncontended we can acquire it here. These tests
* should have already been done at user-level, we just need to be
* sure.
*/
if ((rwstate & ~URW_READERS_MASK) == 0) {
rwstate++;
acquired = 1;
}
} else if (rwstate == 0) {
acquired = 1;
}
/*
* We can only try harder if the lock isn't held by a writer.
*/
/*
* Hmmm, rwstate indicates waiters but there are
* none queued. This could just be the result of a
* spurious wakeup, so let's fix it.
*/
/*
* We now have another chance to acquire the lock
* uncontended, but this is the last chance for a
* writer to acquire the lock without blocking.
*/
rwstate++;
acquired = 1;
} else if (rwstate == 0) {
acquired = 1;
}
/*
* This is the last chance for a reader to acquire
* the lock now, but it can only do so if there is
* no writer of equal or greater priority at the
* head of the queue .
*
* It is also just possible that there is a reader
* at the head of the queue. This may be the result
* of a spurious wakeup or an application failure.
* In this case we only acquire the lock if we have
* equal or greater priority. It is not our job to
* release spurious waiters.
*/
rwstate++;
acquired = 1;
}
}
}
/*
* We're not going to block this time!
*/
locked = 0;
if (acquired) {
/*
* Got the lock!
*/
error = 0;
} else if (try_flag) {
/*
* We didn't get the lock and we're about to block.
* If we're doing a trylock, return EBUSY instead.
*/
} else if (time_error) {
/*
* The SUSV3 POSIX spec is very clear that we should
* get no error from validating the timer (above)
* until we would actually sleep.
*/
error = time_error;
}
goto out_drop;
}
/*
* We're about to block, so indicate what kind of waiter we are.
*/
t->t_writer = 0;
if (rd_wr == WRITE_LOCK) {
t->t_writer = TRW_WANT_WRITE;
}
/*
* Unlock the rwlock's mutex (pagefaults are possible here).
*/
if (mwaiters != 0) {
/*
* Given the locking of mlwpchan around the release of
* the mutex and checking for waiters, the following
* call to lwp_release() can fail ONLY if the lock
* acquirer is interrupted after setting the waiter bit,
* calling lwp_block() and releasing mlwpchan.
* In this case, it could get pulled off the LWP sleep
* queue (via setrun()) before the following call to
* lwp_release() occurs, and the lock requestor will
* update the waiter bit correctly by re-evaluating it.
*/
}
mlocked = 0;
no_fault();
if (mwatched) {
mwatched = 0;
}
if (watched) {
watched = 0;
}
/*
* Put the LWP in an orderly state for debugging.
*/
prstop(PR_REQUESTED, 0);
if (timedwait) {
/*
* If we successfully queue the timeout,
* then don't drop t_delay_lock until
* we are on the sleep queue (below).
*/
mutex_enter(&t->t_delay_lock);
if (lwp_timer_enqueue(&lwpt) != 0) {
mutex_exit(&t->t_delay_lock);
imm_timeout = 1;
}
}
t->t_flag |= T_WAITCVSEM;
/*
* Nothing should happen to cause the LWp to go to sleep until after
* it returns from swtch().
*/
if (timedwait)
mutex_exit(&t->t_delay_lock);
locked = 0;
setrun(t);
swtch();
/*
* We're back, but we need to work out why. Were we interrupted? Did
* we timeout? Were we granted the lock?
*/
t->t_writer = 0;
if (timedwait)
lwp->lwp_asleep = 0;
lwp->lwp_sysabort = 0;
setallwatch();
/*
* If we were granted the lock we don't care about EINTR or ETIME.
*/
if (acquired)
error = 0;
if (t->t_mstate == LMS_USER_LOCK)
(void) new_mstate(t, LMS_SYSTEM);
if (error)
return (0);
/*
* Make sure that the user level lock is dropped before returning
* to the caller.
*/
if (!mlocked) {
mlocked = 1;
}
if (mwaiters != 0) {
/*
* See comment above on lock clearing and lwp_release()
*/
}
mlocked = 0;
no_fault();
if (mwatched)
if (watched)
if (t->t_mstate == LMS_USER_LOCK)
(void) new_mstate(t, LMS_SYSTEM);
if (error)
return (0);
}
/*
* We enter here holding the user-level mutex but, unlike lwp_rwlock_lock(),
* we never drop the lock.
*/
static int
{
volatile int error = 0;
volatile int locked = 0;
volatile int watched = 0;
volatile int no_lwpchan = 1;
/* We only check rw because the mutex is included in it. */
if (no_lwpchan) {
goto out_nodrop;
}
if (locked) {
locked = 0;
}
goto out_nodrop;
}
/* We can only continue for simple USYNC_PROCESS locks. */
if (type != USYNC_PROCESS) {
goto out_nodrop;
}
/* Force Copy-on-write fault incase objects are MAP_PRIVATE. */
/* Convert user level rwlock, "rw", to a unique lwpchan. */
&lwpchan, LWPCHAN_CVPOOL)) {
goto out_nodrop;
}
no_lwpchan = 0;
locked = 1;
/*
* We can resolve multiple readers (except the last reader) here.
* For the last reader or a writer we need lwp_rwlock_release(),
* to which we also delegate the task of copying the new rwstate
* back to userland (see the comment there).
*/
if (rwstate & URW_WRITE_LOCKED)
else if ((rwstate & URW_READERS_MASK) > 0) {
rwstate--;
if ((rwstate & URW_READERS_MASK) == 0)
else
}
locked = 0;
error = 0;
no_fault();
if (watched)
if (error)
return (0);
}
int
{
switch (subcode) {
case 0:
case 1:
case 2:
case 3:
case 4:
return (lwp_rwlock_unlock(rwlp));
}
}
/*
* Return the owner of the user-level s-object.
* Since we can't really do this, return NULL.
*/
/* ARGSUSED */
static kthread_t *
{
}
/*
* Wake up a thread asleep on a user-level synchronization
* object.
*/
static void
lwp_unsleep(kthread_t *t)
{
ASSERT(THREAD_LOCK_HELD(t));
sleepq_unsleep(t);
CL_SETRUN(t);
return;
}
}
panic("lwp_unsleep: thread %p not on sleepq", (void *)t);
}
/*
* Change the priority of a thread asleep on a user-level
* synchronization object. To maintain proper priority order,
* we:
* o dequeue the thread.
* o change its priority.
* o re-enqueue the thread.
* Assumption: the thread is locked on entry.
*/
static void
{
ASSERT(THREAD_LOCK_HELD(t));
sleepq_dequeue(t);
sleepq_insert(sqp, t);
} else
panic("lwp_change_pri: %p not on a sleep queue", (void *)t);
}
/*
* Clean up a locked a robust mutex
*/
static void
{
volatile int locked = 0;
volatile int watched = 0;
if (locked)
goto out;
}
goto out;
}
locked = 1;
}
out:
no_fault();
if (watched)
}
/*
* Register the mutex and initialize the mutex if it is not already
*/
int
{
int error = 0;
volatile int locked = 0;
volatile int watched = 0;
if (type != USYNC_PROCESS_ROBUST)
if (locked)
goto out;
}
/*
* Force Copy-on-write fault if lwp_mutex_t object is
* defined to be MAP_PRIVATE and it was initialized to
* USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
locked = 1;
if (flag & LOCK_INITED) {
locked = 0;
goto out;
}
}
} else {
}
locked = 0;
out:
no_fault();
if (watched)
if (error)
return (0);
}
int
{
int error = 0;
volatile int locked = 0;
volatile int watched = 0;
(void) new_mstate(t, LMS_USER_LOCK);
if (locked)
goto out;
}
no_fault();
(type & USYNC_PROCESS))
if (error)
return (0);
}
/*
* Force Copy-on-write fault if lwp_mutex_t object is
* defined to be MAP_PRIVATE and it was initialized to
* USYNC_PROCESS.
*/
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
locked = 1;
if (type & USYNC_PROCESS_ROBUST) {
if (flag & LOCK_NOTRECOVERABLE) {
goto out;
}
}
if (type & USYNC_PROCESS_ROBUST) {
if (flag & LOCK_OWNERDEAD)
error = EOWNERDEAD;
else if (flag & LOCK_UNMAPPED)
}
}
locked = 0;
out:
if (t->t_mstate == LMS_USER_LOCK)
(void) new_mstate(t, LMS_SYSTEM);
no_fault();
if (watched)
if (error)
return (0);
}
/*
* unlock the mutex and unblock lwps that is trying to acquire this mutex.
* the blocked lwp resumes and retries to acquire the lock.
*/
int
{
volatile int locked = 0;
volatile int watched = 0;
int error = 0;
if (locked)
goto out;
}
no_fault();
if (error)
return (0);
}
/*
* Force Copy-on-write fault if lwp_mutex_t object is
* defined to be MAP_PRIVATE, and type is USYNC_PROCESS
*/
&lwpchan, LWPCHAN_MPPOOL)) {
goto out;
}
locked = 1;
if (type & USYNC_PROCESS_ROBUST) {
}
}
}
/*
* Always wake up an lwp (if any) waiting on lwpchan. The woken lwp will
* re-try the lock in lwp_mutex_timedlock(). The call to lwp_release()
* may fail. If it fails, do not write into the waiter bit.
* The call to lwp_release() might fail due to one of three reasons:
*
* 1. due to the thread which set the waiter bit not actually
* sleeping since it got the lock on the re-try. The waiter
* bit will then be correctly updated by that thread. This
* window may be closed by reading the wait bit again here
* and not calling lwp_release() at all if it is zero.
* 2. the thread which set the waiter bit and went to sleep
* was woken up by a signal. This time, the waiter recomputes
* the wait bit in the return with EINTR code.
* 3. the waiter bit read by lwp_mutex_wakeup() was in
* memory that has been re-used after the lock was dropped.
* In this case, writing into the waiter bit would cause data
* corruption.
*/
if (waiters) {
if ((type & USYNC_PROCESS_ROBUST) &&
(flag & LOCK_NOTRECOVERABLE)) {
}
}
out:
no_fault();
if (watched)
if (error)
return (0);
}