/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* UNIX Device Driver Interface functions
* This file contains the C-versions of putnext() and put().
* Assembly language versions exist for some architectures.
*/
#include <sys/archsystm.h>
/*
* Streams with many modules may create long chains of calls via putnext() which
* may exhaust stack space. When putnext detects that the stack space left is
* too small (less then PUT_STACK_NEEDED), the call chain is broken and
* further processing is delegated to the background thread via call to
* putnext_tail(). Unfortunately there is no generic solution with fixed stack
* size, and putnext() is recursive function, so this hack is a necessary evil.
*
* The redzone value is chosen dependent on the default stack size which is 8K
* on 32-bit kernels and on x86 and 16K on 64-bit kernels. The values are chosen
* empirically. For 64-bit kernels it is 5000 and for 32-bit kernels it is 3000.
* Experiments showed that 2500 is not enough for either 32-bit or 64-bit
* kernels.
*
* The redzone value is a tuneable rather then a constant to allow adjustments
* in the field.
*
* The check in PUT_STACK_NOTENOUGH is taken from segkp_map_red() function. It
* is possible to define it as a generic function exported by seg_kp, but
*
* a) It may sound like an open invitation to use the facility indiscriminately.
* b) It adds extra function call in putnext path.
*
* We keep a global counter `put_stack_notenough' which keeps track how many
* times the stack switching hack was used.
*/
#ifdef _LP64
#else
#endif
#if defined(STACK_GROWTH_DOWN)
#define PUT_STACK_NOTENOUGH() \
#else
#error "STACK_GROWTH_DOWN undefined"
#endif
/*
* function: putnext()
* purpose: call the put routine of the queue linked to qp
*
* Note: this function is written to perform well on modern computer
* architectures by e.g. preloading values into registers and "smearing" out
* code.
*
* A note on the fastput mechanism. The most significant bit of a
* putcount is considered the "FASTPUT" bit. If set, then there is
* nothing stoping a concurrent put from occuring (note that putcounts
* are only allowed on CIPUT perimiters). If, however, it is cleared,
* then we need to take the normal lock path by aquiring the SQLOCK.
* This is a slowlock. When a thread starts exclusiveness, e.g. wants
* writer access, it will clear the FASTPUT bit, causing new threads
* to take the slowlock path. This assures that putcounts will not
* increase in value, so the want-writer does not need to constantly
* aquire the putlocks to sum the putcounts. This does have the
* possibility of having the count drop right after reading, but that
* is no different than aquiring, reading and then releasing. However,
* in this mode, it cannot go up, so eventually they will drop to zero
* and the want-writer can proceed.
*
* If the FASTPUT bit is set, or in the slowlock path we see that there
* are no writers or want-writers, we make the choice of calling the
* putproc, or a "fast-fill_syncq". The fast-fill is a fill with
* immediate intention to drain. This is done because there are
* messages already at the queue waiting to drain. To preserve message
* ordering, we need to put this message at the end, and pickup the
* messages at the beginning. We call the macro that actually
* enqueues the message on the queue, and then call qdrain_syncq. If
* there is already a drainer, we just return. We could make that
* check before calling qdrain_syncq, but it is a little more clear
* to have qdrain_syncq do this (we might try the above optimization
* as this behavior evolves). qdrain_syncq assumes that SQ_EXCL is set
* already if this is a non-CIPUT perimiter, and that an appropriate
* claim has been made. So we do all that work before dropping the
* SQLOCK with our claim.
*
* through to the qfill_syncq, and then tail processing. If state
* has changed in that cycle, or wakeups are needed, it will occur
* there.
*/
void
{
int (*putproc)();
int ix;
} else {
}
/* fastlock: */
if (!((*sqcipcount) & SQ_FASTPUT) ||
goto slowlock;
}
(*sqcipcount)++;
ASSERT(*sqcipcount != 0);
} else {
/*
* We are going to drop SQLOCK, so make a claim to prevent syncq
* from closing.
*/
/*
* If there are writers or exclusive waiters, there is not much
* we can do. Place the message on the syncq and schedule a
* background thread to drain it.
*
* Also if we are approaching end of stack, fill the syncq and
* switch processing to a background thread - see comments on
* top.
*/
"putnext_end:(%p, %p, %p) SQ_EXCL fill",
/*
* NOTE: qfill_syncq will need QLOCK. It is safe to drop
* SQLOCK because positive sq_count keeps the syncq from
* closing.
*/
/*
* NOTE: after the call to qfill_syncq() qp may be
* closed, both qp and sq should not be referenced at
* this point.
*
* This ASSERT is located here to prevent stack frame
* consumption in the DEBUG code.
*/
return;
}
/*
* If not a concurrent perimiter, we need to acquire
* it exclusively. It could not have been previously
* set since we held the SQLOCK before testing
* SQ_GOAWAY above (which includes SQ_EXCL).
* We do this here because we hold the SQLOCK, and need
* to make this state change BEFORE dropping it.
*/
}
}
/*
* We now have a claim on the syncq, we are either going to
* put the message on the syncq and then drain it, or we are
* going to call the putproc().
*/
if (!queued) {
} else {
/*
* If there are no messages in front of us, just call putproc(),
* otherwise enqueue the message and drain the queue.
*/
if (qp->q_syncqmsgs == 0) {
} else {
/*
* We are doing a fill with the intent to
* drain (meaning we are filling because
* there are messages in front of us ane we
* need to preserve message ordering)
* Therefore, put the message on the queue
* and call qdrain_syncq (must be done with
* the QLOCK held).
*/
#ifdef DEBUG
/*
* These two values were in the original code for
* all syncq messages. This is unnecessary in
* the current implementation, but was retained
* in debug mode as it is usefull to know where
* problems occur.
*/
#endif
}
}
/*
* Before we release our claim, we need to see if any
* events were posted. If the syncq is SQ_EXCL && SQ_QUEUED,
* we were responsible for going exclusive and, therefore,
* are resposible for draining.
*/
drain_mask = 0;
} else {
}
/* SQ_EXCL could have been set by qwriter_inner */
/*
* we need SQLOCK to handle
* is needed to decrement sqcipcount.
* SQLOCK has to be grabbed before sqciplock
* for lock ordering purposes.
* after sqcipcount is decremented some lock
* still needs to be held to make sure
* syncq won't get freed on us.
*
* To prevent deadlocks we try to grab SQLOCK and if it
* is held already we drop sqciplock, acquire SQLOCK and
* reacqwire sqciplock again.
*/
}
ASSERT(*sqcipcount != 0);
(*sqcipcount)--;
} else {
ASSERT(*sqcipcount != 0);
(*sqcipcount)--;
return;
}
} else {
}
/*
* The only purpose of this ASSERT is to preserve calling stack
* in DEBUG kernel.
*/
return;
}
/*
* Safe to always drop SQ_EXCL:
* Not SQ_CIPUT means we set SQ_EXCL above
* For SQ_CIPUT SQ_EXCL will only be set if the put
* procedure did a qwriter(INNER) in which case
* nobody else is in the inner perimeter and we
* are exiting.
*
* I would like to make the following assertion:
*
* ASSERT((flags & (SQ_EXCL|SQ_CIPUT)) != (SQ_EXCL|SQ_CIPUT) ||
* sq->sq_count == 0);
*
* which indicates that if we are both putshared and exclusive,
* we became exclusive while executing the putproc, and the only
* claim on the syncq was the one we dropped a few lines above.
* But other threads that enter putnext while the syncq is exclusive
* need to make a claim as they may need to drop SQLOCK in the
* has_writers case to avoid deadlocks. If these threads are
* delayed or preempted, it is possible that the writer thread can
* find out that there are other claims making the (sq_count == 0)
* test invalid.
*/
}
/*
* wrapper for qi_putp entry in module ops vec.
* implements asynchronous putnext().
* Note, that unlike putnext(), this routine is NOT optimized for the
* fastpath. Calling this routine will grab whatever locks are necessary
* to protect the stream head, q_next, and syncq's.
* And since it is in the normal locks path, we do not use putlocks if
* they exist (though this can be changed by swapping the value of
* UseFastlocks).
*/
void
{
int (*putproc)();
int ix;
/* fastlock: */
if (!((*sqcipcount) & SQ_FASTPUT) ||
goto slowlock;
}
(*sqcipcount)++;
ASSERT(*sqcipcount != 0);
} else {
/*
* We are going to drop SQLOCK, so make a claim to prevent syncq
* from closing.
*/
/*
* If there are writers or exclusive waiters, there is not much
* we can do. Place the message on the syncq and schedule a
* background thread to drain it.
*
* Also if we are approaching end of stack, fill the syncq and
* switch processing to a background thread - see comments on
* top.
*/
"putnext_end:(%p, %p, %p) SQ_EXCL fill",
/*
* NOTE: qfill_syncq will need QLOCK. It is safe to drop
* SQLOCK because positive sq_count keeps the syncq from
* closing.
*/
/*
* NOTE: after the call to qfill_syncq() qp may be
* closed, both qp and sq should not be referenced at
* this point.
*
* This ASSERT is located here to prevent stack frame
* consumption in the DEBUG code.
*/
return;
}
/*
* If not a concurrent perimiter, we need to acquire
* it exclusively. It could not have been previously
* set since we held the SQLOCK before testing
* SQ_GOAWAY above (which includes SQ_EXCL).
* We do this here because we hold the SQLOCK, and need
* to make this state change BEFORE dropping it.
*/
}
}
/*
* We now have a claim on the syncq, we are either going to
* put the message on the syncq and then drain it, or we are
* going to call the putproc().
*/
if (!queued) {
} else {
/*
* If there are no messages in front of us, just call putproc(),
* otherwise enqueue the message and drain the queue.
*/
if (qp->q_syncqmsgs == 0) {
} else {
/*
* We are doing a fill with the intent to
* drain (meaning we are filling because
* there are messages in front of us ane we
* need to preserve message ordering)
* Therefore, put the message on the queue
* and call qdrain_syncq (must be done with
* the QLOCK held).
*/
#ifdef DEBUG
/*
* These two values were in the original code for
* all syncq messages. This is unnecessary in
* the current implementation, but was retained
* in debug mode as it is usefull to know where
* problems occur.
*/
#endif
}
}
/*
* Before we release our claim, we need to see if any
* events were posted. If the syncq is SQ_EXCL && SQ_QUEUED,
* we were responsible for going exclusive and, therefore,
* are resposible for draining.
*/
drain_mask = 0;
} else {
}
/* SQ_EXCL could have been set by qwriter_inner */
/*
* we need SQLOCK to handle
* is needed to decrement sqcipcount.
* SQLOCK has to be grabbed before sqciplock
* for lock ordering purposes.
* after sqcipcount is decremented some lock
* still needs to be held to make sure
* syncq won't get freed on us.
*
* To prevent deadlocks we try to grab SQLOCK and if it
* is held already we drop sqciplock, acquire SQLOCK and
* reacqwire sqciplock again.
*/
}
ASSERT(*sqcipcount != 0);
(*sqcipcount)--;
} else {
ASSERT(*sqcipcount != 0);
(*sqcipcount)--;
return;
}
} else {
}
/*
* The only purpose of this ASSERT is to preserve calling stack
* in DEBUG kernel.
*/
return;
}
/*
* Safe to always drop SQ_EXCL:
* Not SQ_CIPUT means we set SQ_EXCL above
* For SQ_CIPUT SQ_EXCL will only be set if the put
* procedure did a qwriter(INNER) in which case
* nobody else is in the inner perimeter and we
* are exiting.
*
* I would like to make the following assertion:
*
* ASSERT((flags & (SQ_EXCL|SQ_CIPUT)) != (SQ_EXCL|SQ_CIPUT) ||
* sq->sq_count == 0);
*
* which indicates that if we are both putshared and exclusive,
* we became exclusive while executing the putproc, and the only
* claim on the syncq was the one we dropped a few lines above.
* But other threads that enter putnext while the syncq is exclusive
* need to make a claim as they may need to drop SQLOCK in the
* has_writers case to avoid deadlocks. If these threads are
* delayed or preempted, it is possible that the writer thread can
* find out that there are other claims making the (sq_count == 0)
* test invalid.
*/
}