/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* UNIX Device Driver Interface functions
*
* This file contains functions that are to be added to the kernel
* to put the interface presented to drivers in conformance with
* the DDI standard. Of the functions added to the kernel, 17 are
* function equivalents of existing macros in sysmacros.h,
* stream.h, and param.h
*
* 17 additional functions -- drv_getparm(), drv_setparm(),
* getrbuf(), freerbuf(),
* getemajor(), geteminor(), etoimajor(), itoemajor(), drv_usectohz(),
* drv_hztousec(), drv_usecwait(), drv_priv(), and kvtoppid() --
* are specified by DDI to exist in the kernel and are implemented here.
*
* Note that putnext() and put() are not in this file. The C version of
* these routines are in uts/common/os/putnext.c and assembly versions
* might exist for some architectures.
*/
#include <sys/types.h>
#include <sys/param.h>
#include <sys/t_lock.h>
#include <sys/time.h>
#include <sys/systm.h>
#include <sys/cpuvar.h>
#include <sys/signal.h>
#include <sys/pcb.h>
#include <sys/user.h>
#include <sys/errno.h>
#include <sys/buf.h>
#include <sys/proc.h>
#include <sys/cmn_err.h>
#include <sys/stream.h>
#include <sys/strsubr.h>
#include <sys/uio.h>
#include <sys/kmem.h>
#include <sys/conf.h>
#include <sys/cred.h>
#include <sys/vnode.h>
#include <sys/file.h>
#include <sys/poll.h>
#include <sys/session.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/esunddi.h>
#include <sys/mkdev.h>
#include <sys/debug.h>
#include <sys/vtrace.h>
/*
* return internal major number corresponding to device
* number (new format) argument
*/
major_t
getmajor(dev_t dev)
{
#ifdef _LP64
return ((major_t)((dev >> NBITSMINOR64) & MAXMAJ64));
#else
return ((major_t)((dev >> NBITSMINOR) & MAXMAJ));
#endif
}
/*
* return external major number corresponding to device
* number (new format) argument
*/
major_t
getemajor(dev_t dev)
{
#ifdef _LP64
return ((major_t)((dev >> NBITSMINOR64) & MAXMAJ64));
#else
return ((major_t)((dev >> NBITSMINOR) & MAXMAJ));
#endif
}
/*
* return internal minor number corresponding to device
* number (new format) argument
*/
minor_t
getminor(dev_t dev)
{
#ifdef _LP64
return ((minor_t)(dev & MAXMIN64));
#else
return ((minor_t)(dev & MAXMIN));
#endif
}
/*
* return external minor number corresponding to device
* number (new format) argument
*/
minor_t
geteminor(dev_t dev)
{
#ifdef _LP64
return ((minor_t)(dev & MAXMIN64));
#else
return ((minor_t)(dev & MAXMIN));
#endif
}
/*
* return internal major number corresponding to external
* major number.
*/
int
etoimajor(major_t emajnum)
{
#ifdef _LP64
if (emajnum >= devcnt)
return (-1); /* invalid external major */
#else
if (emajnum > MAXMAJ || emajnum >= devcnt)
return (-1); /* invalid external major */
#endif
return ((int)emajnum);
}
/*
* return external major number corresponding to internal
* major number argument or -1 if no external major number
* can be found after lastemaj that maps to the internal
* major number. Pass a lastemaj val of -1 to start
* the search initially. (Typical use of this function is
* of the form:
*
* lastemaj = -1;
* while ((lastemaj = itoemajor(imag, lastemaj)) != -1)
* { process major number }
*/
int
itoemajor(major_t imajnum, int lastemaj)
{
if (imajnum >= devcnt)
return (-1);
/*
* if lastemaj == -1 then start from beginning of
* the (imaginary) MAJOR table
*/
if (lastemaj < -1)
return (-1);
/*
* given that there's a 1-1 mapping of internal to external
* major numbers, searching is somewhat pointless ... let's
* just go there directly.
*/
if (++lastemaj < devcnt && imajnum < devcnt)
return (imajnum);
return (-1);
}
/*
* encode external major and minor number arguments into a
* new format device number
*/
dev_t
makedevice(major_t maj, minor_t minor)
{
#ifdef _LP64
return (((dev_t)maj << NBITSMINOR64) | (minor & MAXMIN64));
#else
return (((dev_t)maj << NBITSMINOR) | (minor & MAXMIN));
#endif
}
/*
* cmpdev - compress new device format to old device format
*/
o_dev_t
cmpdev(dev_t dev)
{
major_t major_d;
minor_t minor_d;
#ifdef _LP64
major_d = dev >> NBITSMINOR64;
minor_d = dev & MAXMIN64;
#else
major_d = dev >> NBITSMINOR;
minor_d = dev & MAXMIN;
#endif
if (major_d > OMAXMAJ || minor_d > OMAXMIN)
return ((o_dev_t)NODEV);
return ((o_dev_t)((major_d << ONBITSMINOR) | minor_d));
}
dev_t
expdev(dev_t dev)
{
major_t major_d;
minor_t minor_d;
major_d = ((dev >> ONBITSMINOR) & OMAXMAJ);
minor_d = (dev & OMAXMIN);
#ifdef _LP64
return ((((dev_t)major_d << NBITSMINOR64) | minor_d));
#else
return ((((dev_t)major_d << NBITSMINOR) | minor_d));
#endif
}
/*
* return true (1) if the message type input is a data
* message type, 0 otherwise
*/
#undef datamsg
int
datamsg(unsigned char db_type)
{
return (db_type == M_DATA || db_type == M_PROTO ||
db_type == M_PCPROTO || db_type == M_DELAY);
}
/*
* return a pointer to the other queue in the queue pair of qp
*/
queue_t *
OTHERQ(queue_t *q)
{
return (_OTHERQ(q));
}
/*
* return a pointer to the read queue in the queue pair of qp.
*/
queue_t *
RD(queue_t *q)
{
return (_RD(q));
}
/*
* return a pointer to the write queue in the queue pair of qp.
*/
int
SAMESTR(queue_t *q)
{
return (_SAMESTR(q));
}
/*
* return a pointer to the write queue in the queue pair of qp.
*/
queue_t *
WR(queue_t *q)
{
return (_WR(q));
}
/*
* store value of kernel parameter associated with parm
*/
int
drv_getparm(unsigned int parm, void *valuep)
{
proc_t *p = curproc;
time_t now;
switch (parm) {
case UPROCP:
*(proc_t **)valuep = p;
break;
case PPGRP:
mutex_enter(&p->p_lock);
*(pid_t *)valuep = p->p_pgrp;
mutex_exit(&p->p_lock);
break;
case LBOLT:
*(clock_t *)valuep = ddi_get_lbolt();
break;
case TIME:
if ((now = gethrestime_sec()) == 0) {
timestruc_t ts;
mutex_enter(&tod_lock);
ts = tod_get();
mutex_exit(&tod_lock);
*(time_t *)valuep = ts.tv_sec;
} else {
*(time_t *)valuep = now;
}
break;
case PPID:
*(pid_t *)valuep = p->p_pid;
break;
case PSID:
mutex_enter(&p->p_splock);
*(pid_t *)valuep = p->p_sessp->s_sid;
mutex_exit(&p->p_splock);
break;
case UCRED:
*(cred_t **)valuep = CRED();
break;
default:
return (-1);
}
return (0);
}
/*
* set value of kernel parameter associated with parm
*/
int
drv_setparm(unsigned int parm, unsigned long value)
{
switch (parm) {
case SYSRINT:
CPU_STATS_ADDQ(CPU, sys, rcvint, value);
break;
case SYSXINT:
CPU_STATS_ADDQ(CPU, sys, xmtint, value);
break;
case SYSMINT:
CPU_STATS_ADDQ(CPU, sys, mdmint, value);
break;
case SYSRAWC:
CPU_STATS_ADDQ(CPU, sys, rawch, value);
break;
case SYSCANC:
CPU_STATS_ADDQ(CPU, sys, canch, value);
break;
case SYSOUTC:
CPU_STATS_ADDQ(CPU, sys, outch, value);
break;
default:
return (-1);
}
return (0);
}
/*
* allocate space for buffer header and return pointer to it.
* preferred means of obtaining space for a local buf header.
* returns pointer to buf upon success, NULL for failure
*/
struct buf *
getrbuf(int sleep)
{
struct buf *bp;
bp = kmem_alloc(sizeof (struct buf), sleep);
if (bp == NULL)
return (NULL);
bioinit(bp);
return (bp);
}
/*
* free up space allocated by getrbuf()
*/
void
freerbuf(struct buf *bp)
{
biofini(bp);
kmem_free(bp, sizeof (struct buf));
}
/*
* convert byte count input to logical page units
* (byte counts that are not a page-size multiple
* are rounded down)
*/
pgcnt_t
btop(size_t numbytes)
{
return (numbytes >> PAGESHIFT);
}
/*
* convert byte count input to logical page units
* (byte counts that are not a page-size multiple
* are rounded up)
*/
pgcnt_t
btopr(size_t numbytes)
{
return ((numbytes + PAGEOFFSET) >> PAGESHIFT);
}
/*
* convert size in pages to bytes.
*/
size_t
ptob(pgcnt_t numpages)
{
return (numpages << PAGESHIFT);
}
#define MAXCLOCK_T LONG_MAX
/*
* Convert from system time units (hz) to microseconds.
*
* If ticks <= 0, return 0.
* If converting ticks to usecs would overflow, return MAXCLOCK_T.
* Otherwise, convert ticks to microseconds.
*/
clock_t
drv_hztousec(clock_t ticks)
{
if (ticks <= 0)
return (0);
if (ticks > MAXCLOCK_T / usec_per_tick)
return (MAXCLOCK_T);
return (TICK_TO_USEC(ticks));
}
/*
* Convert from microseconds to system time units (hz), rounded up.
*
* If ticks <= 0, return 0.
* Otherwise, convert microseconds to ticks, rounding up.
*/
clock_t
drv_usectohz(clock_t microsecs)
{
if (microsecs <= 0)
return (0);
return (USEC_TO_TICK_ROUNDUP(microsecs));
}
#ifdef sun
/*
* drv_usecwait implemented in each architecture's machine
* specific code somewhere. For sparc, it is the alternate entry
* to usec_delay (eventually usec_delay goes away). See
* sparc/os/ml/sparc_subr.s
*/
#endif
/*
* bcanputnext, canputnext assume called from timeout, bufcall,
* or esballoc free routines. since these are driven by
* clock interrupts, instead of system calls the appropriate plumbing
* locks have not been acquired.
*/
int
bcanputnext(queue_t *q, unsigned char band)
{
int ret;
claimstr(q);
ret = bcanput(q->q_next, band);
releasestr(q);
return (ret);
}
int
canputnext(queue_t *q)
{
queue_t *qofsq = q;
struct stdata *stp = STREAM(q);
kmutex_t *sdlock;
TRACE_1(TR_FAC_STREAMS_FR, TR_CANPUTNEXT_IN,
"canputnext?:%p\n", q);
if (stp->sd_ciputctrl != NULL) {
int ix = CPU->cpu_seqid & stp->sd_nciputctrl;
sdlock = &stp->sd_ciputctrl[ix].ciputctrl_lock;
mutex_enter(sdlock);
} else
mutex_enter(sdlock = &stp->sd_reflock);
/* get next module forward with a service queue */
q = q->q_next->q_nfsrv;
ASSERT(q != NULL);
/* this is for loopback transports, they should not do a canputnext */
ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(qofsq));
if (!(q->q_flag & QFULL)) {
mutex_exit(sdlock);
TRACE_2(TR_FAC_STREAMS_FR, TR_CANPUTNEXT_OUT,
"canputnext:%p %d", q, 1);
return (1);
}
if (sdlock != &stp->sd_reflock) {
mutex_exit(sdlock);
mutex_enter(&stp->sd_reflock);
}
/* the above is the most frequently used path */
stp->sd_refcnt++;
ASSERT(stp->sd_refcnt != 0); /* Wraparound */
mutex_exit(&stp->sd_reflock);
mutex_enter(QLOCK(q));
if (q->q_flag & QFULL) {
q->q_flag |= QWANTW;
mutex_exit(QLOCK(q));
TRACE_2(TR_FAC_STREAMS_FR, TR_CANPUTNEXT_OUT,
"canputnext:%p %d", q, 0);
releasestr(qofsq);
return (0);
}
mutex_exit(QLOCK(q));
TRACE_2(TR_FAC_STREAMS_FR, TR_CANPUTNEXT_OUT, "canputnext:%p %d", q, 1);
releasestr(qofsq);
return (1);
}
/*
* Open has progressed to the point where it is safe to send/receive messages.
*
* "qprocson enables the put and service routines of the driver
* or module... Prior to the call to qprocson, the put and service
* routines of a newly pushed module or newly opened driver are
* disabled. For the module, messages flow around it as if it
* were not present in the stream... qprocson must be called by
* the first open of a module or driver after allocation and
* initialization of any resource on which the put and service
* routines depend."
*
* Note that before calling qprocson a module/driver could itself cause its
* put or service procedures to be run by using put() or qenable().
*/
void
qprocson(queue_t *q)
{
ASSERT(q->q_flag & QREADR);
/*
* Do not call insertq() if it is a re-open. But if _QINSERTING
* is set, q_next will not be NULL and we need to call insertq().
*/
if ((q->q_next == NULL && WR(q)->q_next == NULL) ||
(q->q_flag & _QINSERTING))
insertq(STREAM(q), q);
}
/*
* Close has reached a point where it can no longer allow put/service
* into the queue.
*
* "qprocsoff disables the put and service routines of the driver
* or module... When the routines are disabled in a module, messages
* flow around the module as if it were not present in the stream.
* qprocsoff must be called by the close routine of a driver or module
* before deallocating any resources on which the driver/module's
* put and service routines depend. qprocsoff will remove the
* queue's service routines from the list of service routines to be
* run and waits until any concurrent put or service routines are
* finished."
*
* Note that after calling qprocsoff a module/driver could itself cause its
* put procedures to be run by using put().
*/
void
qprocsoff(queue_t *q)
{
ASSERT(q->q_flag & QREADR);
if (q->q_flag & QWCLOSE) {
/* Called more than once */
return;
}
disable_svc(q);
removeq(q);
}
/*
* "freezestr() freezes the state of the entire STREAM containing
* the queue pair q. A frozen STREAM blocks any thread
* attempting to enter any open, close, put or service routine
* belonging to any queue instance in the STREAM, and blocks
* any thread currently within the STREAM if it attempts to put
* messages onto or take messages off of any queue within the
* STREAM (with the sole exception of the caller). Threads
* blocked by this mechanism remain so until the STREAM is
* thawed by a call to unfreezestr().
*
* Use strblock to set SQ_FROZEN in all syncqs in the stream (prevents
* further entry into put, service, open, and close procedures) and
* grab (and hold) all the QLOCKs in the stream (to block putq, getq etc.)
*
* Note: this has to be the only code that acquires one QLOCK while holding
* another QLOCK (otherwise we would have locking hirarchy/ordering violations.)
*/
void
freezestr(queue_t *q)
{
struct stdata *stp = STREAM(q);
/*
* Increment refcnt to prevent q_next from changing during the strblock
* as well as while the stream is frozen.
*/
claimstr(RD(q));
strblock(q);
ASSERT(stp->sd_freezer == NULL);
stp->sd_freezer = curthread;
for (q = stp->sd_wrq; q != NULL; q = SAMESTR(q) ? q->q_next : NULL) {
mutex_enter(QLOCK(q));
mutex_enter(QLOCK(RD(q)));
}
}
/*
* Undo what freezestr did.
* Have to drop the QLOCKs before the strunblock since strunblock will
* potentially call other put procedures.
*/
void
unfreezestr(queue_t *q)
{
struct stdata *stp = STREAM(q);
queue_t *q1;
for (q1 = stp->sd_wrq; q1 != NULL;
q1 = SAMESTR(q1) ? q1->q_next : NULL) {
mutex_exit(QLOCK(q1));
mutex_exit(QLOCK(RD(q1)));
}
ASSERT(stp->sd_freezer == curthread);
stp->sd_freezer = NULL;
strunblock(q);
releasestr(RD(q));
}
/*
* Used by open and close procedures to "sleep" waiting for messages to
* arrive. Note: can only be used in open and close procedures.
*
* Lower the gate and let in either messages on the syncq (if there are
* any) or put/service procedures.
*
* If the queue has an outer perimeter this will not prevent entry into this
* syncq (since outer_enter does not set SQ_WRITER on the syncq that gets the
* exclusive access to the outer perimeter.)
*
* Return 0 is the cv_wait_sig was interrupted; otherwise 1.
*
* It only makes sense to grab sq_putlocks for !SQ_CIOC sync queues because
* otherwise put entry points were not blocked in the first place. if this is
* SQ_CIOC then qwait is used to wait for service procedure to run since syncq
* is always SQ_CIPUT if it is SQ_CIOC.
*
* Note that SQ_EXCL is dropped and SQ_WANTEXITWAKEUP set in sq_flags
* atomically under sq_putlocks to make sure putnext will not miss a pending
* wakeup.
*/
int
qwait_sig(queue_t *q)
{
syncq_t *sq, *outer;
uint_t flags;
int ret = 1;
int is_sq_cioc;
/*
* Perform the same operations as a leavesq(sq, SQ_OPENCLOSE)
* while detecting all cases where the perimeter is entered
* so that qwait_sig can return to the caller.
*
* Drain the syncq if possible. Otherwise reset SQ_EXCL and
* wait for a thread to leave the syncq.
*/
sq = q->q_syncq;
ASSERT(sq);
is_sq_cioc = (sq->sq_type & SQ_CIOC) ? 1 : 0;
ASSERT(sq->sq_outer == NULL || sq->sq_outer->sq_flags & SQ_WRITER);
outer = sq->sq_outer;
/*
* XXX this does not work if there is only an outer perimeter.
* The semantics of qwait/qwait_sig are undefined in this case.
*/
if (outer)
outer_exit(outer);
mutex_enter(SQLOCK(sq));
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_ENTER(sq);
}
flags = sq->sq_flags;
/*
* Drop SQ_EXCL and sq_count but hold the SQLOCK
* to prevent any undetected entry and exit into the perimeter.
*/
ASSERT(sq->sq_count > 0);
sq->sq_count--;
if (is_sq_cioc == 0) {
ASSERT(flags & SQ_EXCL);
flags &= ~SQ_EXCL;
}
/*
* Unblock any thread blocked in an entersq or outer_enter.
* Note: we do not unblock a thread waiting in qwait/qwait_sig,
* since that could lead to livelock with two threads in
* qwait for the same (per module) inner perimeter.
*/
if (flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
flags &= ~SQ_WANTWAKEUP;
}
sq->sq_flags = flags;
if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_EXIT(sq);
}
/* drain_syncq() drops SQLOCK */
drain_syncq(sq);
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
entersq(sq, SQ_OPENCLOSE);
return (1);
}
/*
* Sleep on sq_exitwait to only be woken up when threads leave the
* put or service procedures. We can not sleep on sq_wait since an
* outer_exit in a qwait running in the same outer perimeter would
* cause a livelock "ping-pong" between two or more qwait'ers.
*/
do {
sq->sq_flags |= SQ_WANTEXWAKEUP;
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_EXIT(sq);
}
ret = cv_wait_sig(&sq->sq_exitwait, SQLOCK(sq));
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_ENTER(sq);
}
} while (ret && (sq->sq_flags & SQ_WANTEXWAKEUP));
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_EXIT(sq);
}
mutex_exit(SQLOCK(sq));
/*
* Re-enter the perimeters again
*/
entersq(sq, SQ_OPENCLOSE);
return (ret);
}
/*
* Used by open and close procedures to "sleep" waiting for messages to
* arrive. Note: can only be used in open and close procedures.
*
* Lower the gate and let in either messages on the syncq (if there are
* any) or put/service procedures.
*
* If the queue has an outer perimeter this will not prevent entry into this
* syncq (since outer_enter does not set SQ_WRITER on the syncq that gets the
* exclusive access to the outer perimeter.)
*
* It only makes sense to grab sq_putlocks for !SQ_CIOC sync queues because
* otherwise put entry points were not blocked in the first place. if this is
* SQ_CIOC then qwait is used to wait for service procedure to run since syncq
* is always SQ_CIPUT if it is SQ_CIOC.
*
* Note that SQ_EXCL is dropped and SQ_WANTEXITWAKEUP set in sq_flags
* atomically under sq_putlocks to make sure putnext will not miss a pending
* wakeup.
*/
void
qwait(queue_t *q)
{
syncq_t *sq, *outer;
uint_t flags;
int is_sq_cioc;
/*
* Perform the same operations as a leavesq(sq, SQ_OPENCLOSE)
* while detecting all cases where the perimeter is entered
* so that qwait can return to the caller.
*
* Drain the syncq if possible. Otherwise reset SQ_EXCL and
* wait for a thread to leave the syncq.
*/
sq = q->q_syncq;
ASSERT(sq);
is_sq_cioc = (sq->sq_type & SQ_CIOC) ? 1 : 0;
ASSERT(sq->sq_outer == NULL || sq->sq_outer->sq_flags & SQ_WRITER);
outer = sq->sq_outer;
/*
* XXX this does not work if there is only an outer perimeter.
* The semantics of qwait/qwait_sig are undefined in this case.
*/
if (outer)
outer_exit(outer);
mutex_enter(SQLOCK(sq));
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_ENTER(sq);
}
flags = sq->sq_flags;
/*
* Drop SQ_EXCL and sq_count but hold the SQLOCK
* to prevent any undetected entry and exit into the perimeter.
*/
ASSERT(sq->sq_count > 0);
sq->sq_count--;
if (is_sq_cioc == 0) {
ASSERT(flags & SQ_EXCL);
flags &= ~SQ_EXCL;
}
/*
* Unblock any thread blocked in an entersq or outer_enter.
* Note: we do not unblock a thread waiting in qwait/qwait_sig,
* since that could lead to livelock with two threads in
* qwait for the same (per module) inner perimeter.
*/
if (flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
flags &= ~SQ_WANTWAKEUP;
}
sq->sq_flags = flags;
if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_EXIT(sq);
}
/* drain_syncq() drops SQLOCK */
drain_syncq(sq);
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
entersq(sq, SQ_OPENCLOSE);
return;
}
/*
* Sleep on sq_exitwait to only be woken up when threads leave the
* put or service procedures. We can not sleep on sq_wait since an
* outer_exit in a qwait running in the same outer perimeter would
* cause a livelock "ping-pong" between two or more qwait'ers.
*/
do {
sq->sq_flags |= SQ_WANTEXWAKEUP;
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_EXIT(sq);
}
cv_wait(&sq->sq_exitwait, SQLOCK(sq));
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_ENTER(sq);
}
} while (sq->sq_flags & SQ_WANTEXWAKEUP);
if (is_sq_cioc == 0) {
SQ_PUTLOCKS_EXIT(sq);
}
mutex_exit(SQLOCK(sq));
/*
* Re-enter the perimeters again
*/
entersq(sq, SQ_OPENCLOSE);
}
/*
* Used for the synchronous streams entrypoints when sleeping outside
* the perimeters. Must never be called from regular put entrypoint.
*
* There's no need to grab sq_putlocks here (which only exist for CIPUT sync
* queues). If it is CIPUT sync queue put entry points were not blocked in the
* first place by rwnext/infonext which are treated as put entrypoints for
* permiter syncronization purposes.
*
* Consolidation private.
*/
boolean_t
qwait_rw(queue_t *q)
{
syncq_t *sq;
ulong_t flags;
boolean_t gotsignal = B_FALSE;
/*
* Perform the same operations as a leavesq(sq, SQ_PUT)
* while detecting all cases where the perimeter is entered
* so that qwait_rw can return to the caller.
*
* Drain the syncq if possible. Otherwise reset SQ_EXCL and
* wait for a thread to leave the syncq.
*/
sq = q->q_syncq;
ASSERT(sq);
mutex_enter(SQLOCK(sq));
flags = sq->sq_flags;
/*
* Drop SQ_EXCL and sq_count but hold the SQLOCK until to prevent any
* undetected entry and exit into the perimeter.
*/
ASSERT(sq->sq_count > 0);
sq->sq_count--;
if (!(sq->sq_type & SQ_CIPUT)) {
ASSERT(flags & SQ_EXCL);
flags &= ~SQ_EXCL;
}
/*
* Unblock any thread blocked in an entersq or outer_enter.
* Note: we do not unblock a thread waiting in qwait/qwait_sig,
* since that could lead to livelock with two threads in
* qwait for the same (per module) inner perimeter.
*/
if (flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
flags &= ~SQ_WANTWAKEUP;
}
sq->sq_flags = flags;
if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
/* drain_syncq() drops SQLOCK */
drain_syncq(sq);
ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
entersq(sq, SQ_PUT);
return (B_FALSE);
}
/*
* Sleep on sq_exitwait to only be woken up when threads leave the
* put or service procedures. We can not sleep on sq_wait since an
* outer_exit in a qwait running in the same outer perimeter would
* cause a livelock "ping-pong" between two or more qwait'ers.
*/
do {
sq->sq_flags |= SQ_WANTEXWAKEUP;
if (cv_wait_sig(&sq->sq_exitwait, SQLOCK(sq)) <= 0) {
sq->sq_flags &= ~SQ_WANTEXWAKEUP;
gotsignal = B_TRUE;
break;
}
} while (sq->sq_flags & SQ_WANTEXWAKEUP);
mutex_exit(SQLOCK(sq));
/*
* Re-enter the perimeters again
*/
entersq(sq, SQ_PUT);
return (gotsignal);
}
/*
* Asynchronously upgrade to exclusive access at either the inner or
* outer perimeter.
*/
void
qwriter(queue_t *q, mblk_t *mp, void (*func)(), int perim)
{
if (perim == PERIM_INNER)
qwriter_inner(q, mp, func);
else if (perim == PERIM_OUTER)
qwriter_outer(q, mp, func);
else
panic("qwriter: wrong \"perimeter\" parameter");
}
/*
* Schedule a synchronous streams timeout
*/
timeout_id_t
qtimeout(queue_t *q, void (*func)(void *), void *arg, clock_t tim)
{
syncq_t *sq;
callbparams_t *cbp;
timeout_id_t tid;
sq = q->q_syncq;
/*
* you don't want the timeout firing before its params are set up
* callbparams_alloc() acquires SQLOCK(sq)
* qtimeout() can't fail and can't sleep, so panic if memory is not
* available.
*/
cbp = callbparams_alloc(sq, func, arg, KM_NOSLEEP | KM_PANIC);
/*
* the callbflags in the sq use the same flags. They get anded
* in the callbwrapper to determine if a qun* of this callback type
* is required. This is not a request to cancel.
*/
cbp->cbp_flags = SQ_CANCEL_TOUT;
/* check new timeout version return codes */
tid = timeout(qcallbwrapper, cbp, tim);
cbp->cbp_id = (callbparams_id_t)tid;
mutex_exit(SQLOCK(sq));
/* use local id because the cbp memory could be free by now */
return (tid);
}
bufcall_id_t
qbufcall(queue_t *q, size_t size, uint_t pri, void (*func)(void *), void *arg)
{
syncq_t *sq;
callbparams_t *cbp;
bufcall_id_t bid;
sq = q->q_syncq;
/*
* you don't want the timeout firing before its params are set up
* callbparams_alloc() acquires SQLOCK(sq) if successful.
*/
cbp = callbparams_alloc(sq, func, arg, KM_NOSLEEP);
if (cbp == NULL)
return ((bufcall_id_t)0);
/*
* the callbflags in the sq use the same flags. They get anded
* in the callbwrapper to determine if a qun* of this callback type
* is required. This is not a request to cancel.
*/
cbp->cbp_flags = SQ_CANCEL_BUFCALL;
/* check new timeout version return codes */
bid = bufcall(size, pri, qcallbwrapper, cbp);
cbp->cbp_id = (callbparams_id_t)bid;
if (bid == 0) {
callbparams_free(sq, cbp);
}
mutex_exit(SQLOCK(sq));
/* use local id because the params memory could be free by now */
return (bid);
}
/*
* cancel a timeout callback which enters the inner perimeter.
* cancelling of all callback types on a given syncq is serialized.
* the SQ_CALLB_BYPASSED flag indicates that the callback fn did
* not execute. The quntimeout return value needs to reflect this.
* As with out existing callback programming model - callbacks must
* be cancelled before a close completes - so ensuring that the sq
* is valid when the callback wrapper is executed.
*/
clock_t
quntimeout(queue_t *q, timeout_id_t id)
{
syncq_t *sq = q->q_syncq;
clock_t ret;
mutex_enter(SQLOCK(sq));
/* callbacks are processed serially on each syncq */
while (sq->sq_callbflags & SQ_CALLB_CANCEL_MASK) {
sq->sq_flags |= SQ_WANTWAKEUP;
cv_wait(&sq->sq_wait, SQLOCK(sq));
}
sq->sq_cancelid = (callbparams_id_t)id;
sq->sq_callbflags = SQ_CANCEL_TOUT;
if (sq->sq_flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
sq->sq_flags &= ~SQ_WANTWAKEUP;
}
mutex_exit(SQLOCK(sq));
ret = untimeout(id);
mutex_enter(SQLOCK(sq));
if (ret != -1) {
/* The wrapper was never called - need to free based on id */
callbparams_free_id(sq, (callbparams_id_t)id, SQ_CANCEL_TOUT);
}
if (sq->sq_callbflags & SQ_CALLB_BYPASSED) {
ret = 0; /* this was how much time left */
}
sq->sq_callbflags = 0;
if (sq->sq_flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
sq->sq_flags &= ~SQ_WANTWAKEUP;
}
mutex_exit(SQLOCK(sq));
return (ret);
}
void
qunbufcall(queue_t *q, bufcall_id_t id)
{
syncq_t *sq = q->q_syncq;
mutex_enter(SQLOCK(sq));
/* callbacks are processed serially on each syncq */
while (sq->sq_callbflags & SQ_CALLB_CANCEL_MASK) {
sq->sq_flags |= SQ_WANTWAKEUP;
cv_wait(&sq->sq_wait, SQLOCK(sq));
}
sq->sq_cancelid = (callbparams_id_t)id;
sq->sq_callbflags = SQ_CANCEL_BUFCALL;
if (sq->sq_flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
sq->sq_flags &= ~SQ_WANTWAKEUP;
}
mutex_exit(SQLOCK(sq));
unbufcall(id);
mutex_enter(SQLOCK(sq));
/*
* No indication from unbufcall if the callback has already run.
* Always attempt to free it.
*/
callbparams_free_id(sq, (callbparams_id_t)id, SQ_CANCEL_BUFCALL);
sq->sq_callbflags = 0;
if (sq->sq_flags & SQ_WANTWAKEUP) {
cv_broadcast(&sq->sq_wait);
sq->sq_flags &= ~SQ_WANTWAKEUP;
}
mutex_exit(SQLOCK(sq));
}
/*
* Associate the stream with an instance of the bottom driver. This
* function is called by APIs that establish or modify the hardware
* association (ppa) of an open stream. Two examples of such
* post-open(9E) APIs are the dlpi(7p) DL_ATTACH_REQ message, and the
* ndd(1M) "instance=" ioctl(2). This interface may be called from a
* stream driver's wput procedure and from within syncq perimeters,
* so it can't block.
*
* The qassociate() "model" is that it should drive attach(9E), yet it
* can't really do that because driving attach(9E) is a blocking
* operation. Instead, the qassociate() implementation has complex
* dependencies on the implementation behavior of other parts of the
* kernel to ensure all appropriate instances (ones that have not been
* made inaccessible by DR) are attached at stream open() time, and
* that they will not autodetach. The code relies on the fact that an
* open() of a stream that ends up using qassociate() always occurs on
* a minor node created with CLONE_DEV. The open() comes through
* clnopen() and since clnopen() calls ddi_hold_installed_driver() we
* attach all instances and mark them DN_NO_AUTODETACH (given
* DN_DRIVER_HELD is maintained correctly).
*
* Since qassociate() can't really drive attach(9E), there are corner
* cases where the compromise described above leads to qassociate()
* returning failure. This can happen when administrative functions
* that cause detach(9E), such as "update_drv" or "modunload -i", are
* performed on the driver between the time the stream was opened and
* the time its hardware association was established. Although this can
* theoretically be an arbitrary amount of time, in practice the window
* is usually quite small, since applications almost always issue their
* hardware association request immediately after opening the stream,
* and do not typically switch association while open. When these
* corner cases occur, and qassociate() finds the requested instance
* detached, it will return failure. This failure should be propagated
* to the requesting administrative application using the appropriate
* post-open(9E) API error mechanism.
*
* All qassociate() callers are expected to check for and gracefully handle
* failure return, propagating errors back to the requesting administrative
* application.
*/
int
qassociate(queue_t *q, int instance)
{
vnode_t *vp;
major_t major;
dev_info_t *dip;
if (instance == -1) {
ddi_assoc_queue_with_devi(q, NULL);
return (0);
}
vp = STREAM(q)->sd_vnode;
major = getmajor(vp->v_rdev);
dip = ddi_hold_devi_by_instance(major, instance,
E_DDI_HOLD_DEVI_NOATTACH);
if (dip == NULL)
return (-1);
ddi_assoc_queue_with_devi(q, dip);
ddi_release_devi(dip);
return (0);
}
/*
* This routine is the SVR4MP 'replacement' for
* hat_getkpfnum. The only major difference is
* the return value for illegal addresses - since
* sunm_getkpfnum() and srmmu_getkpfnum() both
* return '-1' for bogus mappings, we can (more or
* less) return the value directly.
*/
ppid_t
kvtoppid(caddr_t addr)
{
return ((ppid_t)hat_getpfnum(kas.a_hat, addr));
}
/*
* This is used to set the timeout value for cv_timed_wait() or
* cv_timedwait_sig().
*/
void
time_to_wait(clock_t *now, clock_t time)
{
*now = ddi_get_lbolt() + time;
}