x_call.c revision ae115bc77f6fcde83175c75b4206dc2e50747966
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* Facilities for cross-processor subroutine calls using "mailbox" interrupts.
*
*/
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/mutex_impl.h>
#include <sys/traptrace.h>
};
int, cpuset_t, int);
static int xc_initialized = 0;
void
xc_init()
{
/*
* By making these mutexes type MUTEX_DRIVER, the ones below
* LOCK_LEVEL will be implemented as adaptive mutexes, and the
* ones above LOCK_LEVEL will be spin mutexes.
*/
(void *)ipltospl(XC_MED_PIL));
xc_initialized = 1;
}
#if defined(TRAPTRACE)
/*
* When xc_traptrace is on, put x-call records into the trap trace buffer.
*/
int xc_traptrace;
void
{
if (xc_traptrace == 0)
return;
}
}
#endif
#define CAPTURE_CPU_ARG ~0UL
/*
* X-call interrupt service routine.
*
* arg == X_CALL_MEDPRI - capture cpus.
*
* We're protected against changing CPUs by being a high-priority interrupt.
*/
/*ARGSUSED*/
{
int op;
if (pri == X_CALL_MEDPRI) {
if (arg2val != CAPTURE_CPU_ARG &&
goto unclaimed;
goto unclaimed;
for (;;) {
break;
SMT_PAUSE();
}
return (DDI_INTR_CLAIMED);
}
goto unclaimed;
/*
* Don't invoke a null function.
*/
else
/*
* Acknowledge that we have completed the x-call operation.
*/
if (op != XC_CALL_OP) {
/*
* for (op == XC_SYNC_OP)
* Wait for the initiator of the x-call to indicate
* that all CPUs involved can proceed.
*/
SMT_PAUSE();
SMT_PAUSE();
/*
* Acknowledge that we have received the directive to continue.
*/
}
return (DDI_INTR_CLAIMED);
return (DDI_INTR_UNCLAIMED);
}
/*
* xc_do_call:
*/
static void
int pri,
int sync)
{
/*
* If the pri indicates a low priority lock (below LOCK_LEVEL),
* we must disable preemption to avoid migrating to another CPU
* during the call.
*/
if (pri == X_CALL_LOPRI) {
} else {
pri = X_CALL_HIPRI;
}
/* always grab highest mutex to avoid deadlock */
if (pri == X_CALL_LOPRI)
}
/*
* xc_call: call specified function on all processors
* remotes may continue after service
* we wait here until everybody has completed.
*/
void
int pri,
{
}
/*
* xc_sync: call specified function on all processors
* after doing work, each remote waits until we let
* it continue; send the contiunue after everyone has
* informed us that they are done.
*/
void
int pri,
{
}
/*
* The routines xc_capture_cpus and xc_release_cpus
* can be used in place of xc_sync in order to implement a critical
* code section where all CPUs in the system can be controlled.
* xc_capture_cpus is used to start the critical code section, and
* xc_release_cpus is used to end the critical code section.
*/
/*
* Capture the CPUs specified in order to start a x-call session,
*/
void
{
int cix;
int lcx;
int i;
cpuset_t c;
/*
* Prevent deadlocks where we take an interrupt and are waiting
* for a mutex owned by one of the CPUs that is captured for
* the x-call, while that CPU is waiting for some x-call signal
* to be set by us.
*
* This mutex also prevents preemption, since it raises SPL above
* LOCK_LEVEL (it is a spin-type driver mutex).
*/
/* always grab highest mutex to avoid deadlock */
/*
* Wait for all cpus
*/
for (;;) {
CPUSET_AND(c, cpu_ready_set);
if (CPUSET_ISNULL(c))
break;
SMT_PAUSE();
}
/*
* Store the set of CPUs involved in the x-call session, so that
* xc_release_cpus will know what CPUs to act upon.
*/
/*
* Now capture each CPU in the set and cause it to go into a
* holding pattern.
*/
i = 0;
/*
* In case CPU wasn't ready, but becomes ready later,
* take the CPU out of the set now.
*/
continue;
}
}
i++;
if (i >= ncpus)
break;
}
/*
* Wait here until all remote calls to acknowledge.
*/
i = 0;
SMT_PAUSE();
}
i++;
if (i >= ncpus)
break;
}
}
/*
* Release the CPUs captured by xc_capture_cpus, thus terminating the
* x-call session and exiting the critical section.
*/
void
xc_release_cpus(void)
{
int cix;
int i;
/*
* Allow each CPU to exit its holding pattern.
*/
i = 0;
continue;
/*
* Clear xc_ack since we will be waiting for it
* to be set again after we set XC_DONE.
*/
}
i++;
if (i >= ncpus)
break;
}
}
/*
* Common code to call a specified function on a set of processors.
* sync specifies what kind of waiting is done.
* -1 - no waiting, don't release remotes
* 0 - no waiting, release remotes immediately
* 1 - run service locally w/o waiting for remotes.
*/
static void
int pri,
int sync)
{
int cix;
/*
* Set up the service definition mailbox.
*/
/*
* Request service on all remote processors.
*/
/*
* In case the non-local CPU is not ready but becomes
* ready later, take it out of the set now. The local
* CPU needs to remain in the set to complete the
* requested function.
*/
if (sync > 0)
else
}
}
/*
* Run service locally.
*/
}
if (sync == -1)
return;
/*
* Wait here until all remote calls acknowledge.
*/
SMT_PAUSE();
}
}
if (sync == 0)
return;
/*
* Release any waiting CPUs
*/
}
}
}
/*
* Wait for all CPUs to acknowledge completion before we continue.
* Without this check it's possible (on a VM or hyper-threaded CPUs
* or in the presence of Service Management Interrupts which can all
* cause delays) for the remote processor to still be waiting by
* the time xc_common() is next invoked with the sync flag set
* resulting in a deadlock.
*/
SMT_PAUSE();
}
}
}
}
/*
* xc_trycall: attempt to call specified function on all processors
* remotes may wait for a long time
* we continue immediately
*/
void
{
extern int IGNORE_KERNEL_PREEMPTION;
/*
* If we can grab the mutex, we'll do the cross-call. If not -- if
* someone else is already doing a cross-call -- we won't.
*/
}
}
/*
* Used by the debugger to cross-call the other CPUs, thus causing them to
* enter the debugger. We can't hold locks, so we spin on the cross-call
* lock until we get it. When we get it, we send the cross-call, and assume
* that we successfully stopped the other CPUs.
*/
void
{
extern int IGNORE_KERNEL_PREEMPTION;
int x;
if (!xc_initialized)
return;
for (x = 0; x < 0x400000; x++) {
set, -1);
break;
}
}
}