x_call.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* Facilities for cross-processor subroutine calls using "mailbox" interrupts.
*
*/
#include <sys/machsystm.h>
#include <sys/mutex_impl.h>
};
int, cpuset_t, int);
static int xc_initialized = 0;
extern ulong_t cpu_ready_set;
void
xc_init()
{
/*
* By making these mutexes type MUTEX_DRIVER, the ones below
* LOCK_LEVEL will be implemented as adaptive mutexes, and the
* ones above LOCK_LEVEL will be spin mutexes.
*/
(void *)ipltospl(XC_MED_PIL));
xc_initialized = 1;
}
/*
* Used by the debugger to determine whether or not cross calls have been
* initialized and are safe to use.
*/
int
kdi_xc_initialized(void)
{
return (xc_initialized);
}
#define CAPTURE_CPU_ARG 0xffffffff
/*
* X-call interrupt service routine.
*
* arg == X_CALL_MEDPRI - capture cpus.
*
* We're protected against changing CPUs by being a high-priority interrupt.
*/
/*ARGSUSED*/
{
int op;
if (pri == X_CALL_MEDPRI) {
if (arg2val != CAPTURE_CPU_ARG &&
return (DDI_INTR_UNCLAIMED);
return (DDI_INTR_UNCLAIMED);
for (;;) {
break;
ht_pause();
return_instr();
}
return (DDI_INTR_CLAIMED);
}
return (DDI_INTR_UNCLAIMED);
/*
* When invalidating TLB entries, wait until the initiator changes the
* memory PTE before doing any INVLPG. Otherwise, if the PTE in memory
* hasn't been changed, the processor's TLB Flush filter may ignore
* the INVLPG instruction.
*/
/*
* Don't invoke a null function.
*/
if (!tlbflush)
} else
/*
* Acknowledge that we have completed the x-call operation.
*/
if (op == XC_CALL_OP)
return (DDI_INTR_CLAIMED);
/*
* for (op == XC_SYNC_OP)
* Wait for the initiator of the x-call to indicate
* that all CPUs involved can proceed.
*/
ht_pause();
return_instr();
}
ht_pause();
return_instr();
}
/*
* Flush the TLB, if that's what is requested.
*/
}
/*
* Acknowledge that we have received the directive to continue.
*/
return (DDI_INTR_CLAIMED);
}
/*
* xc_do_call:
*/
static void
int pri,
int sync)
{
/*
* If the pri indicates a low priority lock (below LOCK_LEVEL),
* we must disable preemption to avoid migrating to another CPU
* during the call.
*/
if (pri == X_CALL_LOPRI) {
} else {
pri = X_CALL_HIPRI;
}
/* always grab highest mutex to avoid deadlock */
if (pri == X_CALL_LOPRI)
}
/*
* xc_call: call specified function on all processors
* remotes may continue after service
* we wait here until everybody has completed.
*/
void
int pri,
{
}
/*
* xc_sync: call specified function on all processors
* after doing work, each remote waits until we let
* it continue; send the contiunue after everyone has
* informed us that they are done.
*/
void
int pri,
{
}
/*
* xc_sync_wait: similar to xc_sync(), except that the starting
* cpu waits for all other cpus to check in before running its
* service locally.
*/
void
int pri,
{
}
/*
* The routines xc_capture_cpus and xc_release_cpus
* can be used in place of xc_sync in order to implement a critical
* code section where all CPUs in the system can be controlled.
* xc_capture_cpus is used to start the critical code section, and
* xc_release_cpus is used to end the critical code section.
*/
/*
* Capture the CPUs specified in order to start a x-call session,
*/
void
{
int cix;
int lcx;
int i;
cpuset_t c;
/*
* Prevent deadlocks where we take an interrupt and are waiting
* for a mutex owned by one of the CPUs that is captured for
* the x-call, while that CPU is waiting for some x-call signal
* to be set by us.
*
* This mutex also prevents preemption, since it raises SPL above
* LOCK_LEVEL (it is a spin-type driver mutex).
*/
/* always grab highest mutex to avoid deadlock */
/*
* Wait for all cpus
*/
for (;;) {
CPUSET_AND(c, cpu_ready_set);
if (CPUSET_ISNULL(c))
break;
ht_pause();
}
/*
* Store the set of CPUs involved in the x-call session, so that
* xc_release_cpus will know what CPUs to act upon.
*/
/*
* Now capture each CPU in the set and cause it to go into a
* holding pattern.
*/
i = 0;
/*
* In case CPU wasn't ready, but becomes ready later,
* take the CPU out of the set now.
*/
continue;
}
}
i++;
if (i >= ncpus)
break;
}
/*
* Wait here until all remote calls to complete.
*/
i = 0;
ht_pause();
return_instr();
}
}
i++;
if (i >= ncpus)
break;
}
}
/*
* Release the CPUs captured by xc_capture_cpus, thus terminating the
* x-call session and exiting the critical section.
*/
void
xc_release_cpus(void)
{
int cix;
int i;
/*
* Allow each CPU to exit its holding pattern.
*/
i = 0;
continue;
/*
* Clear xc_ack since we will be waiting for it
* to be set again after we set XC_DONE.
*/
}
i++;
if (i >= ncpus)
break;
}
}
/*
* Common code to call a specified function on a set of processors.
* sync specifies what kind of waiting is done.
* -1 - no waiting, don't release remotes
* 0 - no waiting, release remotes immediately
* 1 - run service locally w/o waiting for remotes.
* 2 - wait for remotes before running locally
*/
static void
int pri,
int sync)
{
int cix;
/*
* Set up the service definition mailbox.
*/
/*
* Request service on all remote processors.
*/
/*
* In case CPU wasn't ready, but becomes ready later,
* take the CPU out of the set now.
*/
if (sync > 0)
else
}
}
/*
* Run service locally if not waiting for remotes.
*/
if (sync == -1)
return;
/*
* Wait here until all remote calls complete.
*/
ht_pause();
return_instr();
}
}
}
/*
* Run service locally if waiting for remotes.
*/
if (sync == 0)
return;
/*
* Release any waiting CPUs
*/
}
}
}
/*
* Wait for all CPUs to acknowledge completion before we continue.
* Without this check it's possible (on a VM or hyper-threaded CPUs
* or in the presence of Service Management Interrupts which can all
* cause delays) for the remote processor to still be waiting by
* the time xc_common() is next invoked with the sync flag set
* resulting in a deadlock.
*/
ht_pause();
return_instr();
}
}
}
}
}
/*
* xc_trycall: attempt to call specified function on all processors
* remotes may wait for a long time
* we continue immediately
*/
void
{
extern int IGNORE_KERNEL_PREEMPTION;
/*
* If we can grab the mutex, we'll do the cross-call. If not -- if
* someone else is already doing a cross-call -- we won't.
*/
}
}
/*
* Used by the debugger to cross-call the other CPUs, thus causing them to
* enter the debugger. We can't hold locks, so we spin on the cross-call
* lock until we get it. When we get it, we send the cross-call, and assume
* that we successfully stopped the other CPUs.
*/
void
{
extern int IGNORE_KERNEL_PREEMPTION;
int x;
for (x = 0; x < 0x400000; x++) {
set, -1);
break;
}
}
}