x_call.c revision a31148363f598def767ac48c5d82e1572e44b935
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Copyright (c) 2010, Intel Corporation.
* All rights reserved.
*/
#include <sys/xc_levels.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/mutex_impl.h>
#include <sys/x86_archext.h>
/*
* Implementation for cross-processor calls via interprocessor interrupts
*
* This implementation uses a message passing architecture to allow multiple
* concurrent cross calls to be in flight at any given time. We use the cmpxchg
* instruction, aka casptr(), to implement simple efficient work queues for
* message passing between CPUs with almost no need for regular locking.
* See xc_extract() and xc_insert() below.
*
* The general idea is that initiating a cross call means putting a message
* on a target(s) CPU's work queue. Any synchronization is handled by passing
* the message back and forth between initiator and target(s).
*
* Every CPU has xc_work_cnt, which indicates it has messages to process.
* This value is incremented as message traffic is initiated and decremented
* with every message that finishes all processing.
*
* The code needs no mfence or other membar_*() calls. The uses of
* casptr(), cas32() and atomic_dec_32() for the message passing are
* implemented with LOCK prefix instructions which are equivalent to mfence.
*
* One interesting aspect of this implmentation is that it allows 2 or more
* CPUs to initiate cross calls to intersecting sets of CPUs at the same time.
* The cross call processing by the CPUs will happen in any order with only
* a guarantee, for xc_call() and xc_sync(), that an initiator won't return
* from cross calls before all slaves have invoked the function.
*
* The reason for this asynchronous approach is to allow for fast global
* TLB shootdowns. If all CPUs, say N, tried to do a global TLB invalidation
* on a different Virtual Address at the same time. The old code required
* N squared IPIs. With this method, depending on timing, it could happen
* with just N IPIs.
*/
/*
* The default is to not enable collecting counts of IPI information, since
* the updating of shared cachelines could cause excess bus traffic.
*/
uint_t xc_collect_enable = 0;
/*
* Values for message states. Here are the normal transitions. A transition
* of "->" happens in the slave cpu and "=>" happens in the master cpu as
* the messages are passed back and forth.
*
* FREE => ASYNC -> DONE => FREE
* FREE => CALL -> DONE => FREE
* FREE => SYNC -> WAITING => RELEASED -> DONE => FREE
*
* The interesing one above is ASYNC. You might ask, why not go directly
* to FREE, instead of DONE. If it did that, it might be possible to exhaust
* the master's xc_free list if a master can generate ASYNC messages faster
* then the slave can process them. That could be handled with more complicated
* handling. However since nothing important uses ASYNC, I've not bothered.
*/
#define XC_MSG_FREE (0) /* msg in xc_free queue */
/*
* We allow for one high priority message at a time to happen in the system.
* This is used for panic, kmdb, etc., so no locking is done.
*/
static volatile cpuset_t xc_priority_set_store;
static xc_data_t xc_priority_data;
/*
* Wrappers to avoid C compiler warnings due to volatile. The atomic bit
* operations don't accept volatile bit vectors - which is a bit silly.
*/
/*
* Decrement a CPU's work count
*/
static void
{
}
/*
* Increment a CPU's work count and return the old value
*/
static int
{
int old;
do {
return (old);
}
/*
* Put a message into a queue. The insertion is atomic no matter
*/
static void
{
/*
* FREE messages should only ever be getting inserted into
* the xc_master CPUs xc_free queue.
*/
do {
}
/*
* Extract a message from a queue. The extraction is atomic only
* when just one thread does extractions from the queue.
* If the queue is empty, NULL is returned.
*/
static xc_msg_t *
{
do {
return (old_head);
return (old_head);
}
/*
* Initialize the machcpu fields used for cross calls
*/
static uint_t xc_initialized = 0;
void
{
int c;
/*
* Allocate message buffers for the new CPU.
*/
for (c = 0; c < max_ncpus; ++c) {
if (plat_dr_support_cpu()) {
/*
* Allocate a message buffer for every CPU possible
* in system, including our own, and add them to our xc
* message queue.
*/
/*
* Add a new message buffer to each existing CPU's free
* list, as well as one for my list for each of them.
* Note: cpu0 is statically inserted into cpu[] array,
* so need to check cpu[c] isn't cpup itself to avoid
* allocating extra message buffers for cpu0.
*/
}
}
if (!plat_dr_support_cpu()) {
/*
* Add one for self messages if CPU hotplug is disabled.
*/
}
if (!xc_initialized)
xc_initialized = 1;
}
void
{
}
}
#define XC_FLUSH_MAX_WAITS 1000
/* Flush inflight message buffers. */
int
{
int i;
/*
* Pause all working CPUs, which ensures that there's no CPU in
* function xc_common().
* This is used to work around a race condition window in xc_common()
* between checking CPU_READY flag and increasing working item count.
*/
start_cpus();
for (i = 0; i < XC_FLUSH_MAX_WAITS; i++) {
break;
}
DELAY(1);
}
for (; i < XC_FLUSH_MAX_WAITS; i++) {
break;
}
DELAY(1);
}
return (i >= XC_FLUSH_MAX_WAITS ? ETIME : 0);
}
/*
* X-call message processing routine. Note that this is used by both
* senders and recipients of messages.
*
* We're protected against changing CPUs by either being in a high-priority
* interrupt, having preemption disabled or by having a raised SPL.
*/
/*ARGSUSED*/
{
uint32_t num_waiting = 0;
while (mcpup->xc_work_cnt != 0) {
/*
* We may have to wait for a message to arrive.
*/
/*
* Alway check for and handle a priority message.
*/
if (mcpup->xc_work_cnt == 0)
return (rc);
}
/*
* wait for a message to arrive
*/
SMT_PAUSE();
}
/*
* process the message
*/
switch (msg->xc_command) {
/*
* ASYNC gives back the message immediately, then we do the
* function and return with no more waiting.
*/
case XC_MSG_ASYNC:
break;
/*
* SYNC messages do the call, then send it back to the master
* in WAITING mode
*/
case XC_MSG_SYNC:
break;
/*
* WAITING messsages are collected by the master until all
* have arrived. Once all arrive, we release them back to
* the slaves
*/
case XC_MSG_WAITING:
break;
msg);
--num_waiting;
}
if (num_waiting != 0)
panic("wrong number waiting");
mcpup->xc_wait_cnt = 0;
break;
/*
* CALL messages do the function and then, like RELEASE,
* send the message is back to master as DONE.
*/
case XC_MSG_CALL:
/*FALLTHROUGH*/
case XC_MSG_RELEASED:
break;
/*
* DONE means a slave has completely finished up.
* Once we collect all the DONE messages, we'll exit
* processing too.
*/
case XC_MSG_DONE:
break;
case XC_MSG_FREE:
break;
default:
break;
}
}
return (rc);
}
/*
* Initiate cross call processing.
*/
static void
{
int c;
int cnt;
int save_spl;
if (!xc_initialized) {
return;
}
/*
* fill in cross call data
*/
/*
* Post messages to all CPUs involved that are CPU_READY
*/
for (c = 0; c < max_ncpus; ++c) {
continue;
continue;
/*
* Fill out a new message.
*/
panic("Ran out of free xc_msg_t's");
/*
* Increment my work count for all messages that I'll
* transition from DONE to FREE.
* Also remember how many XC_MSG_WAITINGs to look for
*/
if (command == XC_MSG_SYNC)
/*
* Increment the target CPU work count then insert the message
* in the target msgbox. If I post the first bit of work
* for the target to do, send an IPI to the target CPU.
*/
if (cnt == 0) {
send_dirint(c, XC_HI_PIL);
if (xc_collect_enable)
++xc_total_cnt;
} else if (xc_collect_enable) {
++xc_multi_cnt;
}
}
}
/*
* Now drop into the message handler until all work is done
*/
}
/*
* Push out a priority cross call.
*/
static void
{
int i;
int c;
/*
* Wait briefly for any previous xc_priority to have finished.
*/
for (c = 0; c < max_ncpus; ++c) {
continue;
/*
* The value of 40000 here is from old kernel code. It
* really should be changed to some time based value, since
* under a hypervisor, there's no guarantee a remote CPU
* is even scheduled.
*/
SMT_PAUSE();
/*
* Some CPU did not respond to a previous priority request. It's
* probably deadlocked with interrupts blocked or some such
* problem. We'll just erase the previous request - which was
* most likely a kmdb_enter that has already expired - and plow
* ahead.
*/
if (BT_TEST(xc_priority_set, c)) {
}
}
/*
* fill in cross call data
*/
/*
* Post messages to all CPUs involved that are CPU_READY
* We'll always IPI, plus bang on the xc_msgbox for i86_mwait()
*/
for (c = 0; c < max_ncpus; ++c) {
continue;
continue;
XC_BT_SET(xc_priority_set, c);
send_dirint(c, XC_HI_PIL);
for (i = 0; i < 10; ++i) {
}
}
}
/*
* Do cross call to all other CPUs with absolutely no waiting or handshaking.
* This should only be used for extraordinary operations, like panic(), which
* need to work, in some fashion, in a not completely functional system.
* All other uses that want minimal waiting should use xc_call_nowait().
*/
void
{
extern int IGNORE_KERNEL_PREEMPTION;
}
/*
* Wrapper for kmdb to capture other CPUs, causing them to enter the debugger.
*/
void
{
extern int IGNORE_KERNEL_PREEMPTION;
if (!xc_initialized)
return;
}
/*
* Invoke function on specified processors. Remotes may continue after
* service with no waiting. xc_call_nowait() may return immediately too.
*/
void
{
}
/*
* Invoke function on specified processors. Remotes may continue after
* service with no waiting. xc_call() returns only after remotes have finished.
*/
void
{
}
/*
* Invoke function on specified processors. Remotes wait until all have
* finished. xc_sync() also waits until all remotes have finished.
*/
void
{
}