wrsmplat.c revision 03831d35f7499c87d51205817c93e9a8d42c4bae
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* Starcat WRSM platform-specific module
*/
#include <sys/int_fmtio.h>
#include <sys/cpu_module.h>
#include <sys/wrsm_config.h>
#include <sys/wrsm_plat_impl.h>
#include <sys/wci_regs.h>
#include <sys/wci_offsets.h>
#include <sys/wci_common.h>
#include <sys/cheetahregs.h>
#include <sys/machcpuvar.h>
#include <sys/wrsmplat.h>
#ifdef DEBUG
#define WP_DEBUG 1
#define WP_QUEUE 2
#define WP_LINK 4
#define WP_THREAD 8
#define WP_MBOX 16
static uint_t wrsm_plat_debug = 0;
#else
#define DPRINTF(a, b) { }
#endif
static void mbox_event_handler(void);
/* Converts a 10-bit safari port id to its 5-bit expander id */
/* IOSRAM Keys for Wildcat */
/*
* Module linkage information for the kernel.
*/
"Sun Fire 15K wrsm platmod %I%"
};
static struct modlinkage modlinkage = {
};
typedef struct wrsmplat_mbox_msg {
void *msg_data;
struct wrsmplat_mbox_msg *next;
/*
* If stop_thread is set for a particular link of a wci, that means that
* the thread trying to bringup that link should give up and exit.
*/
typedef struct wrsmplat_link {
typedef struct wrsmplat_wci {
static struct wrsmplat_state {
} wrsmplat_state = {0};
/*
* version is 8 bits in the low byte of an int, high 4 bits are
* major version number
*/
#define IS_PRE_V3_JAGUAR(cpu) \
/*
*
* wrsmplat_set_asi_cesr_id and wrsmplat_clr_asi_cesr_id set and clear the
* ASI_CESR_ID per per-core registers on Jaguar 2.X processors
*
* The problem is that both Schizo and XMITS(Safari-PCI bridges) require
* bits [5:4] of the safari address to be 0 on a WBIO. They are copied to
* the PCI bus as is, which will cause the write to be issued to the wrong
* address on PCI. This can cause data corruption.
*
* -- TERMINOLOGY --
*
* WCI - WildCat Interface: an asic that implements WildCat Clustering.
* Operation is defined by the WildCat Programmer's Reference
* Manual for WCI-4.1(Part No.: 950-3825-01)
*
* CESR - Cluster Error Status Register: A register with 256 entries to
* record per core WildCat errors(see WCI PRM above)
*
* CESR index: Used by the WCI to store the result of a cluster
* transaction to the appropriate entry in the CESR. Up to and
* including Cheetah+ this number was generated as a function of the
* processors Safari port.(See WCI PRM for how StarCat AXQ deals with
* this).
*
* ASI_CESR_ID: A new per-core register introduced with Jaguar. The cesr
* id is in bits [7:0] of ASI_CESR_ID. This is now required because
* two cpu cores share a safari agent id.
*
* Jaguar 3.X and Panther processors do not require this workaround since
* they address the above problem in hardware by only driving the bits in
* ASI_CESR_ID onto the safari bus for ncslices in [64, 254] and otherwise
* 0 for PCI devices which are mapped into ncslices [1, 63].
*
* The implementation of these functions for Serengeti is null because
* there is a maximum of 2 x 24 = 48 cores on a serengeeti and 48 cores
* can be named by 6 bits of the cesr id name space and so, the low 2 bits
* of ASI_CESR_ID driven to GA[5:4](above) can be always 0 and the
* ASI_CESR_ID can be set by CPU POST. This plan wont work for starcat
* which can have more than 2^6 = 64 cores.
*
* For StarCat we calculate the cesr index in calculate_starcat_cesr_id()
* when the platform specific module is loaded and then use the cpu_id of
* the thread executing send() in wrsm_intr.c so set the ASI_CESR_ID before
* sending a WildCat interrupt and clearing it after the interrupt has
* completed. We can do this because:
*
* - Currently sending a remote interrupt is the only transaction for
* which the wrsm software checks the CESR
*
* - Preemption is disabled during this time so the cpu_id of the
* thread executing this section cant change
*
* -- BIG WARNINGS --
*
* 1) This workaround must be maintained until the business decision that
* WildCat is no longer to be supported on Jaguar 2.X processors is
* taken.
*
* 2) This workaround assumes that the cesr id's are initially 0 from
* POST. They'll be 0 anyway after the first interrupt on a core though.
*
* 3) It is assumed that Jaguar 3.X and Panther processors' POST code will
* set the CESR equivalently to what is done in
* calculate_starcat_cesr_id()
*
*/
void
wrsmplat_set_asi_cesr_id(void)
{
}
}
void
wrsmplat_clr_asi_cesr_id(void) {
asi_cesr_id_wr(0);
}
}
static void
{
if (core == 0) {
/*
* core 0 cesr is same as what axq calulates for
* cheetah
*/
} else if (slot == 0) {
/* for core 1 slot0, use space above last expander */
} else {
/* core 1 slot 1, set bits 1 & 2 */
}
/*
* shuffle bits to work around jaguar error where if
* bits 01234567 are stored in the cesr_id, bit 67012345
* are what gets driven onto the safari bus
*/
}
}
/* support when SC is not available */
static void
{
int i;
wrsmplat_link_t *l;
for (i = 0; i < WRSM_MAX_WCIS; i++) {
mutex_enter(&l->mutex);
while (l->thread_active) {
"stop_thread: wci %d link %d "
"waiting for thread to stop",
l->stop_thread = B_TRUE;
}
mutex_exit(&l->mutex);
"wci %d link %d thread stopped",
break;
}
}
}
static void
{
int i;
wrsmplat_link_t *l;
/* First, look to see if WCI already exists */
for (i = 0; i < WRSM_MAX_WCIS; i++) {
/* Don't start the thread if wci was suspended */
"suspended, can't start thread", wci_id));
return;
}
mutex_enter(&l->mutex);
if (l->thread_active) {
mutex_exit(&l->mutex);
return;
}
l->thread_active = B_TRUE;
l->stop_thread = B_FALSE;
mutex_exit(&l->mutex);
break;
}
}
/* If not, then search for free space and add wci to the list */
if (!found) {
for (i = 0; i < WRSM_MAX_WCIS; i++) {
NULL);
l->thread_active = B_TRUE;
l->stop_thread = B_FALSE;
wci_id));
break;
}
}
}
"start_thread: wci %d, link %d, thread #%d",
/* LINTED - E_NOP_ELSE_STMT */
} else {
}
}
static void
{
int i;
wrsmplat_link_t *l;
for (i = 0; i < WRSM_MAX_WCIS; i++) {
mutex_enter(&l->mutex);
l->thread_active = B_FALSE;
cv_broadcast(&l->cv);
mutex_exit(&l->mutex);
}
}
"exiting for wci %d link %d. There are %d threads left",
thread_exit();
}
static boolean_t
{
int i;
if (wrsmplat_state.thread_stop) {
return (B_TRUE);
}
for (i = 0; i < WRSM_MAX_WCIS; i++) {
}
}
return (B_FALSE);
}
static wrsmplat_mbox_msg_t *dequeue_msg(void);
static void process_message(void);
static void wrsmplat_thread(void *arg);
/* used to start a thread */
typedef struct wrsmplat_thread_data {
int
_init(void)
{
int rc;
MUTEX_DRIVER, NULL);
MUTEX_DRIVER, NULL);
if (rc) {
"rc=%d", rc));
return (rc);
}
if (rc) {
"rc=%d", rc));
return (rc);
}
return (rc);
}
return (0);
}
int
_fini(void)
{
int rc;
return (rc);
}
/*
* This blocks waiting for all the child threads to die
*/
(void) wrsmplat_unreg_callbacks();
return (0);
}
int
{
}
static void
mbox_event_handler(void)
{
int rc;
wrsm_linkisup_msg_t isup_msg = {0};
return;
}
}
wrsmplat_get_node_type(void)
{
return (node_type);
}
int
{
wrsm_uplink_msg_t msg = {0};
return (0);
}
int
{
wrsm_link_msg_t msg = {0};
return (0);
}
int
{
return (EINVAL);
&wrsm_regs);
if (!wrsm_regs)
return (ENXIO);
return (EAGAIN);
return (0);
}
int
{
wrsm_link_led_msg_t msg = {0};
return (0);
}
int
{
int i, rc;
ncslice_bitmask_t allocated = {0};
if (wrsm_use_sgsbbc) {
wrsm_ncslice_claim_msg_t msg = {0};
wrsm_ncslice_claim_msg_t reply = {0};
sizeof (reply));
if (rc) {
"send_mbox_msg failed %d", rc));
return (rc);
}
#ifdef DEBUG
for (i = 0; i < WRSM_MAX_NCSLICES - 1; ++i)
if (WRSM_IN_SET(requested, i)) {
}
#endif
} else {
/*
* We are only allowed to use NC slices 161 to 254.
*/
if (WRSM_IN_SET(requested, i))
WRSMSET_ADD(allocated, i);
}
return (0);
}
int
{
if (length > WIB_SEPROM_MSG_SIZE)
return (EINVAL);
return (0);
}
int
{
if (wrsmplat_state.thread_count != 0) {
"called more than once"));
return (EEXIST);
}
/*
* no need to lock the thread count:
* when using the mailbox : only one thread exists
* not using the mailbox : the wrsmplat thread starts other threads
*/
return (0);
}
int
wrsmplat_unreg_callbacks(void)
{
/* Wake up the message thread */
/*
* Eventually, all of the threads should wake up, notice the
* thread_stop flag is set and exit. When they have all
* finished, the count should go to zero and it is safe to
* proceed. In case of not using SBBC ther is only one thread.
*/
while (wrsmplat_state.thread_count > 0) {
}
return (0);
}
static void
{
wrsmplat_mbox_msg_t *entry, *p;
/* Could be called from interrupt context, so use KM_NOSLEEP */
return;
}
/* Could be called from interrupt context, so use KM_NOSLEEP */
return;
}
else {
p = wrsmplat_state.msg_queue;
p = p->next;
}
msg_type));
msg_type));
}
static wrsmplat_mbox_msg_t *
dequeue_msg(void)
{
return (msg);
}
/* ARGSUSED */
static void
wrsmplat_thread(void *arg)
{
/* LINTED */
while (1) {
/* go to sleep waiting for the next message */
if (wrsmplat_state.thread_stop) {
process_message(); /* finish any waiting messages */
thread_exit();
}
}
}
/*
* Send a message to the SSC and wait for a matching response
*/
static int
{
int rc;
if (rc) {
"mboxsc_putmsg failed %d", rc));
return (rc);
}
"putmsg succeeded transid=%lx", transid));
reply_length));
if (rc) {
"mboxsc_getmsg failed %d", rc));
return (rc);
}
"getmsg succeeded transid=%lx type=%d cmd=%d",
return (0);
}
static void
process_message(void)
{
int rc;
while (entry = dequeue_msg()) {
case LINKISUP: {
wrsm_uplink_msg_t linkdata_msg = {0};
wrsm_uplink_msg_t reply = {0};
if (rc) {
"send_mbox_msg failed for LINKDATA %d",
rc);
break;
"on LINKDATA request");
}
/*
* Call up to wrsm to give notice that the
* link is ready
*/
break;
}
case UPLINK: {
/* this is a message to be sent to SC */
sizeof (wrsmplat_thread_data_t), KM_SLEEP);
if (wrsm_use_sgsbbc) {
sizeof (wrsm_uplink_msg_t), &status,
sizeof (wrsm_status_msg_t))) != 0) {
"failed for UPLINK rc=%d", rc);
"fmnodeid on UPLINK request");
"%d on UPLINK request",
}
} else {
(umsg->wci_port_id,
&thread_data->wrsm_regs);
if (!thread_data->wrsm_regs) {
"UPLINK: failed to get register "
"base"));
return;
}
}
break;
}
case DOWNLINK: {
/* this is a message to be sent to SC */
wrsm_status_msg_t status = {0};
if (wrsm_use_sgsbbc) {
sizeof (wrsm_link_msg_t), &status,
sizeof (wrsm_status_msg_t))) != 0) {
"failed for DOWNLINK rc=%d", rc);
"%d on DOWNLINK request",
}
} else {
(msg->wci_port_id,
&wrsm_regs);
if (wrsm_regs) {
/* LINTED - E_NOP_ELSE_STMT */
} else {
"DOWNLINK: failed to get register "
"base"));
}
}
/* should not call link_down for loopback */
(msg->wci_port_id,
break;
}
case SETLEDSTATE: {
/* this is a message to be sent to SC */
wrsm_status_msg_t status = {0};
if (wrsm_use_sgsbbc) {
sizeof (wrsm_link_led_msg_t), &status,
sizeof (wrsm_status_msg_t))) != 0) {
"failed for SETLEDSTATE rc=%d",
rc);
"%d on SETLEDSTATE request",
}
}
break;
}
case SETSEPROM: {
/* this is a message to be sent to SC */
if (wrsm_use_sgsbbc) {
wrsm_status_msg_t status = {0};
sizeof (wrsm_wib_seprom_msg_t), &status,
sizeof (wrsm_status_msg_t))) != 0) {
"send_mbox_msg failed for "
"SETSEPROM rc=%d", rc);
"%d on SETSEPROM request",
}
}
break;
}
case NCSLICE: {
/* NOT HANDLED HERE */
break;
}
default :
}
if (free_msg)
}
}
static uint64_t
{
}
static void
{
}
/* ARGSUSED */
void
{
/*
* wci_qlim_3req_priority, wci_qlim_2req_priority,
* wci_qlim_sort_ciq, wci_qlim_sort_niq, and wci_qlim_sort_piq
* need to be updated if this cluster node is part of a
* wildcat SSM.
*/
}
/* ARGSUSED */
void
{
}
void
{
#ifdef DEBUG
#endif /* DEBUG */
/* LINTED */
}
#ifdef DEBUG
(void *)pa));
#endif
}
}
/*
* Set the link to in use mode and, turn on the transmitters,
* and clear out the error count.
*/
error_count.val = 0;
/*
* We can send 16 bits to the far side via usr_data_1, it is
* encoded as follows:
* 0-7: gnid
* 8-12: portid
* 13-15: linknum
*/
/*
* Spin waiting for the remote side to set the user_data_2
*/
}
}
/* XXX - just to be sure the link settles down */
}
}
}
/*
* Don't make the callback if the link is supposed to
* be in loopback mode.
*/
} else {
}
}
void
{
switch (link) {
case 0:
break;
case 1:
break;
case 2:
break;
default:
link));
break;
}
/* read back to commit the write */
}
int
{
int checks;
int good_tnid;
int tnid;
if (link_id >= WRSM_LINKS_PER_WCI) {
}
/* Clear error bits corresponding to this link */
sw_esr_mask.val = 0;
case 0:
break;
case 1:
break;
case 2:
break;
}
/* read it back to make sure the write happened */
"Trying to bring up link %d wci %d",
/*
* The max paroli reset setup time is 100ms. Putting the link
* into STATE_SEEK brings the paroli out of reset, so we have
* to wait 100ms before we can expect anything useful from the
* status register
*/
/* Wait for optical_signal_detect to come on */
count = 0;
++count;
if (count >= 200) {
"signal detect link %d wci %d",
return (-1);
}
/* sleep 250ms */
}
}
"Trying tnid %d", tnid));
/*
* There is some probability <1 that the end_status
* will be reported correctly in the link_status
* register. Dave Saterfield guesses ~60% but it
* could be a lot worse if there is much clock skew
* between machines. It should only give false
* negative, not false positive results. So we try a
* few times and if it ever says "ready" then we declare
* success.
*/
break;
}
}
if (good_tnid) {
break;
}
}
#ifdef DEBUG
if (tnid == 16) {
} else {
"link %d wci %d retries %d", tnid,
}
#endif /* DEBUG */
/*
* If we didn't find the right tnid, the link must be in a bad
* state, so reset and try again.
*/
if (!good_tnid)
return (-1);
error_count.val = 0;
#ifndef NEW_PAROLIS_ONLY
#endif /* NEW_PAROLIS_ONLY */
}
statAddr);
continue;
eCntAddr);
/*
* if it's still up and error count is
* not too high, declare success
*/
return (0);
} else {
"high error count %" PRIx64
" status = %" PRIx64,
link_status.val));
return (-1);
}
}
}
}
}
return (-1);
}
/*
* Verifies valid stripegroup. For starcat, we support only 2 way WCI
* boards.
*/
int
{
return (0);
}
return (ENOTSUP);
}
/* expander1 should be even and 1 less than expander 2 */
return (ENOTSUP);
return (0);
}
/*
* Program AXQ nasm tables to route appropriate ncslices to the
* right expander.
* Note: other CPUs are paused when this function is called.
*/
/* ARGSUSED */
void
{
int i;
for (i = 0; i < WRSM_MAX_NCSLICES; ++i) {
continue;
}
"owner type %d slice %d nasm_data %d",
(void) axq_nasm_write_all(i, nasm_data);
}
}
/*
* Enter ncslice update critical section - take AXQ internal lock for Starcat
* before other CPUs are paused. Otherwise a paused thread could be holding
* this lock when axq_nasm_write_all() is called from wrsmplat_ncslice_setup(),
* and if the former attempted to acquire the lock there, deadlock would result.
*/
void
wrsmplat_ncslice_enter(void)
{
}
/*
* Exit ncslice update critical section - release AXQ internal lock for Starcat,
* after other paused CPUs are restarted.
*/
void
wrsmplat_ncslice_exit(void)
{
}
/* Does a cross-trap sync */
void
wrsmplat_xt_sync(int cpu_id)
{
}
void
{
if (!wrsm_use_sgsbbc) {
int i;
int link;
for (i = 0; i < WRSM_MAX_WCIS; i++) {
return;
}
break;
}
}
}
}
}
void
{
/*
* No need to restart link bring-up thread, since driver will
* eventually time-out and retry later.
*/
if (!wrsm_use_sgsbbc) {
int i;
for (i = 0; i < WRSM_MAX_WCIS; i++) {
break;
}
}
}
}