wrsm_nr.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2003 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* Network routing module of the Wildcat RSM driver. This file manages
* routes to nodes. It calculates which links to use to route transactions
* to each remote node, based on the preferred route list for that node.
* It handles failing routes between links and wcis. This includes setting
* up and tearing down wci striping.
*/
#include <sys/tnf_probe.h>
#include <sys/wci_regs.h>
#include <sys/wci_offsets.h>
#include <sys/wci_common.h>
#include <sys/wrsm_nc_impl.h>
#include <sys/wrsm_transport.h>
#include <sys/wrsm_session.h>
#include <sys/wrsm_cmmu.h>
#include <sys/wrsm_intr.h>
#include <sys/wrsm_memseg.h>
#ifdef DEBUG
#define DBG_CONFIG 0x001
#define DBG_ROUTE 0x002
#define DBG_HW 0x004
#define DBG_EVENT 0x008
#define DBG_CONFIG_EXTRA 0x010
#define DBG_ROUTE_EXTRA 0x020
#define DBG_HW_EXTRA 0x040
#define DBG_EVENT_EXTRA 0x080
#define DBG_WARN 0x100
#define DBG_RT_KSTAT 0x200
/*
* Shouldn't actually call cmn_err while cpus are paused -- this could
* cause a kmem_alloc to create the string to print, which might require a
* lock, which if currently taken will never be freed because the owning
* cpu is now paused... ask me how I know. In other words, use these
* debug flags at your own risk.
*/
#define DBG_PAUSECPUS 0x1000
#define DBG_PAUSECPUS_EXTRA 0x2000
#define DBG_PAUSECPUSWARN 0x4000
/*
* 0xff0f - DBG_CONFIG_EXTRA DBG_ROUTE_EXTRA DBG_HW_EXTRA DBG_EVENT_EXTRA
* not turned on
*/
#else /* DEBUG */
#define DPRINTF(a, b) { }
#endif /* DEBUG */
static void nr_reroute_wcis(void *arg);
int wcinum);
wrsm_node_t *node);
int route_counter);
/*
* Freezes number_to_freeze of RAG instances starting at 0th bit.
* Its is required that the CPUs be inactive before calling this function if
* the number of RAGS to freeze (number_to_freeze) is greater than the number
* of RAGs currently frozen.
* We call this function in wrsm_nr_replaceconfig after freezing CPUs
* in order to freeze additional RAG instances. We call this function from
* wrsm_nr_enableconfig without freezing CPUs in order to unfreeze RAG
* instances.
*/
static void
{
int i;
#ifdef DEBUG
#endif
while (wcip) {
continue;
}
#ifdef DEBUG
&(wci_ra_freeze.val));
&(wci_ra_busy.val));
"cnode id %d number of nodes %d, Initial wci_ra_freeze "
"0x%lx initial wci_ra_busy 0x%lx",
wci_ra_busy.val));
#endif /* DEBUG */
/*
* we don't need to worry about saving Instances frozen due to
* error since such an occurence will cause the domain to
* go dead.
*/
wci_ra_freeze.val = 0;
"freeze %d", number_to_freeze));
/* per PRM, bits are to be set starting from 0th bit up */
for (i = 0; i < number_to_freeze; i ++) {
}
#ifdef DEBUG
&(wci_ra_freeze.val));
#endif /* DEBUG */
}
}
/*
* Configuration Functions
*
* (wrsm_nr_replaceconfig(), wrsm_nr_cleanconfig(), wrsm_nr_installconfig(),
* wrsm_nr_enableconfig(), wrsm_nr_initialconfig(), wrsm_nr_removeconfig(),
* wrsm_nr_attachwci(), wrsm_nr_enablecwci() and wrsm_nr_detachwci()) are all
* guaranteed to be single threaded. The config layer causes the NC to
* call these functions. The config later is the only consumer of these and
* it will never call a second function in the NC before the first is
* complete, so none of these functions will be called while another one is
* still running.
*/
/*
* verify that the routing configuration meets certain requirements
*/
int
{
int i, j, k, wnid;
int err;
network->rsm_ctlr_id));
"wrsm_nr_verifyconfig: no routing info\n"));
return (EINVAL);
}
/*
* 2 WCIs are not allowed to be part of the same multi-hop network.
* the local node.
*/
/*
* This wnode refers the local node.
* This is only ok if this is the local
* wnode for this wci (for loopback).
*/
"wrsm_nr_verifyconfig: "
"wci %d: non-local wnode %d"
" goes to this node (%d)\n",
return (EINVAL);
}
}
/*
* Make sure there is an nmembers structure
* for any cnode in the reachable list.
*/
break;
}
/*
* no nmember node for wci
* cnodeid
*/
"wrsm_nr_verifyconfig: "
"wci %d wnode %d: cnode %d "
"doesn't exist in this "
return (EINVAL);
}
}
}
}
/*
* verify that the stripe group configuration is allowed on this
* platform. Also verify that each WCI mentioned in the stripe
* group is in the config.
*/
if ((err = wrsmplat_stripegroup_verify(
"stripegroup_verify failed %d for group %d",
err, i));
return (err);
}
}
}
if (!found) {
"wrsm_nr_verifyconfig: "
"in stripegroup %d, wci %d "
"doesn't exist in this "
"config\n", i, port));
return (EINVAL);
}
}
}
/*
* any node through which forwarding is allowed must be directly
* connected to it from this node - not checked here; enforced
* in nr_cnode_route().
*/
/*
* If the node list is not empty, the local node must be in the list.
*/
have_local = B_TRUE;
break;
}
}
if (!have_local) {
return (EINVAL);
}
}
/*
* Ensure that each hostname is unique
*/
if (i == j) {
continue;
}
return (EINVAL);
}
}
}
/*
* Make sure there is preferred route info for each cnode in the
* config. (Both the nmember list and routing policy list are
* ordered by cnodeid, so the index into each for a particular
* cnodeid should be identical.)
*/
"policy information\n"));
return (EINVAL);
}
return (EINVAL);
}
}
/*
* make sure each preferred route refers to a valid wci or
* stripe group
*/
break;
}
"proute %d uses illegal wci %d\n",
return (EINVAL);
}
} else {
stripe_groups[k]->group_id ==
break;
}
"proute %d uses illegal sg %d\n",
return (EINVAL);
}
}
}
}
/*
* make sure each wci is only in one stripe group.
*/
if (found) {
"wci %d is in more than "
"one stripe group\n",
port));
return (EINVAL);
} else {
}
}
}
}
}
/*
* there must be at least one WCI attached
*/
"attached wci\n"));
return (ENODEV);
}
/*
* checks for conflicts with old config, if there was one
*/
return (WRSM_SUCCESS);
/*
* Verify that any stripe groups in both the old and new config
* have matching info (same number of wcis with the same ids) Note:
* sgs in network and sgs in config are both ordered by group id.
*/
/* sg is not in config; skip it */
continue;
}
if (!sg) {
/* rest of sgs in config are new */
break;
}
/* new sg in config */
continue;
}
/* found matching sg */
"stripe group %d: old nwcis %d, "
return (EINVAL);
}
"%d: wci #%d old port is %d, new port "
return (EINVAL);
}
}
}
/*
* verify that the passed in LC handles match any already saved away
* LC handles
*/
for (i = 0; i < attached_cnt; i++) {
if (!wci)
continue;
/*
* bad LC handle!
*/
"(0x%p) doesn't match new handle (0x%p)\n",
(void *) attached_wcis[i].lcwci);
return (EINVAL);
}
}
/*
* verify that new link config doesn't conflict with old link
* config, and that local wnode hasn't changed.
*/
if (!wci)
continue;
"lc_verifyconfig failed"));
return (EINVAL);
}
}
"local wnodeid has changed from %d "
"to %d cnodeid %d",
return (EINVAL);
}
}
}
return (WRSM_SUCCESS);
}
/*
* set up initial configuration structures for the NR
*/
{
network->rsm_ctlr_id));
KM_SLEEP);
(void) wrsm_tl_add_handler(network,
/*
* Per wci cmmu init is not needed
* on an initial config.
*/
return (B_TRUE);
} else {
return (B_FALSE);
}
}
/*
* Allocate data structures for and store routing info for nodes, wcis,
* stripe groups. Mark old wcis and stripe groups as disabled. Call
* lc_replaceconfig() for each old and new wci.
*/
int
{
int i, j;
network->rsm_ctlr_id));
/* start event thread for this network */
}
/*
* cancel any pending timeout to reconfig wcis
*/
/*
* cancels timeout or waits until it is finished
*/
}
/*
* Cancel any pending timeout to resend passthrough messages
* until the new configuration is installed.
*/
/*
* cancels timeout or waits until it is finished
*/
}
/*
* pause the event thread while modifying data structures it
* uses
*/
#ifdef DEBUG
/*
* Both the list of config wcis and the list of network wcis
* is ordered by safari port id.
*/
}
if (wci)
}
#endif
/*
* Update list of wcis to match config wcis -
* create wci structs for new wcis; disable old wcis; call
* LC for each wci
*/
/*
* this wci doesn't exist in the new config
* mark it as disabled
*/
/* notify LC that LC is not in config */
NULL);
}
}
/* wci is new */
/* mark all wnodes as unreachable */
for (j = 0; j < WRSM_MAX_WNODES; j++) {
}
/*
* add wci to network's list of wcis
* Take rw lock to prevent controller barrier
* code (nr_check_all_*()) from getting confused.
* (All other accesses of the network->nr_wcis
* accesses.)
*/
}
/*
* A wci struct now exists in network struct for this wci;
* replace old config info with new info and move to next
* wci in network list
*/
}
/*
* Any remaining wcis in the network list must no longer
* be part of the configuration. Mark them as disabled.
*/
while (wci) {
/*
* this wci doesn't exist in the new config
* mark it as disabled
*/
/* notify LC that LC is not in config */
}
}
for (i = 0; i < attached_cnt; i++) {
#ifdef DEBUG
if (!wci) {
"wrsm_nr_replaceconfig() "
"no wci for attached_wci #%d (id %d)\n", i,
attached_wcis[i].port));
continue;
}
"wrsm_nr_replaceconfig() "
"no lcwci for attached_wci #%d (id %d)\n", i,
attached_wcis[i].port));
continue;
}
#endif
/*
* new attach - if wrsm_nr_attachwci() fails, the
* LC will not be informed of the new WCI,
* and it won't come up
*/
} else {
/* give LC the replacement configuration */
config);
}
}
#ifdef DEBUG
wci->availability));
}
#endif
/*
* Assumption: All WCI's in a controller will have the
* same number of nodes in their multihop group, hence we
* apply the same rag freeze algorithm to the entire list
* of wcis (network->nr->wcis).
*/
}
/*
* mark disabled cnodes as no longer needing a route
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
}
}
/*
* Free old extended preferred route list if there is one.
*/
/*
* Update the preferred routes for each cnode in the config.
*/
cnodeid));
sizeof (wrsm_node_routeinfo_t), KM_SLEEP);
(sizeof (wrsm_routing_policy_t), KM_SLEEP);
/* Create the route kstat */
}
sizeof (wrsm_routing_policy_t));
}
/*
* For local node, extend the preferred route list to include
* internal loopback routes through all wcis. This guarantees
* there is a loopback route even if a particular wci is removed.
* (It is not permitted to remove the last wci in a controller.)
*/
#ifdef DEBUG
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
panic("missing routinfo (and policy) info for "
}
}
#endif
/*
* Update list of stripe groups to match config stripe groups -
* add sgs, set old sgs to disabled.
*
* Both the list of config stripe groups and the list of network
* stripe groups is ordered by group id.
*/
#ifdef DEBUG
}
if (sg)
}
#endif
/*
* this stripe group doesn't exist in the new config
* mark it as disabled
*/
}
/* sg is new */
}
/*
* add sg to network's list of sgs
*/
}
/*
* An sg now exists in network struct for this stripe
* group; if it was there already, it was already checked
* (in wrsm_nr_verifyconfig()) to be sure it matches the one
* in the new config
*/
}
/*
* Any remaining sgs in the network list must no longer
* be part of the configuration. Mark them as disabled.
*/
while (sg) {
/*
* this sg doesn't exist in the new config
* mark it as disabled
*/
}
#ifdef DEBUG
sg->availability));
}
#endif
/*
* ok to restart event thread
*/
return (WRSM_SUCCESS);
}
{
}
/*
* striped wci. attibutes are the safari portid and one bitmask for a links
* used on all wcis. returns error from nvlist_xxx calls.
*/
int
{
char wcistr[5];
int i, j;
int err;
int link;
int wnodeid;
/*
* linkbitmask is a bitmask of all links for all 4 wci's
* such that bits 0-2 represent link 0, 1, 2 for the first
* wci. bit 3-5 represent links 0, 1, 2 for the second and so on.
*/
uint32_t linkbitmask = 0;
/*
* Copy the current route info into a local data structure
*/
/* Get the wroute for this stripe */
/* this piece should never happen */
wcistr, -1);
if (err != DDI_SUCCESS) {
return (err);
}
continue;
}
if (err != DDI_SUCCESS) {
return (err);
}
/*
* Depending on the route type, the link and
* node information are found in different
* data structures.
*/
"nid_route_inid, stripe=%d", i));
for (j = 0; j < WRSM_MAX_DNIDS; j++) {
/*
* Go through the inid2dnid table.
* The inid2dnid entry is the wnodeid.
*/
wnodeid =
wnode_list[j];
/*
* Since passthrough routes are guaranteed to
* not use link striping we only need to
* loop up the wnode to link mapping once.
*/
link =
if (link != -1) {
(link + (i * WRSM_LINKS_PER_WCI)));
}
}
} else { /* nid_route_wnode */
/* There will be one or two links from this wci */
/* The wroute.id is the wnodeid. */
/* look at links to see which ones lead to wnode */
wnodeid)) {
(link + (i * WRSM_LINKS_PER_WCI)));
}
}
}
}
/*
* in the event cur_rte.wcis < 4 must set to -1 remaining wci
* name-value pairs
*/
while (i < 4) {
wcistr, -1);
if (err != DDI_SUCCESS) {
return (err);
}
i++;
}
}
/*
* system event logger
*/
void
{
int err = DDI_SUCCESS;
KM_SLEEP)) == DDI_SUCCESS) {
switch (eventtype) {
case new_node_route:
"wrsm_nr_logevent: new_node_route event"));
if (err == DDI_SUCCESS) {
}
if (err == DDI_SUCCESS) {
}
if (err == DDI_SUCCESS) {
}
if (err == DDI_SUCCESS) {
}
break;
case lost_node_route:
"lost_node_route and reason is %s", reason));
if (err == DDI_SUCCESS) {
}
if (err == DDI_SUCCESS) {
reason);
}
if (err == DDI_SUCCESS) {
}
break;
case new_config:
"new_config"));
if (err == DDI_SUCCESS) {
}
break;
}
}
if (err != DDI_SUCCESS) {
}
}
/*
* Remove ncslice routes and routing data structures for disabled cnodes;
* reroute cnodes using invalid wnode routes; takedown disabled links.
* Call lc_cleanconfig() on each old and new wci (takes down old links).
* Cause an MH reroute on specified wcis.
*/
int
{
int i, j;
struct wrsm_node_routeinfo *routeinfo;
network->rsm_ctlr_id));
/*
* temporarily stop processing of wnode route events
* while queuing up a bunch of them
*/
/*
* tell the LC to update the link states based on the
* configuration; this will cause the LC to (immediately)
* notify the MH that certain links are down (any that
* are no longer valid in the new config). This should
* cause the MH to (immediately) queue evt_mhdirect events
* for the event thread to process.
*/
/*
* Modify wnodeinfo array of each wci to match new config
* for future route decisions. Remove references to
* disabled cnodes. Request that all cnodes using a no
* longer valid wnode have their routes re-evaluated.
*/
for (i = 0; i < WRSM_MAX_WNODES; i++) {
/* valid wnode in the new config */
/*
* wnodeid to cnodeid mapping changed;
* request rerouting for any nodes
* using wnode, then treat as a newly
* valid wnodeid.
*/
"wci %d wnode %d "
"old cnode %d new cnode %d\n",
}
/* set new values for wnode */
"wci %d add "
"wnode %d cnode %d\n",
}
} else {
/* invalid wnode in this config */
"wci %d remove "
"wnode %d cnode %d\n",
}
}
if (reroute) {
/*
* old wnode is no longer valid, so
* figure out which nodes were using this
* wnode route (to request a reroute later)
*/
for (j = 0; j < WRSM_INID2DNID_ENTRIES;
j++) {
if (WRSM_IN_SET(
wnode_bitmask, i)) {
wci->
users);
}
}
} else {
}
}
}
}
/*
* Request an ncslice reroute on any cnodes using invalid wnodes.
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
/*
* if node was using now invalid wnode, request
* reroute
*/
if (WRSM_IN_SET(lost_route, i)) {
"ctlr %d cleanconfig invalid wnode, "
"node %d check_route\n",
network->rsm_ctlr_id, i));
}
}
/*
* Request that all nodes using a disabled stripe group have their
* ncslice route re-evaluated.
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
"ctlr %d cleanconfig invalid sg, "
"node %d check_route\n",
network->rsm_ctlr_id, i));
}
}
}
}
/*
* force local node loopback route to be calculated
*/
"ctlr %d cleanconfig re-eval loopback route "
"node %d check_route\n",
/*
* restart network event thread
*/
/*
* wait for routes on disabled nodes to be removed
*/
/*
* temporarily stop processing events
* while modifying node routinfo pointers
*/
/*
* At this point, there should be no routes for or references to
* disabled cnodes.
*
* Remove routeinfo for disabled cnodes. If local node is being
* removed (only happens when network/controller is being removed),
* also remove its extended preferred route info.
*/
}
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
/* Delete the route kstat */
/*
* Set routeinfo to NULL after taking lock --
* this is to coordinate with passthrough
* messages. All other accesses to routeinfo
* are by the event thread, which is paused.
*/
sizeof (wrsm_routing_policy_t));
sizeof (wrsm_node_routeinfo_t));
}
}
/*
* Cause an MH reroute of specified "reroute" wcis. The
* reroute request will be queued after any wnode down
* events for this wci caused by the lc_cleanconfig() call.
*/
for (i = 0; i < reroute_cnt; i++) {
KM_SLEEP);
}
}
return (WRSM_SUCCESS);
}
/*
* Make sure all rerouting around old wcis and stripe groups is complete;
* Call lc_installconfig() on each old and new wci (brings up new links).
* Clean up old wci and stripe group data structures. Make sure ncslice
* reroute caused by removal of routes is complete. Recalculate and
* record which nodes are interested in which wnode/stripe-group/PT routes.
*/
int
{
int i, j, k, cindex;
int err;
network->rsm_ctlr_id));
/*
* At this point, no old links or wnodes should be in use, and no
* old cnodes should be using routes.
*/
/*
* set up bitmasks describing which cnodes are interested in
* various wnode routes and PT routes
*/
/*
* zero out old passthrough bitmasks
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
}
/*
* now mark which nodes are interested in which routes
*/
continue;
/*
* for each route, mark in the appropriate passthrough
* node, wci or stripe group that this cnode is a candidate
* user
*/
cindex);
}
}
for (j = 0; j < WRSM_MAX_WNODES; j++) {
cindex) {
interested, cindex);
}
}
} else {
for (k = 0; k < WRSM_MAX_WNODES; k++) {
if (wci->
wci->
cindex) {
wci->
interested, cindex);
}
}
}
}
}
/*
* current.proute was removed during wrsm_nr_replaceconfig,
* so need to recreate current.proute.
*/
}
/*
* Mark all wcis as needing re-evaluation by all cnodes interested
* in any wnode routes on the wci by setting wci's cnode_retry
* field. (Evaluation takes place once check_route is set for
* cnode).
*/
continue;
for (i = 0; i < WRSM_MAX_WNODES; i++) {
continue;
/*
* Add passthrough candidates to list of
* nodes interested in a possible wnode route through
* a particular cnode.
*/
#ifdef DEBUG
"wrsm_nr_installconfig() "
"wci %d wnode %d interested:\n",
if (wrsm_nr_debug & DBG_CONFIG_EXTRA)
#endif
}
#ifdef DEBUG
if (wrsm_nr_debug & DBG_CONFIG_EXTRA)
#endif
}
/*
* Mark all stripe groups as needing a re-evaluation by all cnodes
* that could use stripe groups by setting sg's cnode_retry field.
* (Evaluation takes place once check_route is set for cnode).
*/
continue;
}
#ifdef DEBUG
if (wrsm_nr_debug & DBG_CONFIG_EXTRA)
#endif
}
/*
* Force rerouting of any wcis that are marked as needing a
* reroute. This guarantees that invalid wnode routes are no longer
* in use.
*/
}
}
/*
* Wait until all wci reroutes and ncslice re-routes they cause
* are finished.
*/
/*
* At this point, all WCIs are routing only through wnodes valid in
* both configurations. All ncslices are only using routes valid
* in both configurations. All ncslices on wcis that are in old
* stripe groups are set to use only a single wci. All ncslices
* using a single wci have the no_stripe bit set in their nc2nid
* entry. This means the stripe group bits can be changed without
* affecting any ncslices.
*/
/* may call nr_pause_event_thread() */
return (err);
}
/*
* Tell LC to bringup all links valid to the new configuration on
* each wci. (The LC does not notify the MH of any new up links
* until lc_enableconfig() is called.
*/
}
}
}
/*
* Make sure all cnodes with check_route set due to wci reroutes
* have been re-evaluated and ncslices rerouted.
*/
/*
* remove disabled wcis
*/
while (wci) {
}
/*
* remove wci from linked list
* Take rw lock to prevent controller barrier
* code (nr_check_all_*()) from getting confused.
* (All other accesses of the network->nr_wcis
* accesses.)
*/
} else {
}
}
return (WRSM_SUCCESS);
}
/*
* Allow ncslice routes to use new wnode routes. Force MH reroute on certain
* wcis with new direct wnode routes. Set timeout to ensure MH reroute on
* all wcis with new wnode routes.
*/
int
{
int i;
network->rsm_ctlr_id));
/*
* allow LC to notify the MH of new links that are up
*/
}
}
/* wait for link up notificiations to be processed */
if (reroute_cnt == -1) {
/* reroute all wcis */
nr_reroute_wcis((void *)network);
} else {
/*
* Cause an MH reroute on specified wcis, allowing them
* to use any new, up links. Schedule a timeout to force
* MH reroute on remaining wcis in the near future.
*/
for (i = 0; i < reroute_cnt; i++) {
}
} else {
}
}
/*
* Turn on passthrough forwarding to appropriate nodes,
* and send nodes the latest passthrough list from this
* node.
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
}
if (ptnotify) {
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
}
}
if (network->free_rag_instance) {
/*
* this can be called here with out the security of
* stopping all network transactions because we know that
* nodes have been removed from the network so that
* instances will be either freed or untouched.
* The only time we must stop traffic is if we want to
* freeze an instance that could potentially be busy.
* The call to nr_rag_freeze must come after the call
* wrsm_lc_replaceconfig and after wci's have been removed
* and or added to the configuation.
*/
}
return (WRSM_SUCCESS);
}
/*
* kill off event thread and remove NR related data structures for this
* network
*/
void
{
network->rsm_ctlr_id));
if (nr->event_thread) {
do {
} while (nr->event_thread);
}
}
/*
* Once LC notifies the NR that all links on all wcis are up,
* notify the NC so it can enable the network (allowing new links
* to be used).
*/
void
{
}
}
/*
* Add a route for each wci in the new configuration to the local node's
* list of preferred routes. This guarantees that a loopback route for the
* local node can always be established.
*
* New routes are added by allocating enough space for an array containing
* all the config-supplied routes plus these new routes.
*/
static void
{
int policycount;
int nwcis;
int i;
kmem_alloc(sizeof (wrsm_preferred_route_t *) *
sizeof (wrsm_preferred_route_t *) * policycount);
for (i = 0; i < nwcis; i++) {
policycount++;
}
}
/*
* Remove extended routes from the local node's prefereed route list, and
* point back to the original preferred route list provided by the
* configuration.
*/
static void
{
sizeof (wrsm_preferred_route_t) *
sizeof (wrsm_preferred_route_t *));
}
}
/*
* Turn off striping on wcis in disabled stripe groups, then remove the
* stripe groups.
*
* turn on striping on new stripe groups
*/
static int
{
int i;
int err;
/*
* first, turn off striping on all wcis in disabled stripe
* groups
*/
while (sg) {
!= WRSM_SUCCESS) {
return (err);
}
}
}
/* remove sg from linked list */
} else {
/* this stripe group is ok; skip it */
}
}
/*
* now turn on striping on wcis in new stripe groups
*/
sg->attached_wcis++;
}
/*
* if all wcis in this stripe group
* are attached, stripe them
*/
return (err);
}
}
}
return (WRSM_SUCCESS);
}
/*
* Turn off striping across wcis in the stripe group.
* Note: all wcis must be attached.
*/
static int
{
int i;
#ifdef DEBUG
int j;
#endif
#ifdef DEBUG
/*
* verify that all used nc2nid entries are set to no stripe
*/
for (j = 0; j < WRSM_MAX_CNODES; j++) {
}
#endif
/* modify register to not do wci striping */
&(wci_config.val));
}
return (WRSM_SUCCESS);
}
/*
* Turn on striping across all wcis in the stripe group.
* Note: all wcis must be attached.
*/
static int
{
int i;
int nwcis;
int stripe[4];
#ifdef DEBUG
int j;
#endif
if (nwcis == 1)
stripe[0] = WCI_STRIPE_NONE;
if (nwcis == 2) {
stripe[0] = WCI_STRIPE_2WAY_EVEN;
} else if (nwcis == 4) {
stripe[0] = WCI_STRIPE_4WAY_0;
}
for (i = 0; i < nwcis; i++) {
#ifdef DEBUG
/*
* verify that all used nc2nid entries are
* set to no stripe
*/
for (j = 0; j < WRSM_MAX_CNODES; j++) {
}
#endif
/* modify register to do wci striping */
&(wci_config.val));
}
return (WRSM_SUCCESS);
}
/*
* Timeout function: causes all WCIs with changed routes to perform
* an mh reroute.
*/
static void
nr_reroute_wcis(void *arg)
{
}
}
}
/*
* prepare to use new wci - call lc_replaceconfig(), other WCI
* initialization functions
*/
int
{
int err;
int i;
#ifdef DEBUG
#endif
saf_id));
return (EINVAL);
#ifdef DEBUG
/*
* verify that all nc2nid entries are turned off
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
}
#endif /* DEBUG */
for (i = 0; i < WRSM_MAX_WNODES; i++) {
}
}
if (pause_evt_thread)
if (init_cmmu) {
/*
* Cmmu Manager has not yet been notified of this wci
*/
return (err);
}
}
if (init_cmmu)
return (err);
}
/*
* wci is now initialized and ready for cluster traffic
*/
if (pause_evt_thread)
return (WRSM_SUCCESS);
}
/*
* start using newly attached wci - call lc_installconfig() and
* lc_enableconfig() to bring up links. (For wcis already attached when a
* new config is installed, this function is not called, and the equivalent
* work is done in wrsm_nr_installconfig() and wrsm_nr_enableconfig().)
*/
int
{
int err;
saf_id));
return (ENODEV);
sg->attached_wcis++;
/*
* If a newly attached wci completes a stripe group,
* turn on striping. (For a new config, do this
* during wrsm_nr_installconfig().)
*/
}
if (stripe)
return (err);
}
}
return (WRSM_SUCCESS);
}
/*
* stop using wci in preparation for it being detached
*/
int
{
int err;
int wcicount = 0;
#ifdef DEBUG
int i;
#endif
saf_id));
if (!wci)
return (ENOENT);
if (!lcwci)
return (ENODEV);
return (EBUSY);
}
/*
* Can't remove wci if it is the last one in the controller,
* even if force is requested.
*/
wcicount++;
}
if (wcicount == 1) {
return (EBUSY);
}
/*
* tell the LC to update the link states based on the new
* NULL configuration; this will cause the LC to (immediately)
* notify the MH that all links are down (any that
* are no longer valid in the new config). This should
* cause the MH to (immediately) queue evt_mhdirect events
* for the event thread to process.
*/
/*
* wait for link down events to cause necessary ncslice rerouting
*/
/*
* wait for links to really come down
*/
/*
* turn off striping in the stripe group involving this wci
*/
}
sg->attached_wcis--;
if (unstripe) {
return (err);
}
}
}
/*
* wait for loopback route to be removed
*/
#ifdef DEBUG
/*
* verify that all nc2nid entries are turned off
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
}
#endif /* DEBUG */
return (WRSM_SUCCESS);
}
/*
* determine whether this wci is being used by any ncslice routes
*/
static boolean_t
{
int i;
for (i = 0; i < WRSM_MAX_WNODES; i++) {
#ifdef DEBUG
if (wrsm_nr_debug & DBG_ROUTE_EXTRA) {
"wnode %d route users:\n", i));
}
#endif
}
}
else
return (inuse);
}
/*
* MH calls when there has been a change in link status on this WCI
* (causing a change in wnode route status)
*/
void
{
sizeof (wrsm_mh_reachable_t));
}
/*
* MH calls when it has performed an MH subnet coordinated reroute of the
* wnode routes on this WCI
*/
void
{
sizeof (wrsm_mh_reachable_t));
}
/*
* This is a daemon that processes events for this network. There are
* currently 2 generators of events: a change in configuration (
* initiated by an ioctl), and a change in route state (initiated by
* network wide MH reroute).
*/
static void
{
int i;
int wcis_in_reroute;
"wrsm_nr_event_thread");
/* LINTED: constant in conditional context */
while (1) {
/*
* The event thread consumes all events on the event queue.
* Once the event queue is empty, it looks at each cnode to
* see if it needs to be rerouted. After selecting any new
* routes, it then applies them enmasse, thereby avoiding
* too many interruptions.
*/
if (nr->stop_event_thr) {
"nr_event_thread 0x%p network ctlr %d: "
"stopping\n", (void *)curthread,
network->rsm_ctlr_id));
/* awaken config thread */
nr->wait_pause = 0;
/*
* CALLB_CPR_EXIT() calles mutex_exit() on lock
* passed in above. Therefore, do not call
* mutex_exit() explicitly here.
*/
thread_exit();
}
"nr_event_thread network ctlr %d: sleeping "
"nr_event_thread network ctlr %d: awakened\n",
network->rsm_ctlr_id));
/*
* Check if we're supposed to stop running.
* This shouldn't be called unless the network is
* being removed (all activity on this event queue
* should have stopped at this point).
*/
if (nr->stop_event_thr) {
"nr_event_thread 0x%p network ctlr %d: "
"stopping\n", (void *)curthread,
network->rsm_ctlr_id));
/* awaken config thread */
nr->wait_pause = 0;
/*
* CALLB_CPR_EXIT() calles mutex_exit() on lock
* passed in above. Therefore, do not call
* mutex_exit() explicitly here.
*/
thread_exit();
}
}
nr->event_thr_loopcnt++;
/*
* process all events
*/
}
/*
* recalculate any cnode's ncslice-routes affected
* by events
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
/*
* Calculate route for this node. If
* selected route affects other nodes, go
* back and restart route evaluation for
* previously processed nodes.
*/
if (recalc)
goto recalc;
}
}
/*
* apply calculated routes
*/
/*
* call wrsm_mh_reroute() for any WCIs that are being
* rerouted (must happen after ncslices have been
* routed on any direct connect nodes)
*/
wcis_in_reroute = 0;
}
if (wrsm_mh_reroute(wci))
}
}
/*
* See if thread is waiting to be awakened.
*/
if (nr->wait_eventdrain) {
}
}
if (nr->wait_pause > 0) {
"nr_event_thread network ctlr %d: pausing "
nr->wait_pause--;
do {
"nr_event_thread network ctlr %d: unpausing\n",
network->rsm_ctlr_id));
}
}
}
/*
* function to queue an event structure to the network events queue
*/
void
{
} else {
}
if (release_lock)
}
/*
* calling thread sleeps until the event thread consumes all events
* currently on the queue, re-calculates and applies routes, and
* initiates wci reroutes based on (at least) these events
*/
static void
{
do {
"event queue to drain\n"));
}
/*
* calling thread sleeps until the event thread determines that no
* wcis are in the middle of an MH reroute.
*/
static void
{
do {
"wci reroutes to finish\n"));
}
/*
* cause the event thread to pause after it finishes consuming
* and applying the current set of events on its queue
*/
static void
{
nr->wait_pause++;
do {
"evt thr to pause with wait_pause %d\n", pause));
}
/*
* cause the paused event thread to continue consuming events
*/
static void
{
}
/*
* process the event
* return the event type in case some sort of signalling is needed
*/
static void
{
case wrsm_evt_mhdirect:
/*
* store away new wci wnode route info
*/
#ifdef DEBUG
if (!wci) {
"bad event mhdirect: null wci \n"));
break;
}
#endif
/*
* Only check routes for direct connect nodes; the
* rest will be re-evaluated after wrsm_mh_reroute() is
* performed. wrsm_mh_reroute() is initiated during
* this cycle of the event thread based on the
* wci->reroute_state being set to need_reroute.
*/
/*
* don't change any routes, just record which cnodes
* are affected by changes, and tear down invalid
* routes
*/
} else {
/*
* The network is pending, installed or
* installed-up; a complete wrsm_mh_reroute() will not
* happen automatically (not until network is moved
* to enabled state), and may be delayed for
* some time: force nodes using lost wnode routes
* to reroute using the currently available wnode
* routes.
*/
}
/*
* need to call wrsm_mh_reroute() called for this wci
*/
break;
case wrsm_evt_mhreroute:
/*
* re-evaluate ncslices routes for any cnode using a wnode
* whose route has changed.
*/
#ifdef DEBUG
if (!wci) {
"bad event mhdirect: null wci \n"));
break;
}
#endif
/*
* don't change any routes, just record which cnodes
* are affected by changes, and tear down invalid
* routes
*/
} else {
/*
* re-evaluate all cnodes affected by wnode
* routes changes
*/
}
break;
case wrsm_evt_force_reroute:
/*
* call wrsm_mh_reroute() called for this wci called to cause
* controlled mh_rerouting in a pending/installed/installed-up
* configuration
*/
break;
case wrsm_evt_add_passthrough:
/*
* turn on passthrough capability for this node if
* there is a route to it.
*/
break;
case wrsm_evt_send_ptlist: {
/*
* Send PTLIST message with current list of nodes
* passthrough is provided for to all specified nodes.
*/
int i;
for (i = 0; i < WRSM_MAX_CNODES; i++) {
node);
}
}
}
break;
}
case wrsm_evt_recv_ptlist: {
/*
* received a PTLIST message from some node: process it
*/
if (!node) {
/* ignore requests for non-existant nodes */
"evt_recv_ptlist for non-existent node %d\n",
break;
}
break;
}
case wrsm_evt_wakeup:
/*
* an event to get the event thread to run
*/
break;
case wrsm_evt_sessup:
/*
* initiate a session teardown on a node
*/
break;
case wrsm_evt_sessdown:
/*
* initiate a session teardown on a node
*/
break;
default:
"rsm_ctlr_id %d: unknown event %d",
break;
}
}
/*
* Any cnodes using ncslice routes which involve invalid wnode routes on
* this wci must be rerouted. In some cases, cnode routes should be
* re-evaluated to take advantage of new routes. (This depends on the
* state of the network->availability.) Cnodes the need to do a route
* re-evaluation have the check_route field set to true, which causes
* nr_cnode_route() to be called for this cnode from the event thread.
* (The event thread does this after processing all events in the queue.)
*
* Also, the set of routes each cnode should consider when performing
* a route evaluation is controlled by the cnode_retry fields: the
* route evaluation only considers wcis, stripe groups and PT nodes
* which include this wci in their retry field. (This prevents
* continuously re-evaluating routes that haven't changed since they
* were rejected during the last route evaluation.)
*/
static void
{
int i, wnid;
#ifdef DEBUG
#endif
"nr_wci_routechange(), wci %d, "
if (reachable) {
sizeof (wrsm_mh_reachable_t));
sizeof (wrsm_mh_reachable_t));
}
continue;
#ifdef DEBUG
"nr_wci_routechange(), "
"wnode %d, valid %d, nhops %d stripes %d "
"changed %d cnode %d\n", wnid,
#endif
/*
* If old route change hasn't yet been applied to all
* routes, re-evaluate it.
*/
continue;
/*
* wnode route has changed
*
* For the reroute_direct case, we only allow the direct
* connect route to be established if it was not available.
*
* The reroute_direct case never tears down a route, and
* only sets up a route for the node accessed by the new
* direct route (if needed). So leave route marked as
* changed so it is evaluated again after wrsm_mh_reroute()
* has completed multihop rerouting and reports back.
*
* The reroute_pending is really reroute_direct while
* config is not yet enabled. It only only causes route
* evaluations if a route is lost. This wnode route must
* still be re-evaluated for new routes, so leave it marked
* as changed.
*/
if ((reroute != wci_reroute_direct) &&
(reroute != wci_reroute_pending) &&
(reroute != wci_reroute_force)) {
}
/*
* a route to this wnode is available
*/
#ifdef DEBUG
if (changed) {
"ctlr %d: nr route change wci %d wnode %d "
"(nhops %d)\n",
}
#endif
/*
* cnodes currently using this wnode route in their
* ncslice_route are ok
*
* update the wci and stripe group retry lists that
* include this wnode route: the next time cnodes
* that could use this wnode route perform a route
* re-evaluation, they should reconsider this route
*/
if (reroute == wci_reroute_all) {
/*
* cause route re-evaluation on all cnodes
* that could use this wnode
* (nr_cnode_route() is called for each
* cnode).
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
if (WRSM_IN_SET(
i)) {
(CE_CONT,
"re-eval cnode %d on "
"avail route wci %d "
"wnode %d\n", i,
}
}
} else if (reroute == wci_reroute_direct ||
reroute == wci_reroute_force) {
/*
* If this is a direct route and there is
* currently no ncslice route for this
* node, cause a route re-evaluation for
* this node.
*
* the rest of the re-evaluations will take
* place after the next wrsm_evt_mhreroute
* event.
*/
if (!WRSM_NODE_HAVE_ROUTE(node)) {
(CE_CONT,
"re-eval cnode %d on "
"avail route wci %d "
"wnode %d\n", cnodeid,
}
}
}
continue;
} else {
/*
* there is no longer a route to this wnode
*
* cnode ncslice routes using this wnode route are
* no longer valid
*/
#ifdef DEBUG
if (changed) {
"ctlr %d: nr lost route wci %d "
"wnode %d\n",
}
if (wrsm_nr_debug & DBG_ROUTE_EXTRA) {
"route users:\n"));
}
#endif
for (i = 0; i < WRSM_INID2DNID_ENTRIES; i++) {
if (WRSM_IN_SET(
wnid)) {
}
}
} else {
}
}
}
/*
* if a complete reroute was requested or if configuration
* is in pending state, re-evaluate ncslice routes for all
* cnodes using invalid wnodes
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
if (WRSM_IN_SET(lost_route, i)) {
"re-eval cnode %d on "
"lost route wci %d\n", i,
}
}
} else if (reroute == wci_reroute_disabled) {
/*
* if configuration is currently disabled, teardown routes
* using invalid wnodes (except for the loopback route,
* which is always re-evaluated).
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
if (WRSM_IN_SET(lost_route, i)) {
"nr_wci_routechange(), wci %d cnode %d "
i));
/*
* loopback route must always be
* valid. Recalculate.
*/
"ctlr %d routechange disabled "
"reval loopback "
"node %d check_route\n",
network->rsm_ctlr_id, i));
} else {
route_state !=
}
}
}
}
}
}
/*
* Find the best ncslice route to this node.
* Return whether selected route affects other nodes.
*/
static boolean_t
{
int i;
/*
* Can't set up a route for this node yet. Leave
* check_route set, so that when it is enabled, the next
* time the nr event thread runs it will attempt to set up
* a route.
*/
return (recalc_prev_nodes);
}
/*
* this node is being removed, so don't calculate
* a new route for it
*/
return (recalc_prev_nodes);
}
/*
* Evaluate the preferred routes to see what is available. Always
* consider the current route. If current route has changed,
* re-evaluate all possible ncslice routes; otherwise, only
* re-evaluate ncslice routes where some change has been made to
* the wnode or inid routes since last checked.
*/
/*
* has the current route changed?
*/
/*
* Always re-evaluate passthrough routes, as we don't have
* good information about how each preferred route is
* affected by changes in passthrough routing availability.
*/
} else {
}
/*
* Compare all routes of interest; for each new route, choose
* between it and the current winning route; winning route is
* copied into routeinfo->new_route.
*/
/* found a viable route */
/*
* choose best route based on number
* of stripes
*/
sizeof (ncslice_route_t));
}
} else {
/*
* Order of routes in the preferred routes
* is most important, so use this route,
* and don't consider any routes after this
* one.
*/
"use preferred\n"));
sizeof (ncslice_route_t));
break;
}
}
}
/*
* Selected route is now saved in node's routeinfo->new_route.
*
* if no route found, see if we can look at all wcis, stripe groups,
* passthrough routes; otherwise, remove current route.
*/
/*
* If proute nulled to mark update, get value from new_route.
*/
/*
* The local node should never end up without a route,
* as we're guaranteed to have at least one wci for
* the loopback route.
*/
} else {
/*
* apply new route
*/
/*
* In order to use this route, we
* need to set up the wci to use
* inids for all routes, and force
* all users of this wci to use
* inids. If any nodes with a
* lower cnodeid are affected,
* return a flag to cause routes
* for these nodes to be
* re-evaluated.
*/
cnodeid);
} else {
/*
* only update wci's inid2dnid
* table if this inid entry has
* changed.
*/
}
}
}
}
return (recalc_prev_nodes);
}
/*
* Commit to using inids on this wci.
* Return whether switching to inids affects other nodes.
*/
static boolean_t
{
int i;
for (i = 0; i < WRSM_INID2DNID_ENTRIES; i++) {
/*
* no one should be using inids at this point, as we
* haven't yet switched to using inids!
*/
/* LINTED: E_NOP_IF_STMT */
(CE_NOTE, "ientry->users not "
"empty (wci %d, inid2dnid entry %d)\n",
}
}
}
/*
* If any cnodes using or about to use this wci already had their
* routes calculated, cause their routes to be recalculated.
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
if (WRSM_IN_SET(retrynodes, i)) {
"ctlr %d switch_to_inids "
"node %d check_route\n",
network->rsm_ctlr_id, i));
if (i < cnodeid) {
}
}
}
return (recalc_prev_nodes);
}
/*
* compare node's current ncslice route and a new ncslice route to see if
* they match
*/
static boolean_t
{
int i;
return (B_FALSE);
return (B_FALSE);
return (B_FALSE);
return (B_FALSE);
}
return (B_FALSE);
return (B_TRUE);
}
/*
* only build an ncslice route using this proute if it has changed or if
* force is set to true; otherwise, set it to 0 (indicating there is no
* route using this proute).
*/
{
/*
* Check to see this node should retry evaluating available routes
* on the wci or stripe group for this ncslice route. (This occurs
* if one or more inid2dnid or wnode routes have changed; the wci's
* cnode_retry value includes the nodes that should retry). Remove
* this node from the wci or stripe group's retry list. Note: if
* a wci or stripe group is used in more than one preferred
* route in multi-hop mode, only the first preferred route in the
* list of preferred routes is considered.
*/
}
} else {
}
}
/*
* If this is a passthrough route, always re-evaluate this route.
* (It is difficult to keep track of which ncslice routes should be
* retried when PT access has changed, so don't bother.)
*/
if (wci) {
#ifdef DEBUG
"route on wci %d has %d stripes\n",
}
#endif
} else if (sg) {
#ifdef DEBUG
"route on stripe group %d has %d stripes\n",
"uses wci %d no-stripe from stripe "
"group\n", routep->
}
#endif
}
#ifdef DEBUG
(wrsm_nr_debug & DBG_ROUTE)) {
"route uses passthrough nodes:\n"));
}
}
#endif
}
}
/*
* find a route through a particular wci using the preferred route info
*/
static int
{
int best_inid, most_stripes;
int stripes;
return (0);
}
stripes = 0;
/* just choose one wnode */
/*
* There could be more than one wnode that routes
* to this cnode. (This is not actually allowed in
* the configuration currently.) Always use the
* first wnode found.
*/
"ncslice_one_wci_route() - wnode %d on "
"wci %d stripes %d\n", wnid,
break;
}
}
if (wnid == WRSM_MAX_WNODES) {
/*
* no usable wnode on this wci route to the desired
* cnode
*/
return (0);
}
/*
* only use inid if we have to
*/
/* use wnode */
return (stripes);
}
/*
* Using inids: reserve an inid entry with all dnids set
* to this wnode.
*/
for (i = 0; i < WRSM_MAX_DNIDS; i++) {
}
if (inid > -1) {
/* use inid */
"wci %d using_inids "
return (stripes);
} else {
/* no inid2dnid entry available */
return (0);
}
} else {
/*
* Find routes to max_stripes switches, searching in switch
* list order (preferred order). Use only switches that
* haven't been used by other wcis in this route (if this
* wci is part of a stripe group).
*
* Routemap striping is not allowed in combination with
* passthrough striping, so only use wnode entries with
* stripes == 1.
*/
"find passthrough routes\n"));
dnids = 0;
if (!WRSM_IN_SET(
dest_cnodeid)) {
"passthrough not provided through "
"node %d to node %d\n",
/* skip this switch - no passthrough to cnode */
continue;
}
/* skip this switch - used already */
continue;
}
pt_nodeid) &&
dnids++;
if (dnids == max_stripes) {
/* found enough switches */
(CE_CONT,
"found max_stripes (%d) "
"switches\n", dnids));
break;
}
}
}
}
if (dnids == 0) {
/* no route */
return (0);
}
/*
* found a route
*/
/*
* If just one switch through this wci, can use a
* wnode rather than inid2dnid entry.
*/
/* use wnode */
/* always just one stripe per switch */
return (1);
}
/*
* using inids
*/
/*
* fix up dnid array
*/
/* LINTED: logical expression always true */
if (dnids == 1) {
} else if (dnids == 2) {
} else if (dnids == 3) {
}
/*
* Reserve a new inid entry for this dnid array, or find an
* existing matching entry if there is one.
*/
if (inid > -1) {
/* use inid */
return (stripes);
}
/*
* no inid entry available for the route we created - see
* if one of the existing inid entries can be used instead
*/
most_stripes = 0;
best_inid = -1;
/*
* make sure each wnode (dnid) in the nr.inid2dnid
* entry goes to usable switch
*/
dnids = 0;
for (i = 0; i < WRSM_MAX_DNIDS; i++) {
if (inid_nodeid == pt_nodeid) {
dnids++;
break;
}
}
}
if (dnids == WRSM_MAX_DNIDS) {
/* found a usable entry - all dnids ok */
most_stripes) &&
max_stripes)) {
/* remember the best usable entry */
}
}
}
if (best_inid != -1) {
/* found an inid entry */
return (stripes);
}
/* no inid2dnid entry available */
return (0);
}
}
/*
* build an nclice route using a single wci
*/
static void
{
/*
* found a route
*/
} else {
}
}
}
/*
* build an nclice route using each wci in stripe group
*/
static void
{
int i;
/*
* striping is not enabled -- try just using one
* wci from the stripe group
*/
return;
}
/*
* try to get an equal number of stripes per wci
*/
max_stripes));
if (max_stripes == 0) {
/*
* Too many wcis, can't get a stripe per wci.
* Try using a single wci.
*/
return;
}
/*
* find a route on each wci in the stripe group
*/
totalstripes = 0;
"found %d stripes on wci #%d "
/* found a route on this wci */
totalstripes += stripes;
} else {
}
} else {
/*
* can't use current set of routes - remove routes on
* previous wcis
*/
totalstripes = 0;
if (stripes == 0) {
/*
* There is no route on this wci. Try
* using a single wci from the stripe
* group.
*/
return;
} else {
/*
* It is required that the number of
* stripes per wci be balanced. Retry
* finding routes on all wcis with only as
* many routes as can be found on this wci.
* Start over with the first wci.
*/
"max_stripes per wci = %d\n", max_stripes));
goto retry;
}
}
}
/*
* If balanced stripes aren't required, see if we can get
* additional stripes on any wcis.
*/
if (max_stripes == 0) {
/* found enough stripes */
break;
}
/* remove old route */
totalstripes -=
max_stripes +=
} else {
totalstripes -=
}
/* build a new route with more stripes */
/* found a route on this wci */
totalstripes += stripes;
} else {
}
}
}
}
/*
* build an nclice route using just one wci from a stripe group
*/
static void
{
int i;
"ncslice_build_sg_nostripe_route()\n"));
/*
* wci striping is not required: try using a single
* wci from this stripe group
*/
/*
* store the single route in routep entry 0
*/
/* found a single wci route */
"ncslice_build_sg_nostripe_route() - found route "
} else {
}
return;
}
}
}
/*
* Find a matching inid entry, otherwise find an unused entry and copy
* dnid info into it.
*/
static int
{
int i, j, free_inid, unused_inid;
"ncslice_find_inid() wci %d cnode %d "
"dnids (cnodes) %d (%d) %d (%d) %d (%d) %d (%d)\n",
free_inid = 0;
/*
* Need to set up inid2dnid entries for any valid nc2nid
* entries already using or planning to use a wnode route
* on this WCI prior to looking for free inid entries.
*/
for (i = 0; i < WRSM_MAX_WNODES; i++) {
/*
* This wnode is in use (or about to be).
* Use next inid entry to represent this
* wnode. There will always be at least as
* many inid2dnid entries as wnodes.
*/
/* LINTED: E_NOP_IF_STMT */
(CE_NOTE,
"ientry->users not "
"empty (wci %d, "
"inid2dnid entry %d)\n",
free_inid));
}
/* all dnids in entry use this wnode */
for (j = 0; j < WRSM_MAX_DNIDS; j++) {
ientry->wnode_list[j] = i;
}
"reserved inid "
"%d for wnode %d stripes %d\n", free_inid,
free_inid++;
}
}
}
for (i = 0; i < WRSM_MAX_DNIDS; i++) {
}
free_inid = -1;
unused_inid = -1;
for (i = 0; i < WRSM_INID2DNID_ENTRIES; i++) {
/* inid not set up; remember */
free_inid = i;
free_inid));
continue;
}
/*
* inid's wnode_list doesn't need to match exactly, just
* need the same number of stripes, and same set of
* wnodes
*/
wnode_bitmask)) {
/* found a matching entry */
(CE_CONT, "inid %d matching entry",
i));
return (i);
}
/* no one is using this entry; remember */
unused_inid = i;
}
/* prefer a free inid over an unused inid */
if (free_inid != -1) {
}
if (unused_inid != -1) {
/* copy information into this inid entry */
for (i = 0; i < WRSM_MAX_DNIDS; i++) {
}
#ifdef DEBUG
"inid %d stripes %d cnode_routes",
if (wrsm_nr_debug & DBG_ROUTE)
#endif
} else {
if (switching_to_inids) {
for (i = 0; i < WRSM_INID2DNID_ENTRIES; i++) {
}
}
}
}
return (unused_inid);
}
/*
* Map a wci number, as it appears in the
* node->routeinfo->current_route list of wcis to the stripe number
* that it will use.
*/
static int
wci_to_stripe(int i)
{
ASSERT(i < WRSM_MAX_STRIPEWCIS);
switch (i) {
case 0: return (0);
case 1: return (2);
case 2: return (1);
case 3: return (3);
}
return (0);
}
/*
* Calculate the striping for barriers. The link_stripes field is a bit
* mask of the offsets to read to collect the CESRs of all wcis and links
* providing access to this node.
*
* Called from event thread.
*/
static void
{
int i;
int wci_stripe;
ushort_t link_stripes = 0;
/*
* WCI striping is controlled by bits 7 and 8. This means
* that striping occurs at a granularity of 128 bytes.
* The striping pattern is as follows:
*
* 4-way 2-way 1-way
* bit 8:7 WCI WCI WCI
* 0 A A A
* 1 B A A
* 2 C B A
* 3 D B A
*
* currentp->nwci can be used to turn on the appropriate WCI
* stripes:
* 1: bit 0; 2: bits 0,1; 4: bits 0-3
*
*
*
* for inid2dnid striping, there are always 4 dnids. The
* NR programs the dnids in one of the following patterns:
* 1 way: AAAA, 2 way: ABAB, 3 way: ABCC, 4 way: ABCD
*
* The address bits used for striping are bits 9 and 10.
* The striping occurs at a gramularity of 512 bytes.
*
* The bits are used in reverse from expected order
* (see page 5-224 in the PRM, revsion 1.0, Jun 30, 1999).
* This means the striping pattern is not quite as expected:
*
* INID 4 way 3 way 2 way 1 way
* bit 10:9 entry INID INID INID INID
* 0 0 A A A A
* 1 2 C C A A
* 2 1 B B B A
* 3 3 D C B A
*
* The inid2dnid.stripe field can be used to turn on the appropriate
* stripes:
* 1: bit 0; 2: bits 0,2; 3: bits 0-2; 4: bits 0-3
*
*
*
* For route map striping, there are alway 2 links. The
* possible patterns are:
* 1 way: AA, 2 way: AB
*
* Address bit 9 is used for route map striping.
* The striping occurs at a gramularity of 512 bytes.
* The striping pattern is as follows:
*
* rtmap 2 way 1 way
* bit 9 entry rtmap rtmap
* 0 0 A A
* 1 1 B A
*
* The wnodeinfo.stripes field can be used to turn on the appropriate
* stripes:
* 1: bit 0, 2: bits 0,1
*/
wci_stripe = wci_to_stripe(i);
/* fall through in each case */
case 2:
/* LINTED: E_CASE_FALLTHRU */
case 1:
}
} else {
/* fall through in each case */
case 4:
+ wci_stripe);
/* LINTED: E_CASE_FALLTHRU */
case 3:
+ wci_stripe);
/* LINTED: E_CASE_FALLTHRU */
case 2:
+ wci_stripe);
/* LINTED: E_CASE_FALLTHRU */
case 1:
}
}
}
}
/*
* Check whether any ncslice routes are changing, and if changes require
* traffic to be stopped on the Safari bus. Stop activity on routes that
* are being removed.
*
* Note: the current algorithm removes an old route before adding a new
* route, which means that any time a route changes, traffic must be
* stopped. It might be possible to change the algorithm to change the
* route in one step in the case where the same wci is handling both the
* old and new routes. If this were true, traffic would only need to be
* stopped if the wci handling the route changed (including going from
* striped to non-striped or vice versa).
*/
static boolean_t
{
int i;
#ifdef DEBUG
int x;
#endif
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
#ifdef DEBUG
"ctlr %d: node %d "
"current route wci %d uses "
"inid %d\n",
}
}
#endif
#ifdef DEBUG
"ctlr %d: node %d "
"new route wci %d uses inid %d\n",
} else {
"ctlr %d: node %d new route "
"wci %d no inids\n",
}
}
#endif
if (stop_traffic) {
/*
* already set any necessary booleans
*/
continue;
}
}
#ifdef DEBUG
"ctlr %d: node %d "
"err route wci %d uses inid %d\n",
} else {
"ctlr %d: node %d err route "
"wci %d no inids\n",
}
}
#endif
/*
* Might get here if the error loopback route has
* changed, as well as because of newly losing a route.
*/
/*
* Check whether already using the current
* err loopback route
*/
if (ncslice_routes_match(
continue;
} else {
sizeof (ncslice_route_t));
}
if (WRSM_NODE_HAVE_ROUTE(node)) {
/* previous route was a real one */
}
if (stop_traffic) {
continue;
}
!= 0) {
}
}
"route for node %d",
if (WRSM_NODE_HAVE_ROUTE(node)) {
}
/*
* Route will be completely removed if there is
* one. (There may be no route already, or a real
* route or an error route). It is ok to remove
* this route because the only time this happens is
* when a node is being removed from a config, and
* nodes can only be removed from a config after
* the controller unregisters itself (guaranteeing
* that there will be no client accesses to this
* node). There is no need to pause cpus or wcis
* because they've already stopped using this
* route, but we do want to drain wcis of
* transactions, because although we've turned off
* passthrough, we may need to wait for previous
* in-progress transactions to the ncslice.
*/
/* there is currently a route to remove */
"node %d removing route\n",
} else {
}
}
}
return (route_changes);
}
/*
* Stop all traffic that could be generating new transactions accessing
* ncslice addresses, so that we can safely move ncslice ownership from
* one wci to another. This function is also used by DDI_SUSPEND, to
* stop all incoming traffic.
*/
{
/*
* Stop local cpus from generating nc2nid traffic.
* (pause_cpus does a kpreempt_disable)
* The cpu_pause thread on each cpu does a membar sync
*/
if (!stop_incoming) {
}
/*
* If wcis are generating passthrough traffic or if all incoming
* traffic should be prevented, stop remote
* nodes from generating new traffic by turning on the
* cluster_disable flag in all wcis. (All wcis have the
* same passthrough configuration.) Note: the list of
* lcwcis can't change in the middle, as all cpus are
* stopped.
*
* Note: we don't actually know which wcis are being used
* by remote nodes for passthrough, so there's no way to
* only perform this disabling on some of the wcis.
*/
continue;
&(wci_ca_config.val));
}
/*
* Make sure already arrived passthrough/incoming requests
* have been issued onto the bus.
*
* Look at CAG busy bits; when no CAGs are busy all
* requests have completed.
*/
continue;
do {
&(wci_ca_busy.val));
}
}
/*
* Make sure Request Agents on the wcis having ncslices
* removed are finished servicing requests to moving
* ncslices. Given that all cpus have membar-synced and
* all CAGS are not busy, RAGS should be pretty much idle
* already (some might still be handling a data phase).
*
* For coding simplicity, check all wcis instead of just
* those losing ncslices.
*
* The hardware team advised that we do this.
*/
continue;
do {
&(wci_ra_busy.val));
}
}
static void
{
/*
* Allow remote nodes to generate passthrough/incoming nc2nid traffic.
* Note: list of lcwcis can't change in the middle, as all cpus
* are stopped.
*/
continue;
&(wci_ca_config.val));
}
}
/*
* restart local cpus stopped in stop_ncslice_traffic
* (does a kpreempt_enable)
*/
if (!stop_incoming) {
start_cpus();
}
}
/*
* Modify the hardware to reflect the newly calculated inid2dnids for
* each wci, and the new nc2nid settings described by the ncslice route
* for each node
*/
static void
{
int i;
/*
* If there is no routeinfo for the local node, than it is too
* early to set up routes. Do nothing.
*/
return;
}
/*
* The local node is required to have a loopback route, which is
* used for any ncslice which has lost its route. The local
* loopback route is set up before any other routes, so it is
* guaranteed to be available by the time it is needed. The only
* time it is allowed to go away is when the configuration is being
* removed (in which case the controller unregisters, and dangling
* accesses to remote nodes are guaranteed to not be a problem).
*/
"state is %s; current stripes %d",
/*
* If local node is no_route, then must be removing config,
* and must have already removed routes for all other nodes.
*/
#ifdef DEBUG
for (i = 0; i < WRSM_MAX_CNODES; i++) {
}
#endif
return;
#ifdef DEBUG
} else {
/*
* route_state should never be use_errloopback, as this
* _is_ the errloopback route!
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
}
#endif
}
/*
* check whether any inid2dnids are changing
*/
}
for (i = 0; i < WRSM_MAX_WNODES; i++) {
}
for (i = 0; i < WRSM_INID2DNID_ENTRIES; i++) {
}
}
/*
* Look for changing nc2nid routes. Check whether there were
* existing routes for any of the new routes; replacing an existing
* route requires stopping traffic.
*/
&stop_traffic);
if (!inid_changes && !route_changes)
return;
/*
* Enter platform ncslice update critical section.
*/
if (stop_traffic)
/*
* apply changes to inid2dnid registers if necessary
*/
}
/*
* apply changes to nc2nid registers if necessary
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
}
*node->link_stripesp = 0;
}
/* newly able to communicate with this node */
WRSMSET_ADD(newroute, i);
}
}
}
/*
* Notify platform module of any changes in ncslice ownership.
* (ncslice_responder array is updated by ncslice_add_hw_route()
* and ncslice_remove_hw_route()).
*/
if (stop_traffic)
/*
* Exit platform ncslice update critical section.
*/
/*
* for nodes that have newly acquired routes, do any necessary
* setup and notification
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
"unconfigured");
"no-route");
}
/* lock the struct */
sizeof (ncslice_route_t));
/* unlock the struct */
if (WRSM_IN_SET(newroute, i))
}
}
/*
* notify other nodes if passthrough access has changed
*/
if (ptnotify)
}
/*
* Update the wci inid2dnid registers and enable_inid flag in the
* wci_config registers to match what the data structure says they should
* be.
*/
static void
{
int i, j;
/* update dnids in inid table */
/*
* There are 64 entries in the hardware, organized
* as inid2dnid[dnid][inid]. Each entry is spaced
* at a distance of STRIDE_WCI_INID2DNID_ARRAY bytes
* apart, and is 8 bytes (uint64_t) large.
*/
for (i = 0; i < WRSM_INID2DNID_ENTRIES; i++) {
/* skip over this inid's dnids */
continue;
}
for (j = 0; j < WRSM_MAX_DNIDS; j++) {
WRSM_INID2DNID_ENTRIES * j) +
(STRIDE_WCI_INID2DNID_ARRAY * i);
}
}
/* modify register to use inid table */
&(wci_config.val));
} else {
/* modify register to not use inid table */
&(wci_config.val));
}
}
/*
* stop using the nc2nid entries on wcis specified by this route
*/
static void
{
int i, j;
/* setting launch remote to 0 means all other fields are ignored */
/*
* turn off launch_remote bit for each ncslice on each wci in route
*/
for (i = 0; i < WRSM_NODE_NCSLICES; i++) {
continue;
/*
* for updating AXQ's or SSM WCIs
*/
#ifdef DEBUG
&(wci_nc2nid.val));
wci_nc2nid.val = 0;
#endif
}
}
/*
* Also need to do this for ncslices forwarded to this cnode.
*/
for (i = 0; i < WRSM_MAX_NCSLICES; i++) {
i))
continue;
/*
* for updating AXQ's or SSM WCIs
*/
(i * STRIDE_WCI_NC2NID_ARRAY));
#ifdef DEBUG
&(wci_nc2nid.val));
wci_nc2nid.val = 0;
#endif
}
}
/*
* record that this cnode is no longer using these routes
*/
#ifdef DEBUG
"exist!\n", i));
continue;
"doesn't exist!\n", i));
continue;
}
#endif
} else {
}
}
}
}
/*
* start using the nc2nid entries for ncslices belonging to this node
* on wcis specified by this route
*/
static void
{
int i, j;
wci_nc2nid.val = 0;
} else {
}
/*
*/
for (i = 0; i < WRSM_NODE_NCSLICES; i++) {
continue;
/*
* AXQ's or SSM WCIs
*/
}
}
/*
* Also need to do this for ncslices forwarded to this cnode.
*/
for (i = 0; i < WRSM_MAX_NCSLICES; i++) {
i))
continue;
/*
* AXQ's or SSM WCIs
*/
(i * STRIDE_WCI_NC2NID_ARRAY));
}
}
/*
* record that this cnode is now using these routes
*/
} else {
}
}
}
}
/*
* A route to this node was established (previously there was no
* route). Notify transport layer and passthrough clients.
*
* return whether passthrough notification is needed
*/
static boolean_t
{
return (pt_haveroute(node));
}
/*
* The route to this node was lost:
* tear down all communication to node; notify passthrough
* clients
*/
static void
{
/*
* notify passthrough clients
* etc.etc.
*/
/*
* record new state
*/
}
/*
* about to change routes in the hardware
*/
static void
{
/*
* update local routechange counter
*/
(*network->reroutingp)++;
/*
* notify remote nodes using passthrough through this node of
* changing routes
*/
if (network->passthrough_routes) {
}
}
/*
* finished changing routes in the hardware
*/
static void
{
(*network->route_counterp)++;
(*network->reroutingp)--;
/*
* notify remote nodes using passthrough through this node of
* changing routes
*/
if (network->passthrough_routes) {
}
}
/*
* This routine is called from wrsm_lc_check_lockout() when it notices
* that wci_ra_esr.acc_write_lockout is set. This means that the
* remote node that exports 'ncslice' has its write-lockout bit set
* corresponding to our cnodeid. We have to clear it by doing a read
* from page 1 of that slice (for all stripes) and make sure that all
* currently open barriers fail since this indicates that an
* undetected write error could have occured.
*/
/* ARGSUSED */
void
{
int i, j;
volatile caddr_t lockout_vaddr;
for (i = 0; i < WRSM_MAX_CNODES; i++) {
if (!node)
continue;
/*
* Search the list of ncslices exported by the cnode,
* to see if the cnode exports the failed ncslice.
*/
for (j = 0; j < WRSM_NODE_NCSLICES; j++) {
break;
}
/* If this cnode doesn't export the ncslice, continue */
if (j == WRSM_NODE_NCSLICES)
continue;
/* If haven't yet established a route to node, don't bother */
continue;
}
"nr_clear_lockout: clearing node %d vaddr %p stripes=%d",
stripes != 0;
if (stripes & 1) {
/* Do block read from remote page 1 */
}
/* Advance pointers to next stripe */
}
}
}
/*
* Passthrough related route teardown - remove passthrough forwarding
* to this node, and notify other nodes of this loss of capability.
*/
static void
{
int i;
return;
/*
* stop waiting for this node to finish rerouting
*/
(*network->route_counterp)++;
(*network->reroutingp)--;
} else {
}
/*
* Reroute cnodes that might be using this node as a switch.
* (At some point, we might want to keep better track of which
* nodes really _are_ using a passthrough route through this node,
* to avoid unnecessary route re-evaluations.)
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
}
"ctlr %d pt_noroute "
"node %d check_route\n",
network->rsm_ctlr_id, i));
}
}
/*
* The remote node can't be offering passthrough to anyone if
* there's no route! Zero out pt_provided list, and decrement
* pt_route_counter to be sure the next time we communicate with
* the remote node, we get an updated passthrough list.
*/
/*
* Don't provide passthrough to this node, so skip rest
* of this function.
*/
return;
}
/*
* Update pt_provided list, and immediately notify other nodes of
* loss of passthrough access to this node.
*/
/*
* remove passthrough capability to this node
*/
for (i = 0; i < WRSM_MAX_NCSLICES; i++) {
i)) {
/*
* Note: no lock is needed because passthrough
* operations are single threaded (handled by nr
* event thread, and only passthrough ncslices are
* managed by nr event thread.
*/
network->wrsm_ncslice_users[i]--;
if (network->wrsm_ncslice_users[i] == 0) {
}
}
}
/*
* It shouldn't be necessary to wait for CAG busy bits to clear on
* all wcis to guarantee that they are no longer using this
* passthrough route. Turning off passthrough in the wci should
* just cause CAGs to timeout or return errors.
*/
}
/*
* Passthrough related route setup after a node becomes newly accessible:
* turn on passthrough forwarding to this node if it is allowed, and notify
* other nodes of this capability.
*
* Return whether passthrough notification is needed.
*/
static boolean_t
{
int i;
if (!WRSM_NODE_HAVE_ROUTE(node))
return (B_FALSE);
/*
* let the remote node know about passthrough
* routes the local node currently supplies
*/
}
return (B_FALSE);
}
/*
* add passthrough capability for this node
*/
for (i = 0; i < WRSM_MAX_NCSLICES; i++) {
i)) {
/*
* Note: no lock is needed because passthrough
* operations are single threaded (handled by nr
* event thread, and only passthrough ncslices are
* managed by nr event thread.
*/
if (network->wrsm_ncslice_users[i] == 0) {
#ifdef DEBUG
} else {
#endif
}
network->wrsm_ncslice_users[i]++;
}
}
/*
* update pt_provided list
*/
/*
* request notification of new passthrough routes
*/
return (B_TRUE);
}
/*
* Timeout to send pt_counter to any node that hasn't yet received
* it.
*/
static void
pt_resend_timeout(void *arg)
{
return;
}
nr->pt_retry_timeout_id = 0;
}
/*
* pt_sendptlist to this node failed; schedule a retry
*/
static void
{
return;
}
if (nr->pt_retry_timeout_id == 0) {
/* set up timeout */
} else {
}
}
}
/*
* Send current set of nodes this node provides passthrough access to
* to each other remote node (or to one specified node)
*/
static void
{
int i;
PT_MSGSTRING(msgtype)));
/*
* send message to all remote nodes (or the specified node)
* notifying them of available PT route
*/
/*
* set up message
*/
/*
* take lock to guarantee local pt_provided and pt_route_counter
* match when pt_ptlist_recv_hdlr copies info out
*/
"ctlr %d: sending pt msg with pt_route_counter = %d\n",
if (to_node) {
/*
* queue up an event to retry sending this message
*/
}
} else for (i = 0; i < WRSM_MAX_CNODES; i++) {
!WRSM_NODE_HAVE_ROUTE(node)) {
continue;
}
/*
* queue up an event retry sending this message
*/
}
}
}
/*
* send PTLIST message to specified node; receive and handle reply
* message
*/
static int
{
#ifdef DEBUG
if (wrsm_nr_debug & DBG_ROUTE_EXTRA)
#endif
recvmsg) != WRSM_SUCCESS) {
/*
* This node is not responding (message not delivered or
* response not received). (Transport Layer tears down the
* session if there is a message delivery failure).
*
* stop waiting for this node to finish rerouting
*/
(*network->route_counterp)++;
(*network->reroutingp)--;
} else {
}
return (B_FALSE);
}
#ifdef DEBUG
if (wrsm_nr_debug & DBG_ROUTE_EXTRA)
#endif
"node %d response message %s route_counter %d\n",
/*
* queue up passthrough event to event handler
* to record new passthrough info from this node.
*/
return (B_TRUE);
}
/*
* Handler for the PTLIST message: save new passthrough information
* from sending node, and reply with passthrough information from this
* node (what nodes this node provides passthrough access to).
*
* The transport layer won't call a handler to deliver a message from
* a node unless the transport layer knows that node exists. The transport
* layer is notified when a node is removed by NC, before it removes the
* node pointer in network->nodes. So it's safe to use that pointer in
* the handler.
*/
static boolean_t
{
network->rsm_ctlr_id));
#ifdef DEBUG
if (wrsm_nr_debug & DBG_ROUTE_EXTRA)
#endif
if (cnodeid > WRSM_MAX_CNODES) {
/*
* ignore message! can't send a response to a non-existent
* node
*/
return (B_FALSE);
}
"node %d message %s route_counter %d\n",
/*
* handle passthrough reroute start message
*/
(*network->reroutingp)++;
} else {
}
}
/*
* handle passthrough reroute finish message
*/
(*network->route_counterp)++;
(*network->reroutingp)--;
} else {
}
}
/*
* queue up passthrough event to event handler with latest
* passthrough info from this node
*/
/*
* Always send a response message regardless of the pt_msgtype.
*/
/*
* take lock to guarantee local pt_provided and pt_route_counter
* match
*/
"ctlr %d: sending pt msg with pt_route_counter = %d\n",
#ifdef DEBUG
if (wrsm_nr_debug & DBG_ROUTE_EXTRA)
#endif
/*
* This node is not responding. (Transport Layer tears
* down the session if there is a message delivery
* failure).
*
* stop waiting for this node to finish rerouting
*/
(*network->route_counterp)++;
(*network->reroutingp)--;
} else {
}
/* other side will resend, so don't call pt_retry() here. */
} else {
}
return (B_TRUE);
}
/*
* Update the passthrough routing info for the specified node, and
* cause affected cnodes to re-evaluate their current routes.
*/
static void
int route_counter)
{
int i;
"no routinfo or disabled node %d\n",
/* this node is going away; ignore update message */
return;
}
/*
* list of passthrough nodes hasn't changed
*/
"pt_route_counter unchanged\n"));
#ifdef DEBUG
"pt_provided bitmasks don't match\n"));
#endif
return;
}
/*
* Received new list of nodes this node provides passthrough
* access to.
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
/*
* only re-evaluate routes for nodes interested in using
* this node as a switch
*/
continue;
WRSM_IN_SET(pt_provided, i)) {
/*
* Passthrough access to this node has changed;
* cause route re-evalauation.
*/
continue;
"ctlr %d pt_route_update "
"node %d check_route\n",
network->rsm_ctlr_id, i));
#ifdef DEBUG
if (WRSM_IN_SET(pt_provided, i)) {
"gained access "
"through node %d to node %d\n",
} else {
"lost access through node %d to node %d\n",
}
#endif
}
}
#ifdef DEBUG
"ctlr %d: node %d pt_provided - \n",
if (wrsm_nr_debug & DBG_ROUTE_EXTRA)
#endif
}
/*
* translate from safari port id to wci structure
*/
static wrsm_ncwci_t *
{
return (wci);
}
"safid_to_wci - no wci with id %d!\n", id));
return (NULL);
}
/*
* translate from stripe group id to stripe group structure
*/
static wrsm_nc_strgrp_t *
{
return (sg);
}
"sgid_to_sg - no sg with id %d!\n", sgid));
return (NULL);
}
/*
* add_wrsm_route_kstat
*/
static void
{
int i, j;
char tmp_str[30];
"net",
sizeof (wrsm_route_kstat_t) / sizeof (kstat_named_t),
0);
"rsm ctlr %d to host %s: routes kstat_create failed",
return;
}
/* initialize the named kstats */
for (i = 0; i < WRSM_MAX_WCIS_PER_STRIPE; i++) {
for (j = 0; j < WRSM_MAX_DNIDS; j++) {
}
}
/* save the kstat pointer in the routeinfo struct */
}
static void
{
/*
* The kstat framework guarantees that any outstanding
* calls to the update function will be completed before
* the kstat can be deleted.
*/
}
}
/*
* update the route kstat
*/
static int
{
#ifdef DEBUG
char *fm_node_name;
#endif
int wnodeid;
int link;
int i, j;
/* This is a read=only kstat */
if (rw == KSTAT_WRITE) {
/* Writing is not allowed - log the error */
return (EACCES);
}
/* Get the kstat structure */
/* Get node structure from private part of kstat */
/* always initialize to 0 then count number of links for each wci */
/*
* Initialize the following values to 0, so that there will not be
* bogus values when there is no route
*/
for (i = 0; i < WRSM_MAX_WCIS_PER_STRIPE; i++) {
for (j = 0; j < WRSM_MAX_DNIDS; j++) {
}
}
return (0);
}
#ifdef DEBUG
/* Get the FM node name */
#endif
(CE_CONT, "kstat: node availability not enabled"));
return (0);
}
return (0);
}
(CE_CONT, "kstat: cannot access current route"));
return (0);
}
/* Lock the network router info */
/* Copy the current route info into a local data structure */
&cur_rte, sizeof (ncslice_route_t));
/* Unlock the network router info */
/* do something to indicate this state */
return (0);
}
/* Get the per stripe information */
/* Get the wroute for this stripe */
(CE_CONT, "kstat: NULL wroute.wci encountered"));
continue;
}
/* If nhops==0, set numlinks=0, and forget about links */
/* This must be the loopback route */
continue;
}
/*
* Depending on the route type, the link and node
* information are found in different data structures.
*/
"nid_route_inid, stripe=%d", i));
for (j = 0; j < WRSM_MAX_DNIDS; j++) {
/*
* Go through the inid2dnid table.
* The inid2dnid entry is the wnodeid.
*/
wnodeid =
j, wnodeid));
/*
* Use wnodeid to look up the link.
*/
/*
* Use the wnodeid to look up the node id.
*/
"current stripe total for passthrough %d",
} else {
}
}
} else { /* nid_route_wnode */
/* There will be one or two links from this wci */
/* The wroute.id is the wnodeid. */
/* look at links to see which ones lead to wnode */
j = 0;
wnodeid)) {
link;
"current stripe total for multihop %d",
/* Use the wnodeid to look up the node id. */
j++;
}
}
}
}
return (0);
}
void
{
*num_wcis = *avail_wcis = 0;
/* loop over the wcis and determine which one are available */
(*num_wcis)++;
}
}
/*
* Pause threads and cancel timeouts in NR.
* Disable all incoming traffic.
*/
int
{
return (DDI_SUCCESS);
}
return (DDI_FAILURE);
}
/*
* Pause event thread first. This guarantees that it
* doesn't re-enable the wcis.
*/
return (DDI_FAILURE);
}
/*
* cancel any pending timeout to reconfig wcis
*/
}
/*
* Cancel any pending timeout to resend passthrough messages
*/
/*
* cancels timeout or waits until it is finished
*/
}
if (cancel_reroute) {
(void) untimeout(cancel_reroute);
}
if (cancel_pt_retry) {
(void) untimeout(cancel_pt_retry);
}
return (DDI_SUCCESS);
}
/*
* Restart threads and timeouts in NR.
* Re-enable incoming traffic.
*/
int
{
return (DDI_SUCCESS);
}
return (DDI_FAILURE);
}
/*
* restart cancelled timeout to force an MH reroute on wcis
*/
}
/*
* restart cancelled timeout to retry passthrough message
*/
}
/*
* unpause event thread, and verify that it is running again
*/
return (DDI_SUCCESS);
}
int
{
return (session_id != SESS_ID_INVALID);
}