wrsm_nc.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* This file manages controllers (networks). It coordinate the network
* routing, session and transport modules.
*/
#include <sys/tnf_probe.h>
#include <sys/wrsm_config.h>
#include <sys/wrsm_transport.h>
#include <sys/wrsm_cmmu.h>
#include <sys/wrsm_session.h>
#include <sys/wrsm_memseg.h>
#include <sys/wrsm_nc_impl.h>
#include <sys/wrsm_plugin.h>
#include <sys/wci_common.h>
#include <sys/wrsm_rsmpi.h>
#ifdef DEBUG
#define DBG_CONFIG 0x001
#define DBG_WARN 0x002
#define DBG_CONFIG_EXTRA 0x010
#else
#define DPRINTF(a, b) { }
#endif
static kmutex_t wrsm_networks_lock;
wrsm_node_t **nodep);
/*
* The configuration functions (nc_replaceconfig(), nc_cleanconfig(),
* nc_installconfig(), nc_enableconfig(), nc_initialconfig(),
* nc_removeconfig()) and the newwci() and removewci() functions are all
* guaranteed by the config layer to be single threaded. The config layer
* is the only consumer of these functions, and it will never call a second
* function before the first is complete. The config layer also only
* calls each function in the apprpropriate order, but the functions
* still do a quick check to be sure the NC concept of this controller's
* state is appropriate.
*/
/*
* set up driver communication path to new node
*/
static int
{
int i;
int err;
/*
* allow access to local node's memory through ncslices
* used by the remote node.
*/
/*
* Note: no lock is needed because config operations are single
* threaded, and only imported ncslices are managed during
* configuration.
*/
#ifdef DEBUG
} else {
#endif
}
for (i = 1; i < WRSM_NODE_NCSLICES; i++) {
if (id != 0) {
/*
* Note: no lock is needed because config
* operations are single threaded, and only
* imported ncslices are managed during
* configuration.
*/
#ifdef DEBUG
} else {
#endif
}
}
}
/*
* set up mapping to CESR registers for remote node
* They are visible through page 0 in the ncslice
* exported by the remote note.
*/
/*
* can't allow access to this node if this
* page can't be mapped
*/
"can't set up mapping to ncslice %d cnode %d CESRs",
return (ENOENT);
}
/*
* can't allow access to this node if this
* page can't be mapped
*/
"can't set up mapping to ncslice %d cnode %d CESRs",
return (ENOENT);
}
#ifdef DEBUG
if (wrsm_nc_debug & DBG_CONFIG) {
"mapped lockout page (ncslice %d)"
" for node %d into kernel vaddr 0x%p pa=0x%p\n",
}
#endif
/*
* configure the transport to this node
*/
"wrsm: can't set up transport to cnode %d", cnodeid));
(void) ddi_unmap_regs(wrsm_ncslice_dip,
(void) ddi_unmap_regs(wrsm_ncslice_dip,
node->cesr_vaddr = 0;
node->lockout_vaddr = 0;
return (err);
}
return (WRSM_SUCCESS);
}
/*
* remove node's transport, CESR mapping, then remove node
*/
static void
{
int i;
if (node->cesr_vaddr) {
for (i = 0; i < WRSM_NODE_NCSLICES; i++) {
if (id != 0) {
}
}
}
(void) ddi_unmap_regs(wrsm_ncslice_dip,
(void) ddi_unmap_regs(wrsm_ncslice_dip,
}
}
/*
* Save away new configuration information.
* Notify the NR so it can save the routing configuration and stop rerouting.
*
* On failure, restore old configuration.
*/
int
{
int i, cindex;
int j;
int old_availability;
"local cnodeid %d, config_version %ld, attached_cnt %d\n",
attached_cnt));
"ctlr id\n"));
return (ENXIO);
}
return (EINVAL);
}
"version stamp\n"));
return (EEXIST);
}
"availability\n"));
return (EBUSY);
}
/*
* check for invalid changes in node config info
*/
}
continue;
/*
* all ncslices in old config must also be in new config
*/
for (j = 0; j < WRSM_NODE_NCSLICES; j++) {
"nc_replaceconfig: exported ncslice "
"change in new "
"config for node cnodeid %d\n", cnodeid));
return (EINVAL);
}
}
/*
* check that other configuration stuff hasn't changed
*/
/* these configuration changes not allowed */
"nc_replaceconfig: bad node ncslice info "
" cnodeid %d\n", cnodeid));
return (EINVAL);
}
}
"nc_replaceconfig: no routing config for local node\n"));
return (EINVAL);
}
/*
* check that routing configuration is valid
*/
attached_wcis)) != WRSM_SUCCESS)
return (err);
/*
* configuration looks reasonable
* set new state
*/
if (got_localnode_config) {
} else {
}
for (i = 1; i < WRSM_NODE_NCSLICES; i++) {
break;
}
}
/*
* add new nodes; disable old nodes
*/
#ifdef DEBUG
}
#endif
i = 0;
/*
* the list of nodes in config->members is ordered by cnodeid
*/
"have a config node - cnodeid %d, i %d\n",
cindex, i));
/*
* this is a valid node in new config
*/
/*
* new node
*/
!= WRSM_SUCCESS)
goto err_cleanup;
} else {
/*
* node already existed - update config
* info
*/
}
i++;
} else {
/*
* this node is not valid in the new config
*/
if (node) {
if (err) {
goto err_cleanup;
}
}
}
}
/*
* configure communication to the new nodes
*/
/*
* add routing info
*/
attached_wcis)) != WRSM_SUCCESS) {
goto err_cleanup;
}
return (WRSM_SUCCESS);
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
} else if (oconfig[i]) {
}
}
if (initialnr)
"error %d\n", err));
return (err);
}
/*
* Reconfig transport and other layers to stop supporting old nodes.
* Turn off communication to nodes only in old configuration.
* Initiate link bringdown on links only in old configuration.
* Set up loopback route for local node.
*/
int
{
int i;
int err;
rsm_ctlr_id));
return (ENXIO);
return (EBUSY);
/*
* Clean up old routing configuration - use intersection
* of old and new configurations.
*/
return (err);
}
/*
* remove all nodes that are no longer part of the network
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
/*
* tear down communication path to all nodes that
* are no longer part of network
*/
}
}
return (WRSM_SUCCESS);
}
/*
* Wait for all old resources to stop being used.
* Bring up new links.
*/
int
{
int err = WRSM_SUCCESS;
int i;
rsm_ctlr_id));
"rsm ctlr id\n"));
return (ENXIO);
}
"availability\n"));
return (EBUSY);
}
/*
* Configure each node for communication and set up error
* pages. Skip local node, which has already been configured.
*/
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
return (err);
}
}
/*
* Wait for NR to complete old config clean and to
* start new link bringup
*/
return (err);
}
/*
* check whether all links are up yet
*/
{
rsm_ctlr_id));
"ctlr id\n"));
return (B_FALSE);
}
return (B_TRUE);
else
return (B_FALSE);
}
/*
* start using new routes
*/
int
{
rsm_ctlr_id));
"ctlr id\n"));
return (ENXIO);
}
/* Increment num_reconfigs */
network->num_reconfigs++;
/* cancel enable timeout thread (or wait until it is finished) */
if (network->enable_timeout_id) {
}
/* timeout thread must have enabled network already */
return (WRSM_SUCCESS);
}
}
/*
* internal version of nc_enableconfig() -- may be called from
* enable timeout thread
*/
static int
{
int i;
int err = WRSM_SUCCESS;
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
}
/*
* enable local routing
*/
return (err);
}
int
{
int err;
return (err);
}
/*
* set up page to accept transactions, but return user error
* on all transactions
*/
PAGESIZE)) != DDI_SUCCESS) {
/*
* can't allow access to this node if this
* page can't be mapped
*/
"can't set up mapping to ncslice %d loopback error "
"page on controller %d\n",
err,
network->rsm_ctlr_id));
return (err);
}
PAGESIZE);
"error page pfn 0x%lx\n", *errorpage_pfnp));
return (0);
}
/*
* install and enable a configuration for a new RSM network
*/
int
{
int err;
"local cnodeid %d, attached_cnt %d\n", rsm_ctlr_id,
return (err);
/*
* initialize network->attr (rsm_controller_attr_t) with both the
* default driver values and the rsm_addr_t of the controller
* Note - it is required the cmmu_init take care of initializing
* other fields of the rsm_controller_attr_t structure.
*/
return (err);
}
return (err);
}
!= WRSM_SUCCESS) {
goto err_cleanup;
}
attached_wcis)) != WRSM_SUCCESS) {
goto err_cleanup;
}
/*
* make sure the new config allows a loopback route
*/
goto err_cleanup;
}
goto err_cleanup;
}
goto err_cleanup;
}
/*
* notify RSMPI that there is a new network.
*
* RSMPI clients are kernel modules, and are expected to not mess
* up the network data structure.
*/
goto err_cleanup;
}
/*
* enable local routing - if necessary delay to allow links to come
* up before enabling routes
*/
/* links are all already up - enable now */
(void) nc_enableconfig(network,
} else {
/* set a timeout to enable network later */
"enable network\n"));
}
return (WRSM_SUCCESS);
"failed controller %d installation",
network->rsm_ctlr_id));
if (network->errorpage_tuple)
}
return (err);
}
/*
* stop using an RSM controller, then remove it
*/
int
{
int err;
#ifdef DEBUG
int i;
#endif
rsm_ctlr_id));
return (ENXIO);
/*
* if plugin has controller open, return EBUSY.
* Note, the plugin library (librsmwrsm.so opens the controller
* and keeps it open until completely done. this is how
* the driver knows that the RSMAPI library applications are not
* using the controller.
*/
if (network->is_controller_open) {
(CE_WARN, "nc_removeconfig: FAILED rsm_ctlr_id %d"
" EBUSY in use plugin\n", rsm_ctlr_id));
return (EBUSY);
}
if (network->registered) {
!= RSM_SUCCESS) {
(CE_WARN, "rsm_unregister_controller "
"failed with error %d\n", err));
return (EBUSY);
}
}
if (network->enable_timeout_id) {
network->enable_timeout_id = 0;
}
/*
* initialize and install a null configuration
*/
return (err);
}
/*
* Tear down communication to all nodes;
* stop using all old wcis, wnodes and links.
*/
#ifdef DEBUG
for (i = 0; i < WRSM_MAX_NCSLICES; i++) {
}
#endif
/*
* tear down loopback error page
*/
if (network->errorpage_tuple)
/*
* teardown segments after tearing down sessions
* (in replaceconfig) but before removing WCIs
* and CMMU entries (in nr_installconfig).
*/
/*
* then remove all data structures associated with
* the configuration.
*/
(void) wrsm_nr_installconfig(network);
(void) wrsm_nr_removeconfig(network);
return (WRSM_SUCCESS);
}
/*
* enable sessions on nodes in an RSM controller
*/
int
{
int cindex;
rsm_ctlr_id));
return (ENXIO);
}
}
return (WRSM_SUCCESS);
}
/*
* disable sessions on nodes in an RSM controller
*/
int
{
int cindex;
rsm_ctlr_id));
return (ENXIO);
}
}
return (WRSM_SUCCESS);
}
/*
* If auto-enable has been requested, enable all new links and nodes after
* a timeout, or after all links are up, whichever comes first.
* (Auto-enable is requested if nc_initialconfig() is called to install a
* configuration.)
*
* If auto-enable has not been requested, simply set the network
* availability to installed_up so that a higher level entity (the RSM
* proxy) knows when it is a good time to call nc_enableconfig()).
*/
void
wrsm_nc_config_linksup(void *arg)
{
network->rsm_ctlr_id));
if (network->enable_timeout_id) {
}
/* auto enable of the network has been requested */
"auto enable\n"));
"enabling\n"));
}
} else {
/* don't auto enable - just change network availability */
"enable\n"));
"state "
"to installed_up\n"));
}
}
if (enable) {
/*
* auto-enable was requested - call nc_enableconfig
*/
/*
* let the controller know we've moved to enabled state
*/
}
}
/*
* notify the NC that a WCI in the specified controller is now
* attached, so the NC can start using it
*/
{
int err;
return (ENXIO);
return (EBUSY);
return (EINVAL);
return (err);
}
/*
* notify the NC that a WCI in the specified controller is being
* detached, so the NC stops using it
*/
int
{
return (ENXIO);
return (EINVAL);
return (EBUSY);
/* force removal even if wci is in use */
}
/*
* functions for managing lists wrsm_node_t structures
*/
/*
* find node structure from RSM fmnodeid
*/
{
int index;
while (node) {
return (node);
}
return (NULL);
}
/*
* get cnodeid from fmnodeid
*/
int
{
if (node) {
return (WRSM_SUCCESS);
} else {
return (EBADF);
}
}
/*
* create a new node structure; set default values
*/
static int
wrsm_node_t **nodep)
{
int index;
return (EADDRINUSE);
}
return (EADDRINUSE);
}
/*
* route_umem space was allocated previously by ddi_umem_alloc
* so that the librsmwrsm.c (plugin library) would be able
* to easily mmap in kernel address space. The
* network->node[cnodeid].link_stripesp field is also used
* by the plugin to determine striping
*/
/*
* add to node_hash
*/
return (WRSM_SUCCESS);
}
/*
* remove node structure from network, free it
*/
static void
{
wrsm_node_t **np;
int index;
/*
* remove node from hash table
*/
}
#ifdef DEBUG
#endif
return;
}
/*
* Decrease the number of nodes in the network,
* Since it is decreased we set free_rag_instance to TRUE
* so that the instances can be freed later in installconfig.
*/
}
/*
* functions for managing lists of wrsm_network_t structures
*/
{
return (network);
}
}
return (NULL);
}
{
return (network);
}
/*
* find network structure from controller id
*/
static wrsm_network_t *
{
return (network);
}
return (NULL);
}
/*
* create a new network structure; initialize it
*/
static int
{
network->errorpage_pfn = 0;
network->errorpage_mappings = 0;
network->num_reconfigs = 0;
/*
* wrsm_num_nodes and free_rag_instance are used to determine if we
* should freeze and or release RAG instances. When there
* are more than 8 nodes on a RSM network we must freeze
* RAG instances per hardware requirement (see PRM).
* We release instances when nodes have been removed from the
* network - that is, when free_rag_instance is set to TRUE
*/
network->wrsm_num_nodes = 0;
if (rsmctlr_to_network(rsm_ctlr_id)) {
/* network already exists for this controller */
return (EBUSY);
}
/*
* allocate kernel space that will also be visible to user
* space by a mmap call
*/
&network->route_cookie);
return (EAGAIN);
}
sizeof (uint32_t));
/*
* add to list of wrsm_networks
*/
/* Add controller (rsmpi) kstat */
return (WRSM_SUCCESS);
}
/*
* remove network structure from list of wrsm_networks and free
*/
static void
{
#ifdef DEBUG
int i;
#endif
wrsm_network_t **np;
#ifdef DEBUG
for (i = 0; i < WRSM_MAX_CNODES; i++) {
}
#endif
/* Remove controller (rsmpi) kstat */
"network 0x%p (controller id %d)\n",
return;
}
}
#ifdef DEBUG
#endif
}
int
{
int i, j;
*num_addrs = 0;
j = 0;
for (i = 0; i < WRSM_MAX_CNODES; i++) {
(*num_addrs)++;
if (j < count) {
addr_list[j] = (rsm_addr_t)i;
j++;
}
}
}
return (RSM_SUCCESS);
}
/*
* initialization - returns standard errno errors
*/
void
{
}
/*
* cleanup
*/
int
{
if (wrsm_nc_check() == WRSM_SUCCESS) {
return (WRSM_SUCCESS);
}
return (EBUSY);
}
int
{
if (wrsm_networks) {
return (EBUSY);
}
return (WRSM_SUCCESS);
}
void
{
}
/* Create the controller kstat */
/*
* add_wrsm_ctlr_kstat
*/
static void
{
"net",
sizeof (wrsm_rsmpi_stat_t) / sizeof (kstat_named_t),
0);
"rsm ctlr %d: controller kstat_create failed",
return;
}
/* initialize the named kstats (wrsm specific) */
/* these are defined in rsmpi.h */
}
static void
{
}
static int
{
if (rw == KSTAT_WRITE) {
return (EACCES);
}
/* Update the named values */
if (network->registered) {
} else {
}
/* access private information form the memseg */
/* access private information from the cmmu */
/* check the number of wcis */
return (0);
}
/* Functions provide for Plugin library librsmwrsm.so support for RSMAPI */
int
{
/* if controller isn't part of a configuration, fail */
if (!network) {
return (ENXIO);
}
if (!network->registered) {
return (EBUSY);
}
if (!network->is_controller_open) {
/*
* Only needs to be set to TRUE the first time. controller
* can be opened numerous times, however, it is only closed
* once.
*/
}
return (WRSM_SUCCESS);
}
/*
* network->is_controller_open to FALSE so the remove config will
* know that the the controller device is not in use by the plugin.
*/
void
{
" controller %d", rsm_ctlr_id));
}
/*
* ioctl supplied for plugin library to determine if the export cnode is
* the local cnode - loopback. Returns WRSM_SUCCESS (0) if match
*/
/* ARGSUSED */
int
{
" no valid network for controller %d", minor));
return (ENODEV);
}
flag) != 0) {
} else {
*rval_p = WRSM_SUCCESS;
}
return (*rval_p);
}
/*
* This function translates from nodename to rsm_addr for that node through
* the specified controller. (This is for SunCluster, if they want it...)
*/
int
{
int i;
if (network) {
}
if (!network) {
"no network"));
return (ENODEV);
}
for (i = 0; i < WRSM_MAX_CNODES; i++) {
continue;
/* found matching node */
*addr = (rsm_addr_t)i;
return (0);
}
}
return (ENXIO);
}
int
{
int ret;
if (network) {
}
if (!network) {
return (DDI_SUCCESS);
}
return (ret);
}
int
{
int ret;
if (network) {
}
if (!network) {
return (DDI_SUCCESS);
}
return (ret);
}