mdmn_commd_server.c revision bf85a12b7c81d0745d5a8aff65baeff50006cde9
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <unistd.h>
#include <sys/resource.h>
#include <fcntl.h>
#include <stdio.h>
#include <thread.h>
#include <meta.h>
#include <sdssc.h>
#include <mdmn_changelog.h>
#include "mdmn_subr.h"
/*
* This is the communication daemon for SVM Multi Node Disksets.
* It runs on every node and provides the following rpc services:
* - mdmn_send_svc_2
* - mdmn_work_svc_2
* - mdmn_wakeup_initiator_svc_2
* - mdmn_wakeup_master_svc_2
* - mdmn_comm_lock_svc_2
* - mdmn_comm_unlock_svc_2
* - mdmn_comm_suspend_svc_2
* - mdmn_comm_resume_svc_2
* - mdmn_comm_reinit_set_svc_2
* where send, lock, unlock and reinit are meant for external use,
* work and the two wakeups are for internal use only.
*
* NOTE:
* On every node only one of those xxx_2 functions can be active at the
* same time because the daemon is single threaded.
*
* (not quite true, as mdmn_send_svc_2 and mdmn_work_svc_2 do thr_create()s
* as part of their handlers, so those aspects are multi-threaded)
*
* In case an event occurs that has to be propagated to all the nodes...
*
* One node (the initiator)
* calls the libmeta function mdmn_send_message()
* This function calls the local daemon thru mdmn_send_svc_2.
*
* On the initiator:
* mdmn_send_svc_2()
* - starts a thread -> mdmn_send_to_work() and returns.
* mdmn_send_to_work()
* - sends this message over to the master of the diskset.
* This is done by calling mdmn_work_svc_2 on the master.
* - registers to the initiator_table
* - exits without doing a svc_sendreply() for the call to
* mdmn_send_svc_2. This means that call is blocked until somebody
* (see end of this comment) does a svc_sendreply().
* This means mdmn_send_message() does not yet return.
* - A timeout surveillance is started at this point.
* This means in case the master doesn't reply at all in an
* aproppriate time, an error condition is returned
* to the caller.
*
* On the master:
* mdmn_work_svc_2()
* - starts a thread -> mdmn_master_process_msg() and returns
* mdmn_master_process_msg()
* - logs the message to the change log
* - executes the message locally
* - flags the message in the change log
* - sends the message to mdmn_work_svc_2() on all the
* other nodes (slaves)
* after each call to mdmn_work_svc_2 the thread goes to sleep and
* will be woken up by mdmn_wakeup_master_svc_2() as soon as the
* slave node is done with this message.
* - In case the slave doesn't respond in a apropriate time, an error
* is assumed to ensure the master doesn't wait forever.
*
* On a slave:
* mdmn_work_svc_2()
* - starts a thread -> mdmn_slave_process_msg() and returns
* mdmn_slave_process_msg()
* - processes this message locally by calling the appropriate message
* handler, that creates some result.
* - sends that result thru a call to mdmn_wakeup_master_svc_2() to
* the master.
*
* Back on the master:
* mdmn_wakeup_master_svc_2()
* - stores the result into the master_table.
* - signals the mdmn_master_process_msg-thread.
* - returns
* mdmn_master_process_msg()
* - after getting the results from all nodes
* - sends them back to the initiating node thru a call to
* mdmn_wakeup_initiator_svc_2.
*
* Back on the initiator:
* mdmn_wakeup_initiator_svc_2()
* - calls svc_sendreply() which makes the call to mdmn_send_svc_2()
* return.
* which allows the initial mdmn_send_message() call to return.
*/
char *commdoutfile; /* file name for the above output */
/* want at least 10 MB free space when logging into a file */
/*
* Number of outstanding messages that were initiated by this node.
* If zero, check_timeouts goes to sleep
*/
/* for printing out time stamps */
/* RPC clients for every set and every node and their protecting locks */
/* the descriptors of all possible sets and their protectors */
/* the daemon to daemon communication has to timeout quickly */
/* These indicate if a set has already been setup */
int md_mn_set_inited[MD_MAXSETS];
/* For every set we have a message completion table and protecting mutexes */
/* Stuff to describe the global status of the commd on one node */
#define MD_CGS_INITED 0x0001
/*
* Global verbosity level for the daemon
*/
/*
* libmeta doesn't like multiple threads in metaget_setdesc().
* So we must protect access to it with a global lock
*/
/*
* Need a way to block single message types,
* hence an array with a status for every message type
*/
/* for reading in the config file */
#define MAX_LINE_SIZE 1024
extern char *commd_get_outfile(void);
extern uint_t commd_get_verbosity(void);
/*
* mdmn_clnt_create is a helper function for meta_client_create_retry. It
* merely needs to call clnt_create_timed, and meta_client_create_retry
* will take care of the rest.
*/
/* ARGSUSED */
static CLIENT *
{
time_out));
}
#define FLUSH_DEBUGFILE() \
}
static void
{
char *msg_buf;
if (master_err != MDMNE_ACK) {
"when processing message type %d\n", type);
} else if (slave_result == NULL) {
} else {
"return value from node %d when processing message "
"type %d. Master exitval = %d, Slave exitval = %d\n",
}
}
static void
{
long long avail_bytes;
int warned = 0;
for (; ; ) {
sleep(10);
/* No output file, nothing to do */
continue;
/*
* stat the appropriate filesystem to check for available space.
*/
continue;
}
/*
* If we don't have enough space, we print out a warning.
* And we drop the verbosity level to NULL
* In case the condtion doesn't go away, we don't repeat
* the warning.
*/
if (avail_bytes < MIN_FS_SPACE) {
if (warned) {
continue;
}
"NOT enough space available for logging\n");
"Have %lld bytes, need %lld bytes\n",
warned = 1;
} else {
warned = 0;
}
}
}
/* safer version of clnt_destroy. If clnt is NULL don't do anything */
#define mdmn_clnt_destroy(clnt) { \
if (clnt) \
clnt_destroy(clnt); \
}
/*
* Own version of svc_sendreply that checks the integrity of the transport
* handle and so prevents us from core dumps in the real svc_sendreply()
*/
void
{
"mdmn_svc_sendreply: XPRT_DIED\n");
return;
}
}
/*
* timeout_initiator(set, class)
*
* Alas, I sent a message and didn't get a response back in aproppriate time.
*
* timeout_initiator() takes care for doing the needed svc_sendreply() to the
* calling mdmn_send_message, so that guy doesn't wait forever
* What is done here is pretty much the same as what is done in
* wakeup initiator. The difference is that we cannot provide for any results,
* of course and we set the comm_state to MDMNE_TIMEOUT.
*
* By doing so, mdmn_send_message can decide if a retry would make sense or not.
* It's not our's to decide that here.
*/
void
{
MSGID_ELEMS(mid));
/*
* Give the result the corresponding msgid from the failed message.
*/
/* return to mdmn_send_message() and let it deal with the situation */
}
/*
* check_timeouts - thread
*
* This implements a timeout surveillance for messages sent from the
* initiator to the master.
*
* If a message is started, this thread is triggered thru
* cond_signal(&check_timeout_cv) and we keep track of the numbers of
* messages that are outstanding (messages_on_their_way).
*
* As long as there are messages on their way, this thread never goes to sleep.
* If one is found, it's checked if this message is overdue. In that case,
* timeout_initiator() is called to wakeup the calling mdmn_send_message and
* to clean up the mess.
*
* If the result from the master arrives later, this message is considered
* to be unsolicited. And will be ignored.
*/
void
{
for (; ; ) {
continue;
}
class++) {
mutex_lock(mx);
/* then is the registered time */
then =
}
}
}
/* it's ok to check only once per second */
sleep(1);
/* is there work to do? */
if (messages_on_their_way == 0) {
}
}
}
void
setup_debug(void)
{
char *tmp_dir;
/* Read in the debug-controlling tokens from runtime.cf */
/*
* If the user didn't specify a verbosity level in runtime.cf
* we can safely return here. As we don't intend to printout
* debug messages, we don't need to check for the output file.
*/
if (md_commd_global_verb == 0) {
return;
}
/* if commdout is non-NULL it is an open FILE, we'd better close it */
}
/* setup the debug output */
if (commdoutfile == (char *)NULL) {
/* if no valid file was specified, use the default */
commdoutfile = "/var/run/commd.out";
} else {
/* check if the directory exists and is writable */
"Can't write to specified output file %s,\n"
"using /var/run/commd.out instead\n", commdoutfile);
commdoutfile = "/var/run/commd.out";
}
}
}
}
/*
* mdmn_is_node_dead checks to see if a node is dead using
* the SunCluster infrastructure which is a stable interface.
* If unable to contact SunCuster the node is assumed to be alive.
* Return values:
* 1 - node is dead
* 0 - node is alive
*/
int
{
char *fmt = "/usr/cluster/bin/scha_cluster_get -O NODESTATE_NODE ";
char *cmd;
char buf[10];
int retval = 0;
/* I know that I'm alive */
return (retval);
/* If scha_cluster_get returned DOWN - return dead */
retval = 1;
}
}
return (retval);
}
/*
* global_init()
*
* Perform some global initializations.
*
* the following routines have to call this before operation can start:
* - mdmn_send_svc_2
* - mdmn_work_svc_2
* - mdmn_comm_lock_svc_2
* - mdmn_comm_unlock_svc_2
* - mdmn_comm_suspend_svc_2
* - mdmn_comm_resume_svc_2
* - mdmn_comm_reinit_set_svc_2
*
* This is a single threaded daemon, so it can only be in one of the above
* routines at the same time.
* This means, global_init() cannot be called more than once at the same time.
* Hence, no lock is needed.
*/
void
global_init(void)
{
struct sigaction sighandler;
struct rlimit commd_limit;
/* Do these global initializations only once */
if (md_commd_global_state & MD_CGS_INITED) {
return;
}
(void) sdssc_bind_library();
/* setup the debug options from the config file */
setup_debug();
/* make sure that we don't run out of file descriptors */
"Could not increase the max file descriptors"));
}
/* Make setup_debug() be the action in case of SIGHUP */
sighandler.sa_flags = 0;
__savetime = gethrtime();
/* start a thread that flushes out the debug on a regular basis */
/* global rwlock's / mutex's / cond_t's go here */
/* Make sure the initiator table is initialized correctly */
}
}
/* setup the check for timeouts */
}
/*
* mdmn_init_client(setno, nodeid)
* called if client[setno][nodeid] is NULL
*
* NOTE: Must be called with set_desc_rwlock held as a reader
* NOTE: Must be called with client_rwlock held as a writer
*
* If the rpc client for this node has not been setup for any set, we do it now.
*
* Returns 0 on success (node found in set, rpc client setup)
* -1 if metaget_setdesc failed,
* -2 if node not part of set
* -3 if clnt_create fails
*/
static int
{
/*
* Is the appropriate set_descriptor already initialized ?
* Can't think of a scenario where this is not the case, but we'd better
* check for it anyway.
*/
/* Only one thread is supposed to be in metaget_setdesc() */
return (-1);
}
}
/* first we have to find the node name for this node id */
break; /* we found our node in this set */
}
return (-2);
}
/* Did this node join the diskset? */
return (-2);
}
/* if clnt_create has not been done for that node, do it now */
/*
* While trying to create a connection to a node,
* periodically check to see if the node has been marked
* dead by the SunCluster infrastructure.
* This periodic check is needed since a non-responsive
* rpc.mdcommd (while it is attempting to create a connection
* in the reconfig steps.
*/
(tout < MD_CLNT_CREATE_TOUT)) {
/* Is the node dead? */
"rpc.mdcommd: no client for dead node %s\n",
node->nd_nodename);
break;
} else
}
return (-3);
}
/* this node has the license to send */
/* set the timeout value */
(char *)&FOUR_SECS);
}
return (0);
}
/*
* check_client(setno, nodeid)
*
* must be called with reader lock held for set_desc_rwlock[setno]
* and must be called with reader lock held for client_rwlock[setno]
* if not it upgrades the lock to a writer lock
* and tries to initialize the client.
* Finally it's checked if the client nulled out again due to some race
*
* returns 0 if there is a usable client
* returns MDMNE_RPC_FAIL otherwise
*/
static int
{
int ret = 0;
}
}
return (ret);
}
/*
* mdmn_init_set(setno, todo)
* setno is the number of the set to be initialized.
* todo is one of the MDMN_SET_* thingies or MDMN_SET_READY
* If called with MDMN_SET_READY everything is initialized.
*
* If the set mutexes are already initialized, the caller has to hold
* both set_desc_rwlock[setno] and client_rwlock[setno] as a writer, before
* calling mdmn_init_set()
*/
int
{
int class;
/*
* Check if we are told to setup the mutexes and
* if these are not yet setup
*/
if ((todo & MDMN_SET_MUTEXES) &&
USYNC_THREAD, NULL);
USYNC_THREAD, NULL);
USYNC_THREAD, NULL);
}
}
if ((todo & MDMN_SET_MCT) &&
int fd;
char table_name[32];
filesize = (sizeof (md_mn_mct_t));
/*
* If the mct file exists we map it into memory.
* Otherwise we create an empty file of appropriate
* size and map that into memory.
* The mapped areas are stored in mct[setno].
*/
if (fd < 0) {
"init_set: Can't open MCT\n");
return (-1);
}
/*
* To ensure that the file has the appropriate size,
* we write a byte at the end of the file.
*/
/* at this point we have a file in place that we can mmap */
if (addr == MAP_FAILED) {
"init_set: mmap mct error %d\n",
errno);
return (-1);
}
/* LINTED pointer alignment */
/* finally we initialize the mutexes that protect the mct */
USYNC_THREAD, NULL);
}
}
/*
* Check if we are told to setup the nodes and
* if these are not yet setup
* (Attention: negative logic here compared to above!)
*/
if (((todo & MDMN_SET_NODES) == 0) ||
return (0); /* success */
}
"metasetnosetname(%d) returned NULL\n", setno);
return (MDMNE_NOT_JOINED);
}
/* flush local copy of rpc.metad data */
"metaget_setdesc(%d) returned NULL\n", setno);
return (MDMNE_NOT_JOINED);
}
/*
* if this set is not a multinode set or
* this node didn't join yet the diskset, better don't do anything
*/
if ((MD_MNSET_DESC(sd) == 0) ||
return (MDMNE_NOT_JOINED);
}
"setting up: node=%s, priv_ic=%s, flags=0x%x\n",
"init: %s didn't join set %d\n",
setno);
continue;
}
/* already inited */
continue;
}
/*
* While trying to create a connection to a node,
* periodically check to see if the node has been marked
* dead by the SunCluster infrastructure.
* This periodic check is needed since a non-responsive
* rpc.mdcommd (while it is attempting to create a connection
* in the reconfig steps.
*/
(tout < MD_CLNT_CREATE_TOUT)) {
/* Is the node dead? */
"rpc.mdcommd: no client for dead node %s\n",
node->nd_nodename);
break;
} else
}
/*
* If we cannot connect to a single node
* (maybe because it is down) we mark this node as not
* owned and continue with the next node in the list.
* This is better than failing the entire starting up
* of the commd system.
*/
"WARNING couldn't create client for %s\n"
"Reconfig cycle required\n",
node->nd_nodename);
"WARNING couldn't create client for %s\n"
"Reconfig cycle required\n",
node->nd_nodename);
continue;
}
/* this node has the license to send */
/* set the timeout value */
(char *)&FOUR_SECS);
}
return (0); /* success */
}
void *
mdmn_send_to_work(void *arg)
{
int success;
int try_master;
/* set the sender, so the master knows who to send the results */
mutex_lock(mx);
/*
* combination is free to use.
* If this is not the case, we return CLASS_BUSY forcing the
* initiating send_message call to retry
*/
if (success == MDMNE_CLASS_BUSY) {
"send_to_work: received but locally busy "
"(%d, 0x%llx-%d), set=%d, class=%d, type=%d, "
"active msg=(%d, 0x%llx-%d)\n",
} else {
"send_to_work: received (%d, 0x%llx-%d), "
"set=%d, class=%d, type=%d\n",
}
/* is the rpc client to the master still around ? */
break; /* out of try_master-loop */
}
/*
* Send the request to the work function on the master
* this call will return immediately
*/
/* Everything's Ok? */
/*
* Probably something happened to the daemon on the
* master. Kill the client, and try again...
*/
}
continue;
/* something went wrong, break out */
break; /* out of try_master-loop */
}
/*
* If we are here, we sucessfully delivered the message.
* We register the initiator_table, so that
* wakeup_initiator_2 can do the sendreply with the
* results for us.
*/
/* tell check_timeouts, there's work to do */
break; /* out of try_master-loop */
}
"send_to_work: registered (%d, 0x%llx-%d)\n",
} else {
/* In case of failure do the sendreply now */
/*
* copy the MSGID so that we know _which_ message
* failed (if the transp has got mangled)
*/
"send_to_work: not registered (%d, 0x%llx-%d) cs=%d\n",
/*
* We don't have a timeout registered to wake us up, so we're
* now done with this handle. Release it back to the pool.
*/
}
/* the alloc was done in mdmn_send_svc_2 */
return (NULL);
}
/*
* do_message_locally(msg, result)
* Process a message locally on the master
* Lookup the MCT if the message has already been processed.
* If not, call the handler and store the result
* If yes, retrieve the result from the MCT.
* Return:
* MDMNE_ACK in case of success
* MDMNE_LOG_FAIL if the MCT could not be checked
*/
static int
{
int completed;
result->mmr_exitval = 0;
/* let the sender decide if this is an error or not */
return (MDMNE_NO_HANDLER);
}
if (completed == MDMN_MCT_NOT_DONE) {
/* message not yet processed locally */
"calling handler for (%d,0x%llx-%d) type %d\n",
/*
* Mark the message as being currently processed,
* so we won't start a second handler for it
*/
/* here we actually process the message on the master */
"finished handler for (%d,0x%llx-%d) type %d\n",
/* Mark the message as fully processed, store the result */
} else if (completed == MDMN_MCT_DONE) {
"result for (%d, 0x%llx-%d) from MCT\n",
} else if (completed == MDMN_MCT_IN_PROGRESS) {
"(%d, 0x%llx-%d) is currently being processed\n",
} else {
/* MCT error occurred (should never happen) */
"mdmn_check_completion returned %d "
"for (%d,0x%llx-%d)\n", completed,
return (MDMNE_LOG_FAIL);
}
return (MDMNE_ACK);
}
/*
* do_send_message(msg, node)
*
* Send a message to a given node and wait for a acknowledgment, that the
* message has arrived on the remote node.
* Make sure that the client for the set is setup correctly.
* If no ACK arrives, destroy and recreate the RPC client and retry the
* message one time
* After actually sending wait no longer than the appropriate number of
* before timing out the message.
*
* Note must be called with set_desc_wrlock held in reader mode
*/
static int
{
int err;
int rpc_retries;
int timeout_retries = 0;
/* We try two times to send the message */
rpc_retries = 2;
/*
* if sending the message doesn't succeed the first time due to a
* RPC problem, we retry one time
*/
/* in abort state, we error out immediately */
if (md_commd_global_state & MD_CGS_ABORTED) {
return (MDMNE_ABORT);
}
/* unable to create client? Ignore it */
/*
* In case we cannot establish an RPC client, we
* take this node out of our considerations.
* This will be reset by a reconfig
* cycle that should come pretty soon.
* MNISSUE: Should a reconfig cycle
* be forced on SunCluster?
*/
"WARNING couldn't create client for %s\n"
"Reconfig cycle required\n",
node->nd_nodename);
"WARNING couldn't create client for %s\n",
return (MDMNE_IGNORE_NODE);
}
/* let's be paranoid and check again before sending */
/*
* if this is true, strange enough, we catch our breath,
* and then continue, so that the client is set up
* once again.
*/
sleep(1);
continue;
}
/* send it over, it will return immediately */
"proc_mas: sending (%d,0x%llx-%d) to %d returned "
" 0x%x\n",
} else {
"proc_mas: sending (%d,0x%llx-%d) to %d returned "
" NULL \n",
}
(*ret == MDMNE_THR_CREATE_FAIL)) {
/*
* Something happened to the daemon on the other side.
* Kill the client, and try again.
* check_client() will create a new client
*/
}
/* ... but don't try infinitely */
--rpc_retries;
continue;
}
/*
* If the class is locked on the other node, keep trying.
* This situation will go away automatically,
* if we wait long enough
*/
if (*ret == MDMNE_CLASS_LOCKED) {
sleep(1);
continue;
}
}
return (MDMNE_RPC_FAIL);
}
/* if the slave is in abort state, we just ignore it. */
if (*ret == MDMNE_ABORT) {
"proc_mas: work(%d,0x%llx-%d) returned "
"MDMNE_ABORT\n",
return (MDMNE_IGNORE_NODE);
}
/* Did the remote processing succeed? */
/*
* Some commd failure in the middle of sending the msg
* to the nodes. We don't continue here.
*/
"proc_mas: work(%d,0x%llx-%d) returns %d\n",
return (MDMNE_RPC_FAIL);
}
/*
* When we are here, we have sent the message to the other node and
* we know that node has accepted it.
* We go to sleep and have trust to be woken up by wakeup.
* If we wakeup due to a timeout, or a signal, no result has been
* placed in the appropriate slot.
* If we timeout, it is likely that this is because the node has
* gone away, so we will destroy the client and try it again in the
* expectation that the rpc will fail and we will return
* MDMNE_IGNORE_NODE. If that is not the case, the message must still
* be being processed on the slave. In this case just timeout for 4
* more seconds and then return RPC_FAIL if the message is not complete.
*/
if (err == 0) {
/* everything's fine, return success */
return (MDMNE_ACK);
}
"timeout occured, set=%d, class=%d, "
"msgid=(%d, 0x%llx-%d), timeout_retries=%d\n",
if (timeout_retries == 0) {
/*
* Destroy the client and try the rpc call again
*/
goto retry_rpc;
}
"commd signalled, set=%d, class=%d, "
"msgid=(%d, 0x%llx-%d)\n",
} else {
"cond_reltimedwait err=%d, set=%d, "
"class=%d, msgid=(%d, 0x%llx-%d)\n",
}
/* some failure happened */
return (MDMNE_RPC_FAIL);
}
/*
* before we return we have to
* free_msg(msg); because we are working on a copied message
*/
void
{
int *ret;
int err;
int nmsgs; /* total number of msgs */
int curmsg; /* index of current msg */
uint_t inherit_flags = 0;
"proc_mas: received (%d, 0x%llx-%d) set=%d, class=%d, type=%d\n",
/*
* Put message into the change log unless told otherwise
* Note that we only log original messages.
* If they are generated by some smgen, we don't log them!
* Replay messages aren't logged either.
* Note, that replay messages are unlogged on completion.
*/
"proc_mas: calling log_msg for (%d,0x%llx-%d) type %d\n",
if (err == MDMNE_NULL) {
/* msg logged successfully */
"done log_msg for (%d,0x%llx-%d) type %d\n",
goto proceed;
}
/* Same msg in the slot, proceed */
"already logged (%d,0x%llx-%d) type %d\n",
goto proceed;
}
if (err == MDMNE_LOG_FAIL) {
/* Oh, bad, the log is non functional. */
/*
* Note that the mark_busy was already done by
* mdmn_work_svc_2()
*/
}
if (err == MDMNE_CLASS_BUSY) {
/*
* The log is occupied with a different message
* that needs to be played first.
* We reject the current message with MDMNE_CLASS_BUSY
* because we will proceed with the logged message,
*/
}
"proc_mas: No client for initiator \n");
} else {
}
"proc_mas: couldn't wakeup_initiator \n");
} else {
"wakeup_initiator returned %d\n", *ret);
}
}
if (err == MDMNE_LOG_FAIL) {
/* we can't proceed here */
return;
} else if (err == MDMNE_CLASS_BUSY) {
/* proceed with the logged message */
/*
* The logged message has to have the same class but
* type and sender can be different
*/
"proc_mas: Got new message from change log: "
"(%d,0x%llx-%d) type %d\n",
/* continue normal operation with this message */
}
}
/* no submessages to create, just use the original message */
nmsgs = 1;
} else {
/* some bits are passed on to submessages */
/* some settings for the submessages */
/* Apply the inherited flags */
/*
* Make sure the submessage ID is set correctly
* Note: first submessage has mid_smid of 1 (not 0)
*/
/* need the original class set in msgID (for MCT) */
}
"smgen generated %d submsgs, origclass = %d\n",
nmsgs, orig_class);
}
/*
* This big loop does the following.
* For all messages:
* process message on the master first (a message completion
* table MCT ensures a message is not processed twice)
* in case of an error break out of message loop
* for all nodes -- unless MD_MSGF_NO_BCAST is set --
* send message to node until that succeeds
* merge result -- not yet implemented
* respect MD_MSGF_STOP_ON_ERROR
*/
int break_msg_loop = 0;
int master_err;
int master_exitval = -1;
/* If we are in the abort state, we error out immediately */
if (md_commd_global_state & MD_CGS_ABORTED) {
break; /* out of the message loop */
}
class, orig_class);
/*
* If the current class is different from the original class,
* we have to lock it down.
* The original class is already marked busy.
* At this point we cannot refuse the message because the
* class is busy right now, so we wait until the class becomes
* available again. As soon as something changes for this set
* we will be cond_signal'ed (in mdmn_mark_class_unbusy)
*
*/
if (class != orig_class) {
&mdmn_busy_mutex[setno]);
}
}
if ((master_err != MDMNE_ACK) ||
/*
* if appropriate, unbusy the class and
* break out of the message loop
*/
if (class != orig_class) {
}
break;
}
}
if (master_err == MDMNE_ACK)
/* No broadcast? => next message */
/* if appropriate, unbusy the class */
if (class != orig_class) {
}
continue;
}
/* fake sender, so we get notified when the results are avail */
/*
* register to the master_table. It's needed by wakeup_master to
* wakeup the sleeping thread.
* Access is protected by the class lock: mdmn_mark_class_busy()
*/
/* Send the message to all other nodes */
/* We are master and have already processed the msg */
continue;
}
/* If this node didn't join the disk set, ignore it */
continue;
}
/* If a DIRECTED message, skip non-recipient nodes */
continue;
}
mutex_lock(mx);
/*
* Register the node that is addressed,
* so we can detect unsolicited messages
*/
/*
* Now send it. do_send_message() will return if
* a failure occurs or
* the results are available
*/
/* in abort state, we error out immediately */
if (md_commd_global_state & MD_CGS_ABORTED) {
break;
}
"proc_mas: got result for (%d,0x%llx-%d)\n",
} else if (err == MDMNE_IGNORE_NODE) {
continue; /* send to next node */
}
/*
* If the result is NULL, or err doesn't show success,
* something went wrong with this RPC call.
*/
/*
* If PANIC_WHEN_INCONSISTENT set,
* panic if the master succeeded while
* this node failed
*/
(master_err == MDMNE_ACK))
/* are we supposed to stop in case of error? */
"result (%d,0x%llx-%d) is NULL\n",
break_msg_loop = 1;
break; /* out of node loop first */
} else {
/* send msg to the next node */
continue;
}
}
/*
* Message processed on remote node.
* If PANIC_WHEN_INCONSISTENT set, panic if the
* result is different on this node from the result
* on the master
*/
((master_err != MDMNE_ACK) ||
/*
* At this point we know we have a message that was
* processed on the remote node.
* We now check if the exitval is non zero.
* In that case we discard the previous result and
* rather use the current.
* This means: If a message fails on no node,
* the result from the master will be returned.
* There's currently no such thing as merge of results
* If additionally STOP_ON_ERROR is set, we bail out
*/
if (slave_result->mmr_exitval != 0) {
/* throw away the previously allocated result */
/* copy_result() allocates new memory */
break_msg_loop = 1;
break; /* out of node loop */
}
continue; /* try next node */
} else {
/*
* MNIssue: may want to merge the results
* from all slaves. Currently only report
* the results from the master.
*/
}
} /* End of loop over the nodes */
/* release the current class again */
if (class != orig_class) {
}
/* are we supposed to quit entirely ? */
if (break_msg_loop ||
break; /* out of msg loop */
}
} /* End of loop over the messages */
/*
* If we are here, there's two possibilities:
* - we processed all messages on all nodes without an error.
* In this case we return the result from the master.
* (to be implemented: return the merged result)
* - we encountered an error in which case result has been
* set accordingly already.
*/
if (md_commd_global_state & MD_CGS_ABORTED) {
}
/*
* This message has been processed completely.
* Remove it from the changelog.
* Do this for replay messages too.
* Note that the message is unlogged before waking up the
* initiator. This is done for two reasons.
* 1. Remove a race condition that occurs when back to back
* messages are sent for the same class, the registeration is
* is lost.
* 2. If the initiator died but the action was completed on all the
* the nodes, we want that to be marked "done" quickly.
*/
"proc_mas: calling unlog_msg for (%d,0x%llx-%d) type %d\n",
"proc_mas: done unlog_msg for (%d,0x%llx-%d) type %d\n",
}
/*
* In case of submessages, we increased the submessage ID in the
* result structure. We restore the message ID to the value that
* the initiator is waiting for.
*/
/* if we have an inited client, send result */
"proc_mas: unable to create client for initiator\n");
} else {
sender);
}
"proc_mas: couldn't wakeup initiator\n");
} else {
"proc_mas: wakeup_initiator returned %d\n",
*ret);
}
}
/* Free all submessages, if there were any */
if (nmsgs > 1) {
}
}
/* Free the result */
/*
* We use this ioctl just to get the time in the same format as used in
* the messageID. If it fails, all we get is a bad runtime output.
*/
/* catching possible overflow */
if (usecdiff >= 1000000) {
usecdiff -= 1000000;
secdiff++;
}
"%5d.%06d secs runtime\n",
/* Free the original message */
}
void
{
int completed;
int retries;
"proc_sla: received (%d, 0x%llx-%d) set=%d, class=%d, type=%d\n",
result->mmr_exitval = 0;
/* let the sender decide if this is an error or not */
"proc_sla: No handler for (%d, 0x%llx-%d)\n",
} else {
/* Did we already process this message ? */
if (completed == MDMN_MCT_NOT_DONE) {
/* message not yet processed locally */
"proc_sla: calling handler for (%d, 0x%llx-%d)\n",
/*
* Mark the message as being currently processed,
* so we won't start a second handler for it
*/
"proc_sla: finished handler for (%d, 0x%llx-%d)\n",
/* Mark the message as fully done, store the result */
} else if (completed == MDMN_MCT_DONE) {
/* message processed previously, got result from MCT */
"proc_sla: result for (%d, 0x%llx-%d) from MCT\n",
} else if (completed == MDMN_MCT_IN_PROGRESS) {
/*
* If the message is curruntly being processed,
* we can return here, without sending a result back.
* This will be done by the initial message handling
* thread
*/
"(%d, 0x%llx-%d) is currently being processed\n",
return;
} else {
/* MCT error occurred (should never happen) */
"proc_sla: MCT error for (%d, 0x%llx-%d)\n",
}
}
/*
* At this point we have a result (even in an error case)
* that we return to the master.
*/
while (!successfully_returned && (retries != 0)) {
/*
* If we cannot setup the rpc connection to the master,
* we can't do anything besides logging this fact.
*/
"proc_mas: unable to create client for master\n");
break;
} else {
/*
* if mdmn_wakeup_master_2 returns NULL, it can be that
* the master (or the commd on the master) had died.
* In that case, we destroy the client to the master
* and retry.
* If mdmn_wakeup_master_2 doesn't return MDMNE_ACK,
* the commd on the master is alive but
* something else is wrong,
* in that case a retry doesn't make sense => break out
*/
"proc_sla: wakeup_master returned NULL\n");
/* release reader lock, grab writer lock */
}
retries--;
"retries = %d\n", retries);
continue;
}
"wakeup_master returned %d\n", *ret);
break;
} else { /* Good case */
}
}
}
}
/*
* mdmn_send_svc_2:
* ---------------
* Check that the issuing node is a legitimate one (i.e. is licensed to send
* messages to us), that the RPC request can be staged.
*
* Returns:
* 0 => no RPC request is in-flight, no deferred svc_sendreply()
* 1 => queued RPC request in-flight. Completion will be made (later)
* by a wakeup_initiator_2() [hopefully]
*/
int
{
int err;
/* If we are in the abort state, we error out immediately */
if (md_commd_global_state & MD_CGS_ABORTED) {
return (0);
}
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
"send: received (%d, 0x%llx-%d), set=%d, class=%d, type=%d\n",
/* Check for verbosity related message */
md_mn_verbose_t *d;
md_commd_global_verb = d->mmv_what;
/* everytime the bitmask is set, we reset the timer */
__savetime = gethrtime();
/*
* If local-only-flag is set, we are done here,
* otherwise we pass that message on to the master.
*/
(char *)resultp);
return (0);
}
}
/*
* Are we entering the abort state?
* Here we don't even need to check for MD_MSGF_LOCAL_ONLY, because
* this message cannot be distributed anyway.
* So, it's safe to return immediately.
*/
return (0);
}
/*
* Is this message type blocked?
* If so we return MDMNE_CLASS_LOCKED, immediately
*/
"send: type locked (%d, 0x%llx-%d), set=%d, class=%d, "
return (0);
}
/* Can only use the appropriate mutexes if they are inited */
} else {
}
if (err) {
/* couldn't initialize connections, cannot proceed */
(char *)resultp);
"send: init err = %d\n", err);
return (0);
}
}
"send: class suspended (%d, 0x%llx-%d), set=%d, "
return (0);
}
/* is this rpc request coming from the local node? */
"send: check licence fail(%d, 0x%llx-%d), set=%d, "
return (0);
}
/*
* We allocate a structure that can take two pointers in order to pass
* both the message and the transp into thread_create.
* The free for this alloc is done in mdmn_send_to_work()
*/
/*
* create a thread here that calls work on the master.
* If we are already on the master, this would block if running
* in the same context. (our service is single threaded)(
* Make it a detached thread because it will not communicate with
* anybody thru thr_* mechanisms
*/
NULL);
/*
* We return here without sending results. This will be done by
* mdmn_wakeup_initiator_svc_2() as soon as the results are available.
* Until then the calling send_message will be blocked, while we
* are able to take calls.
*/
return (1);
}
/* ARGSUSED */
int *
{
int err;
int *retval;
/* If we are in the abort state, we error out immediately */
if (md_commd_global_state & MD_CGS_ABORTED) {
*retval = MDMNE_ABORT;
return (retval);
}
/*
* Is this message type blocked?
* If so we return MDMNE_CLASS_LOCKED, immediately.
* This check is performed on master and slave.
*/
return (retval);
}
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* Can only use the appropriate mutexes if they are inited */
} else {
}
if (err) {
return (retval);
}
}
/* is this rpc request coming from a licensed node? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
"work: received (%d, 0x%llx-%d), set=%d, class=%d, type=%d, "
"flags=0x%x\n",
/* Check for various CLASS0 message types */
md_mn_verbose_t *d;
/* for now we ignore set / class in md_mn_verbose_t */
md_commd_global_verb = d->mmv_what;
/* everytime the bitmask is set, we reset the timer */
__savetime = gethrtime();
}
/* check if class is locked via a call to mdmn_comm_lock_svc_2 */
return (retval);
}
/* Check if the class is busy right now. Do it only on the master */
/*
* If the class is currently suspended, don't accept new
* messages, unless they are flagged with an override bit.
*/
"send: set %d is suspended\n", setno);
return (retval);
}
return (retval);
}
/*
* Because the real processing of the message takes time we
* create a thread for it. So the master thread can continue
* to run and accept further messages.
*/
(void *(*)(void *))mdmn_master_process_msg, (void *)msg,
} else {
(void *(*)(void *)) mdmn_slave_process_msg, (void *)msg,
}
if (*retval != 0) {
return (retval);
}
/* Now run the new thread */
"work: done (%d, 0x%llx-%d), set=%d, class=%d, type=%d\n",
return (retval);
}
/* ARGSUSED */
int *
{
int *retval;
int err;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* set not ready means we just crashed are restarted now */
/* Can only use the appropriate mutexes if they are inited */
} else {
}
if (err) {
return (retval);
}
}
/* is this rpc request coming from a licensed node? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
"wake_ini: received (%d, 0x%llx-%d) set=%d, class=%d, type=%d\n",
mutex_lock(mx);
/*
* Search the initiator wakeup table.
* If we find an entry here (which should always be true)
* we are on the initiating node and we wakeup the original
* local rpc call.
*/
"wake_ini: replied (%d, 0x%llx-%d)\n",
} else {
"wakeup initiator: unsolicited message (%d, 0x%llx-%d)\n",
}
/* less work for check_timeouts */
if (messages_on_their_way == 0) {
"Oops, messages_on_their_way < 0 (%d, 0x%llx-%d)\n",
} else {
}
return (retval);
}
/*
* res must be free'd by the thread we wake up
*/
/* ARGSUSED */
int *
{
int *retval;
int err;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* Need to copy the results here, as they are static for RPC */
/* set not ready means we just crashed are restarted now */
/* Can only use the appropriate mutexes if they are inited */
} else {
}
if (err) {
return (retval);
}
}
/* is this rpc request coming from a licensed node? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
"wake_mas: received (%d, 0x%llx-%d) set=%d, class=%d, type=%d "
"from %d\n",
res->mmr_sender);
/*
* The mutex and cv are needed for waking up the thread
* sleeping in mdmn_master_process_msg()
*/
/*
* lookup the master wakeup table
* If we find our message, we are on the master and
* called by a slave that finished processing a message.
* We store the results in the appropriate slot and
* wakeup the thread (mdmn_master_process_msg()) waiting for them.
*/
mutex_lock(mx);
} else {
/* id is correct but wrong sender (I smell a timeout) */
"wakeup master got unsolicited message: "
"(%d, 0x%llx-%d) from %d\n",
*retval = MDMNE_TIMEOUT;
}
} else {
/* id is wrong, smells like a very late timeout */
"wakeup master got unsolicited message: "
"(%d, 0x%llx-%d) from %d, expected (%d, 0x%llx-%d)\n",
}
return (retval);
}
/*
* This is mainly done for debug purpose.
* even in the middle of sending messages to multiple slaves.
* This remains until the user issues a mdmn_comm_unlock_svc_2 for the same
*
* Special messages of class MD_MSG_CLASS0 can never be locked.
* e.g. MD_MN_MSG_VERBOSITY, MD_MN_MSG_ABORT
*
* That means, if MD_MSG_CLASS0 is specified, we lock all classes from
* >= MD_MSG_CLASS1 to < MD_MN_NCLASSES
*
* set must be between 1 and MD_MAXSETS
* class can be:
* MD_MSG_CLASS0 which means all other classes in this case
* or one specific class (< MD_MN_NCLASSES)
*
* Returns:
* MDMNE_ACK on sucess (locking a locked class is Ok)
* MDMNE_EINVAL if a parameter is out of range
*/
/* ARGSUSED */
int *
{
int *retval;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* is this rpc request coming from the local node ? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
/* Perform some range checking */
*retval = MDMNE_EINVAL;
return (retval);
}
if (class != MD_MSG_CLASS0) {
} else {
/* MD_MSG_CLASS0 is used as a wild card for all classes */
}
}
return (retval);
}
/*
* set must be between 1 and MD_MAXSETS
* class can be:
* MD_MSG_CLASS0 which means all other classes in this case (like above)
* or one specific class (< MD_MN_NCLASSES)
*
* Returns:
* MDMNE_ACK on sucess (unlocking an unlocked class is Ok)
* MDMNE_EINVAL if a parameter is out of range
*/
/* ARGSUSED */
int *
{
int *retval;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* is this rpc request coming from the local node ? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
/* Perform some range checking */
*retval = MDMNE_EINVAL;
return (retval);
}
if (class != MD_MSG_CLASS0) {
} else {
/* MD_MSG_CLASS0 is used as a wild card for all classes */
}
}
return (retval);
}
/*
* mdmn_comm_suspend_svc_2(setno, class)
*
* and don't allow new messages to be processed.
*
* Special messages of class MD_MSG_CLASS0 can never be locked.
* e.g. MD_MN_MSG_VERBOSITY
*
* 1 <= setno < MD_MAXSETS or setno == MD_COMM_ALL_SETS
* 1 <= class < MD_MN_NCLASSES or class == MD_COMM_ALL_CLASSES
*
* If class _is_not_ MD_COMM_ALL_CLASSES, then we simply mark this
* one class as being suspended.
* If messages for this class are currently on their way,
* MDMNE_SET_NOT_DRAINED is returned. Otherwise MDMNE_ACK is returned.
*
* If class _is_ MD_COMM_ALL_CLASSES we drain all classes of this set.
* Messages must be generated in ascending order.
* This means, a message cannot create submessages with the same or lower class.
* Draining messages must go from 1 to NCLASSES in order to ensure we don't
* generate a hanging situation here.
* We mark class 1 as being suspended.
* if the class is not busy, we proceed with class 2
* and so on
* if a class *is* busy, we cannot continue here, but return
* MDMNE_SET_NOT_DRAINED.
* We expect the caller to hold on for some seconds and try again.
* When that message, that held the class busy is done in
* mdmn_master_process_msg(), mdmn_mark_class_unbusy() called.
* There it is checked if the class is about to drain.
* In that case it tries to drain all higher classes there.
*
* If setno is MD_COMM_ALL_SETS then we perform this on all possible sets.
* In that case we return MDMNE_SET_NOT_DRAINED if not all sets are
* completely drained.
*
* Returns:
* MDMNE_ACK on sucess (set is drained, no outstanding messages)
* MDMNE_SET_NOT_DRAINED if drain process is started, but there are
* still outstanding messages for this set(s)
* MDMNE_EINVAL if setno is out of range
* MDMNE_NOT_JOINED if the set is not yet initialized on this node
*/
/* ARGSUSED */
int *
{
int *retval;
int failure = 0;
#ifdef NOT_YET_NEEDED
#endif /* NOT_YET_NEEDED */
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* is this rpc request coming from the local node ? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
/* Perform some range checking */
if (setno >= MD_MAXSETS) {
*retval = MDMNE_EINVAL;
return (retval);
}
/* setno == MD_COMM_ALL_SETS means: we walk thru all possible sets. */
if (setno == MD_COMM_ALL_SETS) {
startset = 1;
} else {
}
/* Here we need the mutexes for the set to be setup */
}
/* shall we drain all classes of this set? */
if (oclass == MD_COMM_ALL_CLASSES) {
"suspend: suspending set %d, class %d\n",
if (*retval == MDMNE_SET_NOT_DRAINED) {
failure++;
}
}
} else {
/* only drain one specific class */
"suspend: suspending set=%d class=%d\n",
if (*retval == MDMNE_SET_NOT_DRAINED) {
failure++;
}
}
}
/* If one or more sets are not entirely drained, failure is non-zero */
if (failure != 0) {
"suspend: returning MDMNE_SET_NOT_DRAINED\n");
} else {
}
return (retval);
}
/*
* mdmn_comm_resume_svc_2(setno, class)
*
* Resume processing messages for a given set.
* This incorporates the repeal of a previous suspend operation.
*
* 1 <= setno < MD_MAXSETS or setno == MD_COMM_ALL_SETS
* 1 <= class < MD_MN_NCLASSES or class == MD_COMM_ALL_CLASSES
*
* If class _is_not_ MD_COMM_ALL_CLASSES, then we simply mark this
* one class as being resumed.
*
* If class _is_ MD_COMM_ALL_CLASSES we resume all classes of this set.
*
* If setno is MD_COMM_ALL_SETS then we perform this on all possible sets.
*
* If both setno is MD_COMM_ALL_SETS and class is MD_COMM_ALL_CLASSES we also
* reset any ABORT flag from the global state.
*
* Returns:
* MDMNE_ACK on sucess (resuming an unlocked set is Ok)
* MDMNE_EINVAL if setno is out of range
* MDMNE_NOT_JOINED if the set is not yet initialized on this node
*/
/* ARGSUSED */
int *
{
int *retval;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* is this rpc request coming from the local node ? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
/* Perform some range checking */
if (setno > MD_MAXSETS) {
*retval = MDMNE_EINVAL;
return (retval);
}
if (setno == MD_COMM_ALL_SETS) {
startset = 1;
if (oclass == MD_COMM_ALL_CLASSES) {
/* This is the point where we "unabort" the commd */
}
} else {
}
/* Here we need the mutexes for the set to be setup */
}
if (oclass == MD_COMM_ALL_CLASSES) {
int end_class = 1;
/*
* When SUSPENDing all classes, we go
* from 1 to MD_MN_NCLASSES-1
* The correct reverse action is RESUMing
* from MD_MN_NCLASSES-1 to 1 (or 2)
*/
if (flags & MD_MSCF_DONT_RESUME_CLASS1) {
end_class = 2;
}
/*
* Then mark all classes of this set as no longer
* suspended. This supersedes any previous suspend(1)
* calls and resumes the set entirely.
*/
class --) {
"resume: resuming set=%d class=%d\n",
}
} else {
/*
* In this case only one class is marked as not
* suspended. If a suspend(all) is currently active for
* this set, this class will still be suspended.
* That state will be cleared by a suspend(all)
* (see above)
*/
"resume: resuming set=%d class=%d\n",
}
}
return (retval);
}
/* ARGSUSED */
int *
{
int *retval;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* is this rpc request coming from the local node ? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
/*
* We assume, that all messages have been suspended previously.
*
* As we are modifying lots of clients here we grab the client_rwlock
* in writer mode. This ensures, no new messages come in.
*/
/* This set is no longer initialized */
/* destroy all rpc clients from this set */
}
}
}
return (retval);
}
/*
* This is just an interface for testing purpose.
* Here we can disable single message types.
* If we block a message type, this is valid for all MN sets.
* If a message arrives later, and it's message type is blocked, it will
* be returned immediately with MDMNE_CLASS_LOCKED, which causes the sender to
* resend this message over and over again.
*/
/* ARGSUSED */
int *
{
int *retval;
/* check if the global initialization is done */
if ((md_commd_global_state & MD_CGS_INITED) == 0) {
global_init();
}
/* is this rpc request coming from the local node ? */
*retval = MDMNE_RPC_FAIL;
return (retval);
}
/* Perform some range checking */
*retval = MDMNE_EINVAL;
return (retval);
}
return (retval);
}