etm.c revision 25351652d920ae27c5a56c199da581033ce763f6
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* etm.c FMA Event Transport Module implementation, a plugin of FMD
*
*/
/*
* --------------------------------- includes --------------------------------
*/
#include "etm_xport_api.h"
#include "etm_etm_proto.h"
#include "etm_impl.h"
#include "etm_iosvc.h"
#include "etm_filter.h"
#include "etm_ckpt.h"
#include <pthread.h>
#include <signal.h>
#include <stropts.h>
#include <locale.h>
#include <strings.h>
#include <stdlib.h>
#include <unistd.h>
#include <limits.h>
#include <values.h>
#include <alloca.h>
#include <errno.h>
#include <dlfcn.h>
#include <link.h>
#include <fcntl.h>
#include <time.h>
/*
* ----------------------------- forward decls -------------------------------
*/
static void
static int
static void
etm_send_to_remote_root(void *arg);
static void
etm_recv_from_remote_root(void *arg);
/*
* ------------------------- data structs for FMD ----------------------------
*/
static fmd_hdl_ops_t fmd_ops = {
etm_recv, /* fmdo_recv */
NULL, /* fmdo_timeout */
NULL, /* fmdo_close */
NULL, /* fmdo_stats */
NULL, /* fmdo_gc */
etm_send, /* fmdo_send */
};
static const fmd_prop_t fmd_props[] = {
};
static const fmd_hdl_info_t fmd_info = {
};
/*
* ----------------------- private consts and defns --------------------------
*/
/* misc buffer for variable sized protocol header fields */
static uint32_t
/* try limit for IO operations w/ capped exp backoff sleep on retry */
/*
* Design_Note: ETM will potentially retry forever IO operations that the
* transport fails with EAGAIN (aka EWOULDBLOCK) rather than
* giving up after some number of seconds. This avoids
* dropping FMA events while the service processor is down,
* but at the risk of pending fmdo_recv() forever and
* overflowing FMD's event queue for ETM.
* A future TBD enhancement would be to always recv
* the risk of failure between ETM msg hdr and body,
* assuming the MTU_SZ is large enough.
*/
#define ETM_TRY_BACKOFF_RATE (4)
#define ETM_TRY_BACKOFF_CAP (60)
/* amount to increment protocol transaction id on each new send */
#define ETM_XID_INC (2)
typedef struct etm_resp_q_ele {
} etm_resp_q_ele_t; /* responder queue element */
/*
* ---------------------------- global data ----------------------------------
*/
static fmd_hdl_t
static int
etm_debug_lvl = 0; /* debug level: 0 is off, 1 is on, 2 is more, etc */
static int
static fmd_xprt_t
static pthread_t
static pthread_t
static etm_resp_q_ele_t
static etm_resp_q_ele_t
static uint32_t
etm_resp_q_cur_len = 0; /* cur length (ele cnt) of responder queue */
static uint32_t
etm_resp_q_max_len = 0; /* max length (ele cnt) of responder queue */
static uint32_t
etm_bad_acc_to_sec = 0; /* sleep timeout (in sec) after bad conn accept */
static pthread_mutex_t
static pthread_cond_t
static volatile int
etm_is_dying = 0; /* bool for dying (killing self) */
static uint32_t
etm_xid_cur = 0; /* current transaction id for sends */
static uint32_t
etm_xid_ping = 0; /* xid of last CONTROL msg sent requesting ping */
static uint32_t
etm_xid_ver_negot = 0; /* xid of last CONTROL msg sent requesting ver negot */
static uint32_t
static uint32_t
etm_xid_posted_sa = 0; /* xid of last ALERT msg/event posted OK to syslog */
static uint8_t
static uint32_t
static pthread_mutex_t
static int syslog_facility; /* log(7D) facility (part of priority) */
static int syslog_file = 0; /* log to syslog_logfd */
static int syslog_cons = 0; /* log to syslog_msgfd */
static const struct facility {
const char *fac_name;
int fac_value;
} syslog_facs[] = {
{ "LOG_DAEMON", LOG_DAEMON },
{ "LOG_LOCAL0", LOG_LOCAL0 },
{ "LOG_LOCAL1", LOG_LOCAL1 },
{ "LOG_LOCAL2", LOG_LOCAL2 },
{ "LOG_LOCAL3", LOG_LOCAL3 },
{ "LOG_LOCAL4", LOG_LOCAL4 },
{ "LOG_LOCAL5", LOG_LOCAL5 },
{ "LOG_LOCAL6", LOG_LOCAL6 },
{ "LOG_LOCAL7", LOG_LOCAL7 },
{ NULL, 0 }
};
static struct stats {
/* ETM msg counters */
/* ETM byte counters */
/* ETM [dropped] FMA event counters */
/* ETM protocol failures */
/* IO operation failures */
/* IO operation retries */
/* system and library failures */
/* xport API failures */
/* FMD entry point bad arguments */
/* Alert logging errors */
/* miscellaneous stats */
} etm_stats = {
/* ETM msg counters */
{ "etm_rd_hdr_fmaevent", FMD_TYPE_UINT64,
"ETM fmaevent msg headers rcvd from xport" },
{ "etm_rd_hdr_control", FMD_TYPE_UINT64,
"ETM control msg headers rcvd from xport" },
{ "etm_rd_hdr_alert", FMD_TYPE_UINT64,
"ETM alert msg headers rcvd from xport" },
{ "etm_rd_hdr_response", FMD_TYPE_UINT64,
"ETM response msg headers rcvd from xport" },
{ "etm_rd_body_fmaevent", FMD_TYPE_UINT64,
"ETM fmaevent msg bodies rcvd from xport" },
{ "etm_rd_body_control", FMD_TYPE_UINT64,
"ETM control msg bodies rcvd from xport" },
{ "etm_rd_body_alert", FMD_TYPE_UINT64,
"ETM alert msg bodies rcvd from xport" },
{ "etm_rd_body_response", FMD_TYPE_UINT64,
"ETM response msg bodies rcvd from xport" },
{ "etm_wr_hdr_fmaevent", FMD_TYPE_UINT64,
"ETM fmaevent msg headers sent to xport" },
{ "etm_wr_hdr_control", FMD_TYPE_UINT64,
"ETM control msg headers sent to xport" },
{ "etm_wr_hdr_response", FMD_TYPE_UINT64,
"ETM response msg headers sent to xport" },
{ "etm_wr_body_fmaevent", FMD_TYPE_UINT64,
"ETM fmaevent msg bodies sent to xport" },
{ "etm_wr_body_control", FMD_TYPE_UINT64,
"ETM control msg bodies sent to xport" },
{ "etm_wr_body_response", FMD_TYPE_UINT64,
"ETM response msg bodies sent to xport" },
{ "etm_rd_max_ev_per_msg", FMD_TYPE_UINT64,
"max FMA events per ETM msg from xport" },
{ "etm_wr_max_ev_per_msg", FMD_TYPE_UINT64,
"max FMA events per ETM msg to xport" },
{ "etm_resp_q_cur_len", FMD_TYPE_UINT64,
"cur enqueued response msgs to xport" },
{ "etm_resp_q_max_len", FMD_TYPE_UINT64,
"max enqueable response msgs to xport" },
/* ETM byte counters */
{ "etm_wr_fmd_bytes", FMD_TYPE_UINT64,
"bytes of FMA events sent to FMD" },
{ "etm_rd_fmd_bytes", FMD_TYPE_UINT64,
"bytes of FMA events rcvd from FMD" },
{ "etm_wr_xport_bytes", FMD_TYPE_UINT64,
"bytes of FMA events sent to xport" },
{ "etm_rd_xport_bytes", FMD_TYPE_UINT64,
"bytes of FMA events rcvd from xport" },
{ "etm_magic_drop_bytes", FMD_TYPE_UINT64,
"bytes dropped from xport pre magic num" },
/* ETM [dropped] FMA event counters */
{ "etm_rd_fmd_fmaevent", FMD_TYPE_UINT64,
"FMA events rcvd from FMD" },
{ "etm_wr_fmd_fmaevent", FMD_TYPE_UINT64,
"FMA events sent to FMD" },
{ "etm_rd_drop_fmaevent", FMD_TYPE_UINT64,
"dropped FMA events from xport" },
{ "etm_wr_drop_fmaevent", FMD_TYPE_UINT64,
"dropped FMA events to xport" },
{ "etm_rd_dup_fmaevent", FMD_TYPE_UINT64,
"duplicate FMA events rcvd from xport" },
{ "etm_wr_dup_fmaevent", FMD_TYPE_UINT64,
"duplicate FMA events sent to xport" },
{ "etm_rd_dup_alert", FMD_TYPE_UINT64,
"duplicate ALERTs rcvd from xport" },
{ "etm_wr_dup_alert", FMD_TYPE_UINT64,
"duplicate ALERTs sent to xport" },
{ "etm_enq_drop_resp_q", FMD_TYPE_UINT64,
"dropped response msgs on enq" },
{ "etm_deq_drop_resp_q", FMD_TYPE_UINT64,
"dropped response msgs on deq" },
/* ETM protocol failures */
{ "etm_magic_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid magic num" },
{ "etm_ver_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid protocol version" },
{ "etm_msgtype_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid message type" },
{ "etm_subtype_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid sub type" },
{ "etm_xid_bad", FMD_TYPE_UINT64,
"ETM msgs w/ unmatched xid" },
{ "etm_fmaeventlen_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid FMA event length" },
{ "etm_respcode_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid response code" },
{ "etm_timeout_bad", FMD_TYPE_UINT64,
"ETM msgs w/ invalid timeout value" },
{ "etm_evlens_bad", FMD_TYPE_UINT64,
"ETM msgs w/ too many event lengths" },
/* IO operation failures */
{ "etm_xport_wr_fail", FMD_TYPE_UINT64,
"xport write failures" },
{ "etm_xport_rd_fail", FMD_TYPE_UINT64,
"xport read failures" },
{ "etm_xport_pk_fail", FMD_TYPE_UINT64,
"xport peek failures" },
/* IO operation retries */
{ "etm_xport_wr_retry", FMD_TYPE_UINT64,
"xport write retries" },
{ "etm_xport_rd_retry", FMD_TYPE_UINT64,
"xport read retries" },
{ "etm_xport_pk_retry", FMD_TYPE_UINT64,
"xport peek retries" },
/* system and library failures */
{ "etm_os_nvlist_pack_fail", FMD_TYPE_UINT64,
"nvlist_pack failures" },
{ "etm_os_nvlist_unpack_fail", FMD_TYPE_UINT64,
"nvlist_unpack failures" },
{ "etm_os_nvlist_size_fail", FMD_TYPE_UINT64,
"nvlist_size failures" },
{ "etm_os_pthread_create_fail", FMD_TYPE_UINT64,
"pthread_create failures" },
/* transport API failures */
{ "etm_xport_get_ev_addrv_fail", FMD_TYPE_UINT64,
"xport get event addrv API failures" },
{ "etm_xport_open_fail", FMD_TYPE_UINT64,
"xport open API failures" },
{ "etm_xport_close_fail", FMD_TYPE_UINT64,
"xport close API failures" },
{ "etm_xport_accept_fail", FMD_TYPE_UINT64,
"xport accept API failures" },
{ "etm_xport_open_retry", FMD_TYPE_UINT64,
"xport open API retries" },
/* FMD entry point bad arguments */
{ "etm_fmd_init_badargs", FMD_TYPE_UINT64,
"bad arguments from fmd_init entry point" },
{ "etm_fmd_fini_badargs", FMD_TYPE_UINT64,
"bad arguments from fmd_fini entry point" },
/* Alert logging errors */
{ "etm_log_err", FMD_TYPE_UINT64,
"failed to log message to log(7D)" },
{ "etm_msg_err", FMD_TYPE_UINT64,
"failed to log message to sysmsg(7D)" },
/* miscellaneous stats */
{ "etm_reset_xport", FMD_TYPE_UINT64,
"xport resets after xport API failure" }
};
/*
* -------------------- global data for Root ldom-------------------------
*/
static void *etm_dl_hdl = (void *)NULL;
static const char *etm_dl_path = "libds.so.1";
static int (*etm_ds_fini)(void) = (int (*)(void))NULL;
static pthread_mutex_t
static pthread_t
static etm_proto_v1_ev_hdr_t iosvc_hdr = {
ETM_PROTO_MAGIC_NUM, /* magic number */
ETM_PROTO_V1, /* default to V1, not checked */
ETM_MSG_TYPE_FMA_EVENT, /* Root Domain inteoduces only FMA events */
0, /* sub-type */
0, /* pad */
0, /* add the xid at the Q send time */
0 /* ev_lens, 0-termed, after 1 FMA event */
};
/*
* static iosvc_list
*/
{"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0},
{"", 0}, {"", 0}
};
static etm_iosvc_t io_svc = {
"\0", /* ldom_name */
PTHREAD_COND_INITIALIZER, /* nudges */
PTHREAD_MUTEX_INITIALIZER, /* protects the iosvc msg Q */
NULL, /* iosvc msg Q head */
NULL, /* iosvc msg Q tail */
0, /* msg Q current length */
100, /* msg Q max length */
0, /* current transaction id */
0, /* xid of last event posted to FMD */
DS_INVALID_HDL, /* DS handle */
NULL, /* fmd xprt handle */
NULL, /* tid 4 send to remote RootDomain */
NULL, /* tid 4 recv from remote RootDomain */
PTHREAD_COND_INITIALIZER, /* nudges etm_send_to_remote_root */
PTHREAD_MUTEX_INITIALIZER, /* protects msg_ack_cv */
0, /* flag for start sending msg Q */
0 /* indicate if the ACK has come */
};
static uint32_t
flags; /* flags for fmd_xprt_open */
static etm_async_event_ele_t
static uint32_t
etm_async_q_head = 0; /* ptr to cur head of async event queue */
static uint32_t
etm_async_q_tail = 0; /* ptr to cur tail of async event queue */
static uint32_t
etm_async_q_cur_len = 0; /* cur length (ele cnt) of async event queue */
static uint32_t
/* max length (ele cnt) of async event queue */
static pthread_cond_t
/* nudges async event handler */
static pthread_mutex_t
/* protects async event q */
static ds_ver_t
etm_iosvc_vers[] = { { 1, 0} };
static ds_capability_t
iosvc_caps = {
"ETM", /* svc_id */
etm_iosvc_vers, /* vers */
ETM_NVERS /* number of vers */
};
static void
static void
static ds_ops_t
iosvc_ops = {
etm_iosvc_reg_handler, /* ds_reg_cb */
etm_iosvc_unreg_handler, /* ds_unreg_cb */
NULL, /* ds_data_cb */
NULL /* cb_arg */
};
/*
* -------------------------- support functions ------------------------------
*/
/*
* Design_Note: Each failure worth reporting to FMD should be done using
* a single call to fmd_hdl_error() as it logs an FMA event
* for each call. Also be aware that all the fmd_hdl_*()
* format strings currently use platform specific *printf()
* routines; so "%p" under Solaris does not prepend "0x" to
* the outputted hex digits, while Linux and VxWorks do.
*/
/*
* etm_show_time - display the current time of day (for debugging) using
* the given FMD module handle and annotation string
*/
static void
{
} /* etm_show_time() */
/*
* etm_hexdump - hexdump the given buffer (for debugging) using
* the given FMD module handle
*/
static void
{
int i, j; /* index */
unsigned int n; /* a byte of data for sprintf() */
j = 0;
/*
* Design_Note: fmd_hdl_debug() auto adds a newline if missing;
* hence cb exists to accumulate a longer string.
*/
for (i = 1; i <= byte_cnt; i++) {
n = *bp++;
j += 3;
/* add a newline every 16 bytes or at the buffer's end */
if (((i % 16) == 0) || (i >= byte_cnt)) {
j = 0;
}
} /* for each byte in the buffer */
} /* etm_hexdump() */
/*
* etm_sleep - sleep the caller for the given number of seconds,
* return 0 or -errno value
*
* Design_Note: To avoid interfering with FMD's signal mask (SIGALRM)
* do not use [Solaris] sleep(3C) and instead use
* pthread_cond_wait() or nanosleep(), both of which
* are POSIX spec-ed to leave signal masks alone.
* This is needed for Solaris and Linux (domain and SP).
*/
static int
{
/* errno assumed set by above call */
return (-errno);
}
return (0);
} /* etm_sleep() */
/*
* etm_conn_open - open a connection to the given transport address,
* return 0 and the opened connection handle
* or -errno value
*
* caveats: the err_substr is used in failure cases for calling
* fmd_hdl_error()
*/
static int
{
int nev; /* -errno value */
err_substr, errno);
return (nev);
} else {
return (0);
}
} /* etm_conn_open() */
/*
* etm_conn_close - close the given connection,
* return 0 or -errno value
*
* caveats: the err_substr is used in failure cases for calling
* fmd_hdl_error()
*/
static int
{
int nev; /* -errno value */
err_substr, errno);
return (nev);
} else {
return (0);
}
} /* etm_conn_close() */
/*
* etm_io_op - perform an IO operation on the given connection
* with the given buffer,
* accommodating MTU size and retrying op if needed,
* return how many bytes actually done by the op
* or -errno value
*
* caveats: the err_substr is used in failure cases for calling
* fmd_hdl_error()
*/
static ssize_t
{
ssize_t n; /* gen use */
void *, size_t);
int try_cnt; /* number of tries done */
int sleep_sec; /* exp backoff sleep period in sec */
int sleep_rv; /* ret val from sleeping */
return (-EINVAL);
}
switch (io_op) {
case ETM_IO_OP_RD:
break;
case ETM_IO_OP_WR:
break;
default:
return (-EINVAL);
}
if (byte_cnt == 0) {
return (byte_cnt); /* nop */
}
/* obtain [current] MTU size */
} else {
mtu_sz = n;
}
/* loop until all IO done, try limit exceeded, or real failure */
rv = 0;
try_cnt = 0;
sleep_sec = 0;
/* when give up, return -errno value even if partly done */
(-EAGAIN)) {
try_cnt++;
if (try_cnt > ETM_TRY_MAX_CNT) {
rv = n;
goto func_ret;
}
if (etm_is_dying) {
goto func_ret;
}
goto func_ret;
}
if (etm_debug_lvl >= 1) {
"due to EAGAIN\n", io_op);
}
} /* while trying the io operation */
if (etm_is_dying) {
goto func_ret;
}
if (n < 0) {
rv = n;
goto func_ret;
}
/* avoid spinning CPU when given 0 bytes but no error */
if (n == 0) {
goto func_ret;
}
}
rv += n;
datap += n;
} /* while still have more data */
if (rv < 0) {
err_substr, (int)(-rv));
}
if (etm_debug_lvl >= 3) {
}
return (rv);
} /* etm_io_op() */
/*
* etm_magic_read - read the magic number of an ETM message header
* from the given connection into the given buffer,
* return 0 or -errno value
*
* Design_Note: This routine is intended to help protect ETM from protocol
* framing errors as might be caused by an SP reset / crash in
* the middle of an ETM message send; the connection will be
* read from for as many bytes as needed until the magic number
* is found using a sliding buffer for comparisons.
*/
static int
{
int rv; /* ret val */
int byte_cnt; /* count of bytes read */
int i, j; /* indices into buf5 */
ssize_t n; /* gen use */
rv = 0; /* assume success */
magic_num = 0;
byte_cnt = 0;
j = 0;
/* magic number bytes are sent in network (big endian) order */
while (magic_num != ETM_PROTO_MAGIC_NUM) {
rv = n;
goto func_ret;
}
byte_cnt++;
continue;
}
for (i = 0; i < j; i++) {
} /* for sliding the buffer contents */
}
} /* for reading bytes until find magic number */
}
"first %d of %d bytes:\n", i,
}
if (rv == 0) {
}
return (rv);
} /* etm_magic_read() */
/*
* etm_hdr_read - allocate, read, and validate a [variable sized]
* ETM message header from the given connection,
* return the allocated ETM message header
* (which is guaranteed to be large enough to reuse as a
* RESPONSE msg hdr) and its size
* or NULL and set errno on failure
*/
static void *
{
ssize_t i, n; /* gen use */
int dummy_int; /* dummy var to appease lint */
/* read the magic number which starts the protocol preamble */
errno = (-n);
return (NULL);
}
/* read the rest of the protocol preamble all at once */
ETM_IO_OP_RD)) < 0) {
errno = (-n);
return (NULL);
}
/*
* Design_Note: The magic number was already network decoded; but
* some other preamble fields also need to be decoded,
* specifically pp_xid and pp_timeout. The rest of the
* preamble fields are byte sized and hence need no
* decoding.
*/
/* sanity check the header as best we can */
(int)pp.pp_proto_ver);
return (NULL);
}
if ((dummy_int <= ETM_MSG_TYPE_TOO_LOW) ||
(dummy_int >= ETM_MSG_TYPE_TOO_BIG)) {
return (NULL);
}
/* handle [var sized] hdrs for FMA_EVENT, CONTROL, RESPONSE msgs */
/* sanity check the header's timeout */
return (NULL);
}
/* get all FMA event lengths from the header */
i = -1; /* cnt of length entries preceding 0 */
do {
i++; lenp++;
return (NULL);
}
errno = (-n);
return (NULL);
}
} while (*lenp != 0);
i += 0; /* first len already counted by sizeof(ev_hdr) */
/* sanity check the header's sub type (control selector) */
return (NULL);
}
/* get the control length */
ETM_IO_OP_RD)) < 0) {
errno = (-n);
return (NULL);
}
/* sanity check the header's timeout */
return (NULL);
}
/* get the response code and length */
ETM_IO_OP_RD)) < 0) {
errno = (-n);
return (NULL);
}
/* sanity check the header's protocol version */
return (NULL);
}
/* get the priority and length */
sizeof (sa_hdrp->sa_priority)
ETM_IO_OP_RD)) < 0) {
errno = (-n);
return (NULL);
}
} /* whether we have FMA_EVENT, ALERT, CONTROL, or RESPONSE msg */
/*
* choose a header size that allows hdr reuse for RESPONSE msgs,
* allocate and populate the message header, and
* return alloc size to caller for later free of hdrp
*/
if (etm_debug_lvl >= 3) {
}
return (hdrp);
} /* etm_hdr_read() */
/*
* etm_hdr_write - create and write a [variable sized] ETM message header
* to the given connection appropriate for the given FMA event
* and type of nvlist encoding,
* return the allocated ETM message header and its size
* or NULL and set errno on failure
*/
static void*
{
ssize_t n; /* gen use */
/* allocate and populate the message header for 1 FMA event */
/*
* Design_Note: Although the ETM protocol supports it, we do not (yet)
* such messages are sent with ETM_PROTO_V1_TIMEOUT_NONE.
*/
errno = n;
return (NULL);
}
/* indicate 1 FMA event, network encode its length, and 0-terminate */
/*
* write the network encoded header to the transport, and
* return alloc size to caller for later free
*/
errno = (-n);
return (NULL);
}
return (hdrp);
} /* etm_hdr_write() */
/*
* etm_post_to_fmd - post the given FMA event to FMD
* via a FMD transport API call,
* return 0 or -errno value
*
* caveats: the FMA event (evp) is freed by FMD,
* thus callers of this function should
* immediately discard any ptr they have to the
* nvlist without freeing or dereferencing it
*/
static int
{
if (etm_debug_lvl >= 2) {
}
if (etm_debug_lvl >= 1) {
}
if (etm_debug_lvl >= 2) {
}
return (0);
} /* etm_post_to_fmd() */
/*
* Ideally we would just use syslog(3C) for outputting our messages.
* Unfortunately, as this module is running within the FMA daemon context,
* that would create the situation where this module's openlog() would
* have the monopoly on syslog(3C) for the daemon and all its modules.
* To avoid that situation, this module uses the same logic as the
* syslog-msgs FM module to directly call into the log(7D) and sysmsg(7D)
* devices for syslog and console.
*/
static int
{
char *sysmessage; /* Formatted message */
if ((syslog_file == 0) && (syslog_cons == 0)) {
return (0);
}
if (etm_debug_lvl >= 2) {
}
if (syslog_file) {
"SC Alert: [ID %u FACILITY_AND_PRIORITY] %s", msgid,
body_buf);
}
}
if (syslog_cons) {
"SC Alert: %s\r\n", body_buf);
}
}
if (etm_debug_lvl >= 2) {
}
return (0);
}
/*
* etm_req_ver_negot - send an ETM control message to the other end requesting
* that the ETM protocol version be negotiated/set
*/
static void
{
ssize_t i; /* gen use */
/* populate an ETM control msg to send */
*body_buf++ = ETM_PROTO_V3;
*body_buf++ = ETM_PROTO_V2;
*body_buf++ = ETM_PROTO_V1;
*body_buf++ = '\0';
/*
* open and close a connection to send the ETM control msg
*/
"error: bad ctl dst addrs errno %d\n", errno);
goto func_ret;
}
continue;
}
}
conn);
} /* foreach dst addr */
}
} /* etm_req_ver_negot() */
/*
* etm_iosvc_msg_enq - add element to tail of ETM iosvc msg queue
* etm_iosvc_msg_deq - del element from head of ETM iosvc msg queue
* need to grab the mutex lock before calling this routine
* return >0 for success, or -errno value
*/
static int
{
return (-E2BIG);
}
if (iosvc->msg_q_cur_len == 0) {
} else {
}
iosvc->msg_q_cur_len++;
return (1);
} /* etm_iosvc_msg_enq() */
static int
{
if (iosvc->msg_q_cur_len == 0) {
return (-ENOENT);
}
/*
* free the mem alloc-ed in etm_iosvc_msg_enq()
*/
iosvc->msg_q_cur_len--;
if (iosvc->msg_q_cur_len == 0) {
}
return (1);
} /* etm_iosvc_msg_deq() */
/*
* etm_msg_enq_head():
* enq the msg to the head of the Q.
* If the Q is full, drop the msg at the tail then enq the msg at head.
* need to grab mutex lock iosvc->msg_q_lock before calling this routine.
*/
static void
{
"warning: add to head of a full msg queue."
" Drop the msg at the tail\n");
/*
* drop the msg at the tail
*/
}
/*
* free the msg in iosvc->msg_q_tail->msg
* free the mem pointed to by iosvc->msg_q_tail
*/
iosvc->msg_q_cur_len--;
}
/*
* enq the msg to the head
*/
if (iosvc->msg_q_cur_len == 0) {
} else {
}
iosvc->msg_q_cur_len++;
} /* etm_msg_enq_head() */
/*
* etm_isovc_cleanup():
* clean up what's in the passed-in iosvc struct, including the msg Q.
*/
static void
{
} /* if io svc send thread was created ok */
} /* if root domain recv thread was created */
} /* if fmd-xprt has been opened */
while (iosvc->msg_q_cur_len > 0) {
}
return;
} /* etm_iosvc_cleanup() */
/*
* etm_iosvc_lookup(using ldom_name or ds_hdl when ldom_name is empty)
* not found, create one, add to iosvc_list
*/
{
uint32_t i; /* for loop var */
for (i = 0; i < NUM_OF_ROOT_DOMAINS; i++) {
if (ldom_name[0] == '\0') {
/*
* search by hdl passed in
* the only time this is used is at ds_unreg_cb time.
* there is no ldom name, only the valid ds_hdl.
* find an iosvc with the matching ds_hdl.
* ignore the iosvc_create flag, should never need to
* create an iosvc for ds_unreg_cb
*/
if (etm_debug_lvl >= 2) {
"info: found an iosvc at slot %d w/ ds_hdl %d \n",
i, iosvc_list[i].ds_hdl);
}
if (etm_debug_lvl >= 2) {
"info: found an iosvc w/ ldom_name %s \n",
iosvc_list[i].ldom_name);
}
return (&iosvc_list[i]);
} else {
continue;
}
/*
* this is an non-empty iosvc structure slot
*/
/*
* found an iosvc structure that matches the
* passed in ldom_name, return the ptr
*/
if (etm_debug_lvl >= 2) {
"iosvc at slot %d w/ ds_hdl %d \n",
i, iosvc_list[i].ds_hdl);
"iosvc w/ ldom_name %s \n",
iosvc_list[i].ldom_name);
}
return (&iosvc_list[i]);
} else {
/*
* non-empty slot with no-matching name,
* move on to next slot.
*/
continue;
}
} else {
/*
* found the 1st slot with ldom name being empty
* remember the slot #, will be used for creating one
*/
if (first_empty_slot == -1) {
first_empty_slot = i;
}
}
}
/*
* this is the case we need to add an iosvc at first_empty_slot
* for the ldom_name at iosvc_list[first_empty_slot]
*/
"info: create an iosvc with ldom name %s\n",
i = first_empty_slot;
i, iosvc_list[i].ldom_name);
return (&iosvc_list[i]);
} else {
return (NULL);
}
} /* etm_iosvc_lookup() */
/*
* etm_ckpt_remove:
* remove the ckpt for the iosvc element
*/
static void
int err; /* temp error */
char *buf; /* packed event pointer */
(etm_ldom_type != LDOM_TYPE_CONTROL)) {
return;
}
/* the pointer to the packed event in the etm message */
/* unpack it, then uncheckpoited it */
return;
}
}
/*
* etm_send_ds_msg()
* call ds_send_msg() to send the msg passed in.
* timedcond_wait for the ACK to come back.
* if the ACK doesn't come in the specified time, retrun -EAGAIN.
* other wise, return 1.
*/
int
{
/*
* call ds_send_msg(). Return (-EAGAIN) if not successful
*/
return (-EAGAIN);
}
/*
* wait on the cv for resp msg for cur_send_xid
*/
(void) gettimeofday(&tv, 0);
&timeout);
/*
* check to see if ack_ok is non-zero
* if non-zero, resp msg has been received
*/
/*
* ACK came ok, this send is successful,
* tell the caller ready to send next.
* free mem alloc-ed in
* etm_pack_ds_msg
*/
if (ckpt_remove == B_TRUE &&
}
iosvc->cur_send_xid++;
return (1);
} else {
/*
* the ACK did not come on time
* tell the caller to resend cur_send_xid
*/
return (-EAGAIN);
} /* iosvc->ack_ok != 0 */
} /* etm_send_ds_msg() */
/*
* both events from fmdo_send entry point and from SP are using the
* etm_proto_v1_ev_hdr_t as its header and it will be the same header for all
* Idealy, we should use the hdr coming with the SP FMA event. Since fmdo_send
* entry point can be called before FMA events from SP, we can't rely on
* the SP FMA event hdr. Use the static hdr for packing ds msgs for fmdo_send
* events.
* return >0 for success, or -errno value
* Design assumption: there is one FMA event per ds msg
*/
int
{
char *buf;
char *msg; /* body of msg to be Qed */
} else {
}
/*
* determine hdr_sz if 0, otherwise use the one passed in hdr_sz
*/
if (hdr_sz == 0) {
}
/*
* determine evp size
*/
/* indicate 1 FMA event, no network encoding, and 0-terminate */
/*
* hdr_sz + evsz
* msg will be freed in etm_send_to_remote_root() after ds_send_msg()
*/
/*
* copy hdr, 0 terminate the length vector, and then evp
*/
lenp++;
*lenp = 0;
/*
* decide what to do with the msg:
* if SP ereports (msg_type == SP_MSG), always enq the msg
* if not SP ereports, ie, fmd xprt control msgs, enq it _only_ after
* resource.fm.xprt.run has been sent (which sets start_sending_Q to 1)
*/
/*
* this is the case when the msg needs to be enq-ed
*/
(etm_ldom_type == LDOM_TYPE_CONTROL)) {
}
} else {
/*
* fmd RDWR xprt procotol startup msgs, send it now!
*/
!etm_is_dying) {
evhdrp) < 0) {
continue;
}
}
if (msg_type == FMD_XPRT_RUN_MSG)
}
return (rc);
} /* etm_pack_ds_msg() */
/*
* Design_Note: For all etm_resp_q_*() functions and etm_resp_q_* globals,
* the mutex etm_resp_q_lock must be held by the caller.
*/
/*
* etm_resp_q_enq - add element to tail of ETM responder queue
* etm_resp_q_deq - del element from head of ETM responder queue
*
* return >0 for success, or -errno value
*/
static int
{
if (etm_resp_q_cur_len >= etm_resp_q_max_len) {
return (-E2BIG);
}
if (etm_resp_q_cur_len == 0) {
} else {
}
return (1);
} /* etm_resp_q_enq() */
static int
{
if (etm_resp_q_cur_len == 0) {
return (-ENOENT);
}
if (etm_resp_q_cur_len == 0) {
}
return (1);
} /* etm_resp_q_deq() */
/*
* etm_maybe_enq_response - check the given message header to see
* whether a response has been requested,
* if so then enqueue the given connection
* and header for later transport by the
* responder thread as an ETM response msg,
* return 0 for nop, >0 success, or -errno value
*/
static ssize_t
{
/* bail out now if no response is to be sent */
if (orig_timeout == ETM_PROTO_V1_TIMEOUT_NONE) {
return (0);
} /* if a nop */
if ((orig_msg_type != ETM_MSG_TYPE_FMA_EVENT) &&
(orig_msg_type != ETM_MSG_TYPE_ALERT) &&
(orig_msg_type != ETM_MSG_TYPE_CONTROL)) {
return (-EINVAL);
} /* if inappropriate hdr for a response msg */
/*
* enqueue the msg hdr and nudge the responder thread
* if the responder queue was previously empty
*/
(void) pthread_mutex_lock(&etm_resp_q_lock);
if (etm_resp_q_cur_len == 1)
(void) pthread_cond_signal(&etm_resp_q_cv);
(void) pthread_mutex_unlock(&etm_resp_q_lock);
return (rv);
} /* etm_maybe_enq_response() */
/*
* Design_Note: We rely on the fact that all message types have
* a common protocol preamble; if this fact should
* ever change it may break the code below. We also
* rely on the fact that FMA_EVENT and CONTROL headers
* returned by etm_hdr_read() will be sized large enough
* to reuse them as RESPONSE headers if the remote endpt
* asked for a response via the pp_timeout field.
*/
/*
* etm_send_response - use the given message header and response code
* to construct an appropriate response message,
* and send it back on the given connection,
* return >0 for success, or -errno value
*/
static ssize_t
{
if (etm_debug_lvl >= 2) {
}
/* reuse the given header as a response header */
if ((orig_msg_type == ETM_MSG_TYPE_CONTROL) &&
resp_body[0] = ETM_PROTO_V2;
resp_body[2] = 0;
} /* if should send our/negotiated proto ver in resp body */
/* respond with the proto ver that was negotiated */
/*
* send the whole response msg in one write, header and body;
* avoid the alloc-and-copy if we can reuse the hdr as the msg,
* ie, if the body is empty. update the response stats.
*/
hdr_sz = sizeof (etm_proto_v1_resp_hdr_t);
}
(void) pthread_mutex_lock(&etm_write_lock);
(void) pthread_mutex_unlock(&etm_write_lock);
if (rv < 0) {
goto func_ret;
}
"xid 0x%x code %d len %u\n",
}
if (etm_debug_lvl >= 2) {
}
return (rv);
} /* etm_send_response() */
/*
* etm_reset_xport - reset the transport layer (via fini;init)
* presumably for an error condition we cannot
* otherwise recover from (ex: hung LDC channel)
*
* is idle during an xport reset; we don't want to deadlock
*/
static void
{
(void) etm_xport_fini(hdl);
(void) etm_xport_init(hdl);
} /* etm_reset_xport() */
/*
* etm_handle_new_conn - receive an ETM message sent from the other end via
* the given open connection, pull out any FMA events
* and post them to the local FMD (or handle any ETM
* control or response msg); when done, close the
* connection
*/
static void
{
char *class; /* FMA event class */
ssize_t i, n; /* gen use */
int should_reset_xport; /* bool to reset xport */
int rc; /* return code */
if (etm_debug_lvl >= 2) {
}
should_reset_xport = 0;
resp_code = 0; /* default is success */
enq_rv = 0; /* default is nop, ie, did not enqueue */
/* read a network decoded message header from the connection */
/* errno assumed set by above call */
"bad hdr read errno %d\n", errno);
goto func_ret;
}
/*
* handle the message based on its preamble pp_msg_type
* which is known to be valid from etm_hdr_read() checks
*/
/* allocate buf large enough for whole body / all FMA events */
body_sz = 0;
} /* for summing sizes of all FMA events */
ev_cnt = i;
if (etm_debug_lvl >= 1) {
}
/* read all the FMA events at once */
ETM_IO_OP_RD)) < 0) {
should_reset_xport = (n == -ENOTACTIVE);
goto func_ret;
}
/*
* now that we've read the entire ETM msg from the conn,
* which avoids later ETM protocol framing errors if we didn't,
* if a dup then resend response but skip repost to FMD
*/
"xid 0x%x\n", etm_xid_posted_logged_ev);
goto func_ret;
}
/* unpack each FMA event and post it to FMD */
for (i = 0; i < ev_cnt; i++) {
if ((n = nvlist_unpack((char *)bp,
resp_code = (-n);
"bad event body unpack errno %d\n", n);
if (etm_debug_lvl >= 2) {
"hexdump %d bytes:\n",
}
ui64++;
ui64++;
continue;
}
if (etm_debug_lvl >= 1) {
&class);
class = "NULL";
}
}
"info: evp size before pack ds msg %d\n", evsz);
ldom_name[0] = '\0';
MAX_LDOM_NAME, &did);
/*
* if rc is zero and the ldom_name is not "primary",
* the evp belongs to a root domain, put the evp in an
* outgoing etm queue,
* in all other cases, whether ldom_name is primary or
* can't find a ldom name, call etm_post_to_fmd
*/
/*
* use the ldom_name, guaranteered at this point
* iosvc data.
* add an iosvc struct if can not find one
*/
(void) pthread_mutex_unlock(&iosvc_list_lock);
(void) pthread_mutex_unlock(&iosvc_list_lock);
"error: can't find iosvc for ldom "
"name %s\n", ldom_name);
} else {
resp_code = 0;
/*
* call the new fmd_xprt_log()
*/
}
} else {
/*
* post the fma event to the control fmd
*/
evp);
if (resp_code >= 0) {
}
}
} /* foreach FMA event in the body buffer */
if (etm_debug_lvl >= 1) {
}
/*
* if we have a VER_NEGOT_REQ read the body and validate
* the protocol version set contained therein,
* otherwise we have a PING_REQ (which has no body)
* and we [also] fall thru to the code which sends a
* response msg if the pp_timeout field requested one
*/
should_reset_xport = (n == -ENOTACTIVE);
goto func_ret;
}
/* complain if version set completely incompatible */
for (i = 0; i < body_sz; i++) {
if ((body_buf[i] == ETM_PROTO_V1) ||
(body_buf[i] == ETM_PROTO_V2) ||
(body_buf[i] == ETM_PROTO_V3)) {
break;
}
}
if (i >= body_sz) {
}
} /* if got version set request */
if (etm_debug_lvl >= 1) {
}
should_reset_xport = (n == -ENOTACTIVE);
goto func_ret;
}
/*
* look up the xid to interpret the response body
*
* ping is a nop; for ver negot confirm that a supported
* protocol version was negotiated and remember which one
*/
goto func_ret;
}
if ((body_buf[0] < ETM_PROTO_V1) ||
(body_buf[0] > ETM_PROTO_V3)) {
goto func_ret;
}
etm_resp_ver = body_buf[0];
} /* if have resp to last req to negotiate proto ver */
if (etm_debug_lvl >= 1) {
}
should_reset_xport = (n == -ENOTACTIVE);
goto func_ret;
}
/*
* now that we've read the entire ETM msg from the conn,
* which avoids later ETM protocol framing errors if we didn't,
* if a dup then resend response but skip repost to syslog
*/
"xid 0x%x\n", etm_xid_posted_sa);
goto func_ret;
}
if (resp_code >= 0) {
}
} /* whether we have a FMA_EVENT, CONTROL, RESPONSE or ALERT msg */
if (etm_debug_lvl >= 2) {
}
/*
* if no responder ele was enqueued, close the conn now
* and free the ETM msg hdr; the ETM msg body is not needed
* by the responder thread and should always be freed here
*/
if (enq_rv <= 0) {
conn);
}
}
}
if (should_reset_xport) {
}
} /* etm_handle_new_conn() */
/*
* etm_handle_bad_accept - recover from a failed connection acceptance
*/
static void
{
int should_reset_xport; /* bool to reset xport */
if (should_reset_xport) {
}
} /* etm_handle_bad_accept() */
/*
* etm_server - loop forever accepting new connections
* using the given FMD handle,
* handling any ETM msgs sent from the other side
* via each such connection
*/
static void
etm_server(void *arg)
{
int nev; /* -errno val */
/*
* Restore the checkpointed events and dispatch them before starting to
* receive more events from the sp.
*/
while (!etm_is_dying) {
/* errno assumed set by above call */
if (etm_is_dying) {
break;
}
continue;
}
/* handle the new message/connection, closing it when done */
} /* while accepting new connections until ETM dies */
/* ETM is dying (probably due to "fmadm unload etm") */
} /* etm_server() */
/*
* etm_responder - loop forever waiting for new responder queue elements
* to be enqueued, for each one constructing and sending
* an ETM response msg to the other side, and closing its
* associated connection when appropriate
*
* this thread exists to ensure that the etm_server() thread
* never pends indefinitely waiting on the xport write lock, and is
* hence always available to accept new connections and handle
* incoming messages
*
* this design relies on the fact that each connection accepted and
* returned by the ETM xport layer is unique, and each can be closed
* independently of the others while multiple connections are
* outstanding
*/
static void
etm_responder(void *arg)
{
ssize_t n; /* gen use */
while (!etm_is_dying) {
(void) pthread_mutex_lock(&etm_resp_q_lock);
while (etm_resp_q_cur_len == 0) {
(void) pthread_cond_wait(&etm_resp_q_cv,
if (etm_is_dying) {
(void) pthread_mutex_unlock(&etm_resp_q_lock);
goto func_ret;
}
} /* while the responder queue is empty, wait to be nudged */
/*
* for every responder ele that has been enqueued,
* dequeue and send it as an ETM response msg,
* closing its associated conn and freeing its hdr
*
* enter the queue draining loop holding the responder
* queue lock, but do not hold the lock indefinitely
* (the actual send may pend us indefinitely),
* so that other threads will never pend for long
* trying to enqueue a new element
*/
while (etm_resp_q_cur_len > 0) {
(void) pthread_mutex_unlock(&etm_resp_q_lock);
"errno %d\n", (-n));
}
if (etm_is_dying) {
goto func_ret;
}
(void) pthread_mutex_lock(&etm_resp_q_lock);
} /* while draining the responder queue */
(void) pthread_mutex_unlock(&etm_resp_q_lock);
} /* while awaiting and sending resp msgs until ETM dies */
/* ETM is dying (probably due to "fmadm unload etm") */
(void) pthread_mutex_lock(&etm_resp_q_lock);
if (etm_resp_q_cur_len > 0) {
(int)etm_resp_q_cur_len);
while (etm_resp_q_cur_len > 0) {
}
}
(void) pthread_mutex_unlock(&etm_resp_q_lock);
} /* etm_responder() */
static void *
{
}
static void
{
}
/*
* ---------------------root ldom support functions -----------------------
*/
/*
* use a static array async_event_q instead of dynamicaly allocated mem queue
* for etm_async_q_enq and etm_async_q_deq.
* This is not running in an fmd aux thread, can't use the fmd_hdl_* funcs.
* caller needs to grab the mutex lock before calling this func.
* return >0 for success, or -errno value
*/
static int
{
if (etm_async_q_cur_len >= etm_async_q_max_len) {
/* etm_stats.etm_enq_drop_async_q.fmds_value.ui64++; */
return (-E2BIG);
}
sizeof (*async_e));
if (etm_async_q_tail == etm_async_q_max_len) {
etm_async_q_tail = 0;
}
/* etm_stats.etm_async_q_cur_len.fmds_value.ui64 = etm_async_q_cur_len; */
return (1);
} /* etm_async_q_enq() */
static int
{
if (etm_async_q_cur_len == 0) {
/* etm_stats.etm_deq_drop_async_q.fmds_value.ui64++; */
return (-ENOENT);
}
sizeof (*async_e));
if (etm_async_q_head == etm_async_q_max_len) {
etm_async_q_head = 0;
}
/* etm_stats.etm_async__q_cur_len.fmds_value.ui64 = etm_async_q_cur_len; */
return (1);
} /* etm_async_q_deq */
/*
* ds userland interface ds_reg_cb callback func
*/
/* ARGSUSED */
static void
{
/*
* do version check here.
* checked the ver received here against etm_iosvc_vers here
*/
/*
* can't log an fmd debug msg,
* not running in an fmd aux thread
*/
return;
}
/*
* the callback should have a valid ldom_name
* can't log fmd debugging msg here since this is not in an fmd aux
* thread. log fmd debug msg in etm_async_event_handle()
*/
(void) pthread_mutex_lock(&etm_async_event_q_lock);
(void) etm_async_q_enq(&async_ele);
if (etm_async_q_cur_len == 1)
(void) pthread_cond_signal(&etm_async_event_q_cv);
(void) pthread_mutex_unlock(&etm_async_event_q_lock);
} /* etm_iosvc_reg_handler */
/*
* ds userland interface ds_unreg_cb callback func
*/
/*ARGSUSED*/
static void
{
/*
* fill in async_ele and enqueue async_ele
*/
(void) pthread_mutex_lock(&etm_async_event_q_lock);
(void) etm_async_q_enq(&async_ele);
if (etm_async_q_cur_len == 1)
(void) pthread_cond_signal(&etm_async_event_q_cv);
(void) pthread_mutex_unlock(&etm_async_event_q_lock);
} /* etm_iosvc_unreg_handler */
/*
* ldom event registration callback func
*/
/* ARGSUSED */
static void
{
/*
* the callback will have a valid ldom_name
*/
if (ldom_name)
/*
* fill in async_ele and enq async_ele
*/
switch (event) {
case LDOM_EVENT_BIND:
break;
case LDOM_EVENT_UNBIND:
break;
case LDOM_EVENT_ADD:
break;
case LDOM_EVENT_REMOVE:
break;
default:
/*
* for all other ldom events, do nothing
*/
return;
} /* switch (event) */
(void) pthread_mutex_lock(&etm_async_event_q_lock);
(void) etm_async_q_enq(&async_ele);
if (etm_async_q_cur_len == 1)
(void) pthread_cond_signal(&etm_async_event_q_cv);
(void) pthread_mutex_unlock(&etm_async_event_q_lock);
} /* ldom_event_handler */
/*
* This is running as an fmd aux thread.
* This is the func that actually handle the events, which include:
* 1. ldom events. ldom events are on Control Domain only
* 2. any DS userland callback funcs
* these events are already Q-ed in the async_event_ele_q
* deQ and process the events accordingly
*/
static void
etm_async_event_handler(void *arg)
{
/*
* handle etm is not dying and Q len > 0
*/
while (!etm_is_dying) {
/*
* grab the lock to check the Q len
*/
(void) pthread_mutex_lock(&etm_async_event_q_lock);
while (etm_async_q_cur_len > 0) {
(void) etm_async_q_deq(&async_e);
(void) pthread_mutex_unlock(&etm_async_event_q_lock);
"info: processing an async event type %d ds_hdl"
"info: procssing async evt ldom_name %s\n",
}
/*
* at this point, if async_e.ldom_name is not NULL,
* we have a valid iosvc strcut ptr.
* the only time async_e.ldom_name is NULL is at
* ds_unreg_cb()
*/
switch (async_e.event_type) {
/*
* we have a valid ldom_name,
* etm_lookup_struct(ldom_name)
* do nothing if can't find an iosvc
* no iosvc clean up to do
*/
(void) pthread_mutex_lock(
"error: can't find iosvc for ldom "
"name %s\n",
(void) pthread_mutex_unlock(
break;
}
(void) pthread_mutex_unlock(
break;
/*
* create iosvc if it has not been
* created
* async_e.ds_hdl is invalid
* async_e.ldom_name is valid ldom_name
*/
(void) pthread_mutex_lock(
"error: can't create iosvc for "
"async evnt %d\n",
(void) pthread_mutex_unlock(
break;
}
(void) pthread_mutex_unlock(
break;
if (etm_ldom_type == LDOM_TYPE_CONTROL) {
/*
* find the root ldom name from
*/
MAX_LDOM_NAME) != 0) {
"error: can't find root "
"domain name from did %d\n",
break;
} else {
"info: etm_filter_find_"
"ldom_name returned %s\n",
}
/*
* now we should have a valid
* root domain name.
* lookup the iosvc struct
* associated with the ldom_name
* and init the iosvc struct
*/
(void) pthread_mutex_lock(
"error: can't create iosvc "
"for async evnt %d\n",
(void) pthread_mutex_unlock(
break;
}
iosvc->cur_send_xid = 0;
/*
* open the fmd xprt if it
* hasn't been previously opened
*/
iosvc->start_sending_Q = 0;
"info: before fmd_xprt_open"
"ldom_name is %s\n",
iosvc);
}
iosvc->thr_is_dying = 0;
iosvc);
}
iosvc);
}
(void) pthread_mutex_unlock(
} else {
iosvc->cur_send_xid = 0;
iosvc->start_sending_Q = 0;
/*
* open the fmd xprt if it
* hasn't been previously opened
*/
iosvc);
}
iosvc->thr_is_dying = 0;
iosvc);
}
iosvc);
}
}
break;
/*
* decide which iosvc struct to perform
* this UNREG callback on.
*/
if (etm_ldom_type == LDOM_TYPE_CONTROL) {
(void) pthread_mutex_lock(
/*
* lookup the iosvc struct w/
* ds_hdl
*/
"error: can't find iosvc "
"for async evnt %d\n",
(void) pthread_mutex_unlock(
break;
}
/*
* ds_hdl and fmd_xprt_open
* go hand to hand together
* after unreg_cb,
* ds_hdl is INVALID and
* fmd_xprt is closed.
* the ldom name and the msg Q
* remains in iosvc_list
*/
"info: iosvc w/ ldom_name "
/*
* on Control side.
*/
} /* if send tid was created */
} /* if recv tid was created */
(void) pthread_mutex_unlock(
} else {
/*
* on Root side.
*/
} /* if send tid was created */
} /* if recv tid was created */
}
break;
default:
/*
* for all other events, etm doesn't care.
* already logged an fmd info msg w/
* the event type. Do nothing here.
*/
break;
} /* switch (async_e.event_type) */
if (etm_ldom_type == LDOM_TYPE_CONTROL) {
}
/*
* grab the lock to check the q length again
*/
(void) pthread_mutex_lock(&etm_async_event_q_lock);
if (etm_is_dying) {
break;
}
} /* etm_async_q_cur_len */
/*
* we have the mutex lock at this point, whether
* . q_len == 0
*/
if (!etm_is_dying && etm_async_q_cur_len == 0) {
"info: cond wait on async_event_q_cv\n");
(void) pthread_cond_wait(&etm_async_event_q_cv,
"info: cond wait on async_event_q_cv rtns\n");
}
(void) pthread_mutex_unlock(&etm_async_event_q_lock);
} /* etm_is_dying */
"info: etm async event handler thread exiting\n");
} /* etm_async_event_handler */
/*
* deQ what's in iosvc msg Q
* send iosvc_msgp to the remote io svc ldom by calling ds_send_msg()
* the iosvc_msgp already has the packed msg, which is hdr + 1 fma event
*/
static void
etm_send_to_remote_root(void *arg)
{
"info: send to remote iosvc starting w/ ldom_name %s\n",
/*
* loop forever until etm_is_dying or thr_is_dying
*/
iosvc->start_sending_Q > 0) {
while (iosvc->msg_q_cur_len > 0 &&
&msg_ele);
if (etm_debug_lvl >= 3) {
"ds_hdl before ds_send_msg \n");
}
ev_hdrp = (etm_proto_v1_ev_hdr_t *)
!etm_is_dying) {
/*
* call ds_send_msg() to send the msg,
* wait for the recv end to send the
* resp msg back.
* If resp msg is recv-ed, ack_ok
* will be set to 1.
* otherwise, retry.
*/
continue;
}
break;
}
/*
* if out of the while loop but !ack_ok, ie,
* ds_hdl becomes invalid at some point
* while waiting the resp msg, we need to put
* the msg back to the head of the Q.
*/
(void) pthread_mutex_lock(
&iosvc->msg_q_lock);
/*
* put the msg back to the head of Q.
* If the Q is full at this point,
* drop the msg at the tail, enq this
* msg to the head.
*/
&msg_ele);
(void) pthread_mutex_unlock(
&iosvc->msg_q_lock);
}
/*
*
* grab the lock to check the Q len again
*/
break;
}
} /* while dequeing iosvc msgs to send */
/*
* we have the mutex lock for msg_q_lock at this point
* we are here because
* 1) q_len == 0: then wait on the cv for Q to be filled
* 2) etm_is_dying
*/
iosvc->msg_q_cur_len == 0) {
"info: waiting on msg_q_cv\n");
&iosvc->msg_q_lock);
}
break;
}
} else {
(void) etm_sleep(1);
} /* wait for the start_sendingQ > 0 */
} /* etm_is_dying or thr_is_dying */
} /* etm_send_to_remote_root */
/*
* receive etm msgs from the remote root ldom by calling ds_recv_msg()
* send ACK back by calling ds_send_msg()
*/
static void
etm_recv_from_remote_root(void *arg)
{
/* max msg len */
char *buf; /* ptr to the nvlist */
char *msg; /* ptr to alloc mem */
"info: recv from remote iosvc starting with ldom name %s \n",
/*
* loop forever until etm_is_dying or the thread is dying
*/
"info: ds_hdl is invalid in recv thr\n");
(void) etm_sleep(1);
continue;
}
/*
* for now, there are FMA_EVENT and ACK msg type.
* use FMA_EVENT buf as the maxlen, hdr+1 fma event.
* FMA_EVENT is big enough to hold an ACK msg.
* the actual msg size received is in msg_size.
*/
"info: ds_recv_msg needs mem the size of %d\n",
msg_size);
mem_alloc = 1;
} else if (rc == 0) {
"info: ds_recv_msg received a msg ok\n");
/*
* check the magic # in msg.hdr
*/
"info: bad ds recv on magic\n");
continue;
}
/*
* check the msg type against msg_size to be sure
* that received msg is not a truncated msg
*/
ev_hdrp = (etm_proto_v1_ev_hdr_t *)
"FMA EVENT xid=%d msg_size=%d\n",
if (fma_event_size != msg_size) {
"ev msg size received\n");
continue;
/*
* Simply do nothing. The send side
* will timedcond_wait waiting on the
* resp msg will timeout and
* re-send the same msg.
*/
}
if (etm_debug_lvl >= 3) {
" size %d hdrsz %d evp size %d\n",
}
iosvc->xid_posted_ev) {
/*
* different from last xid posted to
* fmd, post to fmd now.
*/
"info: evp size %d before fmd"
"post\n", evsz);
"info: xid posted to fmd %d"
"\n",
}
}
/*
* ready to send the RESPONSE msg back
* reuse the msg buffer as the response buffer
*/
/*
* send the whole response msg in one send
*/
sizeof (*resp_hdrp)) != 0) {
"info: send response msg failed\n");
} else {
"info: ds send resp msg ok"
"size %d\n", sizeof (*resp_hdrp));
}
"info: ds received respond msg xid=%d"
"info: wrong resp msg size"
"received\n");
"info: resp msg size %d recv resp"
"msg size %d\n",
continue;
}
/*
* is the pp.pp_xid == iosvc->cur_send_xid+1,
* if so, nudge the send routine to send next
*/
"info: ds received resp msg xid=%d "
"doesn't match cur_send_id=%d\n",
continue;
}
(void) pthread_mutex_unlock(
&iosvc->msg_ack_lock);
"info: signaling msg_ack_cv\n");
} else {
/*
* place holder for future msg types
*/
"info: ds received unrecognized msg\n");
}
if (mem_alloc) {
mem_alloc = 0;
}
} else {
if (etm_debug_lvl >= 3) {
"info: ds_recv_msg() failed\n");
}
} /* ds_recv_msg() returns */
} /* etm_is_dying */
/*
* need to free the mem allocated in msg upon exiting the thread
*/
if (mem_alloc) {
mem_alloc = 0;
}
} /* etm_recv_from_remote_root */
/*
* etm_ds_init
* initialize DS services function pointers by calling
* dlopen() followed by dlsym() for each ds func.
* if any dlopen() or dlsym() call fails, return -ENOENT
* return >0 for successs, -ENOENT for failure
*/
static int
{
int rc = 0;
return (-ENOENT);
}
if (etm_ds_svc_reg == NULL) {
"error: failed to dlsym ds_svc_reg() w/ error %s\n",
dlerror());
}
if (etm_ds_clnt_reg == NULL) {
"error: dlsym(ds_clnt_reg) failed w/ errno %d\n", errno);
}
if (etm_ds_send_msg == NULL) {
}
if (etm_ds_recv_msg == NULL) {
}
if (etm_ds_fini == NULL) {
}
(void) dlclose(etm_dl_hdl);
}
return (rc);
} /* etm_ds_init() */
/*
* -------------------------- FMD entry points -------------------------------
*/
/*
* _fmd_init - initialize the transport for use by ETM and start the
* server daemon to accept new connections to us
*
* FMD will read our *.conf and subscribe us to FMA events
*/
void
{
ssize_t n; /* gen use */
char *facname; /* syslog facility property */
int rc; /* funcs return code */
return; /* invalid data in configuration file */
}
/*
* decide the ldom type, do initialization accordingly
*/
return;
}
if (type_mask & LDOM_TYPE_LEGACY) {
/*
* running on a legacy sun4v domain,
* act as the the old sun4v
*/
} else if (type_mask & LDOM_TYPE_CONTROL) {
/*
* looking for libds.so.1.
* If not found, don't do DS registration. As a result,
* there will be no DS callbacks or other DS services.
*/
if (etm_ds_init(hdl) >= 0) {
/*
* ds client registration
*/
&iosvc_ops))) {
"error: ds_clnt_reg(): errno %d\n", rc);
}
} else {
"failed, continue without the DS services");
}
/*
* register for ldom status events
*/
ldom_event_handler, hdl))) {
"error: ldom_register_event():"
" errno %d\n", rc);
}
/*
* create the thread for handling both the ldom status
* change and service events
*/
}
/* setup statistics and properties from FMD */
sizeof (etm_stats) / sizeof (fmd_stat_t),
(fmd_stat_t *)&etm_stats);
"etm_debug_max_ev_cnt %d\n", etm_debug_lvl,
/*
* obtain an FMD transport handle so we can post
* FMA events later
*/
/*
* encourage protocol transaction id to be unique per module
* load
*/
/* init the ETM transport */
if ((n = etm_xport_init(hdl)) != 0) {
(-n));
return;
}
/*
* Cache any properties we use every time we receive an alert.
*/
syslog_file = 0;
}
syslog_cons = 0;
}
if (syslog_file) {
/*
* Look up the value of the "facility" property and
* use it to determine * what syslog LOG_* facility
* value we use to fill in our log_ctl_t.
*/
break;
}
" setting: %s\n", facname);
syslog_file = 0;
} else {
}
}
/*
* start the message responder and the connection acceptance
* server; request protocol version be negotiated after waiting
* a second for the receiver to be ready to start handshaking
*/
(void) etm_sleep(ETM_SLEEP_QUIK);
} else if (type_mask & LDOM_TYPE_ROOT) {
/*
* looking for libds.so.1.
* If not found, don't do DS registration. As a result,
* there will be no DS callbacks or other DS services.
*/
if (etm_ds_init(hdl) < 0) {
"error: dlopen() libds failed, "
"module unregistering\n");
return;
}
/*
* DS service registration
*/
rc);
}
/*
* this thread is created for ds_reg_cb/ds_unreg_cb
*/
/*
* Do not load this module if it is
* . runing on a non-root ldom
* . the domain owns no io devices
*/
"info: non-root ldom, module unregistering\n");
return;
} else {
/*
* place holder, all other cases. unload etm for now
*/
"info: other ldom type, module unregistering\n");
return;
}
} /* _fmd_init() */
/*
* etm_recv - receive an FMA event from FMD and transport it
* to the remote endpoint
*/
/*ARGSUSED*/
void
{
ssize_t i, n; /* gen use */
/*
* if this is running on a Root Domain, ignore the events,
* return right away
*/
if (etm_ldom_type == LDOM_TYPE_ROOT)
return;
buflen = 0;
"event size errno %d class %s\n", n, class);
return;
}
/*
* if the debug limit has been set, avoid excessive traffic,
* for example, an infinite cycle using loopback nodes
*/
if ((etm_debug_max_ev_cnt >= 0) &&
"event %p cnt %llu > debug max %d\n", evp,
return;
}
/* allocate a buffer for the FMA event and nvlist pack it */
/*
* increment the ttl value if the event is from remote (a root domain)
* uncomment this when enabling fault forwarding from Root domains
* to Control domain.
*
* uint8_t ttl;
* if (fmd_event_local(hdl, evp) != FMD_EVF_LOCAL) {
* if (nvlist_lookup_uint8(evp, FMD_EVN_TTL, &ttl) == 0) {
* (void) nvlist_remove(evp, FMD_EVN_TTL, DATA_TYPE_UINT8);
* (void) nvlist_add_uint8(evp, FMD_EVN_TTL, ttl + 1);
* }
* }
*/
NV_ENCODE_XDR, 0)) != 0) {
"event pack errno %d class %s\n", n, class);
return;
}
/* get vector of dst addrs and send the FMA event to each one */
"bad event dst addrs errno %d\n", errno);
return;
}
/* open a new connection to this dst addr */
continue;
}
(void) pthread_mutex_lock(&etm_write_lock);
/* write the ETM message header */
(void) pthread_mutex_unlock(&etm_write_lock);
"bad hdr write errno %d\n", errno);
(void) etm_conn_close(hdl,
"bad conn close per bad hdr wr", conn);
continue;
}
evp);
/* write the ETM message body, ie, the packed nvlist */
"bad io write on event", conn,
(void) pthread_mutex_unlock(&etm_write_lock);
(void) etm_conn_close(hdl,
"bad conn close per bad body wr", conn);
continue;
}
(void) pthread_mutex_unlock(&etm_write_lock);
evp);
/* close the connection */
conn);
} /* foreach dst addr in the vector */
} /* etm_recv() */
/*
* etm_send - receive an FMA event from FMD and enQ it in the iosvc.Q.
* etm_send_to_remote_root() deQ and xprt the FMA events to a
* remote root domain
* return FMD_SEND_SUCCESS for success,
* FMD_SEND_FAILED for error
*/
/*ARGSUSED*/
int
{
/* tell etm_pack_ds_msg() what to do */
char *class; /* nvlist class name */
pack_it = 1;
pack_it = 0;
} else {
if (etm_debug_lvl >= 1) {
"info: evp class= %s in etm_send\n", class);
}
if (etm_ldom_type == LDOM_TYPE_CONTROL) {
iosvc =
/*
* check the flag FORWARDING_FAULTS_TO_CONTROL to
* decide if or not to drop fault subscription
* control msgs
*/
pack_it = 0;
/*
* if (FORWARDING_FAULTS_TO_CONTROL == 1) {
* (void) nvlist_lookup_string(nvl,
* FM_RSRC_XPRT_SUBCLASS, &subclass);
* if (strcmp(subclass, "list.suspect")
* == 0) {
* pack_it = 1;
* msg_action = FMD_XPRT_OTHER_MSG;
* }
* if (strcmp(subclass, "list.repaired")
* == 0) {
* pack_it = 1;
* msg_action = FMD_XPRT_OTHER_MSG;
* }
* }
*/
}
pack_it = 1;
}
} else { /* has to be the root domain ldom */
/*
* drop all ereport and fault subscriptions
* are we dropping too much here, more than just ereport
* and fault subscriptions? need to check
*/
pack_it = 0;
pack_it = 1;
}
}
}
if (pack_it) {
if (etm_debug_lvl >= 1) {
"info: ldom name returned from xprt get specific="
}
/*
* pack the etm msg for the DS library and enq in io_svc->Q
* when the hdrp is NULL, the packing func will use the static
* iosvc_hdr
*/
}
return (FMD_SEND_SUCCESS);
} /* etm_send() */
/*
* _fmd_fini - stop the server daemon and teardown the transport
*/
void
{
ssize_t n; /* gen use */
uint32_t i; /* for loop var */
/* kill the connection server and responder ; wait for them to die */
etm_is_dying = 1;
if (etm_svr_tid != NULL) {
etm_svr_tid = NULL;
} /* if server thread was successfully created */
if (etm_resp_tid != NULL) {
etm_resp_tid = NULL;
} /* if responder thread was successfully created */
if (etm_async_e_tid != NULL) {
} /* if async event handler thread was successfully created */
if ((etm_ldom_type == LDOM_TYPE_LEGACY) ||
(etm_ldom_type == LDOM_TYPE_CONTROL)) {
/* teardown the transport and cleanup syslogging */
if ((n = etm_xport_fini(hdl)) != 0) {
(-n));
}
if (etm_fmd_xprt != NULL) {
}
if (syslog_logfd != -1) {
(void) close(syslog_logfd);
}
if (syslog_msgfd != -1) {
(void) close(syslog_msgfd);
}
}
if (etm_ldom_type == LDOM_TYPE_CONTROL) {
if (ldom_unregister_event(etm_lhp))
/*
* on control side, need to go thru every iosvc struct to
* 1) process remaining events in the iosvc Q:
* for plan A:
* since fmd_xprt_log() already logged in Control D's FMD
* 2) unregister the ds_hdl if valid
* 3) close the fmd_xprt if it has not been closed
*/
for (i = 0; i < NUM_OF_ROOT_DOMAINS; i++) {
/*
* found an iosvc struct for a root domain
*/
iosvc = &iosvc_list[i];
(void) pthread_mutex_lock(&iosvc_list_lock);
(void) pthread_mutex_unlock(&iosvc_list_lock);
} else {
/*
* reach the end of existing iosvc structures
*/
continue;
}
} /* for i<NUM_OF_ROOT_DOMAINS */
} else if (etm_ldom_type == LDOM_TYPE_ROOT) {
} /* if io svc send thread was successfully created */
} /* if io svc receive thread was successfully created */
while (iosvc->msg_q_cur_len > 0) {
}
}
if (etm_ds_fini) {
(*etm_ds_fini)();
(void) dlclose(etm_dl_hdl);
}
} /* _fmd_fini() */