etm.c revision 2ca9f232d5e039a8f2de8723786dbf6248bf9e1e
* ------------------------- data structs for FMD ---------------------------- * ----------------------- private consts and defns -------------------------- /* misc buffer for variable sized protocol header fields */ /* try limit for IO operations w/ capped exp backoff sleep on retry */ * Design_Note: ETM will potentially retry forever IO operations that the * transport fails with EAGAIN (aka EWOULDBLOCK) rather than * giving up after some number of seconds. This avoids * dropping FMA events while the service processor is down, * but at the risk of pending fmdo_recv() forever and * overflowing FMD's event queue for ETM. * A future TBD enhancement would be to always recv * and send each ETM msg in a single read/write() to reduce * the risk of failure between ETM msg hdr and body, * assuming the MTU_SZ is large enough. /* amount to increment protocol transaction id on each new send */ * ---------------------------- global data ---------------------------------- etm_debug_lvl = 0;
/* debug level: 0 is off, 1 is on, 2 is more, etc */ etm_xid_cur = 0;
/* current transaction id for sends */ etm_xid_ping = 0;
/* xid of last CONTROL msg sent requesting ping */ static int syslog_msgfd = -
1;
/* sysmsg(7D) file descriptor */ /* ETM [dropped] FMA event counters */ /* ETM protocol failures */ /* IO operation failures */ /* IO operation retries */ /* system and library failures */ /* FMD entry point bad arguments */ /* Alert logging errors */ /* miscellaneous stats */ "ETM fmaevent msg headers rcvd from xport" },
"ETM control msg headers rcvd from xport" },
"ETM alert msg headers rcvd from xport" },
"ETM response msg headers rcvd from xport" },
"ETM fmaevent msg bodies rcvd from xport" },
"ETM control msg bodies rcvd from xport" },
"ETM alert msg bodies rcvd from xport" },
"ETM response msg bodies rcvd from xport" },
"ETM fmaevent msg headers sent to xport" },
"ETM control msg headers sent to xport" },
"ETM response msg headers sent to xport" },
"ETM fmaevent msg bodies sent to xport" },
"ETM control msg bodies sent to xport" },
"ETM response msg bodies sent to xport" },
"max FMA events per ETM msg from xport" },
"max FMA events per ETM msg to xport" },
"cur enqueued response msgs to xport" },
"max enqueable response msgs to xport" },
"bytes of FMA events sent to FMD" },
"bytes of FMA events rcvd from FMD" },
"bytes of FMA events sent to xport" },
"bytes of FMA events rcvd from xport" },
"bytes dropped from xport pre magic num" },
/* ETM [dropped] FMA event counters */ "FMA events rcvd from FMD" },
"FMA events sent to FMD" },
"dropped FMA events from xport" },
"dropped FMA events to xport" },
"duplicate FMA events rcvd from xport" },
"duplicate FMA events sent to xport" },
"duplicate ALERTs rcvd from xport" },
"duplicate ALERTs sent to xport" },
"dropped response msgs on enq" },
"dropped response msgs on deq" },
/* ETM protocol failures */ "ETM msgs w/ invalid magic num" },
"ETM msgs w/ invalid protocol version" },
"ETM msgs w/ invalid message type" },
"ETM msgs w/ invalid sub type" },
"ETM msgs w/ unmatched xid" },
"ETM msgs w/ invalid FMA event length" },
"ETM msgs w/ invalid response code" },
"ETM msgs w/ invalid timeout value" },
"ETM msgs w/ too many event lengths" },
/* IO operation failures */ "xport write failures" },
/* IO operation retries */ /* system and library failures */ "nvlist_pack failures" },
"nvlist_unpack failures" },
"nvlist_size failures" },
"pthread_create failures" },
/* transport API failures */ "xport get event addrv API failures" },
"xport open API failures" },
"xport close API failures" },
"xport accept API failures" },
"xport open API retries" },
/* FMD entry point bad arguments */ "bad arguments from fmd_recv entry point" },
"bad arguments from fmd_init entry point" },
"bad arguments from fmd_fini entry point" },
/* Alert logging errors */ "failed to log message to log(7D)" },
"failed to log message to sysmsg(7D)" },
/* miscellaneous stats */ "xport resets after xport API failure" }
* -------------------------- support functions ------------------------------ * Design_Note: Each failure worth reporting to FMD should be done using * a single call to fmd_hdl_error() as it logs an FMA event * for each call. Also be aware that all the fmd_hdl_*() * format strings currently use platform specific *printf() * routines; so "%p" under Solaris does not prepend "0x" to * the outputted hex digits, while Linux and VxWorks do. * etm_show_time - display the current time of day (for debugging) using * the given FMD module handle and annotation string * etm_hexdump - hexdump the given buffer (for debugging) using * the given FMD module handle char cb[
80];
/* char buf */ unsigned int n;
/* a byte of data for sprintf() */ * Design_Note: fmd_hdl_debug() auto adds a newline if missing; * hence cb exists to accumulate a longer string. /* add a newline every 16 bytes or at the buffer's end */ if (((i %
16) == 0) || (i >=
byte_cnt)) {
}
/* for each byte in the buffer */ * etm_sleep - sleep the caller for the given number of seconds, * return 0 or -errno value * Design_Note: To avoid interfering with FMD's signal mask (SIGALRM) * do not use [Solaris] sleep(3C) and instead use * pthread_cond_wait() or nanosleep(), both of which * are POSIX spec-ed to leave signal masks alone. * This is needed for Solaris and Linux (domain and SP). /* errno assumed set by above call */ * etm_conn_open - open a connection to the given transport address, * return 0 and the opened connection handle * caveats: the err_substr is used in failure cases for calling int nev;
/* -errno value */ * etm_conn_close - close the given connection, * return 0 or -errno value * caveats: the err_substr is used in failure cases for calling int nev;
/* -errno value */ * etm_io_op - perform an IO operation on the given connection * accommodating MTU size and retrying op if needed, * return how many bytes actually done by the op * caveats: the err_substr is used in failure cases for calling int try_cnt;
/* number of tries done */ int sleep_sec;
/* exp backoff sleep period in sec */ int sleep_rv;
/* ret val from sleeping */ /* obtain [current] MTU size */ /* loop until all IO done, try limit exceeded, or real failure */ /* when give up, return -errno value even if partly done */ "due to EAGAIN\n",
io_op);
}
/* while trying the io operation */ /* avoid spinning CPU when given 0 bytes but no error */ }
/* while still have more data */ * etm_magic_read - read the magic number of an ETM message header * from the given connection into the given buffer, * return 0 or -errno value * Design_Note: This routine is intended to help protect ETM from protocol * framing errors as might be caused by an SP reset / crash in * the middle of an ETM message send; the connection will be * read from for as many bytes as needed until the magic number * is found using a sliding buffer for comparisons. int i, j;
/* indices into buf5 */ rv = 0;
/* assume success */ /* magic number bytes are sent in network (big endian) order */ for (i = 0; i < j; i++) {
}
/* for sliding the buffer contents */ }
/* for reading bytes until find magic number */ "first %d of %d bytes:\n", i,
* etm_hdr_read - allocate, read, and validate a [variable sized] * ETM message header from the given connection, * return the allocated ETM message header * (which is guaranteed to be large enough to reuse as a * RESPONSE msg hdr) and its size * or NULL and set errno on failure int dummy_int;
/* dummy var to appease lint */ /* read the magic number which starts the protocol preamble */ /* read the rest of the protocol preamble all at once */ * Design_Note: The magic number was already network decoded; but * some other preamble fields also need to be decoded, * specifically pp_xid and pp_timeout. The rest of the * preamble fields are byte sized and hence need no /* sanity check the header as best we can */ /* handle [var sized] hdrs for FMA_EVENT, CONTROL, RESPONSE msgs */ /* sanity check the header's timeout */ /* get all FMA event lengths from the header */ i = -
1;
/* cnt of length entries preceding 0 */ i += 0;
/* first len already counted by sizeof(ev_hdr) */ /* sanity check the header's sub type (control selector) */ /* get the control length */ /* sanity check the header's timeout */ /* get the response code and length */ /* sanity check the header's protocol version */ /* get the priority and length */ }
/* whether we have FMA_EVENT, ALERT, CONTROL, or RESPONSE msg */ * choose a header size that allows hdr reuse for RESPONSE msgs, * allocate and populate the message header, and * return alloc size to caller for later free of hdrp * etm_hdr_write - create and write a [variable sized] ETM message header * to the given connection appropriate for the given FMA event * and type of nvlist encoding, * return the allocated ETM message header and its size * or NULL and set errno on failure /* allocate and populate the message header for 1 FMA event */ * Design_Note: Although the ETM protocol supports it, we do not (yet) * such messages are sent with ETM_PROTO_V1_TIMEOUT_NONE. /* indicate 1 FMA event, network encode its length, and 0-terminate */ * write the network encoded header to the transport, and * return alloc size to caller for later free * etm_post_to_fmd - post the given FMA event to FMD * via a FMD transport API call, * return 0 or -errno value * caveats: the FMA event (evp) is freed by FMD, * thus callers of this function should * immediately discard any ptr they have to the * nvlist without freeing or dereferencing it }
/* etm_post_to_fmd() */ * Ideally we would just use syslog(3C) for outputting our messages. * Unfortunately, as this module is running within the FMA daemon context, * that would create the situation where this module's openlog() would * have the monopoly on syslog(3C) for the daemon and all its modules. * To avoid that situation, this module uses the same logic as the * syslog-msgs FM module to directly call into the log(7D) and sysmsg(7D) * devices for syslog and console. "SC Alert: [ID %u FACILITY_AND_PRIORITY] %s",
msgid,
* etm_req_ver_negot - send an ETM control message to the other end requesting /* populate an ETM control msg to send */ body_sz = (
3 +
1);
/* version bytes plus null byte */ * open and close a connection to send the ETM control msg * to any/all of the default dst addrs "error: bad ctl dst addrs errno %d\n",
errno);
}
/* etm_req_ver_negot() */ * Design_Note: For all etm_resp_q_*() functions and etm_resp_q_* globals, * the mutex etm_resp_q_lock must be held by the caller. * etm_resp_q_enq - add element to tail of ETM responder queue * etm_resp_q_deq - del element from head of ETM responder queue * return >0 for success, or -errno value * etm_maybe_enq_response - check the given message header to see * whether a response has been requested, * if so then enqueue the given connection * and header for later transport by the * responder thread as an ETM response msg, * return 0 for nop, >0 success, or -errno value /* bail out now if no response is to be sent */ }
/* if inappropriate hdr for a response msg */ * enqueue the msg hdr and nudge the responder thread * if the responder queue was previously empty }
/* etm_maybe_enq_response() */ * Design_Note: We rely on the fact that all message types have * a common protocol preamble; if this fact should * ever change it may break the code below. We also * rely on the fact that FMA_EVENT and CONTROL headers * returned by etm_hdr_read() will be sized large enough * to reuse them as RESPONSE headers if the remote endpt * asked for a response via the pp_timeout field. * etm_send_response - use the given message header and response code * to construct an appropriate response message, * and send it back on the given connection, * return >0 for success, or -errno value /* reuse the given header as a response header */ /* respond with the proto ver that was negotiated */ * send the whole response msg in one write, header and body; * avoid the alloc-and-copy if we can reuse the hdr as the msg, * ie, if the body is empty. update the response stats. "xid 0x%x code %d len %u\n",
}
/* etm_send_response() */ * etm_reset_xport - reset the transport layer (via fini;init) * presumably for an error condition we cannot * otherwise recover from (ex: hung LDC channel) * is idle during an xport reset; we don't want to deadlock }
/* etm_reset_xport() */ * etm_handle_new_conn - receive an ETM message sent from the other end via * the given open connection, pull out any FMA events * and post them to the local FMD (or handle any ETM * control or response msg); when done, close the char *
class;
/* FMA event class */ enq_rv = 0;
/* default is nop, ie, did not enqueue */ /* read a network decoded message header from the connection */ /* errno assumed set by above call */ "bad hdr read errno %d\n",
errno);
* handle the message based on its preamble pp_msg_type * which is known to be valid from etm_hdr_read() checks /* allocate buf large enough for whole body / all FMA events */ }
/* for summing sizes of all FMA events */ /* read all the FMA events at once */ * now that we've read the entire ETM msg from the conn, * which avoids later ETM protocol framing errors if we didn't, * check for dup msg/xid against last good FMD posting, * if a dup then resend response but skip repost to FMD /* unpack each FMA event and post it to FMD */ for (i = 0; i <
ev_cnt; i++) {
"bad event body unpack errno %d\n", n);
}
/* foreach FMA event in the body buffer */ * if we have a VER_NEGOT_REQ read the body and validate * the protocol version set contained therein, * otherwise we have a PING_REQ (which has no body) * and we [also] fall thru to the code which sends a * response msg if the pp_timeout field requested one /* complain if version set completely incompatible */ }
/* if got version set request */ * look up the xid to interpret the response body * ping is a nop; for ver negot confirm that a supported * protocol version was negotiated and remember which one }
/* if have resp to last req to negotiate proto ver */ * now that we've read the entire ETM msg from the conn, * which avoids later ETM protocol framing errors if we didn't, * check for dup msg/xid against last good syslog posting, * if a dup then resend response but skip repost to syslog }
/* whether we have a FMA_EVENT, CONTROL, RESPONSE or ALERT msg */ * if no responder ele was enqueued, close the conn now * and free the ETM msg hdr; the ETM msg body is not needed * by the responder thread and should always be freed here }
/* etm_handle_new_conn() */ * etm_handle_bad_accept - recover from a failed connection acceptance }
/* etm_handle_bad_accept() */ * etm_server - loop forever accepting new connections * using the given FMD handle, * handling any ETM msgs sent from the other side * via each such connection int nev;
/* -errno val */ /* errno assumed set by above call */ }
/* while accepting new connections until ETM dies */ /* ETM is dying (probably due to "fmadm unload etm") */ * etm_responder - loop forever waiting for new responder queue elements * to be enqueued, for each one constructing and sending * an ETM response msg to the other side, and closing its * associated connection when appropriate * this thread exists to ensure that the etm_server() thread * never pends indefinitely waiting on the xport write lock, and is * hence always available to accept new connections and handle * this design relies on the fact that each connection accepted and * returned by the ETM xport layer is unique, and each can be closed * independently of the others while multiple connections are }
/* while the responder queue is empty, wait to be nudged */ * for every responder ele that has been enqueued, * dequeue and send it as an ETM response msg, * closing its associated conn and freeing its hdr * enter the queue draining loop holding the responder * queue lock, but do not hold the lock indefinitely * (the actual send may pend us indefinitely), * so that other threads will never pend for long * trying to enqueue a new element }
/* while draining the responder queue */ }
/* while awaiting and sending resp msgs until ETM dies */ /* ETM is dying (probably due to "fmadm unload etm") */ * -------------------------- FMD entry points ------------------------------- * _fmd_init - initialize the transport for use by ETM and start the * server daemon to accept new connections to us * FMD will read our *.conf and subscribe us to FMA events const struct facility *
fp;
/* syslog facility matching */ char *
facname;
/* syslog facility property */ return;
/* invalid data in configuration file */ * Do not load this module if it is runing on a guest ldom. /* setup statistics and properties from FMD */ /* obtain an FMD transport handle so we can post FMA events later */ /* encourage protocol transaction id to be unique per module load */ /* init the ETM transport */ * Cache any properties we use every time we receive an alert. * Look up the value of the "facility" property and use it to * determine * what syslog LOG_* facility value we use to * start the message responder and the connection acceptance server; * request protocol version be negotiated after waiting a second * for the receiver to be ready to start handshaking * etm_recv - receive an FMA event from FMD and transport it uint8_t *
buf;
/* tmp buffer for packed FMA event */ "event size errno %d class %s\n", n,
class);
* if the debug limit has been set, avoid excessive traffic, * for example, an infinite cycle using loopback nodes "event %p cnt %llu > debug max %d\n",
evp,
/* allocate a buffer for the FMA event and nvlist pack it */ "event pack errno %d class %s\n", n,
class);
/* get vector of dst addrs and send the FMA event to each one */ "bad event dst addrs errno %d\n",
errno);
/* open a new connection to this dst addr */ "bad conn open on new ev",
addrv[i], &
conn)) < 0) {
/* write the ETM message header */ "bad hdr write errno %d\n",
errno);
"bad conn close per bad hdr wr",
conn);
/* write the ETM message body, ie, the packed nvlist */ "bad io write on event",
conn,
"bad conn close per bad body wr",
conn);
/* close the connection */ }
/* foreach dst addr in the vector */ * _fmd_fini - stop the server daemon and teardown the transport /* kill the connection server and responder ; wait for them to die */ }
/* if server thread was successfully created */ }
/* if responder thread was successfully created */ /* teardown the transport and cleanup syslogging */