fme.c revision 35f59e50e9ef37e6f1bd3b018289d678d5551cab
199767f8919635c4928607450d9e0abb932109ceToomas Soome * CDDL HEADER START
199767f8919635c4928607450d9e0abb932109ceToomas Soome * The contents of this file are subject to the terms of the
199767f8919635c4928607450d9e0abb932109ceToomas Soome * Common Development and Distribution License (the "License").
199767f8919635c4928607450d9e0abb932109ceToomas Soome * You may not use this file except in compliance with the License.
199767f8919635c4928607450d9e0abb932109ceToomas Soome * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
199767f8919635c4928607450d9e0abb932109ceToomas Soome * See the License for the specific language governing permissions
199767f8919635c4928607450d9e0abb932109ceToomas Soome * and limitations under the License.
199767f8919635c4928607450d9e0abb932109ceToomas Soome * When distributing Covered Code, include this CDDL HEADER in each
199767f8919635c4928607450d9e0abb932109ceToomas Soome * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
199767f8919635c4928607450d9e0abb932109ceToomas Soome * If applicable, add the following below this CDDL HEADER, with the
199767f8919635c4928607450d9e0abb932109ceToomas Soome * fields enclosed by brackets "[]" replaced with your own identifying
199767f8919635c4928607450d9e0abb932109ceToomas Soome * information: Portions Copyright [yyyy] [name of copyright owner]
199767f8919635c4928607450d9e0abb932109ceToomas Soome * CDDL HEADER END
199767f8919635c4928607450d9e0abb932109ceToomas Soome * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
199767f8919635c4928607450d9e0abb932109ceToomas Soome * fme.c -- fault management exercise module
199767f8919635c4928607450d9e0abb932109ceToomas Soome * this module provides the simulated fault management exercise.
199767f8919635c4928607450d9e0abb932109ceToomas Soome/* imported from eft.c... */
199767f8919635c4928607450d9e0abb932109ceToomas Soome/* fme under construction is global so we can free it on module abort */
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic int Nextid = 0;
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic int Open_fme_count = 0; /* Count of open FMEs */
199767f8919635c4928607450d9e0abb932109ceToomas Soome/* list of fault management exercises underway */
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct fme {
199767f8919635c4928607450d9e0abb932109ceToomas Soome unsigned long long ull; /* time when fme was created */
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct config *config; /* cooked configuration data */
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct lut *eventtree; /* propagation tree for this FME */
199767f8919635c4928607450d9e0abb932109ceToomas Soome * The initial error report that created this FME is kept in
199767f8919635c4928607450d9e0abb932109ceToomas Soome * two forms. e0 points to the instance tree node and is used
199767f8919635c4928607450d9e0abb932109ceToomas Soome * by fme_eval() as the starting point for the inference
199767f8919635c4928607450d9e0abb932109ceToomas Soome * algorithm. e0r is the event handle FMD passed to us when
199767f8919635c4928607450d9e0abb932109ceToomas Soome * the ereport first arrived and is used when setting timers,
199767f8919635c4928607450d9e0abb932109ceToomas Soome * which are always relative to the time of this initial
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct event *ecurrent; /* ereport under consideration */
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct event *suspects; /* current suspect list */
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct event *psuspects; /* previous suspect list */
199767f8919635c4928607450d9e0abb932109ceToomas Soome int posted_suspects; /* true if we've posted a diagnosis */
199767f8919635c4928607450d9e0abb932109ceToomas Soome int uniqobs; /* number of unique events observed */
199767f8919635c4928607450d9e0abb932109ceToomas Soome int peek; /* just peeking, don't track suspects */
199767f8919635c4928607450d9e0abb932109ceToomas Soome FME_DEFERRED /* don't know yet (k-count not met) */
199767f8919635c4928607450d9e0abb932109ceToomas Soome unsigned long long pull; /* time passed since created */
199767f8919635c4928607450d9e0abb932109ceToomas Soome unsigned long long wull; /* wait until this time for re-eval */
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct event *observations; /* observation list */
199767f8919635c4928607450d9e0abb932109ceToomas Soome struct lut *globals; /* values of global variables */
199767f8919635c4928607450d9e0abb932109ceToomas Soome /* fmd interfacing */
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmd_hdl_t *hdl; /* handle for talking with fmd */
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmd_case_t *fmcase; /* what fmd 'case' we associate with */
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct case_list {
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void fme_eval(struct fme *fmep, fmd_event_t *ffep);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic enum fme_state hypothesise(struct fme *fmep, struct event *ep,
199767f8919635c4928607450d9e0abb932109ceToomas Soome unsigned long long at_latest_by, unsigned long long *pdelay);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct node *eventprop_lookup(struct event *ep, const char *propname);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct node *pathstring2epnamenp(char *path);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void publish_undiagnosable(fmd_hdl_t *hdl, fmd_event_t *ffep,
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmd_case_t *fmcase, nvlist_t *detector, char *arg);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic char *undiag_2reason_str(int ud, char *arg);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void fme_receive_report(fmd_hdl_t *hdl, fmd_event_t *ffep,
199767f8919635c4928607450d9e0abb932109ceToomas Soome const char *eventstring, const struct ipath *ipp, nvlist_t *nvl);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void istat_counter_reset_cb(struct istat_entry *entp,
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void istat_counter_topo_chg_cb(struct istat_entry *entp,
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void serd_reset_cb(struct serd_entry *entp, void *unused,
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void serd_topo_chg_cb(struct serd_entry *entp, void *unused,
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct fme *
199767f8919635c4928607450d9e0abb932109ceToomas Soome * fme_ready -- called when all initialization of the FME (except for
199767f8919635c4928607450d9e0abb932109ceToomas Soome * stats) has completed successfully. Adds the fme to global lists
199767f8919635c4928607450d9e0abb932109ceToomas Soome * and establishes its stats.
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct fme *
199767f8919635c4928607450d9e0abb932109ceToomas Soome Nfmep = NULL; /* don't need to free this on module abort now */
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Rcount", fmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmep->Hcallcount = stats_new_counter(nbuf, "calls to hypothesise()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmep->Ccallcount = stats_new_counter(nbuf, "calls to causes_test()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome stats_new_counter(nbuf, "calls to effects_test()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Marrow", fmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);
199767f8919635c4928607450d9e0abb932109ceToomas Soome out(O_ALTFP|O_VERB2, "newfme: config snapshot contains...");
199767f8919635c4928607450d9e0abb932109ceToomas Soomeextern struct lut *itree_create_dummy(const char *, const struct ipath *);
199767f8919635c4928607450d9e0abb932109ceToomas Soome/* ARGSUSED */
199767f8919635c4928607450d9e0abb932109ceToomas Soomeset_needed_arrows(struct event *ep, struct event *ep2, struct fme *fmep)
199767f8919635c4928607450d9e0abb932109ceToomas Soome/* ARGSUSED */
199767f8919635c4928607450d9e0abb932109ceToomas Soomeunset_needed_arrows(struct event *ep, struct event *ep2, struct fme *fmep)
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void globals_destructor(void *left, void *right, void *arg);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic void clear_arrows(struct event *ep, struct event *ep2, struct fme *fmep);
199767f8919635c4928607450d9e0abb932109ceToomas Soomeprune_propagations(const char *e0class, const struct ipath *e0ipp)
199767f8919635c4928607450d9e0abb932109ceToomas Soome unsigned long long my_delay = TIMEVAL_EVENTUALLY;
199767f8919635c4928607450d9e0abb932109ceToomas Soome Nfmep->eventtree = itree_create_dummy(e0class, e0ipp);
199767f8919635c4928607450d9e0abb932109ceToomas Soome itree_lookup(Nfmep->eventtree, e0class, e0ipp)) == NULL) {
199767f8919635c4928607450d9e0abb932109ceToomas Soome Nfmep->ecurrent = Nfmep->observations = Nfmep->e0;
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Rcount", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome Nfmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Hcall", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome stats_new_counter(nbuf, "calls to hypothesise()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Rcall", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Ccall", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome stats_new_counter(nbuf, "calls to causes_test()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Ecall", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome stats_new_counter(nbuf, "calls to effects_test()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Tcall", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome Nfmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.Marrow", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) sprintf(nbuf, "fme%d.diags", Nfmep->id);
199767f8919635c4928607450d9e0abb932109ceToomas Soome Nfmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);
199767f8919635c4928607450d9e0abb932109ceToomas Soome lut_walk(Nfmep->eventtree, (lut_cb)unset_needed_arrows, (void *)Nfmep);
199767f8919635c4928607450d9e0abb932109ceToomas Soome lut_walk(Nfmep->eventtree, (lut_cb)clear_arrows, (void *)Nfmep);
199767f8919635c4928607450d9e0abb932109ceToomas Soome (void) hypothesise(Nfmep, Nfmep->e0, Nfmep->ull, &my_delay);
199767f8919635c4928607450d9e0abb932109ceToomas Soome lut_walk(Nfmep->eventtree, (lut_cb)set_needed_arrows, (void *)Nfmep);
199767f8919635c4928607450d9e0abb932109ceToomas Soome lut_free(Nfmep->globals, globals_destructor, NULL);
199767f8919635c4928607450d9e0abb932109ceToomas Soomestatic struct fme *
199767f8919635c4928607450d9e0abb932109ceToomas Soomenewfme(const char *e0class, const struct ipath *e0ipp, fmd_hdl_t *hdl,
199767f8919635c4928607450d9e0abb932109ceToomas Soome fmd_case_t *fmcase, fmd_event_t *ffep, nvlist_t *nvl)
199767f8919635c4928607450d9e0abb932109ceToomas Soome * First check if e0ipp is actually in the topology so we can give a
199767f8919635c4928607450d9e0abb932109ceToomas Soome * more useful error message.
199767f8919635c4928607450d9e0abb932109ceToomas Soome platform_units_translate(0, cfgdata->cooked, NULL, NULL,
e0class);
&detector);
return (NULL);
return (NULL);
return (NULL);
return (NULL);
fme_fini(void)
if (Nfmep) {
char *estr;
char *sepptr;
char *estr;
int ocnt;
int elen;
if (elen == 0) {
tmpbuf);
if (pkdlen != 0) {
if (ocnt == 0)
char *sepptr;
char *estr;
int elen;
int init_size;
extern int alloc_total();
char *reason;
goto badcase;
goto badcase;
goto badcase;
sizeof (size_t));
goto badcase;
goto badcase;
goto badcase;
if (elen == 0) {
tmpbuf);
goto badcase;
goto badcase;
goto badcase;
if (rawsz > 0) {
goto badcase;
goto badcase;
goto badcase;
if (f->config)
FREE(f);
return (t == N_DEFECT);
return (t == N_UPSET);
static struct node *
char *ptr;
return (ret);
char *serdname;
char *serdresource;
char *serdclass;
if ((nid =
char *suffix;
char *nserdname;
char *path;
char *ptr;
* eft.conf file values (if Serd_Override is set) or from
if (ptr3)
ptr3++;
serdclass, s);
serdclass, s);
if (!got_n_override) {
NULL);
if (!got_t_override) {
NULL);
const unsigned long long *ullp;
const char *suffix;
int len;
serd_save();
for (i = 0; i < serdincrement; i++) {
if (ippp) {
serdname);
const char *ename;
} *tripped;
nupset = 0;
nupset++;
if (nupset == 0)
ntrip = 0;
ntrip++;
for (i = 0; i < ntrip; i++) {
const char *eventstring;
int prev_verbose;
if (ffep) {
goto retry_lone_ereport;
if (Debug == 0)
Verbose = 0;
return (ntrip);
const char *class)
class);
&detector);
(char *)class);
const char *eventstring)
char *uuid;
while (nvc-- != 0) {
char *path;
path);
istat_save();
serd_save();
istat_save();
serd_save();
int matched = 0;
char *reason;
int prev_verbose;
if (Debug == 0)
Verbose = 0;
matched++;
if (pre_peek_nvp)
if (ffep) {
if (matched)
if (ofmep) {
if (ffep)
if (ffep)
if (ffep) {
static int current_indent;
indent_push(const char *s)
indent_set(const char *s)
current_indent = 0;
indent_push(s);
indent_pop(void)
if (current_indent > 0)
indent(void)
if (!Verbose)
for (i = 0; i < current_indent; i++)
static struct node *
if (n == NULL)
static nvlist_t *
char *failure;
int err, i;
return (NULL);
depth++;
return (NULL);
for (i = 0; i < depth; i++)
if (err != 0) {
goto boom;
if (err != 0) {
goto boom;
if (err != 0) {
goto boom;
pa[i++] = p;
if (err == 0) {
for (i = 0; i < depth; i++)
boom:
for (i = 0; i < depth; i++)
nvlist_free(f);
return (NULL);
struct ipath {
static nvlist_t *
char *failure;
int err, i;
depth++;
for (i = 0; i < depth; i++)
if (err != 0) {
goto boom;
if (err != 0) {
goto boom;
if (err != 0) {
goto boom;
pa[i++] = p;
if (err == 0) {
for (i = 0; i < depth; i++)
boom:
for (i = 0; i < depth; i++)
nvlist_free(f);
return (NULL);
static uint8_t
struct rsl {
rslcmp(const void *a, const void *b)
int rv;
if (rv != 0)
return (rv);
char *pathstr;
pathstr);
int mess_zero_count = 0;
int serd_rval;
if (serd_rval == 0)
rp2++;
rp++;
return (mess_zero_count);
static char *Istatbuf;
static char *Istatbufptr;
static int Istatsz;
int val;
char *str;
int len;
int val;
if (Istat_need_save == 0)
Istatsz = 0;
if (Istatsz == 0) {
Istat_need_save = 0;
int cnt = 0;
char *sname;
char *snamep;
if (np)
if (np)
char *path;
void *unused)
char *path;
istat_fini(void)
static char *Serdbuf;
static char *Serdbufptr;
static int Serdsz;
char *str;
int len;
if (Serd_need_save == 0)
Serdsz = 0;
if (Serdsz == 0) {
Serd_need_save = 0;
int sz;
char *sbuf;
char *sepptr;
char *ptr;
char *namestring;
sepptr++;
serd_save();
char *path;
char *path;
serd_fini(void)
struct node *n;
} else if (fr == 0) {
frcnt++;
cert,
&messval) == 0) {
messval);
&retireval) == 0) {
&responseval) == 0) {
if (allfaulty) {
if (!allfaulty) {
switch (ud) {
case UD_VAL_MISSINGINFO:
case UD_VAL_MISSINGOBS:
case UD_VAL_MISSINGPATH:
case UD_VAL_MISSINGZERO:
case UD_VAL_BADOBS:
case UD_VAL_CFGMISMATCH:
return (UNDIAG_DEFECT_CHKPT);
case UD_VAL_BADEVENTI:
case UD_VAL_BADEVENTPATH:
case UD_VAL_BADEVENTCLASS:
case UD_VAL_INSTFAIL:
case UD_VAL_NOPATH:
case UD_VAL_UNSOLVD:
return (UNDIAG_DEFECT_FME);
case UD_VAL_MAXFME:
return (UNDIAG_DEFECT_LIMIT);
case UD_VAL_UNKNOWN:
return (UNDIAG_DEFECT_UNKNOWN);
switch (ud) {
case UD_VAL_BADEVENTI:
case UD_VAL_BADEVENTPATH:
case UD_VAL_BADEVENTCLASS:
case UD_VAL_INSTFAIL:
case UD_VAL_NOPATH:
case UD_VAL_UNSOLVD:
return (UNDIAG_FAULT_FME);
return (NULL);
const char *ptr;
char *buf;
int with_arg = 0;
switch (ud) {
case UD_VAL_BADEVENTPATH:
case UD_VAL_BADEVENTCLASS:
case UD_VAL_BADEVENTI:
case UD_VAL_BADOBS:
case UD_VAL_CFGMISMATCH:
case UD_VAL_INSTFAIL:
case UD_VAL_MAXFME:
case UD_VAL_MISSINGINFO:
case UD_VAL_MISSINGOBS:
case UD_VAL_MISSINGPATH:
case UD_VAL_MISSINGZERO:
case UD_VAL_NOPATH:
case UD_VAL_UNSOLVD:
case UD_VAL_UNKNOWN:
if (with_arg) {
return (buf);
const char *faultstr;
detector);
B_FALSE);
B_FALSE);
char *pathstr;
const char *faultstr;
pathstr);
B_FALSE);
B_FALSE);
B_FALSE);
B_FALSE);
(void *)fmep);
int mess_zero_count;
int rpcnt;
case FME_CREDIBLE:
if (ffep) {
case FME_WAIT:
case FME_DISPROVED:
static void indent(void);
unsigned long long *pdelay);
indent();
indent();
indent();
indent();
indent();
int count = 0;
unsigned long long my_delay;
int retval = 0;
if (mark == 0) {
keep);
indent();
indent();
indent();
indent();
indent();
indent();
&my_delay);
&my_delay);
indent();
WAIT_EFFECT) {
indent_pop();
indent();
indent();
indent_pop();
return (retval);
static enum fme_state
unsigned long long my_delay;
indent();
indent();
indent();
indent_pop();
return (return_value);
static enum fme_state
int waiting_events;
int credible_events;
int deferred_events;
unsigned long long arrow_delay;
unsigned long long my_delay;
indent();
return (FME_CREDIBLE);
indent();
return (FME_DISPROVED);
indent();
return (FME_WAIT);
indent();
indent();
switch (return_value) {
case FME_CREDIBLE:
case FME_DISPROVED:
case FME_WAIT:
indent_pop();
return (return_value);
credible_events = 0;
waiting_events = 0;
deferred_events = 0;
indent();
indent();
if (n <= credible_events)
&my_delay)) {
case FME_DEFERRED:
case FME_CREDIBLE:
case FME_DISPROVED:
case FME_WAIT:
indent();
indent();
indent_pop();
return (FME_DISPROVED);
} else if (credible_events < n) {
indent();
indent_pop();
return (return_value);
static enum fme_state
unsigned long long my_delay;
int credible_results = 0;
int waiting_results = 0;
indent();
int do_not_follow = 0;
indent();
if (do_not_follow) {
indent();
&my_delay);
switch (fstate) {
case FME_WAIT:
case FME_CREDIBLE:
case FME_DISPROVED:
indent();
indent_pop();
return (FME_DISPROVED);
if (waiting_results != 0) {
indent();
indent_pop();
return (FME_WAIT);
indent();
indent_pop();
return (FME_CREDIBLE);
static enum fme_state
unsigned long long my_delay;
indent();
indent();
indent_pop();
return (FME_DISPROVED);
indent();
indent_pop();
return (FME_DISPROVED);
indent();
indent_pop();
return (FME_DISPROVED);
indent();
indent_pop();
return (FME_WAIT);
indent();
indent_pop();
return (FME_CREDIBLE);
int sz;
char *sbuf;
char *ptr;
char *sepptr;
int val;
istat_save();