fmd_case.c revision cbf75e67acb6c32a2f4884f28a839d59f7988d37
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* FMD Case Subsystem
*
* Diagnosis engines are expected to group telemetry events related to the
* diagnosis of a particular problem on the system into a set of cases. The
* diagnosis engine may have any number of cases open at a given point in time.
* Some cases may eventually be *solved* by associating a suspect list of one
* or more problems with the case, at which point fmd publishes a list.suspect
* event for the case and it becomes visible to administrators and agents.
*
* Every case is named using a UUID, and is globally visible in the case hash.
* Cases are reference-counted, except for the reference from the case hash
* itself. Consumers of case references include modules, which store active
* cases on the mod_cases list, ASRUs in the resource cache, and the RPC code.
*
* Cases obey the following state machine. In states UNSOLVED, SOLVED, and
* CLOSE_WAIT, a case's module refers to the owning module (a diagnosis engine
* or transport) and the case is referenced by the mod_cases list. Once the
* case reaches the CLOSED or REPAIRED states, a case's module changes to refer
* to the root module (fmd.d_rmod) and is deleted from the owner's mod_cases.
*
* +------------+
* +----------| UNSOLVED |
* | +------------+
* | 1 |
* | |
* | +-------v----+
* 2 | | SOLVED |
* | +------------+
* | 3 | 5 |
* +------------+ | |
* | | |
* +-v---v----v-+
* | CLOSE_WAIT |
* +------------+
* | | |
* +-----------+ | +------------+
* | 4 | |
* v +-----v------+ |
* discard | CLOSED | 6 |
* +------------+ |
* | |
* | +------------+
* 7 | |
* +-----v----v-+
* | REPAIRED |
* +------------+
* |
* 8 |
* +-----v------+
* | RESOLVED |
* +------------+
* |
* v
* discard
*
* The state machine changes are triggered by calls to fmd_case_transition()
* from various locations inside of fmd, as described below:
*
* [1] Called by: fmd_case_solve()
* Actions: FMD_CF_SOLVED flag is set in ci_flags
* conviction policy is applied to suspect list
* suspects convicted are marked faulty (F) in R$
* list.suspect event logged and dispatched
*
* [2] Called by: fmd_case_close(), fmd_case_uuclose()
* Actions: diagnosis engine fmdo_close() entry point scheduled
* case discarded upon exit from CLOSE_WAIT
*
* [3] Called by: fmd_case_close(), fmd_case_uuclose(), fmd_xprt_event_uuclose()
* Actions: FMD_CF_ISOLATED flag is set in ci_flags
* suspects convicted (F) are marked unusable (U) in R$
* diagnosis engine fmdo_close() entry point scheduled
* case transitions to CLOSED [4] upon exit from CLOSE_WAIT
*
* [4] Called by: fmd_case_delete() (after fmdo_close() entry point returns)
* Actions: list.isolated event dispatched
* case deleted from module's list of open cases
*
* [5] Called by: fmd_case_repair(), fmd_case_update()
* Actions: FMD_CF_REPAIR flag is set in ci_flags
* diagnosis engine fmdo_close() entry point scheduled
* case transitions to REPAIRED [6] upon exit from CLOSE_WAIT
*
* [6] Called by: fmd_case_delete() (after fmdo_close() entry point returns)
* Actions: suspects convicted are marked non faulty (!F) in R$
* list.repaired or list.updated event dispatched
*
* [7] Called by: fmd_case_repair(), fmd_case_update()
* Actions: FMD_CF_REPAIR flag is set in ci_flags
* suspects convicted are marked non faulty (!F) in R$
* list.repaired or list.updated event dispatched
*
* [8] Called by: fmd_case_uuresolve()
* Actions: list.resolved event dispatched
* case is discarded
*/
#include <alloca.h>
#include <fmd_alloc.h>
#include <fmd_module.h>
#include <fmd_error.h>
#include <fmd_conf.h>
#include <fmd_case.h>
#include <fmd_string.h>
#include <fmd_subr.h>
#include <fmd_protocol.h>
#include <fmd_event.h>
#include <fmd_eventq.h>
#include <fmd_dispq.h>
#include <fmd_buf.h>
#include <fmd_log.h>
#include <fmd_asru.h>
#include <fmd_fmri.h>
#include <fmd_xprt.h>
#include <fmd.h>
static const char *const _fmd_case_snames[] = {
"UNSOLVED", /* FMD_CASE_UNSOLVED */
"SOLVED", /* FMD_CASE_SOLVED */
"CLOSE_WAIT", /* FMD_CASE_CLOSE_WAIT */
"CLOSED", /* FMD_CASE_CLOSED */
"REPAIRED", /* FMD_CASE_REPAIRED */
"RESOLVED" /* FMD_CASE_RESOLVED */
};
fmd_case_hash_create(void)
{
return (chp);
}
/*
* Destroy the case hash. Unlike most of our hash tables, no active references
* are kept by the case hash itself; all references come from other subsystems.
* The hash must be destroyed after all modules are unloaded; if anything was
* present in the hash it would be by definition a reference count leak.
*/
void
{
}
/*
* Take a snapshot of the case hash by placing an additional hold on each
* member in an auxiliary array, and then call 'func' for each case.
*/
void
{
for (i = 0; i < chp->ch_hashlen; i++) {
}
for (i = 0; i < cpc; i++) {
}
}
}
static void
{
}
static void
{
else
break;
}
}
}
}
/*
* Look up the diagcode for this case and cache it in ci_code. If no suspects
* were defined for this case or if the lookup fails, the event dictionary or
* module code is broken, and we set the event code to a precomputed default.
*/
static const char *
{
const char *s;
/*
* delete any existing entry from code hash if it is on it
*/
keyp++;
}
}
/*
* add into hash of solved cases
*/
}
typedef struct {
int *fcl_countp;
int fcl_maxcount;
int *fcl_msgp;
static void
{
boolean_t b;
int state;
return;
&b) == 0 && b == B_FALSE)
if (state & FMD_ASRU_DEGRADED)
if (state & FMD_ASRU_UNUSABLE)
if (state & FMD_ASRU_FAULTY)
if (!(state & FMD_ASRU_PRESENT))
(*entryp->fcl_countp)++;
}
static void
{
}
static void
{
}
static void
{
int *not_faultyp = (int *)arg;
}
/*
* Have we got any suspects with an asru that are still unusable and present?
*/
static void
{
int state;
/*
* if this a proxy case and this suspect doesn't have an local asru
* then state is unknown so we must assume it may still be unusable.
*/
return;
}
return;
}
nvlist_t *
{
const char *code;
int count = 0;
/*
* For each suspect associated with the case, store its fault event
* nvlist in 'nva'. We also look to see if any of the suspect faults
* have asked not to be messaged. If any of them have made such a
* request, propagate that attribute to the composite list.* event.
* Finally, store each suspect's faulty status into the bitmap 'ba'.
*/
(void) fmd_case_mkcode(cp);
/*
* For repair and updated event, we lookup diagcode from dict using key
* "list.repaired" or "list.updated" or "list.resolved".
*/
else
/*
* Use the ci_diag_de if one has been saved (eg for an injected fault).
* Otherwise use the authority for the current module.
*/
return (nvl);
}
static boolean_t
{
int fmri_present = 1;
int new_fmri_present = 1;
fmri_present = 0;
else {
goto done;
goto done;
}
new_fmri_present = 0;
else {
goto done;
goto done;
}
(fmri_present == 0 ||
done:
return (match);
}
static int
{
return (0);
return (0);
return (0);
}
/*
* see if an identical suspect list already exists in the cache
*/
static int
{
int match = 0, match_susp;
uint_t h;
/*
* Find all cases with this code
*/
/*
* only look for any cases (apart from this one)
* whose code and number of suspects match
*/
continue;
continue;
}
/*
* For each suspect in one list, check if there
* is an identical suspect in the other list
*/
match = 1;
match_susp = 0;
match_susp = 1;
break;
}
}
if (match_susp == 0) {
match = 0;
break;
}
}
if (match) {
return (1);
}
}
return (0);
}
/*
* Convict suspects in a case by applying a conviction policy and updating the
* resource cache prior to emitting the list.suspect event for the given case.
* At present, our policy is very simple: convict every suspect in the case.
* In the future, this policy can be extended and made configurable to permit:
*
* - convicting the suspect with the highest FIT rate
* - convicting the suspect with the cheapest FRU
* - convicting the suspect with the FRU that is in a depot's inventory
* - convicting the suspect with the longest lifetime
*
* and so forth. A word to the wise: this problem is significantly harder that
* it seems at first glance. Future work should heed the following advice:
*
* Hacking the policy into C code here is a very bad idea. The policy needs to
* be decided upon very carefully and fundamentally encodes knowledge of what
* suspect list combinations can be emitted by what diagnosis engines. As such
* fmd's code is the wrong location, because that would require fmd itself to
* be updated for every diagnosis engine change, defeating the entire design.
* The FMA Event Registry knows the suspect list combinations: policy inputs
* can be derived from it and used to produce per-module policy configuration.
*
* If the policy needs to be dynamic and not statically fixed at either fmd
* startup or module load time, any implementation of dynamic policy retrieval
* must employ some kind of caching mechanism or be part of a built-in module.
* The fmd_case_convict() function is called with locks held inside of fmd and
* is not a place where unbounded blocking on some inter-process or inter-
* system communication to another service (e.g. another daemon) can occur.
*/
static int
{
(void) fmd_case_mkcode(cp);
else if (cip->ci_precanned)
return (1);
}
/*
* no suspect list already exists - allocate new cache entries
*/
continue;
}
}
return (0);
}
void
{
fmd_event_t *e;
char *class;
if (state == FMD_CASE_CURRENT)
switch (state) {
case FMD_CASE_SOLVED:
/*
* If we already have a code, then case is already solved.
*/
break;
}
if (cip->ci_tv_valid == 0) {
}
break;
}
/*
* For proxy, save some information about the transport
* in the resource cache.
*/
int count = 0;
FMD_XPRT_RDWR) == FMD_XPRT_RDONLY);
}
break;
case FMD_CASE_CLOSE_WAIT:
break;
case FMD_CASE_CLOSED:
break;
case FMD_CASE_REPAIRED:
break;
case FMD_CASE_RESOLVED:
break;
}
}
{
uint_t h;
break;
}
/*
* If deleting bit is set, treat the case as if it doesn't exist.
*/
(void) fmd_set_errno(EFMD_CASE_INVAL);
return ((fmd_case_t *)cip);
}
static fmd_case_impl_t *
{
uint_t h;
return (eip); /* uuid already present */
}
}
return (cip);
}
static void
{
uint_t h;
else
break;
}
fmd_panic("case %p (%s) not found on hash chain %u\n",
}
/*
* delete from code hash if it is on it
*/
}
{
/*
* Calling libuuid: get a clue. The library interfaces cleverly do not
* define any constant for the length of an unparse string, and do not
* permit the caller to specify a buffer length for safety. The spec
* says it will be 36 bytes, but we make it tunable just in case.
*/
/*
* We expect this loop to execute only once, but code it defensively
* against the possibility of libuuid bugs. Keep generating uuids and
* attempting to do a hash insert until we get a unique one.
*/
do {
return ((fmd_case_t *)cip);
}
static void
{
if (cip->ci_proxy_asru)
cip->ci_nsuspects);
if (cip->ci_diag_de)
if (cip->ci_diag_asru)
cip->ci_nsuspects);
}
cip->ci_nsuspects = 0;
}
{
if (state > FMD_CASE_CLOSE_WAIT)
/*
* Insert the case into the global case hash. If the specified UUID is
* already present, check to see if it is an orphan: if so, reclaim it;
* otherwise if it is owned by a different module then return NULL.
*/
/*
* If the ASRU cache is trying to recreate an orphan, then just
* return the existing case that we found without changing it.
*/
/*
* In case the case has already been created from
* a checkpoint file we need to set up code now.
*/
cip);
}
}
/*
* When recreating an orphan case, state passed in may
* either be CLOSED (faulty) or REPAIRED (!faulty). If
* any suspects are still CLOSED (faulty) then the
* overall state needs to be CLOSED.
*/
state == FMD_CASE_CLOSED)
return ((fmd_case_t *)cip);
}
/*
* If the existing case isn't an orphan or is being proxied,
* then we have a UUID conflict: return failure to the caller.
*/
return (NULL);
}
/*
* If the new module is reclaiming an orphaned case, remove
* the case from the root module, switch ci_mod, and then fall
* through to adding the case to the new owner module 'mp'.
*/
/*
* It's possible that fmd crashed or was restarted during a
* previous solve operation between the asru cache being created
* and the ckpt file being updated to SOLVED. Thus when the DE
* recreates the case here from the checkpoint file, the state
* will be UNSOLVED and yet we are having to reclaim because
* the case was in the asru cache. If this happens, revert the
* case back to the UNSOLVED state and let the DE solve it again
*/
if (state == FMD_CASE_UNSOLVED) {
(fmd_case_t *)cip);
cip->ci_codelen = 0;
cip->ci_tv_valid = 0;
}
} else {
/*
* add into hash of solved cases
*/
}
return ((fmd_case_t *)cip);
}
void
{
if (visible) {
}
}
}
void
{
}
void
{
fmd_panic("attempt to hold a deleting case %p (%s)\n",
}
static fmd_case_impl_t *
{
/*
* If the case's "deleting" bit is unset, hold and return case,
* otherwise, return NULL.
*/
} else {
}
return (cip);
}
void
{
else
}
void
{
}
int
{
int new;
else
break;
}
return (new);
}
int
{
int new;
else
break;
}
/*
* If the event is already in the case or the case is already solved,
* there is no reason to save it: just transition it appropriately.
*/
return (new);
}
return (new);
}
void
{
cip->ci_nsuspects++;
}
void
{
boolean_t b;
FM_SUSPECT_MESSAGE, &b) == 0 && b == B_FALSE)
cip->ci_nsuspects++;
}
void
{
}
/*ARGSUSED*/
static void
{
}
/*
* Grab ci_lock and update the case state and set the dirty bit. Then perform
* whatever actions and emit whatever events are appropriate for the state.
* Refer to the topmost block comment explaining the state machine for details.
*/
void
{
fmd_event_t *e;
int resolved = 0;
int any_unusable_and_present = 0;
return; /* already in specified state */
}
switch (state) {
case FMD_CASE_SOLVED:
}
break;
case FMD_CASE_CLOSE_WAIT:
/*
* If the case was never solved, do not change ASRUs.
* If the case was never fmd_case_closed, do not change ASRUs.
* If the case was repaired, do not change ASRUs.
*/
/*
* If an orphaned case transitions to CLOSE_WAIT, the owning
* module is no longer loaded: continue on to CASE_CLOSED.
*/
if (fmd_case_orphaned(cp))
break;
case FMD_CASE_REPAIRED:
/*
* If we've been requested to transition straight on to the
* RESOLVED state (which can happen with fault proxying where a
* list.resolved or a uuresolved is received from the other
* side), or if all suspects are already either usable or not
* present then transition straight to RESOLVED state,
* publishing both the list.repaired and list.resolved. For a
* proxy, if we discover here that all suspects are already
* either usable or not present, notify the diag side instead
* using fmd_xprt_uuresolved().
*/
if (flags & FMD_CF_RESOLVED) {
} else {
}
} else {
break;
break;
}
}
resolved = 1;
break;
case FMD_CASE_RESOLVED:
/*
* For a proxy, no need to check that all suspects are already
* either usable or not present - this request has come from
* the diagnosing side which makes the final decision on this.
*/
resolved = 1;
break;
}
/*
* If all suspects are already either usable or not present then
* carry on, publish list.resolved and discard the case.
*/
if (any_unusable_and_present) {
return;
}
resolved = 1;
break;
}
/*
* If the module has initialized, then publish the appropriate event
* for the new case state. If not, we are being called from the
* checkpoint code during module load, in which case the module's
* _fmd_init() routine hasn't finished yet, and our event dictionaries
* may not be open yet, which will prevent us from computing the event
* code. Defer the call to fmd_case_publish() by enqueuing a PUBLISH
* event in our queue: this won't be processed until _fmd_init is done.
*/
else {
}
if (resolved) {
/*
* If we transitioned to RESOLVED, adjust the reference count to
* reflect our removal from fmd.d_rmod->mod_cases above. If the
* caller has not placed an additional hold on the case, it
* will now be freed.
*/
}
}
/*
* Transition the specified case to *at least* the specified state by first
* re-validating the suspect list using the resource cache. This function is
* employed by the checkpoint code when restoring a saved, solved case to see
* if the state of the case has effectively changed while fmd was not running
* or the module was not loaded.
*/
void
{
int usable = 0; /* are any suspects usable? */
if (!usable) {
flags |= FMD_CF_ISOLATED;
}
}
void
{
}
void
{
}
void
{
}
}
/*
*/
void
{
int count = 0;
count);
}
/*
* fmd_case_update_status() can be called on either the proxy side when a
* list.suspect is received, or on the diagnosing side when an update request
* is received from the proxy. It updates the status in the resource cache.
*/
void
{
int count = 0;
/*
* update status of resource cache entries
*/
&faus);
}
/*
* Called on either the proxy side or the diag side when a repair has taken
* place on the other side but this side may know the asru "contains"
* relationships.
*/
void
{
}
/*
* fmd_case_close_status() is called on diagnosing side when proxy side
* has had a uuclose. It updates the status in the resource cache.
*/
void
{
int count = 0;
/*
* update status of resource cache entries
*/
&facs);
}
/*
* Indicate that the case may need to change state because one or more of the
* ASRUs named as a suspect has changed state. We examine all the suspects
* and if none are still faulty, we initiate a case close transition.
*/
void
{
int faulty = 0;
return; /* update is not appropriate */
}
return; /* already repaired */
}
if (faulty) {
fmd_event_t *e;
char *class;
return; /* one or more suspects are still marked faulty */
}
if (cstate == FMD_CASE_CLOSED)
else
}
/*
* Delete a closed case from the module's case list once the fmdo_close() entry
* point has run to completion. If the case is owned by a transport module,
* tell the transport to proxy a case close on the other end of the transport.
* Transition to the appropriate next state based on ci_flags. This
* function represents the end of CLOSE_WAIT and transitions the case to either
* CLOSED or REPAIRED or discards it entirely because it was never solved;
* refer to the topmost block comment explaining the state machine for details.
*/
void
{
/*
* If the case has been solved, then retain it
* on the root module's case list at least until we're transitioned.
* Otherwise free the case with our final fmd_case_rele() below.
*/
}
/*
* Transition onwards to REPAIRED or CLOSED as originally requested.
* Note that for proxy case if we're transitioning to CLOSED it means
* the case was isolated locally, so call fmd_xprt_uuclose() to notify
* the diagnosing side. No need to notify the diagnosing side if we are
* transitioning to REPAIRED as we only do this when requested to do
* so by the diagnosing side anyway.
*/
}
}
void
{
if (delete_from_asru_cache) {
}
}
/*
* Indicate that the problem corresponding to a case has been repaired by
* clearing the faulty bit on each ASRU named as a suspect. If the case hasn't
* already been closed, this function initiates the transition to CLOSE_WAIT.
* The caller must have the case held from fmd_case_hash_lookup(), so we can
* grab and drop ci_lock without the case being able to be freed in between.
*/
int
{
if (cstate < FMD_CASE_SOLVED) {
return (fmd_set_errno(EFMD_CASE_STATE));
}
return (0); /* already repaired */
}
/*
* if this is a proxied case, send the repair across the transport.
* The remote side will then do the repair and send a list.repaired back
* again such that we can finally repair the case on this side.
*/
return (0);
}
if (cstate == FMD_CASE_CLOSED)
else
return (0);
}
int
{
if (cstate < FMD_CASE_SOLVED) {
return (fmd_set_errno(EFMD_CASE_STATE));
}
return (0); /* already repaired */
}
/*
* if this is a proxied case, send the repair across the transport.
* The remote side will then do the repair and send a list.repaired back
* again such that we can finally repair the case on this side.
*/
return (0);
}
if (cstate == FMD_CASE_CLOSED)
else
return (0);
}
int
{
int rv = 0;
else
break;
}
if (rv != 0)
return (rv);
}
int
{
}
void
{
}
void
{
if (cip->ci_diag_de)
}
void
{
}
/*ARGSUSED*/
void
{
int not_faulty = 0;
int faulty = 0;
fmd_event_t *e;
char *class;
int any_unusable_and_present = 0;
return;
¬_faulty);
/*
* If none of the suspects is faulty, replay the list.repaired.
* If all suspects are already either usable or not present then
* also transition straight to RESOLVED state.
*/
if (!any_unusable_and_present) {
class);
} else {
class);
}
} else if (faulty && not_faulty) {
/*
* if some but not all of the suspects are not faulty, replay
* the list.updated.
*/
}
}
void
{
}