fmd_module.c revision 2a417b235e35bd992c3c76d3eb89d35467286133
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#include <signal.h>
#include <dirent.h>
#include <limits.h>
#include <alloca.h>
#include <unistd.h>
#include <stdio.h>
#include <fmd_string.h>
#include <fmd_alloc.h>
#include <fmd_module.h>
#include <fmd_error.h>
#include <fmd_conf.h>
#include <fmd_dispq.h>
#include <fmd_eventq.h>
#include <fmd_timerq.h>
#include <fmd_subr.h>
#include <fmd_thread.h>
#include <fmd_ustat.h>
#include <fmd_case.h>
#include <fmd_protocol.h>
#include <fmd_buf.h>
#include <fmd_ckpt.h>
#include <fmd_xprt.h>
#include <fmd_topo.h>
#include <fmd.h>
/*
* Template for per-module statistics installed by fmd on behalf of each active
* module. These are used to initialize the per-module mp->mod_stats below.
* NOTE: FMD_TYPE_STRING statistics should not be used here. If they are
* required in the future, the FMD_ADM_MODDSTAT service routine must change.
*/
static const fmd_modstat_t _fmd_modstat_tmpl = {
{
{ "fmd.dispatched", FMD_TYPE_UINT64, "total events dispatched to module" },
{ "fmd.dequeued", FMD_TYPE_UINT64, "total events dequeued by module" },
{ "fmd.prdequeued", FMD_TYPE_UINT64, "protocol events dequeued by module" },
{ "fmd.dropped", FMD_TYPE_UINT64, "total events dropped on queue overflow" },
{ "fmd.wcnt", FMD_TYPE_UINT32, "count of events waiting on queue" },
{ "fmd.wtime", FMD_TYPE_TIME, "total wait time on queue" },
{ "fmd.wlentime", FMD_TYPE_TIME, "total wait length * time product" },
{ "fmd.wlastupdate", FMD_TYPE_TIME, "hrtime of last wait queue update" },
{ "fmd.dtime", FMD_TYPE_TIME, "total processing time after dequeue" },
{ "fmd.dlastupdate", FMD_TYPE_TIME, "hrtime of last event dequeue completion" },
},
{ "fmd.loadtime", FMD_TYPE_TIME, "hrtime at which module was loaded" },
{ "fmd.snaptime", FMD_TYPE_TIME, "hrtime of last statistics snapshot" },
{ "fmd.accepted", FMD_TYPE_UINT64, "total events accepted by module" },
{ "fmd.debugdrop", FMD_TYPE_UINT64, "dropped debug messages" },
{ "fmd.memtotal", FMD_TYPE_SIZE, "total memory allocated by module" },
{ "fmd.memlimit", FMD_TYPE_SIZE, "limit on total memory allocated" },
{ "fmd.buftotal", FMD_TYPE_SIZE, "total buffer space used by module" },
{ "fmd.buflimit", FMD_TYPE_SIZE, "limit on total buffer space" },
{ "fmd.thrtotal", FMD_TYPE_UINT32, "total number of auxiliary threads" },
{ "fmd.thrlimit", FMD_TYPE_UINT32, "limit on number of auxiliary threads" },
{ "fmd.caseopen", FMD_TYPE_UINT64, "cases currently open by module" },
{ "fmd.casesolved", FMD_TYPE_UINT64, "total cases solved by module" },
{ "fmd.caseclosed", FMD_TYPE_UINT64, "total cases closed by module" },
{ "fmd.ckptsave", FMD_TYPE_BOOL, "save checkpoints for module" },
{ "fmd.ckptrestore", FMD_TYPE_BOOL, "restore checkpoints for module" },
{ "fmd.ckptzero", FMD_TYPE_BOOL, "zeroed checkpoint at startup" },
{ "fmd.ckptcnt", FMD_TYPE_UINT64, "number of checkpoints taken" },
{ "fmd.ckpttime", FMD_TYPE_TIME, "total checkpoint time" },
{ "fmd.xprtopen", FMD_TYPE_UINT32, "total number of open transports" },
{ "fmd.xprtlimit", FMD_TYPE_UINT32, "limit on number of open transports" },
{ "fmd.xprtqlimit", FMD_TYPE_UINT32, "limit on transport event queue length" },
};
static void
fmd_module_start(void *arg)
{
fmd_module_t *mp = arg;
fmd_event_t *ep;
fmd_xprt_t *xp;
(void) pthread_mutex_lock(&mp->mod_lock);
if (mp->mod_ops->mop_init(mp) != 0 || mp->mod_error != 0) {
if (mp->mod_error == 0)
mp->mod_error = errno ? errno : EFMD_MOD_INIT;
goto out;
}
if (fmd.d_mod_event != NULL)
fmd_eventq_insert_at_head(mp->mod_queue, fmd.d_mod_event);
ASSERT(MUTEX_HELD(&mp->mod_lock));
mp->mod_flags |= FMD_MOD_INIT;
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
/*
* If the module opened any transports while executing _fmd_init(),
* they are suspended. Now that _fmd_init() is done, wake them up.
*/
for (xp = fmd_list_next(&mp->mod_transports);
xp != NULL; xp = fmd_list_next(xp))
fmd_xprt_xresume(xp, FMD_XPRT_ISUSPENDED);
/*
* Wait for events to arrive by checking mod_error and then sleeping in
* fmd_eventq_delete(). If a NULL event is returned, the eventq has
* been aborted and we continue on to call fini and exit the thread.
*/
while ((ep = fmd_eventq_delete(mp->mod_queue)) != NULL) {
/*
* If the module has failed, discard the event without ever
* passing it to the module and go back to sleep.
*/
if (mp->mod_error != 0) {
fmd_eventq_done(mp->mod_queue);
fmd_event_rele(ep);
continue;
}
mp->mod_ops->mop_dispatch(mp, ep);
fmd_eventq_done(mp->mod_queue);
/*
* Once mop_dispatch() is complete, grab the lock and perform
* any event-specific post-processing. Finally, if necessary,
* checkpoint the state of the module after this event.
*/
fmd_module_lock(mp);
if (FMD_EVENT_TYPE(ep) == FMD_EVT_CLOSE)
fmd_case_delete(FMD_EVENT_DATA(ep));
fmd_ckpt_save(mp);
fmd_module_unlock(mp);
fmd_event_rele(ep);
}
if (mp->mod_ops->mop_fini(mp) != 0 && mp->mod_error == 0)
mp->mod_error = errno ? errno : EFMD_MOD_FINI;
(void) pthread_mutex_lock(&mp->mod_lock);
mp->mod_flags |= FMD_MOD_FINI;
out:
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
}
fmd_module_t *
fmd_module_create(const char *path, const fmd_modops_t *ops)
{
fmd_module_t *mp = fmd_zalloc(sizeof (fmd_module_t), FMD_SLEEP);
char buf[PATH_MAX], *p;
const char *dir;
uint32_t limit;
int err;
(void) strlcpy(buf, fmd_strbasename(path), sizeof (buf));
if ((p = strrchr(buf, '.')) != NULL && strcmp(p, ".so") == 0)
*p = '\0'; /* strip trailing .so from any module name */
(void) pthread_mutex_init(&mp->mod_lock, NULL);
(void) pthread_cond_init(&mp->mod_cv, NULL);
(void) pthread_mutex_init(&mp->mod_stats_lock, NULL);
mp->mod_name = fmd_strdup(buf, FMD_SLEEP);
mp->mod_path = fmd_strdup(path, FMD_SLEEP);
mp->mod_ops = ops;
mp->mod_ustat = fmd_ustat_create();
(void) fmd_conf_getprop(fmd.d_conf, "ckpt.dir", &dir);
(void) snprintf(buf, sizeof (buf),
"%s/%s/%s", fmd.d_rootdir, dir, mp->mod_name);
mp->mod_ckpt = fmd_strdup(buf, FMD_SLEEP);
(void) fmd_conf_getprop(fmd.d_conf, "client.tmrlim", &limit);
mp->mod_timerids = fmd_idspace_create(mp->mod_name, 1, limit + 1);
mp->mod_threads = fmd_idspace_create(mp->mod_name, 0, INT_MAX);
fmd_buf_hash_create(&mp->mod_bufs);
fmd_serd_hash_create(&mp->mod_serds);
mp->mod_topo_current = fmd_topo_hold();
(void) pthread_mutex_lock(&fmd.d_mod_lock);
fmd_list_append(&fmd.d_mod_list, mp);
(void) pthread_mutex_unlock(&fmd.d_mod_lock);
/*
* Initialize the module statistics that are kept on its behalf by fmd.
* These are set up using a template defined at the top of this file.
*/
if ((mp->mod_stats = (fmd_modstat_t *)fmd_ustat_insert(mp->mod_ustat,
FMD_USTAT_ALLOC, sizeof (_fmd_modstat_tmpl) / sizeof (fmd_stat_t),
(fmd_stat_t *)&_fmd_modstat_tmpl, NULL)) == NULL) {
fmd_error(EFMD_MOD_INIT, "failed to initialize per-mod stats");
fmd_module_destroy(mp);
return (NULL);
}
if (nv_alloc_init(&mp->mod_nva_sleep,
&fmd_module_nva_ops_sleep, mp) != 0 ||
nv_alloc_init(&mp->mod_nva_nosleep,
&fmd_module_nva_ops_nosleep, mp) != 0) {
fmd_error(EFMD_MOD_INIT, "failed to initialize nvlist "
"allocation routines");
fmd_module_destroy(mp);
return (NULL);
}
(void) fmd_conf_getprop(fmd.d_conf, "client.evqlim", &limit);
mp->mod_queue = fmd_eventq_create(mp,
&mp->mod_stats->ms_evqstat, &mp->mod_stats_lock, limit);
(void) fmd_conf_getprop(fmd.d_conf, "client.memlim",
&mp->mod_stats->ms_memlimit.fmds_value.ui64);
(void) fmd_conf_getprop(fmd.d_conf, "client.buflim",
&mp->mod_stats->ms_buflimit.fmds_value.ui64);
(void) fmd_conf_getprop(fmd.d_conf, "client.thrlim",
&mp->mod_stats->ms_thrlimit.fmds_value.ui32);
(void) fmd_conf_getprop(fmd.d_conf, "client.xprtlim",
&mp->mod_stats->ms_xprtlimit.fmds_value.ui32);
(void) fmd_conf_getprop(fmd.d_conf, "client.xprtqlim",
&mp->mod_stats->ms_xprtqlimit.fmds_value.ui32);
(void) fmd_conf_getprop(fmd.d_conf, "ckpt.save",
&mp->mod_stats->ms_ckpt_save.fmds_value.bool);
(void) fmd_conf_getprop(fmd.d_conf, "ckpt.restore",
&mp->mod_stats->ms_ckpt_restore.fmds_value.bool);
(void) fmd_conf_getprop(fmd.d_conf, "ckpt.zero",
&mp->mod_stats->ms_ckpt_zeroed.fmds_value.bool);
if (mp->mod_stats->ms_ckpt_zeroed.fmds_value.bool)
fmd_ckpt_delete(mp); /* blow away any pre-existing checkpoint */
/*
* Place a hold on the module and grab the module lock before creating
* the module's thread to ensure that it cannot destroy the module and
* that it cannot call ops->mop_init() before we're done setting up.
* NOTE: from now on, we must use fmd_module_rele() for error paths.
*/
fmd_module_hold(mp);
(void) pthread_mutex_lock(&mp->mod_lock);
mp->mod_stats->ms_loadtime.fmds_value.ui64 = gethrtime();
mp->mod_thread = fmd_thread_create(mp, fmd_module_start, mp);
if (mp->mod_thread == NULL) {
fmd_error(EFMD_MOD_THR, "failed to create thread for %s", path);
(void) pthread_mutex_unlock(&mp->mod_lock);
fmd_module_rele(mp);
return (NULL);
}
/*
* At this point our module structure is nearly finished and its thread
* is starting execution in fmd_module_start() above, which will begin
* by blocking for mod_lock. We now drop mod_lock and wait for either
* FMD_MOD_INIT or mod_error to be set before proceeding.
*/
while (!(mp->mod_flags & FMD_MOD_INIT) && mp->mod_error == 0)
(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
/*
* If the module has failed to initialize, copy its errno to the errno
* of the caller, wait for it to unload, and then destroy it.
*/
if (!(mp->mod_flags & FMD_MOD_INIT)) {
err = mp->mod_error;
(void) pthread_mutex_unlock(&mp->mod_lock);
if (err == EFMD_CKPT_INVAL)
fmd_ckpt_rename(mp); /* move aside bad checkpoint */
/*
* If we're in the background, keep quiet about failure to
* load because a handle wasn't registered: this is a module's
* way of telling us it didn't want to be loaded for some
* reason related to system configuration. If we're in the
* foreground we log this too in order to inform developers.
*/
if (fmd.d_fg || err != EFMD_HDL_INIT) {
fmd_error(EFMD_MOD_INIT, "failed to load %s: %s\n",
path, fmd_strerror(err));
}
fmd_module_unload(mp);
fmd_module_rele(mp);
(void) fmd_set_errno(err);
return (NULL);
}
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
fmd_dprintf(FMD_DBG_MOD, "loaded module %s\n", mp->mod_name);
return (mp);
}
static void
fmd_module_untimeout(fmd_idspace_t *ids, id_t id, fmd_module_t *mp)
{
void *arg = fmd_timerq_remove(fmd.d_timers, ids, id);
/*
* The root module calls fmd_timerq_install() directly and must take
* responsibility for any cleanup of timer arguments that is required.
* All other modules use fmd_modtimer_t's as the arg data; free them.
*/
if (arg != NULL && mp != fmd.d_rmod)
fmd_free(arg, sizeof (fmd_modtimer_t));
}
void
fmd_module_unload(fmd_module_t *mp)
{
fmd_modtopo_t *mtp;
(void) pthread_mutex_lock(&mp->mod_lock);
if (mp->mod_flags & FMD_MOD_QUIT) {
(void) pthread_mutex_unlock(&mp->mod_lock);
return; /* module is already unloading */
}
ASSERT(mp->mod_thread != NULL);
mp->mod_flags |= FMD_MOD_QUIT;
if (mp->mod_queue != NULL)
fmd_eventq_abort(mp->mod_queue);
/*
* Wait for the module's thread to stop processing events and call
* _fmd_fini() and exit. We do this by waiting for FMD_MOD_FINI to be
* set if INIT was set, and then attempting to join with the thread.
*/
while ((mp->mod_flags & (FMD_MOD_INIT | FMD_MOD_FINI)) == FMD_MOD_INIT)
(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
fmd_thread_destroy(mp->mod_thread, FMD_THREAD_JOIN);
mp->mod_thread = NULL;
/*
* Once the module is no longer active, clean up any data structures
* that are only required when the module is loaded.
*/
fmd_module_lock(mp);
if (mp->mod_timerids != NULL) {
fmd_idspace_apply(mp->mod_timerids,
(void (*)())fmd_module_untimeout, mp);
fmd_idspace_destroy(mp->mod_timerids);
mp->mod_timerids = NULL;
}
if (mp->mod_threads != NULL) {
fmd_idspace_destroy(mp->mod_threads);
mp->mod_threads = NULL;
}
(void) fmd_buf_hash_destroy(&mp->mod_bufs);
fmd_serd_hash_destroy(&mp->mod_serds);
while ((mtp = fmd_list_next(&mp->mod_topolist)) != NULL) {
fmd_list_delete(&mp->mod_topolist, mtp);
fmd_topo_rele(mtp->mt_topo);
fmd_free(mtp, sizeof (fmd_modtopo_t));
}
fmd_module_unlock(mp);
fmd_dprintf(FMD_DBG_MOD, "unloaded module %s\n", mp->mod_name);
}
void
fmd_module_destroy(fmd_module_t *mp)
{
fmd_conf_formal_t *cfp = mp->mod_argv;
int i;
ASSERT(MUTEX_HELD(&mp->mod_lock));
if (mp->mod_thread != NULL) {
(void) pthread_mutex_unlock(&mp->mod_lock);
fmd_module_unload(mp);
(void) pthread_mutex_lock(&mp->mod_lock);
}
ASSERT(mp->mod_thread == NULL);
ASSERT(mp->mod_refs == 0);
/*
* Once the module's thread is dead, we can safely remove the module
* from global visibility and by removing it from d_mod_list. Any
* modhash pointers are already gone by virtue of mod_refs being zero.
*/
(void) pthread_mutex_lock(&fmd.d_mod_lock);
fmd_list_delete(&fmd.d_mod_list, mp);
(void) pthread_mutex_unlock(&fmd.d_mod_lock);
if (mp->mod_topo_current != NULL)
fmd_topo_rele(mp->mod_topo_current);
if (mp->mod_nva_sleep.nva_ops != NULL)
nv_alloc_fini(&mp->mod_nva_sleep);
if (mp->mod_nva_nosleep.nva_ops != NULL)
nv_alloc_fini(&mp->mod_nva_nosleep);
/*
* Once the module is no longer processing events and no longer visible
* through any program data structures, we can free all of its content.
*/
if (mp->mod_queue != NULL) {
fmd_eventq_destroy(mp->mod_queue);
mp->mod_queue = NULL;
}
if (mp->mod_ustat != NULL) {
(void) pthread_mutex_lock(&mp->mod_stats_lock);
fmd_ustat_destroy(mp->mod_ustat);
mp->mod_ustat = NULL;
mp->mod_stats = NULL;
(void) pthread_mutex_unlock(&mp->mod_stats_lock);
}
for (i = 0; i < mp->mod_dictc; i++)
fm_dc_closedict(mp->mod_dictv[i]);
fmd_free(mp->mod_dictv, sizeof (struct fm_dc_handle *) * mp->mod_dictc);
if (mp->mod_conf != NULL)
fmd_conf_close(mp->mod_conf);
for (i = 0; i < mp->mod_argc; i++, cfp++) {
fmd_strfree((char *)cfp->cf_name);
fmd_strfree((char *)cfp->cf_default);
}
fmd_free(mp->mod_argv, sizeof (fmd_conf_formal_t) * mp->mod_argc);
fmd_strfree(mp->mod_name);
fmd_strfree(mp->mod_path);
fmd_strfree(mp->mod_ckpt);
nvlist_free(mp->mod_fmri);
fmd_strfree(mp->mod_vers);
fmd_free(mp, sizeof (fmd_module_t));
}
/*
* fmd_module_error() is called after the stack is unwound from a call to
* fmd_module_abort() to indicate that the module has failed. The mod_error
* field is used to hold the error code of the first fatal error to the module.
* An EFMD_MOD_FAIL event is then created and sent to fmd-self-diagnosis.
*/
static void
fmd_module_error(fmd_module_t *mp, int err)
{
fmd_event_t *e;
nvlist_t *nvl;
char *class;
ASSERT(MUTEX_HELD(&mp->mod_lock));
ASSERT(err != 0);
TRACE((FMD_DBG_MOD, "module aborted: err=%d", err));
if (mp->mod_error == 0)
mp->mod_error = err;
if (mp == fmd.d_self)
return; /* do not post event if fmd.d_self itself fails */
/*
* Send an error indicating the module has now failed to fmd.d_self.
* Since the error causing the failure has already been logged by
* fmd_api_xerror(), we do not need to bother logging this event.
* It only exists for the purpose of notifying fmd.d_self that it can
* close the case associated with this module because mod_error is set.
*/
nvl = fmd_protocol_moderror(mp, EFMD_MOD_FAIL, fmd_strerror(err));
(void) nvlist_lookup_string(nvl, FM_CLASS, &class);
e = fmd_event_create(FMD_EVT_PROTOCOL, FMD_HRT_NOW, nvl, class);
fmd_dispq_dispatch(fmd.d_disp, e, class);
}
void
fmd_module_dispatch(fmd_module_t *mp, fmd_event_t *e)
{
const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
fmd_event_impl_t *ep = (fmd_event_impl_t *)e;
fmd_hdl_t *hdl = (fmd_hdl_t *)mp;
fmd_modtimer_t *t;
volatile int err;
/*
* Before calling the appropriate module callback, enter the module as
* if by fmd_module_enter() and establish mod_jmpbuf for any aborts.
*/
(void) pthread_mutex_lock(&mp->mod_lock);
ASSERT(!(mp->mod_flags & FMD_MOD_BUSY));
mp->mod_flags |= FMD_MOD_BUSY;
if ((err = setjmp(mp->mod_jmpbuf)) != 0) {
(void) pthread_mutex_lock(&mp->mod_lock);
fmd_module_error(mp, err);
}
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
/*
* If it's the first time through fmd_module_dispatch(), call the
* appropriate module callback based on the event type. If the call
* triggers an fmd_module_abort(), we'll return to setjmp() above with
* err set to a non-zero value and then bypass this before exiting.
*/
if (err == 0) {
switch (ep->ev_type) {
case FMD_EVT_PROTOCOL:
ops->fmdo_recv(hdl, e, ep->ev_nvl, ep->ev_data);
break;
case FMD_EVT_TIMEOUT:
t = ep->ev_data;
ASSERT(t->mt_mod == mp);
ops->fmdo_timeout(hdl, t->mt_id, t->mt_arg);
break;
case FMD_EVT_CLOSE:
ops->fmdo_close(hdl, ep->ev_data);
break;
case FMD_EVT_STATS:
ops->fmdo_stats(hdl);
fmd_modstat_publish(mp);
break;
case FMD_EVT_GC:
ops->fmdo_gc(hdl);
break;
case FMD_EVT_PUBLISH:
fmd_case_publish(ep->ev_data, FMD_CASE_CURRENT);
break;
case FMD_EVT_TOPO:
fmd_topo_rele(mp->mod_topo_current);
mp->mod_topo_current = (fmd_topo_t *)ep->ev_data;
fmd_topo_addref(mp->mod_topo_current);
ops->fmdo_topo(hdl, mp->mod_topo_current->ft_hdl);
break;
}
}
fmd_module_exit(mp);
}
int
fmd_module_transport(fmd_module_t *mp, fmd_xprt_t *xp, fmd_event_t *e)
{
fmd_event_impl_t *ep = (fmd_event_impl_t *)e;
fmd_hdl_t *hdl = (fmd_hdl_t *)mp;
ASSERT(ep->ev_type == FMD_EVT_PROTOCOL);
return (mp->mod_info->fmdi_ops->fmdo_send(hdl, xp, e, ep->ev_nvl));
}
void
fmd_module_timeout(fmd_modtimer_t *t, id_t id, hrtime_t hrt)
{
fmd_event_t *e;
t->mt_id = id; /* save id in case we need to delete from eventq */
e = fmd_event_create(FMD_EVT_TIMEOUT, hrt, NULL, t);
fmd_eventq_insert_at_time(t->mt_mod->mod_queue, e);
}
/*
* Garbage collection is initiated by a timer callback once per day or at the
* request of fmadm. Purge old SERD entries and send the module a GC event.
*/
void
fmd_module_gc(fmd_module_t *mp)
{
fmd_hdl_info_t *info;
fmd_event_t *e;
if (mp->mod_error != 0)
return; /* do not do anything if the module has failed */
fmd_module_lock(mp);
if ((info = mp->mod_info) != NULL) {
fmd_serd_hash_apply(&mp->mod_serds,
(fmd_serd_eng_f *)fmd_serd_eng_gc, NULL);
}
fmd_module_unlock(mp);
if (info != NULL) {
e = fmd_event_create(FMD_EVT_GC, FMD_HRT_NOW, NULL, NULL);
fmd_eventq_insert_at_head(mp->mod_queue, e);
}
}
void
fmd_module_trygc(fmd_module_t *mp)
{
if (fmd_module_trylock(mp)) {
fmd_serd_hash_apply(&mp->mod_serds,
(fmd_serd_eng_f *)fmd_serd_eng_gc, NULL);
fmd_module_unlock(mp);
}
}
int
fmd_module_contains(fmd_module_t *mp, fmd_event_t *ep)
{
fmd_case_t *cp;
int rv = 0;
fmd_module_lock(mp);
for (cp = fmd_list_next(&mp->mod_cases);
cp != NULL; cp = fmd_list_next(cp)) {
if ((rv = fmd_case_contains(cp, ep)) != 0)
break;
}
if (rv == 0)
rv = fmd_serd_hash_contains(&mp->mod_serds, ep);
fmd_module_unlock(mp);
return (rv);
}
void
fmd_module_setdirty(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
mp->mod_flags |= FMD_MOD_MDIRTY;
(void) pthread_mutex_unlock(&mp->mod_lock);
}
void
fmd_module_setcdirty(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
mp->mod_flags |= FMD_MOD_CDIRTY;
(void) pthread_mutex_unlock(&mp->mod_lock);
}
void
fmd_module_clrdirty(fmd_module_t *mp)
{
fmd_case_t *cp;
fmd_module_lock(mp);
if (mp->mod_flags & FMD_MOD_CDIRTY) {
for (cp = fmd_list_next(&mp->mod_cases);
cp != NULL; cp = fmd_list_next(cp))
fmd_case_clrdirty(cp);
}
if (mp->mod_flags & FMD_MOD_MDIRTY) {
fmd_serd_hash_apply(&mp->mod_serds,
(fmd_serd_eng_f *)fmd_serd_eng_clrdirty, NULL);
fmd_buf_hash_commit(&mp->mod_bufs);
}
(void) pthread_mutex_lock(&mp->mod_lock);
mp->mod_flags &= ~(FMD_MOD_MDIRTY | FMD_MOD_CDIRTY);
(void) pthread_mutex_unlock(&mp->mod_lock);
fmd_module_unlock(mp);
}
void
fmd_module_commit(fmd_module_t *mp)
{
fmd_case_t *cp;
ASSERT(fmd_module_locked(mp));
if (mp->mod_flags & FMD_MOD_CDIRTY) {
for (cp = fmd_list_next(&mp->mod_cases);
cp != NULL; cp = fmd_list_next(cp))
fmd_case_commit(cp);
}
if (mp->mod_flags & FMD_MOD_MDIRTY) {
fmd_serd_hash_apply(&mp->mod_serds,
(fmd_serd_eng_f *)fmd_serd_eng_commit, NULL);
fmd_buf_hash_commit(&mp->mod_bufs);
}
(void) pthread_mutex_lock(&mp->mod_lock);
mp->mod_flags &= ~(FMD_MOD_MDIRTY | FMD_MOD_CDIRTY);
(void) pthread_mutex_unlock(&mp->mod_lock);
mp->mod_gen++;
}
void
fmd_module_lock(fmd_module_t *mp)
{
pthread_t self = pthread_self();
(void) pthread_mutex_lock(&mp->mod_lock);
while (mp->mod_flags & FMD_MOD_LOCK) {
if (mp->mod_owner != self)
(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
else
fmd_panic("recursive module lock of %p\n", (void *)mp);
}
mp->mod_owner = self;
mp->mod_flags |= FMD_MOD_LOCK;
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
}
void
fmd_module_unlock(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
ASSERT(mp->mod_owner == pthread_self());
ASSERT(mp->mod_flags & FMD_MOD_LOCK);
mp->mod_owner = 0;
mp->mod_flags &= ~FMD_MOD_LOCK;
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
}
int
fmd_module_trylock(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
if (mp->mod_flags & FMD_MOD_LOCK) {
(void) pthread_mutex_unlock(&mp->mod_lock);
return (0);
}
mp->mod_owner = pthread_self();
mp->mod_flags |= FMD_MOD_LOCK;
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
return (1);
}
int
fmd_module_locked(fmd_module_t *mp)
{
return ((mp->mod_flags & FMD_MOD_LOCK) &&
mp->mod_owner == pthread_self());
}
int
fmd_module_enter(fmd_module_t *mp, void (*func)(fmd_hdl_t *))
{
volatile int err;
(void) pthread_mutex_lock(&mp->mod_lock);
ASSERT(!(mp->mod_flags & FMD_MOD_BUSY));
mp->mod_flags |= FMD_MOD_BUSY;
if ((err = setjmp(mp->mod_jmpbuf)) != 0) {
(void) pthread_mutex_lock(&mp->mod_lock);
fmd_module_error(mp, err);
}
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
/*
* If it's the first time through fmd_module_enter(), call the provided
* function on the module. If no fmd_module_abort() results, we will
* fall through and return zero. Otherwise we'll longjmp with an err,
* return to the setjmp() above, and return the error to our caller.
*/
if (err == 0 && func != NULL)
(*func)((fmd_hdl_t *)mp);
return (err);
}
void
fmd_module_exit(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
ASSERT(mp->mod_flags & FMD_MOD_BUSY);
mp->mod_flags &= ~FMD_MOD_BUSY;
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
}
/*
* If the client.error policy has been set by a developer, stop or dump core
* based on the policy; if we stop and are resumed we'll continue and execute
* the default behavior to discard events in fmd_module_start(). If the caller
* is the primary module thread, we reach this state by longjmp'ing back to
* fmd_module_enter(), above. If the caller is an auxiliary thread, we cancel
* ourself and arrange for the primary thread to call fmd_module_abort().
*/
void
fmd_module_abort(fmd_module_t *mp, int err)
{
uint_t policy = FMD_CERROR_UNLOAD;
pthread_t tid = pthread_self();
(void) fmd_conf_getprop(fmd.d_conf, "client.error", &policy);
if (policy == FMD_CERROR_STOP) {
fmd_error(err, "stopping after %s in client %s (%p)\n",
fmd_errclass(err), mp->mod_name, (void *)mp);
(void) raise(SIGSTOP);
} else if (policy == FMD_CERROR_ABORT) {
fmd_panic("aborting due to %s in client %s (%p)\n",
fmd_errclass(err), mp->mod_name, (void *)mp);
}
/*
* If the caller is an auxiliary thread, cancel the current thread. We
* prefer to cancel because it affords developers the option of using
* the pthread_cleanup* APIs. If cancellations have been disabled,
* fall through to forcing the current thread to exit. In either case
* we update mod_error (if zero) to enter the failed state. Once that
* is set, further events received by the module will be discarded.
*
* We also set the FMD_MOD_FAIL bit, indicating an unrecoverable error.
* When an auxiliary thread fails, the module is left in a delicate
* state where it is likely not able to continue execution (even to
* execute its _fmd_fini() routine) because our caller may hold locks
* that are private to the module and can no longer be released. The
* FMD_MOD_FAIL bit forces fmd_api_module_lock() to abort if any other
* module threads reach an API call, in an attempt to get them to exit.
*/
if (tid != mp->mod_thread->thr_tid) {
(void) pthread_mutex_lock(&mp->mod_lock);
if (mp->mod_error == 0)
mp->mod_error = err;
mp->mod_flags |= FMD_MOD_FAIL;
(void) pthread_mutex_unlock(&mp->mod_lock);
(void) pthread_cancel(tid);
pthread_exit(NULL);
}
ASSERT(mp->mod_flags & FMD_MOD_BUSY);
longjmp(mp->mod_jmpbuf, err);
}
void
fmd_module_hold(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
TRACE((FMD_DBG_MOD, "hold %p (%s/%u)\n",
(void *)mp, mp->mod_name, mp->mod_refs));
mp->mod_refs++;
ASSERT(mp->mod_refs != 0);
(void) pthread_mutex_unlock(&mp->mod_lock);
}
void
fmd_module_rele(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
TRACE((FMD_DBG_MOD, "rele %p (%s/%u)\n",
(void *)mp, mp->mod_name, mp->mod_refs));
ASSERT(mp->mod_refs != 0);
if (--mp->mod_refs == 0)
fmd_module_destroy(mp);
else
(void) pthread_mutex_unlock(&mp->mod_lock);
}
/*
* Wrapper around libdiagcode's fm_dc_opendict() to load module dictionaries.
* If the dictionary open is successful, the new dictionary is added to the
* mod_dictv[] array and mod_codelen is updated with the new maximum length.
*/
int
fmd_module_dc_opendict(fmd_module_t *mp, const char *dict)
{
struct fm_dc_handle *dcp, **dcv;
char *dictdir, *dictnam, *p;
size_t len;
ASSERT(fmd_module_locked(mp));
dictnam = alloca(strlen(dict) + 1);
(void) strcpy(dictnam, fmd_strbasename(dict));
if ((p = strrchr(dictnam, '.')) != NULL &&
strcmp(p, ".dict") == 0)
*p = '\0'; /* eliminate any trailing .dict suffix */
/*
* If 'dict' is an absolute path, dictdir = $rootdir/`dirname dict`
* If 'dict' is not an absolute path, dictdir = $dictdir/`dirname dict`
*/
if (dict[0] == '/') {
len = strlen(fmd.d_rootdir) + strlen(dict) + 1;
dictdir = alloca(len);
(void) snprintf(dictdir, len, "%s%s", fmd.d_rootdir, dict);
(void) fmd_strdirname(dictdir);
} else {
(void) fmd_conf_getprop(fmd.d_conf, "dictdir", &p);
len = strlen(fmd.d_rootdir) + strlen(p) + strlen(dict) + 3;
dictdir = alloca(len);
(void) snprintf(dictdir, len,
"%s/%s/%s", fmd.d_rootdir, p, dict);
(void) fmd_strdirname(dictdir);
}
fmd_dprintf(FMD_DBG_MOD, "module %s opening %s -> %s/%s.dict\n",
mp->mod_name, dict, dictdir, dictnam);
if ((dcp = fm_dc_opendict(FM_DC_VERSION, dictdir, dictnam)) == NULL)
return (-1); /* errno is set for us */
dcv = fmd_alloc(sizeof (dcp) * (mp->mod_dictc + 1), FMD_SLEEP);
bcopy(mp->mod_dictv, dcv, sizeof (dcp) * mp->mod_dictc);
fmd_free(mp->mod_dictv, sizeof (dcp) * mp->mod_dictc);
mp->mod_dictv = dcv;
mp->mod_dictv[mp->mod_dictc++] = dcp;
len = fm_dc_codelen(dcp);
mp->mod_codelen = MAX(mp->mod_codelen, len);
return (0);
}
/*
* Wrapper around libdiagcode's fm_dc_key2code() that examines all the module's
* dictionaries. We adhere to the libdiagcode return values and semantics.
*/
int
fmd_module_dc_key2code(fmd_module_t *mp,
char *const keys[], char *code, size_t codelen)
{
int i, err;
for (i = 0; i < mp->mod_dictc; i++) {
if ((err = fm_dc_key2code(mp->mod_dictv[i], (const char **)keys,
code, codelen)) == 0 || errno != ENOMSG)
return (err);
}
return (fmd_set_errno(ENOMSG));
}
fmd_modhash_t *
fmd_modhash_create(void)
{
fmd_modhash_t *mhp = fmd_alloc(sizeof (fmd_modhash_t), FMD_SLEEP);
(void) pthread_rwlock_init(&mhp->mh_lock, NULL);
mhp->mh_hashlen = fmd.d_str_buckets;
mhp->mh_hash = fmd_zalloc(sizeof (void *) * mhp->mh_hashlen, FMD_SLEEP);
mhp->mh_nelems = 0;
return (mhp);
}
void
fmd_modhash_destroy(fmd_modhash_t *mhp)
{
fmd_module_t *mp, *nmp;
uint_t i;
for (i = 0; i < mhp->mh_hashlen; i++) {
for (mp = mhp->mh_hash[i]; mp != NULL; mp = nmp) {
nmp = mp->mod_next;
mp->mod_next = NULL;
fmd_module_rele(mp);
}
}
fmd_free(mhp->mh_hash, sizeof (void *) * mhp->mh_hashlen);
(void) pthread_rwlock_destroy(&mhp->mh_lock);
fmd_free(mhp, sizeof (fmd_modhash_t));
}
static void
fmd_modhash_loaddir(fmd_modhash_t *mhp, const char *dir,
const fmd_modops_t *ops, const char *suffix)
{
char path[PATH_MAX];
struct dirent *dp;
const char *p;
DIR *dirp;
if ((dirp = opendir(dir)) == NULL)
return; /* failed to open directory; just skip it */
while ((dp = readdir(dirp)) != NULL) {
if (dp->d_name[0] == '.')
continue; /* skip "." and ".." */
p = strrchr(dp->d_name, '.');
if (p != NULL && strcmp(p, ".conf") == 0)
continue; /* skip .conf files */
if (suffix != NULL && (p == NULL || strcmp(p, suffix) != 0))
continue; /* skip files with the wrong suffix */
(void) snprintf(path, sizeof (path), "%s/%s", dir, dp->d_name);
(void) fmd_modhash_load(mhp, path, ops);
}
(void) closedir(dirp);
}
void
fmd_modhash_loadall(fmd_modhash_t *mhp, const fmd_conf_path_t *pap,
const fmd_modops_t *ops, const char *suffix)
{
int i;
for (i = 0; i < pap->cpa_argc; i++)
fmd_modhash_loaddir(mhp, pap->cpa_argv[i], ops, suffix);
}
void
fmd_modhash_apply(fmd_modhash_t *mhp, void (*func)(fmd_module_t *))
{
fmd_module_t *mp, *np;
uint_t i;
(void) pthread_rwlock_rdlock(&mhp->mh_lock);
for (i = 0; i < mhp->mh_hashlen; i++) {
for (mp = mhp->mh_hash[i]; mp != NULL; mp = np) {
np = mp->mod_next;
func(mp);
}
}
(void) pthread_rwlock_unlock(&mhp->mh_lock);
}
void
fmd_modhash_tryapply(fmd_modhash_t *mhp, void (*func)(fmd_module_t *))
{
fmd_module_t *mp, *np;
uint_t i;
if (mhp == NULL || pthread_rwlock_tryrdlock(&mhp->mh_lock) != 0)
return; /* not initialized or couldn't grab lock */
for (i = 0; i < mhp->mh_hashlen; i++) {
for (mp = mhp->mh_hash[i]; mp != NULL; mp = np) {
np = mp->mod_next;
func(mp);
}
}
(void) pthread_rwlock_unlock(&mhp->mh_lock);
}
void
fmd_modhash_dispatch(fmd_modhash_t *mhp, fmd_event_t *ep)
{
fmd_module_t *mp;
uint_t i;
fmd_event_hold(ep);
(void) pthread_rwlock_rdlock(&mhp->mh_lock);
for (i = 0; i < mhp->mh_hashlen; i++) {
for (mp = mhp->mh_hash[i]; mp != NULL; mp = mp->mod_next) {
/*
* If FMD_MOD_INIT is set but MOD_FINI, MOD_QUIT, and
* mod_error are all zero, then the module is active:
* enqueue the event in the corresponding event queue.
*/
(void) pthread_mutex_lock(&mp->mod_lock);
if ((mp->mod_flags & (FMD_MOD_INIT | FMD_MOD_FINI |
FMD_MOD_QUIT)) == FMD_MOD_INIT && !mp->mod_error) {
/*
* If the event we're dispatching is of type
* FMD_EVT_TOPO and there are already redundant
* FMD_EVT_TOPO events in this module's queue,
* then drop those before adding the new one.
*/
if (FMD_EVENT_TYPE(ep) == FMD_EVT_TOPO)
fmd_eventq_drop_topo(mp->mod_queue);
fmd_eventq_insert_at_time(mp->mod_queue, ep);
}
(void) pthread_mutex_unlock(&mp->mod_lock);
}
}
(void) pthread_rwlock_unlock(&mhp->mh_lock);
fmd_event_rele(ep);
}
fmd_module_t *
fmd_modhash_lookup(fmd_modhash_t *mhp, const char *name)
{
fmd_module_t *mp;
uint_t h;
(void) pthread_rwlock_rdlock(&mhp->mh_lock);
h = fmd_strhash(name) % mhp->mh_hashlen;
for (mp = mhp->mh_hash[h]; mp != NULL; mp = mp->mod_next) {
if (strcmp(name, mp->mod_name) == 0)
break;
}
if (mp != NULL)
fmd_module_hold(mp);
else
(void) fmd_set_errno(EFMD_MOD_NOMOD);
(void) pthread_rwlock_unlock(&mhp->mh_lock);
return (mp);
}
fmd_module_t *
fmd_modhash_load(fmd_modhash_t *mhp, const char *path, const fmd_modops_t *ops)
{
char name[PATH_MAX], *p;
fmd_module_t *mp;
int tries = 0;
uint_t h;
(void) strlcpy(name, fmd_strbasename(path), sizeof (name));
if ((p = strrchr(name, '.')) != NULL && strcmp(p, ".so") == 0)
*p = '\0'; /* strip trailing .so from any module name */
(void) pthread_rwlock_wrlock(&mhp->mh_lock);
h = fmd_strhash(name) % mhp->mh_hashlen;
/*
* First check to see if a module is already present in the hash table
* for this name. If so, the module is already loaded: skip it.
*/
for (mp = mhp->mh_hash[h]; mp != NULL; mp = mp->mod_next) {
if (strcmp(name, mp->mod_name) == 0)
break;
}
if (mp != NULL) {
(void) pthread_rwlock_unlock(&mhp->mh_lock);
(void) fmd_set_errno(EFMD_MOD_LOADED);
return (NULL);
}
/*
* fmd_module_create() will return a held (as if by fmd_module_hold())
* module. We leave this hold in place to correspond to the hash-in.
*/
while ((mp = fmd_module_create(path, ops)) == NULL) {
if (tries++ != 0 || errno != EFMD_CKPT_INVAL) {
(void) pthread_rwlock_unlock(&mhp->mh_lock);
return (NULL); /* errno is set for us */
}
}
mp->mod_hash = mhp;
mp->mod_next = mhp->mh_hash[h];
mhp->mh_hash[h] = mp;
mhp->mh_nelems++;
(void) pthread_rwlock_unlock(&mhp->mh_lock);
return (mp);
}
int
fmd_modhash_unload(fmd_modhash_t *mhp, const char *name)
{
fmd_module_t *mp, **pp;
uint_t h;
(void) pthread_rwlock_wrlock(&mhp->mh_lock);
h = fmd_strhash(name) % mhp->mh_hashlen;
pp = &mhp->mh_hash[h];
for (mp = *pp; mp != NULL; mp = mp->mod_next) {
if (strcmp(name, mp->mod_name) == 0)
break;
else
pp = &mp->mod_next;
}
if (mp == NULL) {
(void) pthread_rwlock_unlock(&mhp->mh_lock);
return (fmd_set_errno(EFMD_MOD_NOMOD));
}
*pp = mp->mod_next;
mp->mod_next = NULL;
ASSERT(mhp->mh_nelems != 0);
mhp->mh_nelems--;
(void) pthread_rwlock_unlock(&mhp->mh_lock);
fmd_module_unload(mp);
fmd_module_rele(mp);
return (0);
}
void
fmd_modstat_publish(fmd_module_t *mp)
{
(void) pthread_mutex_lock(&mp->mod_lock);
ASSERT(mp->mod_flags & FMD_MOD_STSUB);
mp->mod_flags |= FMD_MOD_STPUB;
(void) pthread_cond_broadcast(&mp->mod_cv);
while (mp->mod_flags & FMD_MOD_STPUB)
(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
(void) pthread_mutex_unlock(&mp->mod_lock);
}
int
fmd_modstat_snapshot(fmd_module_t *mp, fmd_ustat_snap_t *uss)
{
fmd_event_t *e;
int err;
/*
* Grab the module lock and wait for the STSUB bit to be clear. Then
* set it to indicate we are a subscriber and everyone else must wait.
*/
(void) pthread_mutex_lock(&mp->mod_lock);
while (mp->mod_error == 0 && (mp->mod_flags & FMD_MOD_STSUB))
(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
if (mp->mod_error != 0) {
(void) pthread_mutex_unlock(&mp->mod_lock);
return (fmd_set_errno(EFMD_HDL_ABORT));
}
mp->mod_flags |= FMD_MOD_STSUB;
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
/*
* Create a stats pseudo-event and dispatch it to the module, forcing
* it to next execute its custom snapshot routine (or the empty one).
*/
e = fmd_event_create(FMD_EVT_STATS, FMD_HRT_NOW, NULL, NULL);
fmd_eventq_insert_at_head(mp->mod_queue, e);
/*
* Grab the module lock and then wait on mod_cv for STPUB to be set,
* indicating the snapshot routine is completed and the module is idle.
*/
(void) pthread_mutex_lock(&mp->mod_lock);
while (mp->mod_error == 0 && !(mp->mod_flags & FMD_MOD_STPUB))
(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
if (mp->mod_error != 0) {
(void) pthread_mutex_unlock(&mp->mod_lock);
return (fmd_set_errno(EFMD_HDL_ABORT));
}
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
/*
* Update ms_snaptime and take the actual snapshot of the various
* statistics while the module is quiescent and waiting for us.
*/
(void) pthread_mutex_lock(&mp->mod_stats_lock);
if (mp->mod_stats != NULL) {
mp->mod_stats->ms_snaptime.fmds_value.ui64 = gethrtime();
err = fmd_ustat_snapshot(mp->mod_ustat, uss);
} else
err = fmd_set_errno(EFMD_HDL_ABORT);
(void) pthread_mutex_unlock(&mp->mod_stats_lock);
/*
* With the snapshot complete, grab the module lock and clear both
* STSUB and STPUB, permitting everyone to wake up and continue.
*/
(void) pthread_mutex_lock(&mp->mod_lock);
ASSERT(mp->mod_flags & FMD_MOD_STSUB);
ASSERT(mp->mod_flags & FMD_MOD_STPUB);
mp->mod_flags &= ~(FMD_MOD_STSUB | FMD_MOD_STPUB);
(void) pthread_cond_broadcast(&mp->mod_cv);
(void) pthread_mutex_unlock(&mp->mod_lock);
return (err);
}
struct topo_hdl *
fmd_module_topo_hold(fmd_module_t *mp)
{
fmd_modtopo_t *mtp;
ASSERT(fmd_module_locked(mp));
mtp = fmd_zalloc(sizeof (fmd_modtopo_t), FMD_SLEEP);
mtp->mt_topo = mp->mod_topo_current;
fmd_topo_addref(mtp->mt_topo);
fmd_list_prepend(&mp->mod_topolist, mtp);
return (mtp->mt_topo->ft_hdl);
}
int
fmd_module_topo_rele(fmd_module_t *mp, struct topo_hdl *hdl)
{
fmd_modtopo_t *mtp;
ASSERT(fmd_module_locked(mp));
for (mtp = fmd_list_next(&mp->mod_topolist); mtp != NULL;
mtp = fmd_list_next(mtp)) {
if (mtp->mt_topo->ft_hdl == hdl)
break;
}
if (mtp == NULL)
return (-1);
fmd_list_delete(&mp->mod_topolist, mtp);
fmd_topo_rele(mtp->mt_topo);
fmd_free(mtp, sizeof (fmd_modtopo_t));
return (0);
}