avs/rdc/sndrsyncd.c

	sndrsyncd.c revision fcf3ce441efd61da9bb2884968af01cb7c1452cc
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/wait.h>
#include <stdio.h>
#include <errno.h>
#include <limits.h>
#include <fcntl.h>
#include <strings.h>
#include <stdlib.h>
#include <unistd.h>
#include <pthread.h>
#include <thread.h>

#include <locale.h>
#include <langinfo.h>
#include <libintl.h>
#include <stdarg.h>

#include <sys/nsctl/rdc_io.h>
#include <sys/nsctl/rdc_ioctl.h>
#include <sys/nsctl/rdc_prot.h>

#include <sys/nsctl/cfg.h>

#include <sys/unistat/spcs_s.h>
#include <sys/unistat/spcs_s_u.h>
#include <sys/unistat/spcs_errors.h>

#include <sys/nsctl/librdc.h>

#include "rdcadm.h"


#define RDCADM "/usr/sbin/sndradm"
#define IIADM "/usr/sbin/iiadm"

#define UPDATE "update"
#define NOUPDATE "noupdate"

#define RESYNC_SLEEP    (3 * 60)    /* Three minutes */
#define MAIN_SLEEP  (5 * 60)    /* Five minutes */
#define CFG_WAIT_SLEEP  (5)     /* 5 sec */

#define MAXHOSTS 1024
mutex_t cfglock = DEFAULTMUTEX;
#define LOCKCFG() (void) mutex_lock(&cfglock);
#define UNLOCKCFG() (void) mutex_unlock(&cfglock);

typedef struct host_list_s {
    char *hosts[MAXHOSTS];
    int numhosts;
    int configured[MAXHOSTS];
    mutex_t hosts_mutex;
} host_list_t;

host_list_t *host_list;

extern char *basename(char *);
int rdc_maxsets;
char *program;

static int clustered = 0;

int isnewhost(char *host);
void *wait_sync_event();
void *wait_link_down(void *host);
void rdc_sync(char *tohost);
void remove_from_hostlist(char *host);
void sync_start(char *master);
void sync_complete(char *master);
void cleanup_hostlist();
void group_start(char *group);
void group_complete(char *group);


void
init_host_list(void)
{
    host_list = calloc(1, sizeof (host_list_t));
    if (host_list == NULL) {
        spcs_log("sndr", NULL,
            gettext("host list not initialized, cannot run"));
        rdc_err(NULL, gettext("host list not initialized, cannot run"));
    }
    (void) mutex_init(&host_list->hosts_mutex, USYNC_THREAD, NULL);
}

/* ARGSUSED */
#ifdef lint
void
sndrsyncd_lintmain(argc, argv)
#else
int
main(argc, argv)
#endif
int argc;
char **argv;
{
    rdc_status_t *rdc_info;
    int size;
    int i;
    pid_t pid;
    spcs_s_info_t ustatus;
    int rc, trc;
    int first = 0;
    char *required;

    (void) setlocale(LC_ALL, "");
    (void) textdomain("rdc");

    ustatus = spcs_s_ucreate();

    program = basename(argv[0]);

    init_host_list();

    rc = rdc_check_release(&required);
    if (rc < 0) {
        rdc_err(NULL,
            gettext("unable to determine the current "
            "Solaris release: %s\n"), strerror(errno));
        /* NOTREACHED */
    } else if (rc == FALSE) {
        rdc_err(NULL,
            gettext("incorrect Solaris release (requires %s)\n"),
            required);
        /* NOTREACHED */
    }

    clustered = cfg_iscluster();
    if (clustered < 0) {
        rdc_err(NULL, gettext("unable to ascertain environment"));
    }

    rdc_maxsets = rdc_get_maxsets();
    if (rdc_maxsets == -1) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to get maxsets value from kernel"),
            program);
        rdc_err(NULL,
            gettext("unable to get maxsets value from kernel"));
    }
    size = sizeof (rdc_status_t) + (sizeof (rdc_set_t) * (rdc_maxsets - 1));
    rdc_info = malloc(size);
    if (rdc_info == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_err(NULL,
            gettext("unable to allocate %ld bytes"), size);
    }
    bzero(rdc_info, size);

    rdc_info->nset = rdc_maxsets;

    /*
     * Fork off a child that becomes the daemon.
     */
    if ((pid = fork()) > 0)
        exit(0);
    else if (pid < 0) {
        spcs_log("sndr", NULL,
            gettext("%s: cannot fork: %s"),
            program, strerror(errno));
        rdc_err(NULL, gettext("cannot fork: %s\n"),
            strerror(errno));
    }

    /*
     * In child - become daemon.
     */

    for (i = 0; i < 3; i++)
        (void) close(i);

    (void) open("/dev/console", O_WRONLY|O_APPEND);
    (void) dup(0);
    (void) dup(0);
    (void) close(0);

    (void) setpgrp();

    (void) setlocale(LC_ALL, "");
    (void) textdomain("rdc");

    /* launch a thread to wait for sync start and sync stop events */

    if ((trc = thr_create(NULL, 0, wait_sync_event, NULL,
        THR_BOUND|THR_DETACHED, NULL)) != 0) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to create thread wait_sync_event"),
            program);
        rdc_warn(NULL,
            gettext("%s unable to create thread wait_sync_event"),
            program);
    } else {
#ifdef DEBUG
        spcs_log("sndr", NULL,
            gettext("%s: thread wait_sync_event started"), program);
#endif
        ;
    }

    for (;;) {
        if (!first) {
            first++;
            (void) sleep(15);
        } else
            (void) sleep(MAIN_SLEEP);

        bzero(rdc_info, size);
        rdc_info->nset = rdc_maxsets;
        if (RDC_IOCTL(RDC_STATUS, rdc_info, 0, 0, 0, 0, ustatus)
            != SPCS_S_OK) {
            spcs_log("sndr", &ustatus,
                gettext("%s: status ioctl"),
                program);
            rdc_warn(&ustatus, gettext("status ioctl"));
            continue;
        }

        cleanup_hostlist(rdc_info); /* remove non-existent hosts */

        /*
         * Check all enabled sets to see if a new remote host has
         * appeared.
         */
        for (i = 0; i < rdc_maxsets; i++) {
            if (!(rdc_info->rdc_set[i].flags & RDC_ENABLED))
                continue;
            /* spawn a new thread for each new host found */
            if (isnewhost(rdc_info->rdc_set[i].secondary.intf)) {
                /*
                 * right now, we could be here before
                 * the database did the write for this set
                 * I could check the lock on the database
                 * but I am just going to give up some time here
                 * instead. Why do the allocations etc, etc
                 * if the set is enabled in the kernel and not
                 * in the config, we know that this set has the
                 * lock. Why bother adding more contention to
                 * the lock.
                 * this is a daemon, afterall. its got time
                 */
                (void) sleep(CFG_WAIT_SLEEP);

                spcs_log("sndr", NULL,
                    gettext("%s: new host found (%s) starting "
                    "its autosync thread"), program,
                    rdc_info->rdc_set[i].secondary.intf);

                trc = thr_create(NULL, 0, wait_link_down,
                    (void *) rdc_info->rdc_set[i].\
secondary.intf, THR_BOUND|THR_DETACHED, NULL);

                if (trc != 0) {
                    spcs_log("sndr", NULL,
                        gettext(
                        "%s create new autosync "
                        "thread failed"), program);
                    rdc_warn(NULL, gettext(
                        "%s create new autosync "
                        "thread failed"), program);
                }
            }
        }
    }
    /* NOTREACHED */
}


/*
 * The kernel wakes up this function every time it detects the link to the
 * specified host has dropped.
 */
void *
wait_link_down(void *thehost)
{
    char *host = (char *)thehost;
    char tmphost[MAX_RDC_HOST_SIZE] = { '\0' };
    spcs_s_info_t ustatus;

    if (host)
        strncpy(tmphost, host, MAX_RDC_HOST_SIZE);

    ustatus = spcs_s_ucreate();

    /* Never give up */
    for (;;) {
#ifdef DEBUG
        spcs_log("sndr", NULL,
            gettext("%s: awaiting link down ioctl for %s"),
            program, host[0] == '\0' ? tmphost : host);
#endif
        if (RDC_IOCTL(RDC_LINK_DOWN, host, 0, 0, 0, 0, ustatus)
            != SPCS_S_OK) {
            spcs_log("sndr", &ustatus,
                gettext("%s: link down ioctl"),
                program);
            rdc_warn(&ustatus, gettext("link down ioctl"));
            continue;
        }
#ifdef DEBUG

        spcs_log("sndr", NULL,
            gettext("%s: received link down ioctl for %s"),
            program, host[0] == '\0' ? tmphost : host);
#endif
        rdc_sync(host[0] == '\0' ? tmphost : host);
    }
    /* LINTED */
}


/*
 * Called when the link to the specified host has dropped.
 * For all Remote Mirror sets using the link that have autosync on,
 * issue rdcadm -u commands until they complete successfully.
 */
void
rdc_sync(char *tohost)
{
    rdc_set_t *rdc_set = NULL;
    int *sync_done = NULL;
    int sets = 0;
    int syncs_done = 0;
    char cmd[256];
    rdc_config_t parms = { 0 };
    spcs_s_info_t ustatus;
    int i;
    int setnumber;
    int numfound = 0;
    char buf[CFG_MAX_BUF];
    char key[CFG_MAX_KEY];
    CFGFILE *cfg = NULL;
    int size;
    int first = 0;
    int death = 0;
    int cfglocked = 0;

    ustatus = spcs_s_ucreate();

    size = sizeof (rdc_set_t) * rdc_maxsets;
    rdc_set = malloc(size);
    if (rdc_set == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_warn(NULL,
            gettext("unable to allocate %ld bytes"), size);
        goto done;
    }
    bzero(rdc_set, size);
    size = sizeof (int) * rdc_maxsets;
    sync_done = malloc(size);
    if (sync_done == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_warn(NULL,
            gettext("unable to allocate %ld bytes"), size);
        goto done;
    }
    bzero(sync_done, size);

    /*
     * Get all sndr entries with shost matching tohost, and save the
     * details in an array.
     */
    for (i = 0; i < rdc_maxsets; i++) {
        setnumber = i + 1;
        bzero(buf, sizeof (buf));
        bzero(key, sizeof (key));

        (void) snprintf(key, sizeof (key), "sndr.set%d.shost",
            setnumber);

        if (!cfglocked) {
            LOCKCFG();
            if ((cfg = cfg_open(NULL)) == NULL) {
                spcs_log("sndr", NULL,
                    gettext("%s: error opening config"),
                    program);

                rdc_warn(NULL,
                    gettext("error opening config"));
                UNLOCKCFG();
                goto done;
            }

            if (!cfg_lock(cfg, CFG_RDLOCK)) {
                spcs_log("sndr", NULL,
                    gettext("%s: error locking config"),
                    program);
                rdc_warn(NULL, gettext("error locking config"));
                goto done;
            }
        }

        cfglocked = 1;

        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
            if (numfound == 0) /* no matching hosts */
                death = 1; /* thread will exit */
            break;
        }
        if (strcmp(buf, tohost) != 0)
            continue;

        numfound++;
        strncpy(rdc_set[sets].secondary.intf, buf, MAX_RDC_HOST_SIZE);

        /* Got a matching entry */

        (void) snprintf(key, sizeof (key), "sndr.set%d.phost",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        strncpy(rdc_set[sets].primary.intf, buf, MAX_RDC_HOST_SIZE);

        (void) snprintf(key, sizeof (key), "sndr.set%d.primary",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        strncpy(rdc_set[sets].primary.file, buf, NSC_MAXPATH);

        (void) snprintf(key, sizeof (key), "sndr.set%d.secondary",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        strncpy(rdc_set[sets].secondary.file, buf, NSC_MAXPATH);

        parms.command = RDC_CMD_STATUS;
        bcopy((void *)(&rdc_set[sets]), (void *)(&parms.rdc_set[0]),
            sizeof (rdc_set_t));

        /*
         * release cfg before diving into the kernel
         * this prevents a possible deadlock when doing
         * a reverse sync whick will wake up the sync_event
         * thread which will try and iiadm -c and hang
         * because we still have the cfg_lock. the timed
         * wait cv in the kernel will fail the sync and things
         * will undeadlock.
         */

        cfg_close(cfg);
        cfg = NULL;
        cfglocked = 0;
        UNLOCKCFG();

        if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
            continue;
        }
        if ((parms.rdc_set[0].autosync == 0) ||
            (!(parms.rdc_set[0].flags & RDC_LOGGING))) {
            continue;
        }

        /* Found a suitable set with autosync on, in logging mode */
        sets++;
    }

    if (cfg) {
        cfg_close(cfg);
        cfg = NULL;
        UNLOCKCFG();
    }

    if (sets == 0) {
#ifdef DEBUG
        spcs_log("sndr", NULL,
            gettext("%s: no sets requiring autosync found for %s"),
            program, tohost);
#endif
        if (death) {
            spcs_log("sndr", NULL,
                gettext("%s: autosync thread stopping for %s "
                "(host deconfigured)"), program, tohost);
        }
        goto done;
    }

    /* Keep issuing rdcadm -u commands until they have all completed */
    for (;;) {
        if (!first)
            first++;
        else
            (void) sleep(RESYNC_SLEEP);

        /* Issue rdcadm -u commands for all remaining sets */
        for (i = 0; i < sets; i++) {
            if (sync_done[i])
                continue;

            /*
             * Need to check if autosync was turned off for a set
             * while we were sleeping. We could have the case where
             * an update sync failed and autosync was disabled
             * while we were sleeping and didn't detect the disable.
             * See BugID 4814213.
             */
            parms.command = RDC_CMD_STATUS;
            bcopy((void *)(&rdc_set[i]),
                (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
            if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
                ustatus) < 0) {
                spcs_log("sndr", &ustatus, gettext("%s: "
                    "status not available for %s:%s, stopping "
                    "this autosync attempt"), program, tohost,
                    rdc_set[i].secondary.file);
                sync_done[i] = 1;
                syncs_done++;
                continue;
            }
            if (!(parms.rdc_set[0].autosync)) {
#ifdef DEBUG
    spcs_log("sndr", NULL, gettext("%s: autosync disabled during sleep, "
        "stopping attempt for set %s:%s"), program, tohost,
        rdc_set[i].secondary.file);
#endif
                sync_done[i] = 1;
                syncs_done++;
                continue;
            }

            (void) sprintf(cmd, "%s -un %s:%s", RDCADM, tohost,
                rdc_set[i].secondary.file);
            spcs_log("sndr", NULL,
                gettext("%s: issuing update sync for %s:%s"),
                program, tohost, rdc_set[i].secondary.file);
            (void) system(cmd);
        }

        /* Issue rdcadm -w commands to wait for updates to finish */
        for (i = 0; i < sets; i++) {
            if (sync_done[i])
                continue;

            (void) sprintf(cmd, "%s -wn %s:%s", RDCADM, tohost,
                rdc_set[i].secondary.file);
            spcs_log("sndr", NULL,
                gettext("%s: issuing wait for %s:%s"),
                program, tohost, rdc_set[i].secondary.file);

            (void) system(cmd);

            parms.command = RDC_CMD_STATUS;
            bcopy((void *)(&rdc_set[i]),
                (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));

            if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
                ustatus) < 0) {
                spcs_log("sndr", &ustatus,
                    gettext("%s: status not available for "
                    "%s:%s, stopping this autosync attempt"),
                    program, tohost, rdc_set[i].secondary.file);
                sync_done[i] = 1;
                syncs_done++;
                continue;
            }
            /* Check if completed OK, failed or autosync off */
            if (!(parms.rdc_set[0].autosync) ||
                !(parms.rdc_set[0].flags & RDC_LOGGING) &&
                !(parms.rdc_set[0].flags & RDC_SYNCING)) {
                sync_done[i] = 1;
                syncs_done++;
            }
        }

        if (syncs_done == sets)
            break;      /* All completed OK */
    }

done:
    if (cfg) {
        cfg_close(cfg);
        UNLOCKCFG();
    }
    spcs_s_ufree(&ustatus);
    if (sync_done)
        free(sync_done);
    if (rdc_set)
        free(rdc_set);
    if (death) { /* bye bye */
        /*
         * if perhaps we lost some race, lets remove this entry from
         * the list. Then, if something did go wrong, and we did kill
         * a valid thread, it will be detected on the next go around
         * of the thread who is looking for new hosts to spawn threads
         */

        remove_from_hostlist(tohost);
        thr_exit(0);
    }

    (void) sleep(RESYNC_SLEEP);
}

/*
 * Wait for notification by the kernel of a sync start or a sync completed OK
 */
void *
wait_sync_event()
{
    spcs_s_info_t ustatus;
    char master[NSC_MAXPATH];
    char group[NSC_MAXPATH];
    int state;

    ustatus = spcs_s_ucreate();

    master[0] = '\0';
    group[0] = '\0';

    /* Never give up */
    for (;;) {
        /* Kernel tells us which volume and group the event is for */
        state = RDC_IOCTL(RDC_SYNC_EVENT, master, group, 0, 0, 0,
            ustatus);
        if (state < SPCS_S_OK) {
            if (errno != EAGAIN) {
                spcs_log("sndr", &ustatus,
                    gettext("%s: update ioctl"),
                    program);
                rdc_warn(&ustatus, gettext("update ioctl"));
                continue;
            }
            master[0] = '\0';
            continue;
        }

        /*
         * If target is mounted at the start of a sync or reverse sync,
         * return a negative ack.
         */
        if ((state == RDC_SYNC_START || state == RDC_RSYNC_START) &&
            mounted(master)) {
            spcs_log("sndr", NULL,
                gettext("%s: %s has a file system mounted"),
                program, master);
            rdc_warn(NULL,
                gettext("%s has a file system mounted"),
                master);
            master[0] = '\0';   /* negative ack */
            continue;
        }

        switch (state) {
        case RDC_SYNC_START:
            if (group[0])
                group_start(group);
            else
                sync_start(master);
            break;

        case RDC_SYNC_DONE:
            if (group[0])
                group_complete(group);
            else
                sync_complete(master);
            break;

        default:
            break;
        }
    }
    /* LINTED */
}


/*
 * A sync has completed OK to a volume not belonging to a group.
 * Set the state of the ndr_ii config entry to "update".
 */
void
sync_complete(char *master)
{
    CFGFILE *cfg = NULL;
    char buf[CFG_MAX_BUF];
    char key[CFG_MAX_KEY];
    int i;
    int setnumber;
    int sev;

    LOCKCFG();
    if ((cfg = cfg_open(NULL)) == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: error opening config"),
            program);
        rdc_warn(NULL, gettext("error opening config"));
        UNLOCKCFG();
        return;
    }
    if (!cfg_lock(cfg, CFG_WRLOCK)) {
        spcs_log("sndr", NULL,
            gettext("%s: error locking config"),
            program);
        rdc_warn(NULL, gettext("error locking config"));
        cfg_close(cfg);
        UNLOCKCFG();
        return;
    }

    /* get ndr_ii entries until a match is found */
    for (i = 0; ; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        if (strcmp(buf, master) != 0)
            continue;

        /* Found the matching entry */

        /*
         * Set state to "update" so that starting another sync will
         * cause a new Point-in-Time Copy snapshot to be taken.
         */
        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
            setnumber);
        if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
            (cfg_commit(cfg) < 0)) {
            spcs_log("sndr", NULL,
                gettext("%s: unable to update \"%s\" "
                "in configuration storage: %s"),
                program, buf, cfg_error(&sev));
            rdc_warn(NULL,
                gettext("unable to update \"%s\" "
                "in configuration storage: %s"),
                buf, cfg_error(&sev));
        }
        break;
    }

    cfg_close(cfg);
    UNLOCKCFG();
}


/*
 * Starting a sync to the specified master volume.
 * Check the ndr_ii config entries to see if a Point-in-Time Copy
 * snapshot should be taken.
 */
void
sync_start(char *master)
{
    char cmd[256];
    char buf[CFG_MAX_BUF];
    char key[CFG_MAX_KEY];
    CFGFILE *cfg = NULL;
    int i;
    int setnumber;
    int found;
    int sev;
    char shadow[NSC_MAXPATH];
    char bitmap[NSC_MAXPATH];
    char *ctag = NULL;

    LOCKCFG();
    if ((cfg = cfg_open(NULL)) == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: error opening config"),
            program);
        rdc_warn(NULL,
            gettext("error opening config"));
        UNLOCKCFG();
        return;
    }
    if (!cfg_lock(cfg, CFG_RDLOCK)) {
        spcs_log("sndr", NULL,
            gettext("%s: error locking config"),
            program);
        rdc_warn(NULL, gettext("error locking config"));
        cfg_close(cfg);
        UNLOCKCFG();
        return;
    }

    found = 0;
    /* get ndr_ii entries until a match is found */
    for (i = 0; ; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        if (strcmp(buf, master) != 0)
            continue;

        /* Got a matching entry */

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        strncpy(shadow, buf, NSC_MAXPATH);

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        strncpy(bitmap, buf, NSC_MAXPATH);

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;

        /*
         * If an PIT snapshot has already been taken, and syncing did
         * not complete, the state will be "noupdate", to indicate we
         * should not take another one at this point.
         */
        if (strcmp(buf, NOUPDATE) != 0)
            found = 1;

        break;
    }

    if (!found) {
        cfg_close(cfg);
        UNLOCKCFG();
        return;
    }

    found = 0;
    /* get ii entries until a match is found */
    for (i = 0; ; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key), "ii.set%d.shadow",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        if (strcmp(buf, shadow) != 0)
            continue;

        /* Matching shadow found, so ii already enabled */
        found = 1;
        break;
    }

    if (found) {
        /* Already PIT enabled, so just take a snapshot */

        /* Get cluster tag of matching entry */
        (void) snprintf(key, sizeof (key), "ii.set%d.cnode", setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
            if ((strlen(buf) == 0) || (buf[0] == '-'))
                    ctag = "-C local";
                else
                    ctag = "";
        (void) sprintf(cmd, "%s %s -u s %s", IIADM, ctag, shadow);
    } else {
        /*
         * If clustered, need to enable PIT Copy sets in the same
         * cluster as the Remote Mirror set
         */

        if (clustered) {
            /* Find a RM set with master as the local volume */

            for (i = 0; i < rdc_maxsets; i++) {
                setnumber = i + 1;
                (void) snprintf(key, sizeof (key),
                    "sndr.set%d.phost", setnumber);
                if (cfg_get_cstring(cfg, key, buf,
                    CFG_MAX_BUF) < 0)
                    break;

                if (self_check(buf))
                    (void) snprintf(key, sizeof (key),
                        "sndr.set%d.primary", setnumber);
                else
                    (void) snprintf(key, sizeof (key),
                        "sndr.set%d.secondary", setnumber);
                if (cfg_get_cstring(cfg, key, buf,
                    CFG_MAX_BUF) < 0)
                    break;

                if (strcmp(buf, master) != 0)
                    continue;

                /* Get cluster tag of matching entry */

                (void) snprintf(key, sizeof (key),
                    "sndr.set%d.cnode", setnumber);
                if (cfg_get_cstring(cfg, key, buf,
                    CFG_MAX_BUF) < 0)
                    break;
                if ((strlen(buf) == 0) || (buf[0] == '-'))
                    ctag = strdup("local");
                else
                    ctag = strdup(buf);
                break;
            }
        }

        /* Not already enabled, so enable a dependent */
        if (ctag) {
            (void) sprintf(cmd, "%s -C %s -e dep %s %s %s", IIADM,
                ctag, master, shadow, bitmap);
            free(ctag);
        } else
            (void) sprintf(cmd, "%s -e dep %s %s %s", IIADM, master,
                shadow, bitmap);
    }

    cfg_close(cfg);

    if (system(cmd) != 0) {
        spcs_log("sndr", NULL,
            gettext("Point-in-Time Copy snapshot failed for %s %s %s."
            " Please check validity of ndr_ii entry"),
            master, shadow, bitmap);
        cfg_close(cfg);
        UNLOCKCFG();
        return;
    }

    /*
     * PIT Copy enable or update was fine, so update the ndr_ii entry
     * to "noupdate", to prevent invalid point in time copies.
     */

    if ((cfg = cfg_open(NULL)) == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: error opening config"),
            program);
        rdc_warn(NULL,
            gettext("error opening config"));
        UNLOCKCFG();
        return;
    }
    if (!cfg_lock(cfg, CFG_WRLOCK)) {
        spcs_log("sndr", NULL,
            gettext("%s: error locking config"),
            program);
        rdc_warn(NULL, gettext("error locking config"));
        cfg_close(cfg);
        UNLOCKCFG();
        return;
    }

    /* get ndr_ii entries until a match is found */
    for (i = 0; ; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        if (strcmp(buf, shadow) != 0)
            continue;

        /* Found the matching entry */

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
            setnumber);
        if ((cfg_put_cstring(cfg, key, NOUPDATE,
            strlen(NOUPDATE)) < 0) || (cfg_commit(cfg) < 0)) {
            spcs_log("sndr", NULL,
                gettext("%s: unable to update \"%s\" "
                "in configuration storage: %s"),
                program, buf, cfg_error(&sev));
            rdc_warn(NULL,
                gettext("unable to update \"%s\" "
                "in configuration storage: %s"),
                buf, cfg_error(&sev));
        }
        break;
    }
    cfg_close(cfg);
    UNLOCKCFG();
}

void
cleanup_hostlist(rdc_status_t *rdc_info)
{
    int i, j, k;
    char *host, *exhost;


    (void) mutex_lock(&host_list->hosts_mutex);
    for (i = 0; i < host_list->numhosts; i++) {
        int found = 0;
        for (j = 0; (j < rdc_maxsets) && !found; j++) {
            if (!rdc_info->rdc_set[j].flags & RDC_ENABLED)
                continue;
            if ((!host_list->configured[i]) ||
                (host_list->hosts[i] == '\0')) {
                (void) mutex_unlock(&host_list->hosts_mutex);
                return;
            }

            host = rdc_info->rdc_set[j].secondary.intf;
            if (strcmp(host_list->hosts[i], host) == 0)
                found++;
        }
        if (j == rdc_maxsets) {
            /*
             * this set is not in the kernel, so remove from list
             */
            exhost = host_list->hosts[i];
            if (exhost) {
                free(exhost);
                exhost = NULL;
            }

            k = i;
            while (k < host_list->numhosts) {
                host_list->hosts[k] = k < host_list->numhosts - 1 ?
                host_list->hosts[k+1] : NULL;
                k++;
            }
            host_list->numhosts--;

            bcopy(&host_list->configured[i+1],
                &host_list->configured[i],
                (MAXHOSTS - i + 1) * sizeof (int));
            host_list->configured[MAXHOSTS - 1] = 0;
        }
    }
    (void) mutex_unlock(&host_list->hosts_mutex);
}

/*
 * explicity remove a host from the host list
 * also update the configured array
 * called in rdc_sync, just before exiting a thread.
 */
void
remove_from_hostlist(char *host)
{
    int i, k;
    char *exhost;

    /* why bother? */
    if ((!host) || (host[0] == '\0'))
        return;

    (void) mutex_lock(&host_list->hosts_mutex);
    for (i = 0; i < host_list->numhosts; i++) {
        if (strcmp(host, host_list->hosts[i]) == 0) { /* found it */
            exhost = host_list->hosts[i];
            if (exhost) {
                free(exhost);
                exhost = NULL;
            }
            k = i;
            while (k < host_list->numhosts) {
                host_list->hosts[k] = k < host_list->numhosts - 1 ?
                host_list->hosts[k+1] : NULL;
                k++;
            }
            host_list->numhosts--;
            bcopy(&host_list->configured[i+1],
                &host_list->configured[i],
                (MAXHOSTS - i + 1) * sizeof (int));
            host_list->configured[MAXHOSTS - 1] = 0;
        }

    }
    (void) mutex_unlock(&host_list->hosts_mutex);
}
/*
 * Check to see if this host isn't in our list, so needs a new rdcsyncd proc
 */
int
isnewhost(char *host)
{
    int i;
    int new;

    if (self_check(host)) {
        return (0);
    }

    (void) mutex_lock(&host_list->hosts_mutex);
    new = 1;
    for (i = 0; i < MAXHOSTS; i++) {
        if (host_list->configured[i] == 0) {
            host_list->configured[i] = 1;
            host_list->hosts[i] = strdup(host);
            host_list->numhosts++;
            break;
        }
        if (strcmp(host, host_list->hosts[i]) == 0) {
            new = 0;
            break;
        }
    }
    (void) mutex_unlock(&host_list->hosts_mutex);
    if (i == MAXHOSTS)
        new = 0;
    return (new);
}


/*
 * Look for a matching volume name in our remembered list.
 */
int
volume_match(char *buf, char **volume_list, int volumes)
{
    int i;
    char *vol;

    for (i = 0; i < volumes; i++) {
        vol = volume_list[i];
        if (strcmp(buf, vol) == 0) {
            return (1);
        }
    }
    return (0);
}


/*
 * A sync has completed to a group. We can only update the ndr_ii entries
 * if all the members of the group have completed their syncs OK.
 * It would be bad to allow some members of the group to have PIT Copy snapshots
 * taken and others not, as they need to be consistent.
 */
void
group_complete(char *group)
{
    char **volumes = NULL;
    spcs_s_info_t ustatus;
    rdc_config_t parms = { 0 };
    char buf[CFG_MAX_BUF];
    char key[CFG_MAX_KEY];
    CFGFILE *cfg = NULL;
    int i;
    int setnumber;
    int found;
    int replicating = 0;
    char primary[NSC_MAXPATH];
    char secondary[NSC_MAXPATH];
    char phost[MAX_RDC_HOST_SIZE];
    char shost[MAX_RDC_HOST_SIZE];
    rdc_set_t *rdc_set;
    int sev;
    char *local_file;
    int size;

    ustatus = spcs_s_ucreate();

    size = sizeof (char *) * rdc_maxsets;
    volumes = malloc(size);
    if (volumes == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_warn(NULL,
            gettext("unable to allocate %ld bytes"), size);
        goto done;
    }
    bzero(volumes, size);

    /*
     * If all members of this group are replicating
     * set ii_ndr state to "update". Otherwise leave them alone.
     */
    LOCKCFG();
    if ((cfg = cfg_open(NULL)) == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: error opening lconfig"),
            program);
        rdc_warn(NULL, gettext("error opening config"));
        UNLOCKCFG();
        goto done;
    }

    if (!cfg_lock(cfg, CFG_RDLOCK)) {
        spcs_log("sndr", NULL,
            gettext("%s: error locking config"),
            program);
        rdc_warn(NULL, gettext("error locking config"));
        goto done;
    }

    found = 0;

    /* get all RM entries, with a matching group, that are replicating */
    for (i = 0; i < rdc_maxsets; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.group", setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;

        if (strcmp(buf, group) != 0)
            continue;

        /* Found a matching entry */

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.primary", setnumber);
        if (cfg_get_cstring(cfg, key, primary, sizeof (primary)) < 0)
            break;
        strcpy(parms.rdc_set->primary.file, primary);

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.phost", setnumber);
        if (cfg_get_cstring(cfg, key, phost, sizeof (phost)) < 0)
            break;
        strcpy(parms.rdc_set->primary.intf, phost);

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.secondary", setnumber);
        if (cfg_get_cstring(cfg, key, secondary,
                sizeof (secondary)) < 0)
            break;
        strcpy(parms.rdc_set->secondary.file, secondary);

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.shost", setnumber);
        if (cfg_get_cstring(cfg, key, shost, sizeof (shost)) < 0)
            break;
        strcpy(parms.rdc_set->secondary.intf, shost);

        parms.command = RDC_CMD_STATUS;
        if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
            continue;
        }

        /* We found a matching set */
        found++;

        if (self_check(phost))
            local_file = primary;
        else
            local_file = secondary;

        rdc_set = &parms.rdc_set[0];
        if (!(rdc_set->flags & RDC_LOGGING) &&
            !(rdc_set->flags & RDC_SYNCING)) {
            volumes[replicating] = strdup(local_file);
            if (volumes[replicating] == NULL) {
                size = strlen(local_file);
                spcs_log("sndr", NULL,
                    gettext("%s: unable to allocate %ld bytes"),
                    program, size);
                rdc_warn(NULL,
                    gettext("unable to allocate %ld bytes"),
                    size);
                goto done;
            }
            /* We remember all replicating sets */
            replicating++;
        } else
            break;      /* Not all replicating, so done */
    }

    if (found != replicating)
        goto done;

    /* All replicating, so update ndr_ii state fields */

    cfg_unlock(cfg);

    if (!cfg_lock(cfg, CFG_WRLOCK)) {
        spcs_log("sndr", NULL,
            gettext("%s: error locking lconfig"),
            program);
        rdc_warn(NULL, gettext("error locking config"));
        goto done;
    }

    /*
     * Search through the ndr_ii entries for entries
     * that match the saved secondary volume names.
     * Set state to "update".
     */

    for (i = 0; ; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;

        if (!volume_match(buf, volumes, found)) {
            continue;
        }

        /* Got a matching entry */

        (void) snprintf(key, sizeof (key),
            "ndr_ii.set%d.state", setnumber);
        if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
            (cfg_commit(cfg) < 0)) {
            spcs_log("sndr", NULL,
                gettext("%s: unable to update \"%s\" "
                "in configuration storage: %s"),
                program, buf, cfg_error(&sev));
            rdc_warn(NULL,
                gettext("unable to update \"%s\" "
                "in configuration storage: %s"),
                buf, cfg_error(&sev));
        }
    }


done:
    if (cfg) {
        cfg_close(cfg);
        UNLOCKCFG();
    }
    spcs_s_ufree(&ustatus);
    if (volumes) {
        for (i = 0; i < replicating; i++)
            free(volumes[i]);
        free(volumes);
    }
}


/*
 * Sync started to a member of a group.
 * If all members of the group are in ndr_ii state "update" then take an PIT
 * snapshot on all of them. This will provide a consistent point-in-time
 * copy until whatever syncs take place are all completed.
 */
void
group_start(char *group)
{
    char **masters = NULL;
    char **shadows = NULL;
    char **bitmaps = NULL;
    char cmd[256];
    char buf[CFG_MAX_BUF];
    char key[CFG_MAX_KEY];
    CFGFILE *cfg = NULL;
    int i;
    int j;
    int setnumber;
    int found;
    int sndr_sets = 0;
    int update_needed = 0;
    int sev;
    char *ctag = NULL;
    int commit = 0;
    int size;

    size = sizeof (char *) * rdc_maxsets;
    masters = malloc(size);
    if (masters == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_warn(NULL,
            gettext("unable to allocate %ld bytes"), size);
        goto done;
    }
    bzero(masters, size);
    shadows = malloc(size);
    if (shadows == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_warn(NULL,
            gettext("unable to allocate %ld bytes"), size);
        goto done;
    }
    bzero(shadows, size);
    bitmaps = malloc(size);
    if (bitmaps == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: unable to allocate %ld bytes"),
            program, size);
        rdc_warn(NULL,
            gettext("unable to allocate %ld bytes"), size);
        goto done;
    }
    bzero(bitmaps, size);

    LOCKCFG();
    if ((cfg = cfg_open(NULL)) == NULL) {
        spcs_log("sndr", NULL,
            gettext("%s: error opening config"),
            program);
        rdc_warn(NULL,
            gettext("error opening config"));
        UNLOCKCFG();
        goto done;
    }

    if (!cfg_lock(cfg, CFG_WRLOCK)) {
        spcs_log("sndr", NULL,
            gettext("%s: error locking config"),
            program);
        rdc_warn(NULL, gettext("error locking config"));
        goto done;
    }

    /* Now get all Remote Mirror entries with a matching group */
    for (i = 0; i < rdc_maxsets; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.group", setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;

        if (strcmp(buf, group) != 0)
            continue;

        /* Found a matching entry */

        (void) snprintf(key, sizeof (key),
            "sndr.set%d.phost", setnumber);
        if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
            break;

        if (self_check(buf)) {
            (void) snprintf(key, sizeof (key), "sndr.set%d.primary",
                setnumber);
        } else {
            (void) snprintf(key, sizeof (key),
                "sndr.set%d.secondary", setnumber);
        }
        if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
            break;

        masters[sndr_sets] = strdup(buf);
        if (masters[sndr_sets] == NULL) {
            size = strlen(buf);
            spcs_log("sndr", NULL,
                gettext("%s: unable to allocate %ld bytes"),
                program, size);
            rdc_warn(NULL,
                gettext("unable to allocate %ld bytes"), size);
            goto done;
        }
        sndr_sets++;

        if (ctag == NULL && clustered) {
            /* Get cluster tag of matching entry */

            (void) snprintf(key, sizeof (key), "sndr.set%d.cnode",
                setnumber);
            if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
                ctag = strdup(buf);
        }
    }

    /*
     * Search through the ndr_ii entries for entries
     * that match the saved local volume names and are in "update" state.
     */

    update_needed = 0;

    for (i = 0; ; i++) {
        setnumber = i + 1;

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;

        if (!volume_match(buf, masters, sndr_sets))
            continue;

        /* Got a matching entry */

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
            break;
        shadows[update_needed] = strdup(buf);
        if (shadows[update_needed] == NULL) {
            size = strlen(buf);
            spcs_log("sndr", NULL,
                gettext("%s: unable to allocate %ld bytes"),
                program, size);
            rdc_warn(NULL,
                gettext("unable to allocate %ld bytes"), size);
            goto done;
        }

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
            break;
        }
        bitmaps[update_needed] = strdup(buf);
        if (bitmaps[update_needed] == NULL) {
            size = strlen(buf);
            spcs_log("sndr", NULL,
                gettext("%s: unable to allocate %ld bytes"),
                program, size);
            rdc_warn(NULL,
                gettext("unable to allocate %ld bytes"), size);
            goto done;
        }

        (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
            setnumber);
        if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
            break;
        }
        if (strcmp(buf, UPDATE) != 0) {
            break;
        }

        update_needed++;
    }

    if (update_needed != sndr_sets) {
#ifdef DEBUG
        spcs_log("sndr", NULL,
            gettext("%s: group sync: no Point-in-Time Copy snapshot "
                "for %s"), program, group);
#endif
        goto done;
    }

    /* All RM sets in the group have an ndr_ii entry in "update" state */

    /* Issue PIT Copy snapshot commands for all sets in the group */
    for (j = 0; j < sndr_sets; j++) {
        found = 0;

        /* get ii entries until a match is found */
        for (i = 0; ; i++) {
            setnumber = i + 1;

            (void) snprintf(key, sizeof (key), "ii.set%d.shadow",
                setnumber);
            if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
                break;
            if (strcmp(buf, shadows[j]) != 0)
                continue;

            /* Matching shadow found, so ii already enabled */
            found = 1;
            break;
        }

        if (commit)
            if (cfg_commit(cfg) < 0)
                rdc_warn(NULL, gettext("commit config error"));
        cfg_close(cfg);

        if (found) {
            (void) sprintf(cmd, "%s -u s %s", IIADM, shadows[j]);
        } else {
            if (ctag) {
                (void) sprintf(cmd, "%s -C %s -e dep %s %s %s",
                    IIADM, ctag, masters[j], shadows[j],
                    bitmaps[j]);
                free(ctag);
                ctag = NULL;
            } else
                (void) sprintf(cmd, "%s -e dep %s %s %s", IIADM,
                    masters[j], shadows[j], bitmaps[j]);
        }

        if (system(cmd) != 0) {
            spcs_log("sndr", NULL,
                gettext("%s: group sync: Point-in-Time Copy"
                " snapshot failed for %s"),
                program, masters[j]);

            goto done;
        }

        if ((cfg = cfg_open(NULL)) == NULL) {
            spcs_log("sndr", NULL,
                gettext("%s: error opening config"),
                program);
            rdc_warn(NULL,
                gettext("error opening config"));
            goto done;
        }
        if (!cfg_lock(cfg, CFG_WRLOCK)) {
            spcs_log("sndr", NULL,
                gettext("%s: error locking config"),
                program);
            rdc_warn(NULL, gettext("error locking config"));
            goto done;
        }
        commit = 0;

        /* PIT enable or update was fine, so update the ndr_ii entry */

        /* get ndr_ii entries until a match is found */
        for (i = 0; ; i++) {
            setnumber = i + 1;

            (void) snprintf(key, sizeof (key),
                "ndr_ii.set%d.shadow", setnumber);
            if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
                break;
            if (strcmp(buf, shadows[j]) != 0)
                continue;

            /* Found the matching entry */

            (void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
                setnumber);
            if (cfg_put_cstring(cfg, key, NOUPDATE,
                strlen(NOUPDATE)) < 0) {
                spcs_log("sndr", NULL,
                    gettext("%s: unable to update \"%s\" "
                    "in configuration storage: %s"),
                    program, buf, cfg_error(&sev));
                rdc_warn(NULL,
                    gettext("unable to update \"%s\" "
                    "in configuration storage: %s"),
                    buf, cfg_error(&sev));
            } else
                commit = 1;
            break;
        }
    }

    if (commit)
        if (cfg_commit(cfg) < 0)
            rdc_warn(NULL, gettext("commit config error"));

    spcs_log("sndr", NULL,
        gettext("%s: group sync: Point-in-Time Copy snapshots completed "
            "for %s"), program, group);

done:
    if (ctag)
        free(ctag);

    if (cfg) {
        cfg_close(cfg);
        UNLOCKCFG();
    }

    if (masters) {
        for (i = 0; i < sndr_sets; i++) {
            if (masters[i])
                free(masters[i]);
        }
        free(masters);
    }

    if (shadows) {
        for (i = 0; i < update_needed; i++) {
            if (shadows[i])
                free(shadows[i]);
        }
        free(shadows);
    }

    if (bitmaps) {
        for (i = 0; i < update_needed; i++) {
            if (bitmaps[i])
                free(bitmaps[i]);
        }
        free(bitmaps);
    }
}