usr.lib/ilbd/ilbd_main.c

	ilbd_main.c revision dbed73cbda2229fd1aa6dc5743993cae7f0a7ee9
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * The core of ilbd daemon is a single-threaded event loop using
 * event completion framework; it receives requests from client using
 * the libilb functions, handles timeouts, initiates health checks, and
 * populates the kernel state.
 *
 * The daemon has the following privileges (in addition to the basic ones):
 *
 *  PRIV_PROC_OWNER, PRIV_NET_ICMPACCESS,
 *  PRIV_SYS_IP_CONFIG, PRIV_PROC_AUDIT
 *
 * The aforementioned  privileges will be specified in the SMF manifest.
 *
 * AF_UNIX socket is used for IPC between libilb and this daemon as
 * both processes will run on the same machine.
 *
 * To do health check, the daemon will create a timer for every health
 * check probe. Each of these timers will be  associated with the
 * event port. When a timer goes off, the daemon will initiate a
 * pipe to a separate process to execute the specific health check
 * probe. This new process will run with the same user-id as that of
 * ilbd daemon and will inherit all the privileges from the ilbd
 * daemon parent process except the following:
 *
 * PRIV_PROC_OWNER, PRIV_PROC_AUDIT
 *
 * All health checks, will be implemented as external methods
 * (binary or script). The following arguments will be passed
 * to external methods:
 *
 *  $1  VIP (literal IPv4 or IPv6 address)
 *  $2  Server IP (literal IPv4 or IPv6 address)
 *  $3  Protocol (UDP, TCP as a string)
 *  $4  The load balance mode, "DSR", "NAT", "HALF_NAT"
 *  $5  Numeric port range
 *  $6  maximum time (in seconds) the method
 * should wait before returning failure. If the method runs for
 * longer, it may be killed, and the test considered failed.
 *
 * Upon success, a health check method should print the RTT to the
 * it finds to its STDOUT for ilbd to consume.  The implicit unit
 * is microseconds but only the number needs to be printed.  If it
 * cannot find the RTT, it should print 0.  If the method decides
 * that the server is dead, it should print -1 to its STDOUT.
 *
 * By default, an user-supplied health check probe process will
 * also run with the same set of privileges as ILB's built-in
 * probes.  If the administrator has an user-supplied health check
 * program that requires a larger privilege set, he/she will have
 * to implement setuid program.
 *
 * Each health check will have a timeout, such that if the health
 * check process is hung, it will be killed after the timeout interval
 * and the daemon will notify the kernel ILB engine of the server's
 * unresponsiveness, so that load distribution can be appropriately
 * adjusted.  If on the other hand the health check is successful
 * the timeout timer is cancelled.
 */

#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include <libgen.h>
#include <fcntl.h>
#include <stddef.h>
#include <signal.h>
#include <port.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <sys/note.h>
#include <sys/resource.h>
#include <unistd.h>
#include <sys/socket.h>
#include <errno.h>
#include <ucred.h>
#include <priv_utils.h>
#include <net/if.h>
#include <libilb.h>
#include <assert.h>
#include <inet/ilb.h>
#include <libintl.h>
#include <fcntl.h>
#include <rpcsvc/daemon_utils.h>
#include "libilb_impl.h"
#include "ilbd.h"

/*
 * NOTE: The following needs to be kept up to date.
 */
#define ILBD_VERSION    "1.0"
#define ILBD_COPYRIGHT  \
    "Copyright 2009 Sun Microsystems, Inc.  All rights reserved.\n" \
    "Use is subject to license terms.\n"

/*
 * Global reply buffer to client request.  Note that ilbd is single threaded,
 * so a global buffer is OK.  If ilbd becomes multi-threaded, this needs to
 * be changed.
 */
static uint32_t reply_buf[ILBD_MSG_SIZE / sizeof (uint32_t)];

static void
ilbd_free_cli(ilbd_client_t *cli)
{
    (void) close(cli->cli_sd);
    if (cli->cli_cmd == ILBD_SHOW_NAT)
        ilbd_show_nat_cleanup();
    if (cli->cli_cmd == ILBD_SHOW_PERSIST)
        ilbd_show_sticky_cleanup();
    if (cli->cli_saved_reply != NULL)
        free(cli->cli_saved_reply);
    free(cli->cli_pw_buf);
    free(cli);
}

static void
ilbd_reset_kernel_state(void)
{
    ilb_status_t    rc;
    ilb_name_cmd_t  kcmd;

    kcmd.cmd = ILB_DESTROY_RULE;
    kcmd.flags = ILB_RULE_ALLRULES;
    kcmd.name[0] = '\0';

    rc = do_ioctl(&kcmd, 0);
    if (rc != ILB_STATUS_OK)
        logdebug("ilbd_reset_kernel_state: do_ioctl failed: %s",
            strerror(errno));
}

/* Signal handler to do clean up. */
/* ARGSUSED */
static void
ilbd_cleanup(int sig)
{
    (void) remove(SOCKET_PATH);
    ilbd_reset_kernel_state();
    exit(0);
}

/*
 * Create a socket and return it to caller.  If there is a failure, this
 * function calls exit(2).  Hence it always returns a valid listener socket.
 *
 * Note that this function is called before ilbd becomes a daemon.  So
 * we call perror(3C) to print out error message directly so that SMF can
 * catch them.
 */
static int
ilbd_create_client_socket(void)
{
    int         s;
    mode_t          omask;
    struct sockaddr_un  sa;
    int         sobufsz;

    s = socket(PF_UNIX, SOCK_SEQPACKET, 0);
    if (s == -1) {
        perror("ilbd_create_client_socket: socket to"
            " client failed");
        exit(errno);
    }
    if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
        perror("ilbd_create_client_socket: fcntl(FD_CLOEXEC)");
        exit(errno);
    }

    sobufsz = ILBD_MSG_SIZE;
    if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, &sobufsz,
        sizeof (sobufsz)) != 0) {
        perror("ilbd_creat_client_socket: setsockopt(SO_SNDBUF) "
            "failed");
        exit(errno);
    }
    if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, &sobufsz,
        sizeof (sobufsz)) != 0) {
        perror("ilbd_creat_client_socket: setsockopt(SO_RCVBUF) "
            "failed");
        exit(errno);
    }

    /*
     * since everybody can talk to us, we need to open up permissions
     * we check peer privileges on a per-operation basis.
     * This is no security issue as long as we're single-threaded.
     */
    omask = umask(0);

    /* just in case we didn't clean up properly after last exit */
    (void) remove(SOCKET_PATH);

    bzero(&sa, sizeof (sa));
    sa.sun_family = AF_UNIX;
    (void) strlcpy(sa.sun_path, SOCKET_PATH, sizeof (sa.sun_path));

    if (bind(s, (struct sockaddr *)&sa, sizeof (sa)) != 0) {
        perror("ilbd_create_client_socket(): bind to client"
            " socket failed");
        exit(errno);
    }

    /* re-instate old umask */
    (void) umask(omask);

#define QLEN    16

    if (listen(s, QLEN) != 0) {
        perror("ilbd_create_client_socket: listen to client"
            " socket failed");
        exit(errno);
    }

    (void) signal(SIGHUP, SIG_IGN);
    (void) signal(SIGPIPE, SIG_IGN);
    (void) signal(SIGSTOP, SIG_IGN);
    (void) signal(SIGTSTP, SIG_IGN);
    (void) signal(SIGTTIN, SIG_IGN);
    (void) signal(SIGTTOU, SIG_IGN);

    (void) signal(SIGINT, ilbd_cleanup);
    (void) signal(SIGTERM, ilbd_cleanup);
    (void) signal(SIGQUIT, ilbd_cleanup);

    return (s);
}

/*
 * Return the minimum size of a given request.  The returned size does not
 * include the variable part of a request.
 */
static size_t
ilbd_cmd_size(const ilb_comm_t *ic)
{
    size_t cmd_sz;

    cmd_sz = sizeof (*ic);
    switch (ic->ic_cmd) {
    case ILBD_RETRIEVE_SG_NAMES:
    case ILBD_RETRIEVE_RULE_NAMES:
    case ILBD_RETRIEVE_HC_NAMES:
    case ILBD_CMD_OK:
        break;
    case ILBD_CMD_ERROR:
        cmd_sz += sizeof (ilb_status_t);
        break;
    case ILBD_RETRIEVE_SG_HOSTS:
    case ILBD_CREATE_SERVERGROUP:
    case ILBD_DESTROY_SERVERGROUP:
    case ILBD_DESTROY_RULE:
    case ILBD_ENABLE_RULE:
    case ILBD_DISABLE_RULE:
    case ILBD_RETRIEVE_RULE:
    case ILBD_DESTROY_HC:
    case ILBD_GET_HC_INFO:
    case ILBD_GET_HC_SRVS:
        cmd_sz += sizeof (ilbd_name_t);
        break;
    case ILBD_ENABLE_SERVER:
    case ILBD_DISABLE_SERVER:
    case ILBD_ADD_SERVER_TO_GROUP:
    case ILBD_REM_SERVER_FROM_GROUP:
        cmd_sz += sizeof (ilb_sg_info_t);
        break;
    case ILBD_SRV_ADDR2ID:
    case ILBD_SRV_ID2ADDR:
        cmd_sz += sizeof (ilb_sg_info_t) + sizeof (ilb_sg_srv_t);
        break;
    case ILBD_CREATE_RULE:
        cmd_sz += sizeof (ilb_rule_info_t);
        break;
    case ILBD_CREATE_HC:
        cmd_sz += sizeof (ilb_hc_info_t);
        break;
    case ILBD_SHOW_NAT:
    case ILBD_SHOW_PERSIST:
        cmd_sz += sizeof (ilb_show_info_t);
        break;
    }

    return (cmd_sz);
}

/*
 * Given a request and its size, check that the size is big enough to
 * contain the variable part of a request.
 */
static ilb_status_t
ilbd_check_req_size(ilb_comm_t *ic, size_t ic_sz)
{
    ilb_status_t rc = ILB_STATUS_OK;
    ilb_sg_info_t *sg_info;
    ilbd_namelist_t *nlist;

    switch (ic->ic_cmd) {
    case ILBD_CREATE_SERVERGROUP:
    case ILBD_ENABLE_SERVER:
    case ILBD_DISABLE_SERVER:
    case ILBD_ADD_SERVER_TO_GROUP:
    case ILBD_REM_SERVER_FROM_GROUP:
        sg_info = (ilb_sg_info_t *)&ic->ic_data;

        if (ic_sz < ilbd_cmd_size(ic) + sg_info->sg_srvcount *
            sizeof (ilb_sg_srv_t)) {
            rc = ILB_STATUS_EINVAL;
        }
        break;
    case ILBD_ENABLE_RULE:
    case ILBD_DISABLE_RULE:
    case ILBD_DESTROY_RULE:
        nlist = (ilbd_namelist_t *)&ic->ic_data;

        if (ic_sz < ilbd_cmd_size(ic) + nlist->ilbl_count *
            sizeof (ilbd_name_t)) {
            rc = ILB_STATUS_EINVAL;
        }
        break;
    }
    return (rc);
}

/*
 * this function *relies* on a complete message/data struct
 * being passed in (currently via the SOCK_SEQPACKET socket type).
 *
 * Note that the size of ip is at most ILBD_MSG_SIZE.
 */
static ilb_status_t
consume_common_struct(ilb_comm_t *ic, size_t ic_sz, ilbd_client_t *cli,
    int ev_port)
{
    ilb_status_t    rc;
    struct passwd   *ps;
    size_t      rbufsz;
    ssize_t     ret;
    boolean_t   standard_reply = B_TRUE;
    ilbd_name_t name;

    /*
     * cli_ev must be overridden during handling of individual commands,
     * if there's a special need; otherwise, leave this for
     * the "default" case
     */
    cli->cli_ev = ILBD_EVENT_REQ;

    ps = &cli->cli_pw;
    rbufsz = ILBD_MSG_SIZE;

    /* Sanity check on the size of the static part of a request. */
    if (ic_sz < ilbd_cmd_size(ic)) {
        rc = ILB_STATUS_EINVAL;
        goto out;
    }

    switch (ic->ic_cmd) {
    case ILBD_CREATE_SERVERGROUP: {
        ilb_sg_info_t sg_info;

        /*
         * ilbd_create_sg() only needs the sg_name field.  But it
         * takes in a ilb_sg_info_t because it is used as a callback
         * in ilbd_walk_sg_pgs().
         */
        (void) strlcpy(sg_info.sg_name, (char *)&(ic->ic_data),
            sizeof (sg_info.sg_name));
        rc = ilbd_create_sg(&sg_info, ev_port, ps,
            cli->cli_peer_ucredp);
        break;
    }

    case ILBD_DESTROY_SERVERGROUP:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_destroy_sg(name, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_ADD_SERVER_TO_GROUP:
        if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
            break;
        rc = ilbd_add_server_to_group((ilb_sg_info_t *)&ic->ic_data,
            ev_port, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_REM_SERVER_FROM_GROUP:
        if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
            break;
        rc = ilbd_rem_server_from_group((ilb_sg_info_t *)&ic->ic_data,
            ev_port, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_ENABLE_SERVER:
        if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
            break;
        rc = ilbd_enable_server((ilb_sg_info_t *)&ic->ic_data, ps,
            cli->cli_peer_ucredp);
        break;

    case ILBD_DISABLE_SERVER:
        if ((rc = ilbd_check_req_size(ic, ic_sz)) != ILB_STATUS_OK)
            break;
        rc = ilbd_disable_server((ilb_sg_info_t *)&ic->ic_data, ps,
            cli->cli_peer_ucredp);
        break;

    case ILBD_SRV_ADDR2ID:
        rc = ilbd_address_to_srvID((ilb_sg_info_t *)&ic->ic_data,
            reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_SRV_ID2ADDR:
        rc = ilbd_srvID_to_address((ilb_sg_info_t *)&ic->ic_data,
            reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_RETRIEVE_SG_HOSTS:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_retrieve_sg_hosts(name, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_RETRIEVE_SG_NAMES:
    case ILBD_RETRIEVE_RULE_NAMES:
    case ILBD_RETRIEVE_HC_NAMES:
        rc = ilbd_retrieve_names(ic->ic_cmd, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_CREATE_RULE:
        rc = ilbd_create_rule((ilb_rule_info_t *)&ic->ic_data, ev_port,
            ps, cli->cli_peer_ucredp);
        break;

    case ILBD_DESTROY_RULE:
        /* Copy the name to ensure that name is NULL terminated. */
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_destroy_rule(name, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_ENABLE_RULE:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_enable_rule(name, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_DISABLE_RULE:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_disable_rule(name, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_RETRIEVE_RULE:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_retrieve_rule(name, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_CREATE_HC:
        rc = ilbd_create_hc((ilb_hc_info_t *)&ic->ic_data, ev_port, ps,
            cli->cli_peer_ucredp);
        break;

    case ILBD_DESTROY_HC:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_destroy_hc(name, ps, cli->cli_peer_ucredp);
        break;

    case ILBD_GET_HC_INFO:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_get_hc_info(name, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_GET_HC_SRVS:
        (void) strlcpy(name, (char *)&(ic->ic_data), sizeof (name));
        rc = ilbd_get_hc_srvs(name, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_SHOW_NAT:
        rc = ilbd_show_nat(cli, ic, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    case ILBD_SHOW_PERSIST:
        rc = ilbd_show_sticky(cli, ic, reply_buf, &rbufsz);
        if (rc == ILB_STATUS_OK)
            standard_reply = B_FALSE;
        break;

    default:
        logdebug("consume_common_struct: unknown command");
        rc = ILB_STATUS_INVAL_CMD;
        break;
    }

out:
    /*
     * The message exchange is always in pairs, request/response.  If
     * a transaction requires multiple exchanges, the client will send
     * in multiple requests to get multiple responses.  The show-nat and
     * show-persist request are examples of this.  The end of transaction
     * is marked with ic_flags set to ILB_COMM_END.
     */

    /* This is the standard reply. */
    if (standard_reply) {
        if (rc == ILB_STATUS_OK)
            ilbd_reply_ok(reply_buf, &rbufsz);
        else
            ilbd_reply_err(reply_buf, &rbufsz, rc);
    }

    if ((ret = send(cli->cli_sd, reply_buf, rbufsz, 0)) != rbufsz) {
        if (ret == -1) {
            if (errno != EWOULDBLOCK) {
                logdebug("consume_common_struct: send: %s",
                    strerror(errno));
                rc = ILB_STATUS_SEND;
                goto err_out;
            }
            /*
             * The reply is blocked, save the reply.  handle_req()
             * will associate the event port for the re-send.
             */
            assert(cli->cli_saved_reply == NULL);
            if ((cli->cli_saved_reply = malloc(rbufsz)) == NULL) {
                /*
                 * Set the error to ILB_STATUS_SEND so that
                 * handle_req() will free the client.
                 */
                logdebug("consume_common_struct: failure to "
                    "allocate memory to save reply");
                rc = ILB_STATUS_SEND;
                goto err_out;
            }
            bcopy(reply_buf, cli->cli_saved_reply, rbufsz);
            cli->cli_saved_size = rbufsz;
            return (ILB_STATUS_EWOULDBLOCK);
        }
    }
err_out:
    return (rc);
}

/*
 * Accept a new client request.  A struct ilbd_client_t is allocated to
 * store the client info.  The accepted socket is port_associate() with
 * the given port.  And the allocated ilbd_client_t struct is passed as
 * the user pointer.
 */
static void
new_req(int ev_port, int listener, void *ev_obj)
{
    struct sockaddr sa;
    int     sa_len;
    int     new_sd;
    int     sflags;
    ilbd_client_t   *cli;
    int     res;
    uid_t       uid;

    sa_len = sizeof (sa);
    if ((new_sd = accept(listener, &sa, &sa_len)) == -1) {
        /* don't log if we're out of file descriptors */
        if (errno != EINTR && errno != EMFILE)
            logperror("new_req: accept failed");
        goto done;
    }

    /* Set the new socket to be non-blocking. */
    if ((sflags = fcntl(new_sd, F_GETFL, 0)) == -1) {
        logperror("new_req: fcntl(F_GETFL)");
        goto clean_up;
    }
    if (fcntl(new_sd, F_SETFL, sflags | O_NONBLOCK) == -1) {
        logperror("new_req: fcntl(F_SETFL)");
        goto clean_up;
    }
    if (fcntl(new_sd, F_SETFD, FD_CLOEXEC) == -1) {
        logperror("new_req: fcntl(FD_CLOEXEC)");
        goto clean_up;
    }
    if ((cli = calloc(1, sizeof (ilbd_client_t))) == NULL) {
        logerr("new_req: malloc(ilbd_client_t)");
        goto clean_up;
    }
    res = getpeerucred(new_sd, &cli->cli_peer_ucredp);
    if (res == -1) {
        logperror("new_req: getpeerucred failed");
        goto clean_up;
    }
    if ((uid = ucred_getruid(cli->cli_peer_ucredp)) == (uid_t)-1) {
        logperror("new_req: ucred_getruid failed");
        goto clean_up;
    }
    cli->cli_pw_bufsz = (size_t)sysconf(_SC_GETPW_R_SIZE_MAX);
    if ((cli->cli_pw_buf = malloc(cli->cli_pw_bufsz)) == NULL) {
        free(cli);
        logerr("new_req: malloc(cli_pw_buf)");
        goto clean_up;
    }
    if (getpwuid_r(uid, &cli->cli_pw, cli->cli_pw_buf,
        cli->cli_pw_bufsz) == NULL) {
        free(cli->cli_pw_buf);
        free(cli);
        logperror("new_req: invalid user");
        goto clean_up;
    }
    cli->cli_ev = ILBD_EVENT_REQ;
    cli->cli_sd = new_sd;
    cli->cli_cmd = ILBD_BAD_CMD;
    cli->cli_saved_reply = NULL;
    cli->cli_saved_size = 0;
    if (port_associate(ev_port, PORT_SOURCE_FD, new_sd, POLLRDNORM,
        cli) == -1) {
        logperror("new_req: port_associate(cli) failed");
        free(cli->cli_pw_buf);
        free(cli);
clean_up:
        (void) close(new_sd);
    }

done:
    /* Re-associate the listener with the event port. */
    if (port_associate(ev_port, PORT_SOURCE_FD, listener, POLLRDNORM,
        ev_obj) == -1) {
        logperror("new_req: port_associate(listener) failed");
        exit(1);
    }
}

static void
handle_req(int ev_port, ilbd_event_t event, ilbd_client_t *cli)
{
    /* All request should be smaller than ILBD_MSG_SIZE */
    union {
        ilb_comm_t  ic;
        uint32_t    buf[ILBD_MSG_SIZE / sizeof (uint32_t)];
    } ic_u;
    int rc = ILB_STATUS_OK;
    ssize_t r;

    if (event == ILBD_EVENT_REQ) {
        /*
         * Something is wrong with the client since there is a
         * pending reply, the client should not send us another
         * request.  Kill this client.
         */
        if (cli->cli_saved_reply != NULL) {
            logerr("handle_req: misbehaving client, more than one "
                "outstanding request");
            rc = ILB_STATUS_INTERNAL;
            goto err_out;
        }

        /*
         * Our socket is message based so we should be able
         * to get the request in one single read.
         */
        r = recv(cli->cli_sd, (void *)ic_u.buf, sizeof (ic_u.buf), 0);
        if (r < 0) {
            if (errno != EINTR) {
                logperror("handle_req: read failed");
                rc = ILB_STATUS_READ;
                goto err_out;
            }
            /*
             * If interrupted, just re-associate the cli_sd
             * with the port.
             */
            goto done;
        }
        cli->cli_cmd = ic_u.ic.ic_cmd;

        rc = consume_common_struct(&ic_u.ic, r, cli, ev_port);
        if (rc == ILB_STATUS_EWOULDBLOCK)
            goto blocked;
        /* Fatal error communicating with client, free it. */
        if (rc == ILB_STATUS_SEND)
            goto err_out;
    } else {
        assert(event == ILBD_EVENT_REP_OK);
        assert(cli->cli_saved_reply != NULL);

        /*
         * The reply to client was previously blocked, we will
         * send again.
         */
        if (send(cli->cli_sd, cli->cli_saved_reply,
            cli->cli_saved_size, 0) != cli->cli_saved_size) {
            if (errno != EWOULDBLOCK) {
                logdebug("handle_req: send: %s",
                    strerror(errno));
                rc = ILB_STATUS_SEND;
                goto err_out;
            }
            goto blocked;
        }
        free(cli->cli_saved_reply);
        cli->cli_saved_reply = NULL;
        cli->cli_saved_size = 0;
    }
done:
    /* Re-associate with the event port for more requests. */
    cli->cli_ev = ILBD_EVENT_REQ;
    if (port_associate(ev_port, PORT_SOURCE_FD, cli->cli_sd,
        POLLRDNORM, cli) == -1) {
        logperror("handle_req: port_associate(POLLRDNORM)");
        rc = ILB_STATUS_INTERNAL;
        goto err_out;
    }
    return;

blocked:
    /* Re-associate with the event port. */
    cli->cli_ev = ILBD_EVENT_REP_OK;
    if (port_associate(ev_port, PORT_SOURCE_FD, cli->cli_sd, POLLWRNORM,
        cli) == -1) {
        logperror("handle_req: port_associate(POLLWRNORM)");
        rc = ILB_STATUS_INTERNAL;
        goto err_out;
    }
    return;

err_out:
    ilbd_free_cli(cli);
}

static void
i_ilbd_read_config(int ev_port)
{
    logdebug("i_ilbd_read_config: port %d", ev_port);
    (void) ilbd_walk_sg_pgs(ilbd_create_sg, &ev_port, NULL);
    (void) ilbd_walk_hc_pgs(ilbd_create_hc, &ev_port, NULL);
    (void) ilbd_walk_rule_pgs(ilbd_create_rule, &ev_port, NULL);
}

/*
 * main event loop for ilbd
 * asserts that argument 'listener' is a server socket ready to accept() on.
 */
static void
main_loop(int listener)
{
    port_event_t        p_ev;
    int         ev_port, ev_port_obj;
    ilbd_event_obj_t    ev_obj;
    ilbd_timer_event_obj_t  timer_ev_obj;

    ev_port = port_create();
    if (ev_port == -1) {
        logperror("main_loop: port_create failed");
        exit(-1);
    }
    ilbd_hc_timer_init(ev_port, &timer_ev_obj);

    ev_obj.ev = ILBD_EVENT_NEW_REQ;
    if (port_associate(ev_port, PORT_SOURCE_FD, listener, POLLRDNORM,
        &ev_obj) == -1) {
        logperror("main_loop: port_associate failed");
        exit(1);
    }

    i_ilbd_read_config(ev_port);
    ilbd_hc_timer_update(&timer_ev_obj);

    _NOTE(CONSTCOND)
    while (B_TRUE) {
        int r;
        ilbd_event_t event;
        ilbd_client_t *cli;

        r = port_get(ev_port, &p_ev, NULL);
        if (r == -1) {
            if (errno == EINTR)
                continue;
            logperror("main_loop: port_get failed");
            break;
        }

        ev_port_obj = p_ev.portev_object;
        event = ((ilbd_event_obj_t *)p_ev.portev_user)->ev;

        switch (event) {
        case ILBD_EVENT_TIMER:
            ilbd_hc_timeout();
            break;

        case ILBD_EVENT_PROBE:
            ilbd_hc_probe_return(ev_port, ev_port_obj,
                p_ev.portev_events,
                (ilbd_hc_probe_event_t *)p_ev.portev_user);
            break;

        case ILBD_EVENT_NEW_REQ:
            assert(ev_port_obj == listener);
            /*
             * An error happens in the listener.  Exit
             * for now....
             */
            if (p_ev.portev_events & (POLLHUP|POLLERR)) {
                logerr("main_loop: listener error");
                exit(1);
            }
            new_req(ev_port, ev_port_obj, &ev_obj);
            break;

        case ILBD_EVENT_REP_OK:
        case ILBD_EVENT_REQ:
            cli = (ilbd_client_t *)p_ev.portev_user;
            assert(ev_port_obj == cli->cli_sd);

            /*
             * An error happens in the newly accepted
             * client request.  Clean up the client.
             * this also happens when client closes socket,
             * so not necessarily a reason for alarm
             */
            if (p_ev.portev_events & (POLLHUP|POLLERR)) {
                ilbd_free_cli(cli);
                break;
            }

            handle_req(ev_port, event, cli);
            break;

        default:
            logerr("main_loop: unknown event %d", event);
            exit(EXIT_FAILURE);
            break;
        }

        ilbd_hc_timer_update(&timer_ev_obj);
    }
}

static void
i_ilbd_setup_lists(void)
{
    i_setup_sg_hlist();
    i_setup_rule_hlist();
    i_ilbd_setup_hc_list();
}

/*
 * Usage message - call only during startup. it will print its
 * message on stderr and exit
 */
static void
Usage(char *name)
{
    (void) fprintf(stderr, gettext("Usage: %s [-d|--debug]\n"), name);
    exit(1);
}

static void
print_version(char *name)
{
    (void) printf("%s %s\n", basename(name), ILBD_VERSION);
    (void) printf(gettext(ILBD_COPYRIGHT));
    exit(0);
}

/*
 * Increase the file descriptor limit for handling a lot of health check
 * processes (each requires a pipe).
 *
 * Note that this function is called before ilbd becomes a daemon.  So
 * we call perror(3C) to print out error message directly so that SMF
 * can catch them.
 */
static void
set_rlim(void)
{
    struct rlimit rlp;

    if (getrlimit(RLIMIT_NOFILE, &rlp) == -1) {
        perror("ilbd: getrlimit");
        exit(errno);
    }
    rlp.rlim_cur = rlp.rlim_max;
    if (setrlimit(RLIMIT_NOFILE, &rlp) == -1) {
        perror("ilbd: setrlimit");
        exit(errno);
    }
}

int
main(int argc, char **argv)
{
    int s;
    int c;

    (void) setlocale(LC_ALL, "");
#if !defined(TEXT_DOMAIN)
#define TEXT_DOMAIN "SYS_TEST"
#endif
    static const char daemon_dir[] = DAEMON_DIR;

    (void) textdomain(TEXT_DOMAIN);

    while ((c = getopt(argc, argv, ":V?d(debug)")) != -1) {
        switch ((char)c) {
        case '?': Usage(argv[0]);
            /* not reached */
            break;
        case 'V': print_version(argv[0]);
            /* not reached */
            break;
        case 'd': ilbd_enable_debug();
            break;
        default: Usage(argv[0]);
            /* not reached */
            break;
        }
    }

    /*
     * Whenever the daemon starts, it needs to start with a clean
     * slate in the kernel. We need sys_ip_config privilege for
     * this.
     */
    ilbd_reset_kernel_state();

    /* Increase the limit on the number of file descriptors. */
    set_rlim();

    /*
     * ilbd daemon starts off as root, just so it can create
     * /var/run/daemon if one does not exist. After that is done
     * the daemon switches to "daemon" uid. This is similar to what
     * rpcbind does.
     */
    if (mkdir(daemon_dir, DAEMON_DIR_MODE) == 0 || errno == EEXIST) {
        (void) chmod(daemon_dir, DAEMON_DIR_MODE);
        (void) chown(daemon_dir, DAEMON_UID, DAEMON_GID);
    } else {
        perror("main: mkdir failed");
        exit(errno);
    }
    /*
     * Now lets switch ilbd as uid = daemon, gid = daemon with a
     * trimmed down privilege set
     */
    if (__init_daemon_priv(PU_RESETGROUPS | PU_LIMITPRIVS | PU_INHERITPRIVS,
        DAEMON_UID, DAEMON_GID, PRIV_PROC_OWNER, PRIV_PROC_AUDIT,
        PRIV_NET_ICMPACCESS, PRIV_SYS_IP_CONFIG, NULL) == -1) {
        (void) fprintf(stderr, "Insufficient privileges\n");
        exit(EXIT_FAILURE);
    }

    /*
     * Opens a PF_UNIX socket to the client. No privilege needed
     * for this.
     */
    s = ilbd_create_client_socket();

    /*
     * Daemonify if ilbd is not running with -d option
     * Need proc_fork privilege for this
     */
    if (!is_debugging_on()) {
        logdebug("daemonizing...");
        if (daemon(0, 0) != 0) {
            logperror("daemon failed");
            exit(EXIT_FAILURE);
        }
    }
    (void) priv_set(PRIV_OFF, PRIV_INHERITABLE, PRIV_PROC_OWNER,
        PRIV_PROC_AUDIT, NULL);

    /* if daemonified then set up syslog */
    if (!is_debugging_on())
        openlog("ilbd", LOG_PID, LOG_DAEMON);

    i_ilbd_setup_lists();

    main_loop(s);

    /*
     * if we come here, then we experienced an error or a shutdown
     * indicator, so clean up after ourselves.
     */
    logdebug("main(): terminating");

    (void) remove(SOCKET_PATH);
    ilbd_reset_kernel_state();

    return (0);
}