usr.bin/nca/ncab2clf.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident   "%Z%%M% %I% %E% SMI"

/*
 *
 *  Converts binary log files to CLF (Common Log Format).
 *
 */

#include <stdlib.h>
#include <unistd.h>
#include <strings.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdio.h>
#include <locale.h>
#include <errno.h>
#include <time.h>
#include <synch.h>
#include <syslog.h>

#ifndef TRUE
#define TRUE    1
#endif  /* TRUE */

#ifndef FALSE
#define FALSE   0
#endif  /* FALSE */

#include "ncadoorhdr.h"
#include "ncalogd.h"

extern char *gettext();

typedef enum    {   /* Boolean type */
    false = 0,
    true  = 1
} bool;

static const char *const
g_method_strings[8] = {
    "UNKNOWN",
    "OPTIONS",
    "GET",
    "HEAD",
    "POST",
    "PUT",
    "DELETE",
    "TRACE"
};

/* Short month strings */
static const char * const sMonthStr [12] = {
    "Jan",
    "Feb",
    "Mar",
    "Apr",
    "May",
    "Jun",
    "Jul",
    "Aug",
    "Sep",
    "Oct",
    "Nov",
    "Dec",
};

#define SEC_PER_MIN     (60)
#define SEC_PER_HOUR        (60*60)
#define SEC_PER_DAY     (24*60*60)
#define SEC_PER_YEAR        (365*24*60*60)
#define LEAP_TO_70      (70/4)

#define KILO_BYTE       (1024)
#define MEGA_BYTE       (KILO_BYTE * KILO_BYTE)
#define GIGA_BYTE       (KILO_BYTE * MEGA_BYTE)

#define CLF_DATE_BUF_LENGTH (128)
#define OUTFILE_BUF_SIZE    (256 * KILO_BYTE)

static bool g_enable_directio = true;
static ssize_t  g_invalid_count = 0;
static ssize_t  g_skip_count = 0;
static char *g_start_time_str = NULL;

/* init value must match logd & NCA kmod */
static ssize_t  g_n_log_upcall = 0;

/* input binary file was written in 64k chunks by default  */
static ssize_t  g_infile_blk_size = NCA_DEFAULT_LOG_BUF_SIZE;

/* num of output records, by default infinite */
static ssize_t  g_out_records = -1;

/* start time for log output, default none (i.e. output all) */
static struct tm g_start_time;

/*
 * http_version(version)
 *
 * Returns out the string of a given http version
 */

static char *
http_version(int http_ver)
{
    char    *ver_num;

    switch (http_ver) {
    case HTTP_0_9:
    case HTTP_0_0:
        ver_num = "HTTP/0.9";
        break;
    case HTTP_ERR:
    case HTTP_1_0:
        ver_num = "HTTP/1.0";
        break;
    case HTTP_1_1:
        ver_num = "HTTP/1.1";
        break;
    default:
        ver_num = "HTTP/unknown";
    }

    return (ver_num);
}

static bool
valid_version(int http_ver)
{
    switch (http_ver) {
    case HTTP_0_9:
    case HTTP_0_0:
    case HTTP_1_0:
    case HTTP_1_1:
        return (true);
    default:
        break;
    }

    return (false);
}

static bool
valid_method(int method)
{
    switch (method) {
    case NCA_OPTIONS:
    case NCA_GET:
    case NCA_HEAD:
    case NCA_POST:
    case NCA_PUT:
    case NCA_DELETE:
    case NCA_TRACE:
        return (true);
    default:
        break;
    }

    return (false);
}

/*
 * http_method
 *
 *   Returns the method string for the given method.
 */

static char *
http_method(int method)
{
    if (method < sizeof (g_method_strings) / sizeof (g_method_strings[0]))
        return ((char *)(g_method_strings[method]));
    else
        return ((char *)(g_method_strings[0]));
}

/* sMonth: Return short month string */

static const char *
sMonth(int index)
{
    return (sMonthStr[index]);
}

/*
 * Debug formatting routine.  Returns a character string representation of the
 * addr in buf, of the form xxx.xxx.xxx.xxx.  This routine takes the address
 * as a pointer.  The "xxx" parts including left zero padding so the final
 * string will fit easily in tables.  It would be nice to take a padding
 * length argument instead.
 */

static char *
ip_dot_saddr(uchar_t *addr, char *buf)
{
    (void) sprintf(buf, "%03d.%03d.%03d.%03d",
        addr[0] & 0xFF, addr[1] & 0xFF, addr[2] & 0xFF, addr[3] & 0xFF);
    return (buf);
}

/*
 * Debug formatting routine.  Returns a character string representation of the
 * addr in buf, of the form xxx.xxx.xxx.xxx.  This routine takes the address
 * in the form of a ipaddr_t and calls ip_dot_saddr with a pointer.
 */

static char *
ip_dot_addr(ipaddr_t addr, char *buf)
{
    return (ip_dot_saddr((uchar_t *)&addr, buf));
}

static int
http_clf_date(char *buf, int bufsize, time_t t)
{
    struct tm   local_time;
    long        time_zone_info;
    char        sign;

    if (localtime_r(&t, &local_time) == NULL)
        return (0);

    if (g_start_time.tm_year > 0 &&
        (local_time.tm_year < g_start_time.tm_year ||
        (local_time.tm_year == g_start_time.tm_year &&
        local_time.tm_mon < g_start_time.tm_mon ||
        (local_time.tm_mon == g_start_time.tm_mon &&
        local_time.tm_mday < g_start_time.tm_mday ||
        (local_time.tm_mday == g_start_time.tm_mday &&
        local_time.tm_hour < g_start_time.tm_hour ||
        (local_time.tm_hour == g_start_time.tm_hour &&
        local_time.tm_min < g_start_time.tm_min ||
        (local_time.tm_min == g_start_time.tm_min &&
        local_time.tm_sec < g_start_time.tm_sec))))))) {
        /* clf record before the specified start time */
        return (1);
    }

    if (local_time.tm_isdst)
        time_zone_info = -timezone + SEC_PER_HOUR;
    else
        time_zone_info = -timezone;

    if (time_zone_info < 0) {
        sign = '-';
        time_zone_info = -time_zone_info;
    } else {
        sign = '+';
    }

    (void) snprintf(buf, bufsize,
        "[%02d/%s/%04d:%02d:%02d:%02d %c%02ld%02ld]",
        local_time.tm_mday, sMonth(local_time.tm_mon),
        1900 + local_time.tm_year, local_time.tm_hour,
        local_time.tm_min, local_time.tm_sec,
        sign, time_zone_info / SEC_PER_HOUR,
        time_zone_info % SEC_PER_HOUR);

    return (0);
}

/*
 * xmalloc(size)
 * Abort if malloc fails
 */

static void *
xmalloc(size_t size)
{
    void *p;

    if (! size)
        size = 1;

    if ((p = malloc(size)) == NULL) {
        syslog(LOG_ERR, gettext("Error: ncab2clf: Out of memory\n"));
        abort();
    }

    return (p);
}

/*
 * xstrdup(string)
 *   duplicate string
 */

static char *
xstrdup(const char *string)
{
    char    *new_string;

    if (string) {
        new_string = xmalloc(strlen(string) + 1);
        (void) strcpy(new_string, string);

        return (new_string);
    }

    return (NULL);
}

static void
usage()
{
    (void) fprintf(stderr, gettext(
        "\nncab2clf [-Dhv] [-b <block-size>] [-i <binary-log-file>] "
            "[-n <n>]\n"
        "    [-o <output-file>] [-s <date/time>]\n"
        "\tconverts a NCA binary log file to HTTP CLF"
        " (Common Log Format)\n\n"
        "\t-b <block-size>\n"
        "\t\tinput file blocking size in KB\n"
        "\t\t- default is 64K bytes\n"
        "\t-D\tdisable directio on <output-file-name>\n"
        "\t-h\tthis usage message\n"
        "\t-i <binary-log-file>\n"
        "\t\tspecify input file\n"
        "\t-n <n>\n"
        "\t\toutput <n> CLF records\n"
        "\t-o <output-file>\n"
        "\t\tspecify output file\n"
        "\t-s <date/time>\n"
        "\t\tskip any records before <date/time>\n"
        "\t\t- <date/time> may be in CLF format\n"
        "\t\t- <date/time> may be in time format as specified "
            "by touch(1)\n"
        "\t-v\tverbose output\n"
        "\tNote: if no <output-file> - output goes to standard output\n"
        "\tNote: if no <binary-log-file> - input is taken from standard "
            "input\n"));

    exit(3);
}

/*
 * atoi_for2(p, value)
 *   - stores the numerical value of the two digit string p into value
 *   - return TRUE upon success and FALSE upon failure
 */

static int
atoi_for2(char *p, int *value) {

    *value = (*p - '0') * 10 + *(p+1) - '0';
    if ((*value < 0) || (*value > 99))
        return (FALSE);
    return (TRUE);
}

/*
 * parse_time(t, tm)
 *   - parses the string t to retrieve the UNIX time format as specified by
 *     touch(1).
 *   - return TRUE upon success and FALSE upon failure
 */

static int
parse_time(char *t, struct tm *tm)
{
    int     century = 0;
    int     seconds = 0;
    time_t      when;
    char        *p;

    /*
     * time in the following format (defined by the touch(1) spec):
     *  [[CC]YY]MMDDhhmm[.SS]
     */
    if ((p = strchr(t, '.')) != NULL) {
        if (strchr(p+1, '.') != NULL)
            return (FALSE);
        if (!atoi_for2(p+1, &seconds))
            return (FALSE);
        *p = '\0';
    }

    when = time(0);
    bzero(tm, sizeof (struct tm));
    tm->tm_year = localtime(&when)->tm_year;

    switch (strlen(t)) {
        case 12:    /* CCYYMMDDhhmm */
            if (!atoi_for2(t, &century))
                return (FALSE);
            t += 2;
            /* FALLTHROUGH */
        case 10:    /* YYMMDDhhmm */
            if (!atoi_for2(t, &tm->tm_year))
                return (FALSE);
            t += 2;
            if (century == 0) {
                if (tm->tm_year < 69)
                    tm->tm_year += 100;
            } else
                tm->tm_year += (century - 19) * 100;
            /* FALLTHROUGH */
        case 8:     /* MMDDhhmm */
            if (!atoi_for2(t, &tm->tm_mon))
                return (FALSE);
            tm->tm_mon--;
            t += 2;

            if (!atoi_for2(t, &tm->tm_mday))
                return (FALSE);
            t += 2;

            if (!atoi_for2(t, &tm->tm_hour))
                return (FALSE);
            t += 2;

            if (!atoi_for2(t, &tm->tm_min))
                return (FALSE);

            tm->tm_sec = seconds;
            break;
        default:
            return (FALSE);
    }

    return (TRUE);
}

static void
close_files(int ifd, int ofd)
{
    if (ifd != STDIN_FILENO)
        (void) close(ifd);

    if (ofd != STDOUT_FILENO)
        (void) close(ofd);
}

/*
 * Read the requested number of bytes from the given file descriptor
 */

static ssize_t
read_n_bytes(int fd, char *buf, ssize_t bufsize)
{
    ssize_t num_to_read = bufsize;
    ssize_t num_already_read = 0;
    ssize_t i;

    while (num_to_read > 0) {

        i = read(fd, &(buf[num_already_read]), num_to_read);
        if (i < 0) {
            if (errno == EINTR)
                continue;
            else
                (void) fprintf(stderr, gettext(
                    "Error: ncab2clf: "
                    "reading input file: %s\n"),
                    strerror(errno));
                return (-1);    /* some wierd interrupt */
        }

        if (i == 0)
            break;

        num_already_read += i;
        num_to_read -= i;
    }

    return (num_already_read);
}

/*
 * Write the requested number of bytes to the given file descriptor
 */

static ssize_t
write_n_bytes(int fd, char *buf, ssize_t bufsize)
{
    ssize_t num_to_write = bufsize;
    ssize_t num_written = 0;
    ssize_t i;

    while (num_to_write > 0) {

        i = write(fd, &(buf[num_written]), num_to_write);
        if (i < 0) {
            if (errno == EINTR)
                continue;
            else
                (void) fprintf(stderr, gettext(
                    "Error: ncab2clf: "
                    "writing output file: %s\n"),
                    strerror(errno));
                return (-1);    /* some wierd interrupt */
        }

        num_written += i;
        num_to_write -= i;
    }

    return (num_written);
}

/* do constraint checks and determine if it's a valid header */

static bool
is_valid_header(void *ibuf)
{
    nca_log_buf_hdr_t   *h;
    nca_log_stat_t      *s;

    h = (nca_log_buf_hdr_t *)ibuf;

    /* Do some validity checks on ibuf */

    if (((h->nca_loghdr).nca_version != NCA_LOG_VERSION1) ||
        ((h->nca_loghdr).nca_op != log_op)) {
        return (false);
    }

    s = &(h->nca_logstats);

    if (g_n_log_upcall == 0) {
        g_n_log_upcall = s->n_log_upcall;
    } else {
        if ((++g_n_log_upcall) != (ssize_t)s->n_log_upcall) {
            (void) fprintf(stderr, gettext(
                "Warning: ncab2clf:"
                " expected record number (%d) is"
                " different from the one seen (%d)\n."
                " Resetting the expected record"
                " number.\n"), g_n_log_upcall, s->n_log_upcall);

            g_n_log_upcall = s->n_log_upcall;
        }
    }

    return (true);
}

/* convert input binary buffer into CLF */

static int
b2clf_buf(
    void    *ibuf,
    char    *obuf,
    ssize_t isize,
    ssize_t osize,
    ssize_t *out_size)
{
    nca_log_buf_hdr_t   *h;
    nca_log_stat_t      *s;
    nca_request_log_t   *r;

    char    *br;
    void    *er;
    char    ip_buf[64];
    ssize_t max_input_size, num_bytes_read;
    int n_recs;
    bool    error_seen;

    ssize_t count;
    char    clf_timebuf[CLF_DATE_BUF_LENGTH];
    char    *method;
    char    *http_version_string;
    char    *ruser;
    char    *req_url;
    char    *remote_ip;

    h = (nca_log_buf_hdr_t *)ibuf;
    s = &(h->nca_logstats);
    r = (nca_request_log_t *)(&(h[1]));

    /* OK, it's a valid buffer which we can use, go ahead and convert it */

    max_input_size = (ssize_t)isize - sizeof (nca_log_buf_hdr_t);

    *out_size = 0;
    error_seen = false;
    num_bytes_read = 0;
    for (n_recs = 0; n_recs < s->n_log_recs; n_recs++) {

        /* Make sure there is enough space in the output buffer */

        if ((*out_size >= osize) ||
                (num_bytes_read >= max_input_size)) {
            error_seen = true;
            break;
        }

        if (http_clf_date(clf_timebuf, sizeof (clf_timebuf),
            ((time_t)r->start_process_time))) {
            /* A start time was speced and we're not there yet */
            ++g_skip_count;
            goto skip;
        }

        /* Only logs valid HTTP ops */

        if ((! valid_method((int)r->method)) ||
                (! valid_version((int)r->version))) {
            ++g_invalid_count;
            goto skip;
        }

        method = http_method((int)r->method);
        http_version_string = http_version((int)r->version);

        remote_ip = ip_dot_addr(r->remote_host, (char *)&ip_buf);
        if (r->remote_user_len) {
            ruser = NCA_REQLOG_RDATA(r, remote_user);
        } else {
            ruser = "-";
        }

        if (r->request_url_len) {
            req_url = NCA_REQLOG_RDATA(r, request_url);
        } else {
            req_url = "UNKNOWN";
        }

        count = (ssize_t)snprintf(&(obuf[*out_size]), osize - *out_size,
                "%s %s %s %s \"%s %s %s\" %d %d\n",
                ((remote_ip) ? remote_ip : "-"),
                /* should be remote_log_name */
                "-",
                ruser,
                clf_timebuf,
                method,
                req_url,
                http_version_string,
                r->response_status,
                r->response_len);

        *out_size += count;
    skip:
        br = (char *)r;
        er = ((char *)r) + NCA_LOG_REC_SIZE(r);

        /*LINTED*/
        r = (nca_request_log_t *)NCA_LOG_ALIGN(er);
        num_bytes_read += (ssize_t)(((char *)r) - br);
        if (g_out_records > 0 && --g_out_records == 0)
            break;
    }

    if (error_seen) {
        (void) fprintf(stderr, gettext(
            "Error: ncab2clf: "
            "Input buffer not fully converted.\n"));

        if (n_recs != s->n_log_recs)
            (void) fprintf(stderr, gettext(
                "Warning: ncab2clf: "
                "Converted only %d of %d records\n"),
                n_recs, s->n_log_recs);
    }

    return (0);
}

static int
b2clf(int ifd, int ofd)
{
    char    *ibuf;
    char    *obuf;
    bool    error_seen;
    bool    eof_seen;
    ssize_t num_iterations, ni, nh, no, olen;

    nca_log_buf_hdr_t   *h;
    nca_log_stat_t      *s;

    ibuf = xmalloc(g_infile_blk_size);
    obuf = xmalloc(OUTFILE_BUF_SIZE);
    error_seen = false;

    eof_seen = false;
    num_iterations = 0;
    while (! eof_seen && g_out_records != 0) {
        ++num_iterations;

        nh = ni = no = 0;

        /* read the binary header first */
        nh = read_n_bytes(ifd, ibuf, sizeof (nca_log_buf_hdr_t));
        if (nh != sizeof (nca_log_buf_hdr_t)) {
            eof_seen = true;
            break;
        }

        if (! is_valid_header(ibuf)) {
            (void) fprintf(stderr, gettext(
                "Error: ncab2clf: "
                "Can't convert the input data to CLF\n"));
            continue;
        }

        /* read the data to be converted */
        /* LINTED */
        h = (nca_log_buf_hdr_t *)ibuf;
        s = &(h->nca_logstats);

        if (s->n_log_size == 0)
            continue;

        ni = read_n_bytes(ifd, &(ibuf[nh]), (ssize_t)s->n_log_size);
        if (ni < 0) {
            error_seen = true;
            break;
        } else if (ni < (ssize_t)s->n_log_size) {
            eof_seen = true;
        }

        if (ni == 0)
            break;

        /* convert binary input into text output */

        if (b2clf_buf(ibuf, obuf, ni + nh, OUTFILE_BUF_SIZE, &olen)) {
            (void) fprintf(stderr, gettext(
                "Error: ncab2clf: "
                "Can't convert the input data to CLF\n"));
            error_seen = true;
            break;
        }

        /* write out the text data */
        no = write_n_bytes(ofd, obuf, olen);
        if (no != olen) {
            error_seen = true;
            break;
        }

        bzero(ibuf, nh + ni);
        bzero(obuf, no);
    }

    free(ibuf);
    free(obuf);

    if (error_seen)
        return (-1);

    return (0);
}


int
main(int argc, char **argv)
{
    int c;
    int ifd;        /* input fd - binary log file */
    int ofd;
    struct tm t;

    char    *infile = NULL;  /* input file name */
    char    *outfile = NULL; /* output file name */

    char    monstr[64];

    (void) setlocale(LC_ALL, "");

#if !defined(TEXT_DOMAIN)   /* Should be defined by cc -D */
#define TEXT_DOMAIN "SYS_TEST"
#endif

    (void) textdomain(TEXT_DOMAIN);

    /* parse any arguments */
    while ((c = getopt(argc, argv, "hvDi:o:b:n:s:")) != EOF) {
        switch (c) {
        case 'h':
            usage();
            break;
        case 'i':
            infile = xstrdup(optarg);
            break;
        case 'D':
            g_enable_directio = false;
            break;
        case 'o':
            outfile = xstrdup(optarg);
            break;
        case 'b':
            g_infile_blk_size = (KILO_BYTE * atoi(optarg));
            break;
        case 'n':
            g_out_records = atoi(optarg);
            break;
        case 's':
            g_start_time_str = strdup(optarg);
            bzero(&t, sizeof (t));
            if (sscanf(optarg, "%d/%3s/%d:%d:%d:%d", &t.tm_mday,
                &monstr[0], &t.tm_year, &t.tm_hour, &t.tm_min,
                &t.tm_sec) == 6) {
                /* Valid CLF time (e.g. 06/Apr/2001:09:14:14) */
                t.tm_mon = 0;
                do {
                    if (strcasecmp(monstr,
                        sMonthStr[t.tm_mon]) == 0)
                        break;
                } while (t.tm_mon++ < 12);
                t.tm_year -= 1900;
                g_start_time = t;
            } else if (parse_time(optarg, &t)) {
                g_start_time = t;
            } else {
                (void) fprintf(stderr,
                    gettext("Error: ncab2clf:"
                    " %s: unrecognized date/time.\n"),
                    optarg);
            }
            break;
        case 'v':
            (void) fprintf(stderr, gettext("Error: ncab2clf: "
                "verbose functionality not yet supported\n"));
            exit(3);
            break;
        case '?':
            usage();
            break;
        }
    }

    /* set up the input stream */

    if (infile) {

        if ((ifd = open(infile, O_RDONLY)) < 0) {
            (void) fprintf(stderr,
                gettext("Error: ncab2clf: "
                "Failure to open binary log file %s: %s\n"),
                infile, strerror(errno));
            exit(1);
        }

    } else {
        ifd = STDIN_FILENO;
    }

    /* set up the output stream */

    if (outfile) {

        if ((ofd = open(outfile, O_WRONLY|O_CREAT, 0644)) < 0) {
            (void) fprintf(stderr, gettext(
                "Error: ncab2clf: "
                "Failure to open output file %s: %s\n"),
                outfile, strerror(errno));
            exit(1);
        }

        /* Enable directio on output stream if specified */

        if (g_enable_directio)
            (void) directio(ofd, DIRECTIO_ON);

    } else {
        ofd = STDOUT_FILENO;
    }

    if ((b2clf(ifd, ofd) != 0)) {
        close_files(ifd, ofd);
        exit(2);
    }

    close_files(ifd, ofd);

    if (g_invalid_count) {
        (void) fprintf(stderr, gettext("Warning: ncab2clf: %d"
        " number of invalid log records encountered in binary input"
        " file were skipped\n"), g_invalid_count);
    }
    if (g_skip_count) {
        (void) fprintf(stderr, gettext("Warning: ncab2clf:"
            " %d log records in binary input file before %s"
            " were skipped\n"),
            g_skip_count, g_start_time_str);
    }

    return (0);
}