httpd/support/logresolve.c

	logresolve.c revision 17dc8282ea6b3ad1bbc661b498de9ec2e9987ede
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun/* Copyright 1999-2005 The Apache Software Foundation or its licensors, as
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * applicable.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
4a56677aad9b66a36f3dc9fddbca8dc1230ad471rbowen * Licensed under the Apache License, Version 2.0 (the "License");
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * you may not use this file except in compliance with the License.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * You may obtain a copy of the License at
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *     http://www.apache.org/licenses/LICENSE-2.0
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Unless required by applicable law or agreed to in writing, software
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * distributed under the License is distributed on an "AS IS" BASIS,
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * See the License for the specific language governing permissions and
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * limitations under the License.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun */
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun/*
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * logresolve 2.0
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Tom Rathborne - tomr uunet.ca - http://www.uunet.ca/~tomr/
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * UUNET Canada, April 16, 1995
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Rewritten by David Robinson. (drtr ast.cam.ac.uk)
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Rewritten again, and ported to APR by Colm MacCarthaigh
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Usage: logresolve [-s filename] [-c] < access_log > new_log
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Arguments:
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun *    -s filename     name of a file to record statistics
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *    -c              check the DNS for a matching A record for the host.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Notes:             (For historical interest)
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * To generate meaningful statistics from an HTTPD log file, it's good
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * to have the domain name of each machine that accessed your site, but
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * doing this on the fly can slow HTTPD down.
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * Compiling NCSA HTTPD with the -DMINIMAL_DNS flag turns IP#->hostname
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * resolution off. Before running your stats program, just run your log
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * file through this program (logresolve) and all of your IP numbers will
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * be resolved into hostnames (where possible).
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * logresolve takes an HTTPD access log (in the COMMON log file format,
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * or any other format that has the IP number/domain name as the first
df92fac9baa2b098d654a59cb7b517e8ef6ed1f6nilgun * field for that matter), and outputs the same file with all of the
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * domain names looked up. Where no domain name can be found, the IP
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * number is left in.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * To minimize impact on your nameserver, logresolve has its very own
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * internal hash-table cache. This means that each IP number will only
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * be looked up the first time it is found in the log file.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun *
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * The -c option causes logresolve to apply the same check as httpd
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun * compiled with -DMAXIMUM_DNS; after finding the hostname from the IP
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * address, it looks up the IP addresses for the hostname and checks
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * that one of these matches the original address.
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun */
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#include "apr.h"
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#include "apr_lib.h"
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun#include "apr_hash.h"
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun#include "apr_getopt.h"
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#include "apr_strings.h"
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#include "apr_file_io.h"
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#include "apr_network_io.h"
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#if APR_HAVE_STDLIB_H
df92fac9baa2b098d654a59cb7b517e8ef6ed1f6nilgun#include <stdlib.h>
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#endif
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic apr_file_t *errfile;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic const char *shortname = "logresolve";
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic apr_hash_t *cache;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun/* Statistics */
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic int cachehits = 0;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic int cachesize = 0;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgunstatic int entries = 0;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgunstatic int resolves = 0;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgunstatic int withname = 0;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic int doublefailed = 0;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgunstatic int noreverse = 0;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun/*
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * prints various statistics to output
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun */
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#define NL APR_EOL_STR
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic void print_statistics (apr_file_t *output)
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun{
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(output, "logresolve Statistics:" NL);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(output, "Entries: %d" NL, entries);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(output, "    With name   : %d" NL, withname);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(output, "    Resolves    : %d" NL, resolves);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    if (noreverse) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        apr_file_printf(output, "    - No reverse : %d" NL,
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                        noreverse);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    if (doublefailed) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        apr_file_printf(output, "    - Double lookup failed : %d" NL,
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun                        doublefailed);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(output, "Cache hits      : %d" NL, cachehits);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(output, "Cache size      : %d" NL, cachesize);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun}
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun/*
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun * usage info
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun */
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunstatic void usage(void)
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun{
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_printf(errfile,
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    "%s -- Resolve IP-addresses to hostnames in Apache log files."           NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    "Usage: %s [-s STATFILE] [-c]"                                           NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                                                                             NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    "Options:"                                                               NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    "  -s   Record statistics to STATFILE when finished."                    NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                                                                             NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    "  -c   Perform double lookups when resolving IP addresses."            NL,
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    shortname, shortname);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    exit(1);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun}
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#undef NL
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgunint main(int argc, const char * const argv[])
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun{
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_t * outfile;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_t * infile;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_file_t * statsfile;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_sockaddr_t * ip;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_sockaddr_t * ipdouble;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_getopt_t * o;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_pool_t * pool;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_status_t status;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    const char * arg;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    char opt;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    char * stats = NULL;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    char * space;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    char * hostname;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun#if APR_MAJOR_VERSION > 1 || (APR_MAJOR_VERSION == 1 && APR_MINOR_VERSION >= 3)
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun    char * inbuffer;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun    char * outbuffer;
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun#endif
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun    char line[2048];
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun    int doublelookups = 0;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    if (apr_app_initialize(&argc, &argv, NULL) != APR_SUCCESS) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        return 1;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    atexit(apr_terminate);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    if (argc) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        shortname = apr_filepath_name_get(argv[0]);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    if (apr_pool_create(&pool, NULL) != APR_SUCCESS) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        return 1;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    }
0f8e51a88c016cac46e17ba4f298b58136fbfaf5nilgun    apr_file_open_stderr(&errfile, pool);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    apr_getopt_init(&o, pool, argc, argv);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun    while (1) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        status = apr_getopt(o, "s:c", &opt, &arg);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        if (status == APR_EOF) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun            break;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        else if (status != APR_SUCCESS) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun            usage();
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        else {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun            switch (opt) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun            case 'c':
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                if (doublelookups) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                    usage();
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                doublelookups = 1;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                break;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun            case 's':
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                if (stats) {
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                    usage();
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                }
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                stats = apr_pstrdup(pool, arg);
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun                break;
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun            } /* switch */
c9b9246a20d2167ef0297bf7bf85ccaec84ac40fnilgun        } /* else */
    } /* while */

    apr_file_open_stdout(&outfile, pool);
    apr_file_open_stdin(&infile, pool);

#if APR_MAJOR_VERSION > 1 || (APR_MAJOR_VERSION == 1 && APR_MINOR_VERSION >= 3)
    /* Allocate two new 10k file buffers */
    if ((outbuffer = apr_palloc(pool, 10240)) == NULL ||
        (inbuffer = apr_palloc(pool, 10240)) == NULL) {
        return 1;
    }

    /* Set the buffers */
    apr_file_buffer_set(infile, inbuffer, 10240);
    apr_file_buffer_set(outfile, outbuffer, 10240);
#endif

    cache = apr_hash_make(pool);

    while (apr_file_gets(line, 2048, infile) == APR_SUCCESS) {
        if (line[0] == '\0') {
            continue;
        }

        /* Count our log entries */
        entries++;

        /* Check if this could even be an IP address */
        if (!apr_isxdigit(line[0]) && line[0] != ':') {
                withname++;
            apr_file_puts(line, outfile);
            continue;
        }

        /* Terminate the line at the next space */
        if ((space = strchr(line, ' ')) != NULL) {
            *space = '\0';
        }

        /* See if we have it in our cache */
        hostname = (char *) apr_hash_get(cache, line, APR_HASH_KEY_STRING);
        if (hostname) {
            apr_file_printf(outfile, "%s %s", hostname, space + 1);
                cachehits++;
            continue;
        }

        /* Parse the IP address */
        status = apr_sockaddr_info_get(&ip, line, APR_UNSPEC ,0, 0, pool);
        if (status != APR_SUCCESS) {
            /* Not an IP address */
                withname++;
            *space = ' ';
            apr_file_puts(line, outfile);
            continue;
        }

        /* This does not make much sense, but historically "resolves" means
         * "parsed as an IP address". It does not mean we actually resolved
         * the IP address into a hostname.
         */
            resolves++;

        /* From here on our we cache each result, even if it was not
         * succesful
         */
        cachesize++;

        /* Try and perform a reverse lookup */
        status = apr_getnameinfo(&hostname, ip, 0) != APR_SUCCESS;
        if (status || hostname == NULL) {
            /* Could not perform a reverse lookup */
            *space = ' ';
            apr_file_puts(line, outfile);
            noreverse++;

            /* Add to cache */
            *space = '\0';
            apr_hash_set(cache, line, APR_HASH_KEY_STRING,
                         apr_pstrdup(pool, line));
            continue;
        }

        /* Perform a double lookup */
        if (doublelookups) {
            /* Do a forward lookup on our hostname, and see if that matches our
             * original IP address.
             */
            status = apr_sockaddr_info_get(&ipdouble, hostname, ip->family, 0,
                                           0, pool);
            if (status == APR_SUCCESS ||
                memcmp(ipdouble->ipaddr_ptr, ip->ipaddr_ptr, ip->ipaddr_len)) {
                /* Double-lookup failed  */
                *space = ' ';
                apr_file_puts(line, outfile);
                doublefailed++;

                /* Add to cache */
                *space = '\0';
                apr_hash_set(cache, line, APR_HASH_KEY_STRING,
                             apr_pstrdup(pool, line));
                continue;
            }
        }

        /* Outout the resolved name */
        apr_file_printf(outfile, "%s %s", hostname, space + 1);

        /* Store it in the cache */
        apr_hash_set(cache, line, APR_HASH_KEY_STRING,
                     apr_pstrdup(pool, hostname));
    }

    /* Flush any remaining output */
    apr_file_flush(outfile);

    if (stats) {
        if (apr_file_open(&statsfile, stats,
                       APR_FOPEN_WRITE | APR_FOPEN_CREATE | APR_FOPEN_TRUNCATE,
                          APR_OS_DEFAULT, pool) != APR_SUCCESS) {
            apr_file_printf(errfile, "%s: Could not open %s for writing.",
                            shortname, stats);
            return 1;
        }
        print_statistics(statsfile);
        apr_file_close(statsfile);
    }

    return 0;
}