src/lib/iostream-rawlog.c

	iostream-rawlog.c revision b367011da1a2ba3e9113dfbc601aaa4b6b04317d
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen/* Copyright (c) 2011-2013 Dovecot authors, see the included COPYING file */
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "lib.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "hostpid.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "ioloop.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "buffer.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "str.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "write-full.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "time-util.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "istream.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "ostream.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "iostream-private.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "iostream-rawlog-private.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "istream-rawlog.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "ostream-rawlog.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include "iostream-rawlog.h"
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include <unistd.h>
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#include <fcntl.h>
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen#define RAWLOG_MAX_LINE_LEN 8192
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenstatic int
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenrawlog_write(struct rawlog_iostream *rstream, const void *data, size_t size)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen{
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if (rstream->rawlog_fd == -1)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        return -1;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if (write_full(rstream->rawlog_fd, data, size) < 0) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        i_error("rawlog_istream.write(%s) failed: %m",
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            rstream->rawlog_path);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        iostream_rawlog_close(rstream);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        return -1;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    return 0;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen}
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainenstatic int
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainenrawlog_write_timestamp(struct rawlog_iostream *rstream, bool line_ends)
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen{
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen    unsigned char data[MAX_INT_STRLEN + 6 + 1 + 3];
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen    buffer_t buf;
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_TIMESTAMP) == 0)
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen        return 0;
e815af0640b38444b31eadfaa1673bcb422e1573Timo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    buffer_create_from_data(&buf, data, sizeof(data));
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    str_printfa(&buf, "%lu.%06u ",
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            (unsigned long)ioloop_timeval.tv_sec,
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            (unsigned int)ioloop_timeval.tv_usec);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) != 0) {
14175321ddb88619015866978c05a27786ca4814Timo Sirainen        str_append_c(&buf, rstream->input ? 'I' : 'O');
14175321ddb88619015866978c05a27786ca4814Timo Sirainen        str_append_c(&buf, line_ends ? ':' : '>');
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        str_append_c(&buf, ' ');
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    return rawlog_write(rstream, buf.data, buf.used);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen}
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenvoid iostream_rawlog_init(struct rawlog_iostream *rstream,
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen              enum iostream_rawlog_flags flags, bool input)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen{
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    rstream->flags = flags;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    rstream->input = input;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) != 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        rstream->buffer = buffer_create_dynamic(default_pool, 1024);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen}
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenstatic void
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Siraineniostream_rawlog_write_unbuffered(struct rawlog_iostream *rstream,
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen                 const unsigned char *data, size_t size)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen{
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    size_t i, start;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if (!rstream->line_continued) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (rawlog_write_timestamp(rstream, TRUE) < 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            return;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    for (start = 0, i = 1; i < size; i++) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (data[i-1] == '\n') {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            if (rawlog_write(rstream, data + start, i - start) < 0 ||
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen                rawlog_write_timestamp(rstream, TRUE) < 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen                return;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            start = i;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if (start != size) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (rawlog_write(rstream, data + start, size - start) < 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            return;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    rstream->line_continued = data[size-1] != '\n';
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen}
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenvoid iostream_rawlog_write(struct rawlog_iostream *rstream,
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen               const unsigned char *data, size_t size)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen{
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    const unsigned char *p;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    size_t pos;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    bool line_ends;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5478609e31a7665ee108ded988a309673f221aa1Timo Sirainen    if (size == 0)
5478609e31a7665ee108ded988a309673f221aa1Timo Sirainen        return;
5478609e31a7665ee108ded988a309673f221aa1Timo Sirainen
5478609e31a7665ee108ded988a309673f221aa1Timo Sirainen    io_loop_time_refresh();
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) == 0) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        iostream_rawlog_write_unbuffered(rstream, data, size);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        return;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    while (rstream->rawlog_fd != -1 && size > 0) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        p = memchr(data, '\n', size);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (p != NULL) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            line_ends = TRUE;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            pos = p-data + 1;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        } else if (rstream->buffer->used + size < RAWLOG_MAX_LINE_LEN) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            buffer_append(rstream->buffer, data, size);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            return;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        } else {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            line_ends = FALSE;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            pos = size;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (rawlog_write_timestamp(rstream, line_ends) < 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            break;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (rstream->buffer->used > 0) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            if (rawlog_write(rstream, rstream->buffer->data,
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen                     rstream->buffer->used) < 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen                break;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            buffer_set_used_size(rstream->buffer, 0);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (rawlog_write(rstream, data, pos) < 0)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            break;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        data += pos;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        size -= pos;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen}
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenvoid iostream_rawlog_close(struct rawlog_iostream *rstream)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen{
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_AUTOCLOSE) != 0 &&
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        rstream->rawlog_fd != -1) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        if (close(rstream->rawlog_fd) < 0) {
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen            i_error("rawlog_istream.close(%s) failed: %m",
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen                rstream->rawlog_path);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    }
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    rstream->rawlog_fd = -1;
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    i_free_and_null(rstream->rawlog_path);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen    if (rstream->buffer != NULL)
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen        buffer_free(&rstream->buffer);
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen}
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainen
5fe5ea74285e2fc0fbf7568c53f251aa894650fbTimo Sirainenint iostream_rawlog_create(const char *dir, struct istream **input,
               struct ostream **output)
{
    static unsigned int counter = 0;
    const char *timestamp, *prefix;

    timestamp = t_strflocaltime("%Y%m%d-%H%M%S", ioloop_time);

    counter++;
    prefix = t_strdup_printf("%s/%s.%s.%u", dir, timestamp, my_pid, counter);
    return iostream_rawlog_create_prefix(prefix, input, output);
}

int iostream_rawlog_create_prefix(const char *prefix, struct istream **input,
                  struct ostream **output)
{
    const char *in_path, *out_path;
    struct istream *old_input;
    struct ostream *old_output;
    int in_fd, out_fd;

    in_path = t_strdup_printf("%s.in", prefix);
    in_fd = open(in_path, O_CREAT | O_APPEND | O_WRONLY, 0600);
    if (in_fd == -1) {
        i_error("creat(%s) failed: %m", in_path);
        return -1;
    }

    out_path = t_strdup_printf("%s.out", prefix);
    out_fd = open(out_path, O_CREAT | O_APPEND | O_WRONLY, 0600);
    if (out_fd == -1) {
        i_error("creat(%s) failed: %m", out_path);
        i_close_fd(&in_fd);
        (void)unlink(in_path);
        return -1;
    }

    old_input = *input;
    old_output = *output;

    *input = i_stream_create_rawlog(old_input, in_path, in_fd,
                    IOSTREAM_RAWLOG_FLAG_AUTOCLOSE |
                    IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    *output = o_stream_create_rawlog(old_output, out_path, out_fd,
                     IOSTREAM_RAWLOG_FLAG_AUTOCLOSE |
                     IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    i_stream_unref(&old_input);
    o_stream_unref(&old_output);
    return 0;
}

int iostream_rawlog_create_path(const char *path, struct istream **input,
                struct ostream **output)
{
    struct istream *old_input;
    struct ostream *old_output;
    int fd;

    fd = open(path, O_CREAT | O_APPEND | O_WRONLY, 0600);
    if (fd == -1) {
        i_error("creat(%s) failed: %m", path);
        return -1;
    }

    old_input = *input;
    old_output = *output;
    *input = i_stream_create_rawlog(old_input, path, fd,
                    IOSTREAM_RAWLOG_FLAG_BUFFERED |
                    IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    *output = o_stream_create_rawlog(old_output, path, fd,
                     IOSTREAM_RAWLOG_FLAG_AUTOCLOSE |
                     IOSTREAM_RAWLOG_FLAG_BUFFERED |
                     IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    i_stream_unref(&old_input);
    o_stream_unref(&old_output);
    return 0;
}