src/lib/iostream-rawlog.c

	iostream-rawlog.c revision 75157863d633300d814fe630590ccb5310b9fdc2
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen/* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen
53febe02364e20b2b95594ca8e7906a37973be01Timo Sirainen#include "lib.h"
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen#include "hostpid.h"
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen#include "ioloop.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "buffer.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "str.h"
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen#include "net.h"
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen#include "write-full.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "time-util.h"
677e22747b82bf15b339e31d1d0106d62bf806daTimo Sirainen#include "istream.h"
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen#include "ostream.h"
52f6112882fb2a4b7b921fc45640f4a9d748bae7Stephan Bosch#include "iostream-private.h"
654c60f1741fd195878d74a30df90bf130649d64Timo Sirainen#include "iostream-rawlog-private.h"
654c60f1741fd195878d74a30df90bf130649d64Timo Sirainen#include "istream-rawlog.h"
7f4fa37676bac8efcf4e2ac706172b1bad779a8aMartti Rannanjärvi#include "ostream-rawlog.h"
677e22747b82bf15b339e31d1d0106d62bf806daTimo Sirainen#include "iostream-rawlog.h"
677e22747b82bf15b339e31d1d0106d62bf806daTimo Sirainen
654c60f1741fd195878d74a30df90bf130649d64Timo Sirainen#include <unistd.h>
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen#include <fcntl.h>
fa3fc0b4c821874ccc56a1512604f661b411d3a4Aki Tuomi
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen#define RAWLOG_MAX_LINE_LEN 8192
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
a966016e605eea27e02d73ff1412632cd684d770Timo Sirainenstatic void
bc10373fc050eb9dd23f6ed5ee8207d0e4d142eeTimo Sirainenrawlog_write_timestamp(struct rawlog_iostream *rstream, bool line_ends)
9d1526ac8bcec9aff3c3a32f092ee2f3da2760b7Timo Sirainen{
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    unsigned char data[MAX_INT_STRLEN + 6 + 1 + 3];
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    buffer_t buf;
9d1526ac8bcec9aff3c3a32f092ee2f3da2760b7Timo Sirainen
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_TIMESTAMP) == 0)
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen        return;
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen
62300a38f91227b9de043a9a8ec1d4f1978e1138Timo Sirainen    buffer_create_from_data(&buf, data, sizeof(data));
a35cbba04d0a2823da98e693bd09a051addffdb2Timo Sirainen    str_printfa(&buf, "%lu.%06u ",
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen            (unsigned long)ioloop_timeval.tv_sec,
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen            (unsigned int)ioloop_timeval.tv_usec);
654c60f1741fd195878d74a30df90bf130649d64Timo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) != 0) {
cb951d3282610a9a0960230865bc5f3e3347b203Timo Sirainen        str_append_c(&buf, rstream->input ? 'I' : 'O');
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen        str_append_c(&buf, line_ends ? ':' : '>');
b863b7e3fccf75f90f613b875f02fd1781e14e5eTimo Sirainen        str_append_c(&buf, ' ');
b863b7e3fccf75f90f613b875f02fd1781e14e5eTimo Sirainen    }
1795e934ebcd58175d3b5bbdd811b13c7889efa3Timo Sirainen    o_stream_nsend(rstream->rawlog_output, buf.data, buf.used);
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen}
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainenvoid iostream_rawlog_init(struct rawlog_iostream *rstream,
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen              enum iostream_rawlog_flags flags, bool input)
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen{
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen    rstream->flags = flags;
654c60f1741fd195878d74a30df90bf130649d64Timo Sirainen    rstream->input = input;
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) != 0)
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen        rstream->buffer = buffer_create_dynamic(default_pool, 1024);
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen}
b863b7e3fccf75f90f613b875f02fd1781e14e5eTimo Sirainen
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainenstatic void
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Siraineniostream_rawlog_write_buffered(struct rawlog_iostream *rstream,
e1b13a5c1507c2583fc6bd07e690037be8a802a7Timo Sirainen                   const unsigned char *data, size_t size)
e1b13a5c1507c2583fc6bd07e690037be8a802a7Timo Sirainen{
26ada766cfff6ebfb91d100936118cf8aa51e3aeTimo Sirainen    const unsigned char *p;
f4526ec0e620af27bc027955f7a56cfe187c212fTimo Sirainen    size_t pos;
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen    bool line_ends;
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen    while (size > 0) {
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen        p = memchr(data, '\n', size);
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen        if (p != NULL) {
9563e6542b7a8a13b13a0ef62ec68b0383b99293Timo Sirainen            line_ends = TRUE;
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen            pos = p-data + 1;
7dd64d2faf200cb8b87569cbc3206b54f95c9ff5Aki Tuomi        } else if (rstream->buffer->used + size < RAWLOG_MAX_LINE_LEN) {
7dd64d2faf200cb8b87569cbc3206b54f95c9ff5Aki Tuomi            buffer_append(rstream->buffer, data, size);
7dd64d2faf200cb8b87569cbc3206b54f95c9ff5Aki Tuomi            break;
b200bc3875fa06d42c8619865cc306c3297fcaccAki Tuomi        } else {
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen            line_ends = FALSE;
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen            pos = size;
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen        }
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen
b200bc3875fa06d42c8619865cc306c3297fcaccAki Tuomi        rawlog_write_timestamp(rstream, line_ends);
b200bc3875fa06d42c8619865cc306c3297fcaccAki Tuomi        if (rstream->buffer->used > 0) {
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen            o_stream_nsend(rstream->rawlog_output,
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen                       rstream->buffer->data,
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen                       rstream->buffer->used);
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen            buffer_set_used_size(rstream->buffer, 0);
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen        }
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen        o_stream_nsend(rstream->rawlog_output, data, pos);
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen
c0699d9e0842a53c047fcbd695fa93ce435baa8fTimo Sirainen        data += pos;
4c7caf83adc56d54e2ecec0a803f5ca9b6d0498dTimo Sirainen        size -= pos;
667f09546abd3d3636b277fec9dba8622cb3d644Timo Sirainen    }
06d1238dd2a6b87af5ad1fad53d7b9b74ad93bfaTimo Sirainen}
acbd82bd456316a32f61df67fc41bbff681119c7Timo Sirainen
acbd82bd456316a32f61df67fc41bbff681119c7Timo Sirainenstatic void
9fd7e883292dab58f3c209f0ec6a8cd11e52075cTimo Siraineniostream_rawlog_write_unbuffered(struct rawlog_iostream *rstream,
d0cc826dab58ed8e30e1ccd7cb6ad2f44b391a00Timo Sirainen                 const unsigned char *data, size_t size)
f0339f522dc9c8e2e8a29ef9a3f937c431c6bd1bTimo Sirainen{
f0339f522dc9c8e2e8a29ef9a3f937c431c6bd1bTimo Sirainen    size_t i, start;
d0cc826dab58ed8e30e1ccd7cb6ad2f44b391a00Timo Sirainen
acbd82bd456316a32f61df67fc41bbff681119c7Timo Sirainen    if (!rstream->line_continued)
fba3d682d6bad6a9530b254d6fd07dc5577be7feTimo Sirainen        rawlog_write_timestamp(rstream, TRUE);
23bdbb7b1831785c6ba6df190f6369da882d2b9dTimo Sirainen
d0cc826dab58ed8e30e1ccd7cb6ad2f44b391a00Timo Sirainen    for (start = 0, i = 1; i < size; i++) {
a321662b406acc9c9e6fe52332236740d449f591Timo Sirainen        if (data[i-1] == '\n') {
a5563dc790a44bb58860d74479a24349f593d68fTimo Sirainen            o_stream_nsend(rstream->rawlog_output,
f740f218bbb1a98cea08dd82b601a8dd50f35093Timo Sirainen                       data + start, i - start);
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen            rawlog_write_timestamp(rstream, TRUE);
a5563dc790a44bb58860d74479a24349f593d68fTimo Sirainen            start = i;
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen        }
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen    }
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen    if (start != size) {
fdbcd3027578d35cb6e12b44c06a46905024e300Timo Sirainen        o_stream_nsend(rstream->rawlog_output,
fdbcd3027578d35cb6e12b44c06a46905024e300Timo Sirainen                   data + start, size - start);
3a53aff14d327af2819b4755864b3571703683b8Timo Sirainen    }
3a53aff14d327af2819b4755864b3571703683b8Timo Sirainen    rstream->line_continued = data[size-1] != '\n';
797d20c35779d68f2542d3e287c1a869048c8f26Timo Sirainen}
a35cbba04d0a2823da98e693bd09a051addffdb2Timo Sirainen
797d20c35779d68f2542d3e287c1a869048c8f26Timo Sirainenvoid iostream_rawlog_write(struct rawlog_iostream *rstream,
797d20c35779d68f2542d3e287c1a869048c8f26Timo Sirainen               const unsigned char *data, size_t size)
7dd64d2faf200cb8b87569cbc3206b54f95c9ff5Aki Tuomi{
7dd64d2faf200cb8b87569cbc3206b54f95c9ff5Aki Tuomi    if (size == 0 || rstream->rawlog_output == NULL)
797d20c35779d68f2542d3e287c1a869048c8f26Timo Sirainen        return;
88c580cdff2656df007fdfe5bd8ca8a465ee605aTimo Sirainen
88c580cdff2656df007fdfe5bd8ca8a465ee605aTimo Sirainen    io_loop_time_refresh();
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen
acbd82bd456316a32f61df67fc41bbff681119c7Timo Sirainen    o_stream_cork(rstream->rawlog_output);
acbd82bd456316a32f61df67fc41bbff681119c7Timo Sirainen    if ((rstream->flags & IOSTREAM_RAWLOG_FLAG_BUFFERED) != 0)
acbd82bd456316a32f61df67fc41bbff681119c7Timo Sirainen        iostream_rawlog_write_buffered(rstream, data, size);
23ad3970cd22adb6c875caf95fa791548849f0d1Timo Sirainen    else
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen        iostream_rawlog_write_unbuffered(rstream, data, size);
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen    o_stream_uncork(rstream->rawlog_output);
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen    if (o_stream_nfinish(rstream->rawlog_output) < 0) {
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen        i_error("write(%s) failed: %s",
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen            o_stream_get_name(rstream->rawlog_output),
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen            o_stream_get_error(rstream->rawlog_output));
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen        iostream_rawlog_close(rstream);
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen    }
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen}
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen
c35e049bf1b9bbee97551bcb029e53a10cafd761Timo Sirainenvoid iostream_rawlog_close(struct rawlog_iostream *rstream)
c35e049bf1b9bbee97551bcb029e53a10cafd761Timo Sirainen{
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen    if (rstream->rawlog_output != NULL)
149299c7d5136a8fb425ef3cf8953026a1358002Timo Sirainen        o_stream_unref(&rstream->rawlog_output);
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen    if (rstream->buffer != NULL)
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen        buffer_free(&rstream->buffer);
de02255bbe97d5d5c54ca1054bb7c9304ee025a1Timo Sirainen}

static void
iostream_rawlog_create_fd(int fd, const char *path, struct istream **input,
              struct ostream **output)
{
    struct istream *old_input;
    struct ostream *old_output;

    old_input = *input;
    old_output = *output;
    *input = i_stream_create_rawlog(old_input, path, fd,
                    IOSTREAM_RAWLOG_FLAG_BUFFERED |
                    IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    *output = o_stream_create_rawlog(old_output, path, fd,
                     IOSTREAM_RAWLOG_FLAG_AUTOCLOSE |
                     IOSTREAM_RAWLOG_FLAG_BUFFERED |
                     IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    i_stream_unref(&old_input);
    o_stream_unref(&old_output);
}

static int
iostream_rawlog_try_create_tcp(const char *path,
                   struct istream **input, struct ostream **output)
{
    const char *p, *host;
    struct ip_addr *ips;
    unsigned int port, ips_count;
    int ret, fd;

    /* tcp:host:port */
    if (strncmp(path, "tcp:", 4) != 0)
        return 0;
    path += 4;

    if (strchr(path, '/') != NULL)
        return 0;
    if ((p = strchr(path, ':')) == NULL)
        return 0;
    if (str_to_uint(p+1, &port) < 0 || port == 0 || port > 65535)
        return 0;
    host = t_strdup_until(path, p);

    ret = net_gethostbyname(host, &ips, &ips_count);
    if (ret != 0) {
        i_error("net_gethostbyname(%s) failed: %s", host,
            net_gethosterror(ret));
        return -1;
    }
    fd = net_connect_ip_blocking(&ips[0], port, NULL);
    if (fd == -1) {
        i_error("connect(%s:%u) failed: %m", net_ip2addr(&ips[0]), port);
        return -1;
    }
    iostream_rawlog_create_fd(fd, path, input, output);
    return 1;
}

int iostream_rawlog_create(const char *dir, struct istream **input,
               struct ostream **output)
{
    static unsigned int counter = 0;
    const char *timestamp, *prefix;
    struct stat st;
    int ret;

    if ((ret = iostream_rawlog_try_create_tcp(dir, input, output)) != 0)
        return ret < 0 ? -1 : 0;
    if (stat(dir, &st) < 0) {
        if (errno != ENOENT && errno != EACCES)
            i_error("rawlog: stat(%s) failed: %m", dir);
        return -1;
    }

    timestamp = t_strflocaltime("%Y%m%d-%H%M%S", ioloop_time);

    counter++;
    prefix = t_strdup_printf("%s/%s.%s.%u", dir, timestamp, my_pid, counter);
    return iostream_rawlog_create_prefix(prefix, input, output);
}

int iostream_rawlog_create_prefix(const char *prefix, struct istream **input,
                  struct ostream **output)
{
    const char *in_path, *out_path;
    struct istream *old_input;
    struct ostream *old_output;
    int in_fd, out_fd;

    in_path = t_strdup_printf("%s.in", prefix);
    in_fd = open(in_path, O_CREAT | O_APPEND | O_WRONLY, 0600);
    if (in_fd == -1) {
        i_error("creat(%s) failed: %m", in_path);
        return -1;
    }

    out_path = t_strdup_printf("%s.out", prefix);
    out_fd = open(out_path, O_CREAT | O_APPEND | O_WRONLY, 0600);
    if (out_fd == -1) {
        i_error("creat(%s) failed: %m", out_path);
        i_close_fd(&in_fd);
        (void)unlink(in_path);
        return -1;
    }

    old_input = *input;
    old_output = *output;

    *input = i_stream_create_rawlog(old_input, in_path, in_fd,
                    IOSTREAM_RAWLOG_FLAG_AUTOCLOSE |
                    IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    *output = o_stream_create_rawlog(old_output, out_path, out_fd,
                     IOSTREAM_RAWLOG_FLAG_AUTOCLOSE |
                     IOSTREAM_RAWLOG_FLAG_TIMESTAMP);
    i_stream_unref(&old_input);
    o_stream_unref(&old_output);
    return 0;
}

int iostream_rawlog_create_path(const char *path, struct istream **input,
                struct ostream **output)
{
    int ret, fd;

    if ((ret = iostream_rawlog_try_create_tcp(path, input, output)) != 0)
        return ret < 0 ? -1 : 0;
    fd = open(path, O_CREAT | O_APPEND | O_WRONLY, 0600);
    if (fd == -1) {
        i_error("creat(%s) failed: %m", path);
        return -1;
    }
    iostream_rawlog_create_fd(fd, path, input, output);
    return 0;
}

void iostream_rawlog_create_from_stream(struct ostream *rawlog_output,
                    struct istream **input,
                    struct ostream **output)
{
    const enum iostream_rawlog_flags rawlog_flags =
        IOSTREAM_RAWLOG_FLAG_BUFFERED |
        IOSTREAM_RAWLOG_FLAG_TIMESTAMP;
    struct istream *old_input;
    struct ostream *old_output;

    if (input != NULL) {
        old_input = *input;
        *input = i_stream_create_rawlog_from_stream(old_input,
                rawlog_output, rawlog_flags);
        i_stream_unref(&old_input);
    }
    if (output != NULL) {
        old_output = *output;
        *output = o_stream_create_rawlog_from_stream(old_output,
                rawlog_output, rawlog_flags);
        o_stream_unref(&old_output);
    }
    if (input != NULL && output != NULL)
        o_stream_ref(rawlog_output);
}