src/lib/iostream-temp.c

	iostream-temp.c revision b3f4c31f1533e25380f49f77d5bb1251bf43db2a
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "lib.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "buffer.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "str.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "safe-mkstemp.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "write-full.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "istream-private.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "ostream-private.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include "iostream-temp.h"
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#include <unistd.h>
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen#define IOSTREAM_TEMP_MAX_BUF_SIZE (1024*128)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstruct temp_ostream {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    struct ostream_private ostream;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    char *temp_path_prefix;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    enum iostream_temp_flags flags;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    struct istream *dupstream;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    uoff_t dupstream_offset, dupstream_start_offset;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    buffer_t *buf;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    int fd;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    bool fd_tried;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    uoff_t fd_size;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen};
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic void
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Siraineno_stream_temp_close(struct iostream_private *stream,
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            bool close_parent ATTR_UNUSED)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    struct temp_ostream *tstream = (struct temp_ostream *)stream;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (tstream->fd != -1)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        i_close_fd(&tstream->fd);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (tstream->buf != NULL)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        buffer_free(&tstream->buf);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    i_free(tstream->temp_path_prefix);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
2ed75e0243a9845eb4f92436ff6016906a3189e4Timo Sirainenstatic int o_stream_temp_move_to_fd(struct temp_ostream *tstream)
2ed75e0243a9845eb4f92436ff6016906a3189e4Timo Sirainen{
2ed75e0243a9845eb4f92436ff6016906a3189e4Timo Sirainen    string_t *path;
2ed75e0243a9845eb4f92436ff6016906a3189e4Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (tstream->fd_tried)
2ed75e0243a9845eb4f92436ff6016906a3189e4Timo Sirainen        return -1;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    tstream->fd_tried = TRUE;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    path = t_str_new(128);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    str_append(path, tstream->temp_path_prefix);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    tstream->fd = safe_mkstemp_hostpid(path, 0600, (uid_t)-1, (gid_t)-1);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (tstream->fd == -1) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        i_error("safe_mkstemp(%s) failed: %m", str_c(path));
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return -1;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (unlink(str_c(path)) < 0) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        i_error("unlink(%s) failed: %m", str_c(path));
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        i_close_fd(&tstream->fd);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return -1;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (write_full(tstream->fd, tstream->buf->data, tstream->buf->used) < 0) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        i_error("write(%s) failed: %m", str_c(path));
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        i_close_fd(&tstream->fd);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return -1;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    tstream->fd_size = tstream->buf->used;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    buffer_free(&tstream->buf);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    return 0;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic ssize_t
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Siraineno_stream_temp_fd_sendv(struct temp_ostream *tstream,
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen               const struct const_iovec *iov, unsigned int iov_count)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    size_t bytes = 0;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    unsigned int i;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    for (i = 0; i < iov_count; i++) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        if (write_full(tstream->fd, iov[i].iov_base, iov[i].iov_len) < 0) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            tstream->ostream.ostream.stream_errno = errno;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            return -1;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        bytes += iov[i].iov_len;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        tstream->ostream.ostream.offset += iov[i].iov_len;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    tstream->fd_size += bytes;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    return bytes;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic ssize_t
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Siraineno_stream_temp_sendv(struct ostream_private *stream,
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            const struct const_iovec *iov, unsigned int iov_count)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    struct temp_ostream *tstream = (struct temp_ostream *)stream;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    ssize_t ret = 0;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    unsigned int i;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    tstream->flags &= ~IOSTREAM_TEMP_FLAG_TRY_FD_DUP;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    if (tstream->fd != -1)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        return o_stream_temp_fd_sendv(tstream, iov, iov_count);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    for (i = 0; i < iov_count; i++) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        if (tstream->buf->used + iov[i].iov_len > IOSTREAM_TEMP_MAX_BUF_SIZE) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            if (o_stream_temp_move_to_fd(tstream) == 0) {
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen                return o_stream_temp_fd_sendv(tstream, iov+i,
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen                                  iov_count-i);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen            /* failed to move to temp fd, just keep it in memory */
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        buffer_append(tstream->buf, iov[i].iov_base, iov[i].iov_len);
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        ret += iov[i].iov_len;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        stream->ostream.offset += iov[i].iov_len;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    }
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    return ret;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen}
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainenstatic int o_stream_temp_dup_cancel(struct temp_ostream *tstream)
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen{
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    struct istream *input;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    uoff_t size = tstream->dupstream_offset -
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen        tstream->dupstream_start_offset;
5c2d695acf9f95ae0dcdda89c4d2391ceda4d672Timo Sirainen    off_t ret;

    i_stream_seek(tstream->dupstream, tstream->dupstream_start_offset);

    input = i_stream_create_limit(tstream->dupstream, size);
    do {
        ret = io_stream_copy(&tstream->ostream.ostream,
                     input, IO_BLOCK_SIZE);
    } while (input->v_offset < tstream->dupstream_offset && ret > 0);
    if (ret < 0 && tstream->ostream.ostream.stream_errno == 0) {
        i_assert(input->stream_errno != 0);
        tstream->ostream.ostream.stream_errno = input->stream_errno;
    }
    i_stream_destroy(&input);
    i_stream_unref(&tstream->dupstream);
    return ret < 0 ? -1 : 0;
}

static int o_stream_temp_dup_istream(struct temp_ostream *outstream,
                     struct istream *instream)
{
    uoff_t in_size;
    off_t ret;

    if (!instream->readable_fd || i_stream_get_fd(instream) == -1)
        return 0;

    if (i_stream_get_size(instream, TRUE, &in_size) <= 0) {
        if (outstream->dupstream != NULL)
            return o_stream_temp_dup_cancel(outstream);
        return 0;
    }

    if (outstream->dupstream == NULL) {
        outstream->dupstream = instream;
        outstream->dupstream_start_offset = instream->v_offset;
        i_stream_ref(outstream->dupstream);
    } else {
        if (outstream->dupstream != instream ||
            outstream->dupstream_offset != instream->v_offset ||
            outstream->dupstream_offset > in_size)
            return o_stream_temp_dup_cancel(outstream);
    }
    ret = in_size - instream->v_offset;
    i_stream_seek(instream, in_size);
    outstream->dupstream_offset = instream->v_offset;
    return ret;
}

static off_t o_stream_temp_send_istream(struct ostream_private *_outstream,
                    struct istream *instream)
{
    struct temp_ostream *outstream = (struct temp_ostream *)_outstream;
    uoff_t orig_offset;
    int ret;

    if ((outstream->flags & IOSTREAM_TEMP_FLAG_TRY_FD_DUP) != 0) {
        orig_offset = outstream->dupstream_offset;
        if ((ret = o_stream_temp_dup_istream(outstream, instream)) > 0)
            return outstream->dupstream_offset - orig_offset;
        if (ret < 0)
            return -1;
        outstream->flags &= ~IOSTREAM_TEMP_FLAG_TRY_FD_DUP;
    }
    return io_stream_copy(&outstream->ostream.ostream,
                  instream, IO_BLOCK_SIZE);
}

static int
o_stream_temp_write_at(struct ostream_private *stream,
               const void *data, size_t size, uoff_t offset)
{
    struct temp_ostream *tstream = (struct temp_ostream *)stream;

    if (tstream->fd == -1) {
        i_assert(stream->ostream.offset == tstream->buf->used);
        buffer_write(tstream->buf, offset, data, size);
        stream->ostream.offset = tstream->buf->used;
    } else {
        if (pwrite_full(tstream->fd, data, size, offset) < 0) {
            stream->ostream.stream_errno = errno;
            i_close_fd(&tstream->fd);
            return -1;
        }
        if (tstream->fd_size < offset + size)
            tstream->fd_size = offset + size;
    }
    return 0;
}

struct ostream *iostream_temp_create(const char *temp_path_prefix,
                     enum iostream_temp_flags flags)
{
    struct temp_ostream *tstream;
    struct ostream *output;

    tstream = i_new(struct temp_ostream, 1);
    tstream->ostream.sendv = o_stream_temp_sendv;
    tstream->ostream.send_istream = o_stream_temp_send_istream;
    tstream->ostream.write_at = o_stream_temp_write_at;
    tstream->ostream.iostream.close = o_stream_temp_close;
    tstream->temp_path_prefix = i_strdup(temp_path_prefix);
    tstream->flags = flags;
    tstream->buf = buffer_create_dynamic(default_pool, 8192);
    tstream->fd = -1;

    output = o_stream_create(&tstream->ostream, NULL, -1);
    o_stream_set_name(output, "(temp iostream)");
    return output;
}

static void iostream_temp_buf_destroyed(buffer_t *buf)
{
    buffer_free(&buf);
}

struct istream *iostream_temp_finish(struct ostream **output,
                     size_t max_buffer_size)
{
    struct temp_ostream *tstream =
        (struct temp_ostream *)(*output)->real_stream;
    struct istream *input, *input2;
    uoff_t abs_offset, size;
    int fd;

    if (tstream->dupstream != NULL && !tstream->dupstream->closed) {
        abs_offset = tstream->dupstream->real_stream->abs_start_offset +
            tstream->dupstream_start_offset;
        size = tstream->dupstream_offset -
            tstream->dupstream_start_offset;
        fd = dup(i_stream_get_fd(tstream->dupstream));
        if (fd == -1)
            input = i_stream_create_error_str(errno, "dup() failed: %m");
        else {
            input2 = i_stream_create_fd_autoclose(&fd, max_buffer_size);
            i_stream_seek(input2, abs_offset);
            input = i_stream_create_limit(input2, size);
            i_stream_unref(&input2);
        }
        i_stream_set_name(input, t_strdup_printf(
            "(Temp file in %s, from %s)", tstream->temp_path_prefix,
            i_stream_get_name(tstream->dupstream)));
        i_stream_unref(&tstream->dupstream);
    } else if (tstream->dupstream != NULL) {
        /* return the original failed stream. */
        input = tstream->dupstream;
    } else if (tstream->fd != -1) {
        int fd = tstream->fd;
        input = i_stream_create_fd_autoclose(&tstream->fd, max_buffer_size);
        i_stream_set_name(input, t_strdup_printf(
            "(Temp file fd %d in %s, %"PRIuUOFF_T" bytes)",
            fd, tstream->temp_path_prefix, tstream->fd_size));
    } else {
        input = i_stream_create_from_data(tstream->buf->data,
                          tstream->buf->used);
        i_stream_set_name(input, t_strdup_printf(
            "(Temp buffer in %s, %"PRIuSIZE_T" bytes)",
            tstream->temp_path_prefix, tstream->buf->used));
        i_stream_add_destroy_callback(input, iostream_temp_buf_destroyed,
                          tstream->buf);
        tstream->buf = NULL;
    }
    o_stream_destroy(output);
    return input;
}