src/lib/istream.c

	istream.c revision d66ef20c30fee728899ee168c75fcc5ff8fbdac1
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen/* Copyright (c) 2002-2011 Dovecot authors, see the included COPYING file */
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen#include "lib.h"
767431e5084a037c4dbefdf30ebfa03c84b1f449Timo Sirainen#include "ioloop.h"
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen#include "str.h"
1c633f71ec2060e5bfa500a97f34cd881a958ecdTimo Sirainen#include "istream-private.h"
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
a8fe899601735459641edae975c0fa08be8482e2Timo Sirainenvoid i_stream_set_name(struct istream *stream, const char *name)
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen{
5fb3f13537dffd15a31e997da133a721c0728af8Timo Sirainen    i_free(stream->real_stream->iostream.name);
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen    stream->real_stream->iostream.name = i_strdup(name);
bb25bed75eefd011138ebf1b8e033fc8ef55ca74Timo Sirainen}
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainenconst char *i_stream_get_name(struct istream *stream)
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen{
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    while (stream->real_stream->iostream.name == NULL) {
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen        stream = stream->real_stream->parent;
e09c7dc961cb9cab04ec7cc79215c2f6318fbde0Timo Sirainen        if (stream == NULL)
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen            return "";
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    }
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    return stream->real_stream->iostream.name;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen}
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
62d0db14d2c5008758983c28d242ec158baabf9eTimo Sirainenvoid i_stream_destroy(struct istream **stream)
62d0db14d2c5008758983c28d242ec158baabf9eTimo Sirainen{
62d0db14d2c5008758983c28d242ec158baabf9eTimo Sirainen    i_stream_close(*stream);
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    i_stream_unref(stream);
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen}
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainenvoid i_stream_ref(struct istream *stream)
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen{
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    io_stream_ref(&stream->real_stream->iostream);
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen}
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainenvoid i_stream_unref(struct istream **stream)
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen{
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    struct istream_private *_stream = (*stream)->real_stream;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    if (_stream->iostream.refcount == 1) {
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen        if (_stream->line_str != NULL)
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen            str_free(&_stream->line_str);
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    }
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    io_stream_unref(&(*stream)->real_stream->iostream);
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    *stream = NULL;
10b8040903b1d1591f1d44552ff466c8789b8814Timo Sirainen}
10b8040903b1d1591f1d44552ff466c8789b8814Timo Sirainen
0a9cb42cbb135e3200cbfbb657820304cca8ecb8Timo Sirainen#undef i_stream_set_destroy_callback
0a9cb42cbb135e3200cbfbb657820304cca8ecb8Timo Sirainenvoid i_stream_set_destroy_callback(struct istream *stream,
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen                   istream_callback_t *callback, void *context)
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen{
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen    struct iostream_private *iostream = &stream->real_stream->iostream;
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    iostream->destroy_callback = callback;
cdfdb67422891a44fc7d9ace6bc1a00185fd3528Timo Sirainen    iostream->destroy_context = context;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen}
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainenvoid i_stream_unset_destroy_callback(struct istream *stream)
1d082a46e1676e7ec13928d588c4a25e062713ccTimo Sirainen{
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen    struct iostream_private *iostream = &stream->real_stream->iostream;
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen
7358272563d8ef77366447708ab0e58c0cff4151Timo Sirainen    iostream->destroy_callback = NULL;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    iostream->destroy_context = NULL;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen}
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen
7358272563d8ef77366447708ab0e58c0cff4151Timo Sirainenint i_stream_get_fd(struct istream *stream)
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen{
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen    struct istream_private *_stream = stream->real_stream;
885a3c2287ae3e5827aa580ea06b231de38abb47Timo Sirainen
a8281b7c770f4a9a842b19303083fc7f6859e756Timo Sirainen    return _stream->fd;
a8281b7c770f4a9a842b19303083fc7f6859e756Timo Sirainen}
29f138b4b9bc037b21dfaa6b8e458943a99d5db2Timo Sirainen
29f138b4b9bc037b21dfaa6b8e458943a99d5db2Timo Sirainenvoid i_stream_close(struct istream *stream)
29f138b4b9bc037b21dfaa6b8e458943a99d5db2Timo Sirainen{
7358272563d8ef77366447708ab0e58c0cff4151Timo Sirainen    io_stream_close(&stream->real_stream->iostream);
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen    stream->closed = TRUE;
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    if (stream->stream_errno == 0)
23878bd03d1de531e3261a25598beec621351910Timo Sirainen        stream->stream_errno = ENOENT;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen}
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainenvoid i_stream_set_init_buffer_size(struct istream *stream, size_t size)
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen{
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    stream->real_stream->init_buffer_size = size;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen}
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainen
1db62753d9e3b5d71018889c8ef0a3722a307455Timo Sirainenvoid i_stream_set_max_buffer_size(struct istream *stream, size_t max_size)
ad58b50aef8125981ebdbc89513236558bcccf60Timo Sirainen{
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    io_stream_set_max_buffer_size(&stream->real_stream->iostream, max_size);
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen}
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainensize_t i_stream_get_max_buffer_size(struct istream *stream)
0f9a8663b0ff6fe30389d02284a2b002c40914ebTimo Sirainen{
bd417d416988d11a6b555b9aa57779e7ed976951Timo Sirainen    return stream->real_stream->max_buffer_size;
a9efdb661eb7a8a33aacfdcc3486dcc675a21543Timo Sirainen}
a9efdb661eb7a8a33aacfdcc3486dcc675a21543Timo Sirainen
fab850a6aee4aaef4f4795bd7946807a3ba45041Timo Sirainenvoid i_stream_set_return_partial_line(struct istream *stream, bool set)
bd417d416988d11a6b555b9aa57779e7ed976951Timo Sirainen{
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen    stream->real_stream->return_nolf_line = set;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen}
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainenstatic void i_stream_update(struct istream_private *stream)
5685e60e62a8e0d368bd28a1526056f97bbba022Timo Sirainen{
5685e60e62a8e0d368bd28a1526056f97bbba022Timo Sirainen    if (stream->parent == NULL)
5685e60e62a8e0d368bd28a1526056f97bbba022Timo Sirainen        stream->access_counter++;
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen    else {
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen        stream->access_counter =
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainen            stream->parent->real_stream->access_counter;
c14c5561e85853d91280235a7611b6050feaebb2Timo Sirainen        stream->parent_expected_offset = stream->parent->v_offset;
c14c5561e85853d91280235a7611b6050feaebb2Timo Sirainen    }
c14c5561e85853d91280235a7611b6050feaebb2Timo Sirainen}
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen
72bc08129fb0aaec8144cc183a998ccc426fef9eTimo Sirainenssize_t i_stream_read(struct istream *stream)
2cc88ff507e244faa63683f804833b321a62c665Timo Sirainen{
71df09024cea5f2faa93da3bb9513ee96ba6bf22Timo Sirainen    struct istream_private *_stream = stream->real_stream;
0779e926687b319fe1bcc0f1010ba7f88023e789Timo Sirainen    size_t old_size;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen    ssize_t ret;
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen    if (unlikely(stream->closed)) {
97437f768d1a3e6134fed1971202803fd250eef2Timo Sirainen        errno = stream->stream_errno;
97afa073e3e1e0301dc41173ec34beb08edcce50Timo Sirainen        return -1;
97afa073e3e1e0301dc41173ec34beb08edcce50Timo Sirainen    }
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
b8b085f7bc6f1c0367802a9f00062bbbd981690dTimo Sirainen    stream->eof = FALSE;
b8b085f7bc6f1c0367802a9f00062bbbd981690dTimo Sirainen    stream->stream_errno = 0;
94ba4820927b906b333e39445c1508a29387c3aaTimo Sirainen
b932ee7fbbec6e79b777dcc7ba613b9e99f8337bTimo Sirainen    if (_stream->parent != NULL)
b932ee7fbbec6e79b777dcc7ba613b9e99f8337bTimo Sirainen        i_stream_seek(_stream->parent, _stream->parent_expected_offset);
cf63dc8723b971cc80638fccbf494d961cbafc7fTimo Sirainen
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    old_size = _stream->pos - _stream->skip;
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    ret = _stream->read(_stream);
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    switch (ret) {
23878bd03d1de531e3261a25598beec621351910Timo Sirainen    case -2:
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        i_assert(_stream->skip != _stream->pos);
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        break;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen    case -1:
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        if (stream->stream_errno != 0) {
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen            /* error handling should be easier if we now just
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen               assume the stream is now at EOF */
bb25bed75eefd011138ebf1b8e033fc8ef55ca74Timo Sirainen            stream->eof = TRUE;
bb25bed75eefd011138ebf1b8e033fc8ef55ca74Timo Sirainen            errno = stream->stream_errno;
5fbccc935e3f7b916aa7c6e302a212821072e83aTimo Sirainen        } else {
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen            i_assert(stream->eof);
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen        }
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen        break;
6eb7938cd366fc087b39fc9a901e7de426131384Timo Sirainen    case 0:
6eb7938cd366fc087b39fc9a901e7de426131384Timo Sirainen        i_assert(!stream->blocking);
6eb7938cd366fc087b39fc9a901e7de426131384Timo Sirainen        break;
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen    default:
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen        i_assert(ret > 0);
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen        i_assert(_stream->skip < _stream->pos);
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen        i_assert((size_t)ret+old_size == _stream->pos - _stream->skip);
138495d02aa177230a9f1eaf90b720b4ce0f6544Timo Sirainen        break;
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen    }
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen    i_stream_update(_stream);
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen    return ret;
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen}
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainenssize_t i_stream_read_copy_from_parent(struct istream *istream)
2a15ce3abe14099b94535f6dfc2d4ee023a7c455Timo Sirainen{
2c57ebc900742bd1119ef011b77b4910c4660cfaTimo Sirainen    struct istream_private *stream = istream->real_stream;
2c57ebc900742bd1119ef011b77b4910c4660cfaTimo Sirainen    size_t pos;
2c57ebc900742bd1119ef011b77b4910c4660cfaTimo Sirainen    ssize_t ret;
2c57ebc900742bd1119ef011b77b4910c4660cfaTimo Sirainen
eed1ec3ac96fddb8d9e4fa2af6e760ee42801fb8Timo Sirainen    stream->pos -= stream->skip;
94ba4820927b906b333e39445c1508a29387c3aaTimo Sirainen    stream->skip = 0;
a6ab8f00351265e35b79f3a22b1f5a4978ae5c35Timo Sirainen
    stream->buffer = i_stream_get_data(stream->parent, &pos);
    if (pos > stream->pos)
        ret = 0;
    else do {
        if ((ret = i_stream_read(stream->parent)) == -2)
            return -2;

        stream->istream.stream_errno = stream->parent->stream_errno;
        stream->istream.eof = stream->parent->eof;
        stream->buffer = i_stream_get_data(stream->parent, &pos);
        /* check again, in case the parent stream had been seeked
           backwards and the previous read() didn't get us far
           enough. */
    } while (pos <= stream->pos && ret > 0);

    ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) :
        (ret == 0 ? 0 : -1);
    stream->pos = pos;
    i_assert(ret != -1 || stream->istream.eof ||
         stream->istream.stream_errno != 0);
    return ret;
}

void i_stream_skip(struct istream *stream, uoff_t count)
{
    struct istream_private *_stream = stream->real_stream;
    size_t data_size;

    data_size = _stream->pos - _stream->skip;
    if (count <= data_size) {
        /* within buffer */
        stream->v_offset += count;
        _stream->skip += count;
        return;
    }

    /* have to seek forward */
    count -= data_size;
    _stream->skip = _stream->pos;
    stream->v_offset += data_size;

    if (unlikely(stream->closed))
        return;

    stream->stream_errno = 0;
    _stream->seek(_stream, stream->v_offset + count, FALSE);
}

static bool i_stream_can_optimize_seek(struct istream_private *stream)
{
    if (stream->parent == NULL)
        return TRUE;

    /* use the fast route only if the parent stream hasn't been changed */
    if (stream->access_counter !=
        stream->parent->real_stream->access_counter)
        return FALSE;

    return i_stream_can_optimize_seek(stream->parent->real_stream);
}

void i_stream_seek(struct istream *stream, uoff_t v_offset)
{
    struct istream_private *_stream = stream->real_stream;

    if (v_offset >= stream->v_offset &&
        i_stream_can_optimize_seek(_stream))
        i_stream_skip(stream, v_offset - stream->v_offset);
    else {
        if (unlikely(stream->closed))
            return;

        stream->eof = FALSE;
        _stream->seek(_stream, v_offset, FALSE);
    }
    i_stream_update(_stream);
}

void i_stream_seek_mark(struct istream *stream, uoff_t v_offset)
{
    struct istream_private *_stream = stream->real_stream;

    if (unlikely(stream->closed))
        return;

    stream->eof = FALSE;
    _stream->seek(_stream, v_offset, TRUE);
    i_stream_update(_stream);
}

void i_stream_sync(struct istream *stream)
{
    struct istream_private *_stream = stream->real_stream;

    if (unlikely(stream->closed))
        return;

    if (_stream->sync != NULL) {
        _stream->sync(_stream);
        i_stream_update(_stream);
    }
}

const struct stat *i_stream_stat(struct istream *stream, bool exact)
{
    struct istream_private *_stream = stream->real_stream;

    if (unlikely(stream->closed))
        return NULL;

    return _stream->stat(_stream, exact);
}

int i_stream_get_size(struct istream *stream, bool exact, uoff_t *size_r)
{
    struct istream_private *_stream = stream->real_stream;

    if (unlikely(stream->closed))
        return -1;

    return _stream->get_size(_stream, exact, size_r);
}

bool i_stream_have_bytes_left(const struct istream *stream)
{
    const struct istream_private *_stream = stream->real_stream;

    return !stream->eof || _stream->skip != _stream->pos;
}

bool i_stream_is_eof(struct istream *stream)
{
    const struct istream_private *_stream = stream->real_stream;

    if (_stream->skip == _stream->pos)
        (void)i_stream_read(stream);
    return !i_stream_have_bytes_left(stream);
}

uoff_t i_stream_get_absolute_offset(struct istream *stream)
{
    return stream->real_stream->abs_start_offset + stream->v_offset;
}

static char *i_stream_next_line_finish(struct istream_private *stream, size_t i)
{
    char *ret;
    size_t end;

    if (i > 0 && stream->buffer[i-1] == '\r')
        end = i - 1;
    else
        end = i;

    if (stream->w_buffer != NULL) {
        /* modify the buffer directly */
        stream->w_buffer[end] = '\0';
        ret = (char *)stream->w_buffer + stream->skip;
    } else {
        /* use a temporary string to return it */
        if (stream->line_str == NULL)
            stream->line_str = str_new(default_pool, 256);
        str_truncate(stream->line_str, 0);
        str_append_n(stream->line_str, stream->buffer + stream->skip,
                 end - stream->skip);
        ret = str_c_modifiable(stream->line_str);
    }

    if (i < stream->pos)
        i++;
    stream->istream.v_offset += i - stream->skip;
    stream->skip = i;
    return ret;
}

static char *i_stream_last_line(struct istream_private *_stream)
{
    if (_stream->istream.eof && _stream->skip != _stream->pos &&
        _stream->return_nolf_line) {
        /* the last line is missing LF and we want to return it. */
        return i_stream_next_line_finish(_stream, _stream->pos);
    }
    return NULL;
}

char *i_stream_next_line(struct istream *stream)
{
    struct istream_private *_stream = stream->real_stream;
    const unsigned char *pos;

    if (_stream->skip >= _stream->pos) {
        stream->stream_errno = 0;
        return NULL;
    }

    pos = memchr(_stream->buffer + _stream->skip, '\n',
             _stream->pos - _stream->skip);
    if (pos != NULL) {
        return i_stream_next_line_finish(_stream,
                         pos - _stream->buffer);
    } else {
        return i_stream_last_line(_stream);
    }
}

char *i_stream_read_next_line(struct istream *stream)
{
    char *line;

    for (;;) {
        line = i_stream_next_line(stream);
        if (line != NULL)
            break;

        if (i_stream_read(stream) <= 0)
            return i_stream_last_line(stream->real_stream);
    }
    return line;
}

const unsigned char *
i_stream_get_data(const struct istream *stream, size_t *size_r)
{
    const struct istream_private *_stream = stream->real_stream;

    if (_stream->skip >= _stream->pos) {
        *size_r = 0;
        return NULL;
    }

        *size_r = _stream->pos - _stream->skip;
        return _stream->buffer + _stream->skip;
}

unsigned char *i_stream_get_modifiable_data(const struct istream *stream,
                        size_t *size_r)
{
    const struct istream_private *_stream = stream->real_stream;

    if (_stream->skip >= _stream->pos || _stream->w_buffer == NULL) {
        *size_r = 0;
        return NULL;
    }

        *size_r = _stream->pos - _stream->skip;
        return _stream->w_buffer + _stream->skip;
}

int i_stream_read_data(struct istream *stream, const unsigned char **data_r,
               size_t *size_r, size_t threshold)
{
    ssize_t ret = 0;
    bool read_more = FALSE;

    do {
        *data_r = i_stream_get_data(stream, size_r);
        if (*size_r > threshold)
            return 1;

        /* we need more data */
        ret = i_stream_read(stream);
        if (ret > 0)
            read_more = TRUE;
    } while (ret > 0);

    *data_r = i_stream_get_data(stream, size_r);
    if (ret == -2)
        return -2;

    if (ret == 0) {
        /* need to read more */
        i_assert(!stream->blocking);
        return 0;
    }
    if (stream->eof) {
        if (read_more) {
            /* we read at least some new data */
            return 0;
        }
    } else {
        i_assert(stream->stream_errno != 0);
    }
    return -1;
}

void i_stream_compress(struct istream_private *stream)
{
    memmove(stream->w_buffer, stream->w_buffer + stream->skip,
        stream->pos - stream->skip);
    stream->pos -= stream->skip;

    stream->skip = 0;
}

void i_stream_grow_buffer(struct istream_private *stream, size_t bytes)
{
    size_t old_size;

    i_assert(stream->max_buffer_size > 0);

    old_size = stream->buffer_size;

    stream->buffer_size = stream->pos + bytes;
    if (stream->buffer_size <= stream->init_buffer_size)
        stream->buffer_size = stream->init_buffer_size;
    else
        stream->buffer_size = nearest_power(stream->buffer_size);

    if (stream->buffer_size > stream->max_buffer_size)
        stream->buffer_size = stream->max_buffer_size;

    if (stream->buffer_size <= old_size)
        stream->buffer_size = old_size;
    else {
        stream->w_buffer = i_realloc(stream->w_buffer, old_size,
                         stream->buffer_size);
        stream->buffer = stream->w_buffer;
    }
}

bool i_stream_get_buffer_space(struct istream_private *stream,
                   size_t wanted_size, size_t *size_r)
{
    i_assert(wanted_size > 0);

    if (wanted_size > stream->buffer_size - stream->pos) {
        if (stream->skip > 0) {
            /* remove the unused bytes from beginning of buffer */
                        i_stream_compress(stream);
        } else if (stream->max_buffer_size == 0 ||
               stream->buffer_size < stream->max_buffer_size) {
            /* buffer is full - grow it */
            i_stream_grow_buffer(stream, I_STREAM_MIN_SIZE);
        }
    }

    if (size_r != NULL)
        *size_r = stream->buffer_size - stream->pos;
    return stream->pos != stream->buffer_size;
}

bool i_stream_add_data(struct istream *_stream, const unsigned char *data,
               size_t size)
{
    struct istream_private *stream = _stream->real_stream;
    size_t size2;

    (void)i_stream_get_buffer_space(stream, size, &size2);
    if (size > size2)
        return FALSE;

    memcpy(stream->w_buffer + stream->pos, data, size);
    stream->pos += size;
    return TRUE;
}

static void
i_stream_default_set_max_buffer_size(struct iostream_private *stream,
                     size_t max_size)
{
    struct istream_private *_stream = (struct istream_private *)stream;

    _stream->max_buffer_size = max_size;
    if (_stream->parent != NULL)
        i_stream_set_max_buffer_size(_stream->parent, max_size);
}

static void i_stream_default_destroy(struct iostream_private *stream)
{
    struct istream_private *_stream = (struct istream_private *)stream;

    i_free(_stream->w_buffer);
    if (_stream->parent != NULL)
        i_stream_unref(&_stream->parent);
}

void i_stream_default_seek(struct istream_private *stream,
               uoff_t v_offset, bool mark ATTR_UNUSED)
{
    size_t available;

    if (stream->istream.v_offset > v_offset)
        i_panic("stream doesn't support seeking backwards");

    while (stream->istream.v_offset < v_offset) {
        (void)i_stream_read(&stream->istream);

        available = stream->pos - stream->skip;
        if (available == 0) {
            stream->istream.stream_errno = ESPIPE;
            return;
        }
        if (available <= v_offset - stream->istream.v_offset)
            i_stream_skip(&stream->istream, available);
        else {
            i_stream_skip(&stream->istream,
                      v_offset - stream->istream.v_offset);
        }
    }
}

static const struct stat *
i_stream_default_stat(struct istream_private *stream, bool exact ATTR_UNUSED)
{
    return &stream->statbuf;
}

static int
i_stream_default_get_size(struct istream_private *stream,
              bool exact, uoff_t *size_r)
{
    const struct stat *st;

    st = stream->stat(stream, exact);
    if (st == NULL)
        return -1;
    if (st->st_size == -1)
        return 0;

    *size_r = st->st_size;
    return 1;
}

struct istream *
i_stream_create(struct istream_private *_stream, struct istream *parent, int fd)
{
    _stream->fd = fd;
    if (parent != NULL) {
        _stream->access_counter = parent->real_stream->access_counter;
        _stream->parent = parent;
        _stream->parent_start_offset = parent->v_offset;
        _stream->parent_expected_offset = parent->v_offset;
        _stream->abs_start_offset = parent->v_offset +
            parent->real_stream->abs_start_offset;
        i_stream_ref(parent);
    }
    _stream->istream.real_stream = _stream;

    if (_stream->iostream.destroy == NULL)
        _stream->iostream.destroy = i_stream_default_destroy;
    if (_stream->seek == NULL) {
        i_assert(!_stream->istream.seekable);
        _stream->seek = i_stream_default_seek;
    }
    if (_stream->stat == NULL)
        _stream->stat = i_stream_default_stat;
    if (_stream->get_size == NULL)
        _stream->get_size = i_stream_default_get_size;
    if (_stream->iostream.set_max_buffer_size == NULL) {
        _stream->iostream.set_max_buffer_size =
            i_stream_default_set_max_buffer_size;
    }
    if (_stream->init_buffer_size == 0)
        _stream->init_buffer_size = I_STREAM_MIN_SIZE;

    memset(&_stream->statbuf, 0, sizeof(_stream->statbuf));
    _stream->statbuf.st_size = -1;
    _stream->statbuf.st_atime =
        _stream->statbuf.st_mtime =
        _stream->statbuf.st_ctime = ioloop_time;

    io_stream_init(&_stream->iostream);
    return &_stream->istream;
}