istream-concat.c revision 5a0ac2e5ef482016e00575a7dce83f52c1704732
76b43e4417bab52e913da39b5f5bc2a130d3f149Timo Sirainen/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "lib.h"
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen#include "buffer.h"
eff552f5fdc275c940c4c709eeeddb833bc51b40Timo Sirainen#include "istream-internal.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen#include "istream-concat.h"
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainenstruct concat_istream {
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen struct istream_private istream;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen struct istream **input, *cur_input;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen uoff_t *input_size;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen unsigned int cur_idx, unknown_size_idx;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen size_t prev_stream_left, prev_skip;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen};
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstatic void i_stream_concat_close(struct iostream_private *stream)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen{
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen struct concat_istream *cstream = (struct concat_istream *)stream;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen unsigned int i;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen for (i = 0; cstream->input[i] != NULL; i++)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen i_stream_close(cstream->input[i]);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen}
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstatic void i_stream_concat_destroy(struct iostream_private *stream)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen{
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen struct concat_istream *cstream = (struct concat_istream *)stream;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen unsigned int i;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen for (i = 0; cstream->input[i] != NULL; i++)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen i_stream_unref(&cstream->input[i]);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen i_free(cstream->input);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen i_free(cstream->input_size);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen i_free(cstream->istream.w_buffer);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen}
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstatic void
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Siraineni_stream_concat_set_max_buffer_size(struct iostream_private *stream,
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen size_t max_size)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen{
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen struct concat_istream *cstream = (struct concat_istream *)stream;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen unsigned int i;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen cstream->istream.max_buffer_size = max_size;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen for (i = 0; cstream->input[i] != NULL; i++)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen i_stream_set_max_buffer_size(cstream->input[i], max_size);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen}
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainenstatic void i_stream_concat_read_next(struct concat_istream *cstream)
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen{
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen const unsigned char *data;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen size_t data_size, size;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen i_assert(cstream->cur_input->eof);
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen data = i_stream_get_data(cstream->cur_input, &data_size);
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen cstream->cur_idx++;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen cstream->cur_input = cstream->input[cstream->cur_idx];
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen i_stream_seek(cstream->cur_input, 0);
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen if (cstream->prev_stream_left > 0 || cstream->istream.pos == 0) {
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen cstream->prev_stream_left += data_size;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen i_assert(cstream->prev_stream_left ==
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen cstream->istream.pos - cstream->istream.skip);
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen return;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen }
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen
73e19ec2d5069ea125dcd1ede5d8a70f701fd9a8Timo Sirainen /* we already verified that the data size is less than the
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen maximum buffer size */
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen cstream->istream.pos = 0;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen if (!i_stream_get_buffer_space(&cstream->istream, data_size, &size))
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen i_unreached();
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen i_assert(size >= data_size);
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen cstream->prev_stream_left = data_size;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen memcpy(cstream->istream.w_buffer, data, data_size);
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen cstream->istream.skip = 0;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen cstream->istream.pos = data_size;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen}
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainenstatic ssize_t i_stream_concat_read(struct istream_private *stream)
ccfc6441cadb577084daeb1f0aa3dd7bdfa2a220Timo Sirainen{
137ea7ca34005345aa2304a940149b7f3774d727Timo Sirainen struct concat_istream *cstream = (struct concat_istream *)stream;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen const unsigned char *data;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen size_t size, pos, cur_pos, bytes_skipped;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen ssize_t ret;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen bool last_stream;
2521482f3f897c83f7d5a2f9e17fe99fa4ba2cbeTimo Sirainen
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen if (cstream->cur_input == NULL) {
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen stream->istream.eof = TRUE;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen return -1;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen }
49e513d090753ccbf95560b2f3a21f081a5b6c51Timo Sirainen
73e19ec2d5069ea125dcd1ede5d8a70f701fd9a8Timo Sirainen i_assert(stream->skip >= cstream->prev_skip);
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen bytes_skipped = stream->skip - cstream->prev_skip;
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen if (cstream->prev_stream_left == 0) {
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen /* no need to worry about buffers, skip everything */
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen i_assert(cstream->prev_skip == 0);
12aad74464367f7e8be11eafe1af985bf7b1adecTimo Sirainen } else if (bytes_skipped < cstream->prev_stream_left) {
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen /* we're still skipping inside buffer */
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen cstream->prev_stream_left -= bytes_skipped;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen bytes_skipped = 0;
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen } else {
965ed6ea3fc8f7637bd0d159d2fdb283a191ce34Timo Sirainen /* done with the buffer */
91e4199476cb2add8143c18583fa57e1decfea88Timo Sirainen bytes_skipped -= cstream->prev_stream_left;
0727e38ac12efb8963a339daf56255e2be1f29fcTimo Sirainen cstream->prev_stream_left = 0;
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen }
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen i_stream_skip(cstream->cur_input, bytes_skipped);
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen stream->pos -= bytes_skipped;
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen stream->skip -= bytes_skipped;
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen cur_pos = stream->pos - stream->skip - cstream->prev_stream_left;
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen data = i_stream_get_data(cstream->cur_input, &pos);
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen if (pos > cur_pos)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen ret = 0;
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen else {
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen /* need to read more */
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen i_assert(cur_pos == pos);
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen ret = i_stream_read(cstream->cur_input);
747e77e3ab073a8e9e69c7a3e71b4593c5655d03Timo Sirainen if (ret == -2 || ret == 0)
747e77e3ab073a8e9e69c7a3e71b4593c5655d03Timo Sirainen return ret;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen if (ret == -1 && stream->istream.stream_errno != 0) {
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen stream->istream.stream_errno =
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen cstream->cur_input->stream_errno;
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen return -1;
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen }
dd93aba1901a457346990f49c54a738947dc7128Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen /* we either read something or we're at EOF */
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen last_stream = cstream->input[cstream->cur_idx+1] == NULL;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (ret == -1 && !last_stream) {
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (stream->pos >= stream->max_buffer_size)
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen return -2;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen i_stream_concat_read_next(cstream);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen cstream->prev_skip = stream->skip;
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen return i_stream_concat_read(stream);
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen }
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen
08aea01ef9a9d20703e0fcf8618e6195c0037a44Timo Sirainen stream->istream.eof = cstream->cur_input->eof && last_stream;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen i_assert(ret != -1 || stream->istream.eof);
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen data = i_stream_get_data(cstream->cur_input, &pos);
eff552f5fdc275c940c4c709eeeddb833bc51b40Timo Sirainen }
eff552f5fdc275c940c4c709eeeddb833bc51b40Timo Sirainen
eff552f5fdc275c940c4c709eeeddb833bc51b40Timo Sirainen if (cstream->prev_stream_left == 0) {
eff552f5fdc275c940c4c709eeeddb833bc51b40Timo Sirainen stream->buffer = data;
eff552f5fdc275c940c4c709eeeddb833bc51b40Timo Sirainen stream->pos -= stream->skip;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen stream->skip = 0;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen } else if (pos == cur_pos) {
e9503210d3521a6833ed62dc332fc42ffb0e7a13Timo Sirainen stream->buffer = stream->w_buffer;
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen } else {
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen stream->buffer = stream->w_buffer;
b42f37ae6f65ed986315b6885568d32115e589b1Timo Sirainen if (!i_stream_get_buffer_space(stream, pos - cur_pos, &size))
25757faf029c369a8318349dafe952e2358df1d8Timo Sirainen return -2;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen if (pos > size)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen pos = size;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen memcpy(stream->w_buffer + stream->pos,
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen data + cur_pos, pos - cur_pos);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen pos += stream->skip + cstream->prev_stream_left;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) :
51ead2f4c04ee85615d23c453924633b9ed8a4c2Timo Sirainen (ret == 0 ? 0 : -1);
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen stream->pos = pos;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen cstream->prev_skip = stream->skip;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen return ret;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen}
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenstatic unsigned int
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainenfind_v_offset(struct concat_istream *cstream, uoff_t *v_offset)
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen{
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen const struct stat *st;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen unsigned int i;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen for (i = 0; cstream->input[i] != NULL; i++) {
51ead2f4c04ee85615d23c453924633b9ed8a4c2Timo Sirainen if (*v_offset == 0) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen /* seek to beginning of this stream */
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen break;
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen }
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen if (i == cstream->unknown_size_idx) {
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen /* we'll need to figure out this stream's size */
43a66a0b16299bd4f7615acd85e98bd3832c54d5Timo Sirainen st = i_stream_stat(cstream->input[i], TRUE);
if (st == NULL) {
cstream->istream.istream.stream_errno =
cstream->input[i]->stream_errno;
return (unsigned int)-1;
}
/* @UNSAFE */
cstream->input_size[i] = st->st_size;
cstream->unknown_size_idx = i + 1;
}
if (*v_offset < cstream->input_size[i])
break;
*v_offset -= cstream->input_size[i];
}
return i;
}
static void i_stream_concat_seek(struct istream_private *stream,
uoff_t v_offset, bool mark ATTR_UNUSED)
{
struct concat_istream *cstream = (struct concat_istream *)stream;
stream->istream.v_offset = v_offset;
stream->skip = stream->pos = 0;
cstream->prev_stream_left = 0;
cstream->prev_skip = 0;
cstream->cur_idx = find_v_offset(cstream, &v_offset);
if (cstream->cur_idx == (unsigned int)-1) {
cstream->cur_input = NULL;
return;
}
cstream->cur_input = cstream->input[cstream->cur_idx];
if (cstream->cur_input != NULL)
i_stream_seek(cstream->cur_input, v_offset);
}
static const struct stat *
i_stream_concat_stat(struct istream_private *stream, bool exact ATTR_UNUSED)
{
struct concat_istream *cstream = (struct concat_istream *)stream;
uoff_t v_offset = (uoff_t)-1;
unsigned int i;
/* make sure we have all sizes */
(void)find_v_offset(cstream, &v_offset);
stream->statbuf.st_size = 0;
for (i = 0; i < cstream->unknown_size_idx; i++)
stream->statbuf.st_size += cstream->input_size[i];
return &stream->statbuf;
}
struct istream *i_stream_create_concat(struct istream *input[])
{
struct concat_istream *cstream;
unsigned int count;
size_t max_buffer_size = I_STREAM_MIN_SIZE;
bool blocking = TRUE, seekable = TRUE;
/* if any of the streams isn't blocking or seekable, set ourself also
nonblocking/nonseekable */
for (count = 0; input[count] != NULL; count++) {
size_t cur_max = input[count]->real_stream->max_buffer_size;
if (cur_max > max_buffer_size)
max_buffer_size = cur_max;
if (!input[count]->blocking)
blocking = FALSE;
if (!input[count]->seekable)
seekable = FALSE;
i_stream_ref(input[count]);
}
i_assert(count != 0);
cstream = i_new(struct concat_istream, 1);
cstream->input = i_new(struct istream *, count + 1);
cstream->input_size = i_new(uoff_t, count + 1);
memcpy(cstream->input, input, sizeof(*input) * count);
cstream->cur_input = cstream->input[0];
i_stream_seek(cstream->cur_input, 0);
cstream->istream.iostream.close = i_stream_concat_close;
cstream->istream.iostream.destroy = i_stream_concat_destroy;
cstream->istream.iostream.set_max_buffer_size =
i_stream_concat_set_max_buffer_size;
cstream->istream.max_buffer_size = max_buffer_size;
cstream->istream.read = i_stream_concat_read;
cstream->istream.seek = i_stream_concat_seek;
cstream->istream.stat = i_stream_concat_stat;
cstream->istream.istream.readable_fd = FALSE;
cstream->istream.istream.blocking = blocking;
cstream->istream.istream.seekable = seekable;
return i_stream_create(&cstream->istream, NULL, -1);
}