istream-lzma.c revision 0dffa25d211be541ee3c953b23566a1a990789df
0N/A/* Copyright (c) 2010-2016 Dovecot authors, see the included COPYING file */
0N/A
0N/A#include "lib.h"
0N/A
0N/A#ifdef HAVE_LZMA
0N/A
115N/A#include "istream-private.h"
0N/A#include "istream-zlib.h"
0N/A#include <lzma.h>
0N/A
0N/A#define CHUNK_SIZE (1024*64)
0N/A
115N/A#define LZMA_MEMORY_LIMIT (1024*1024*80)
116N/A
0N/Astruct lzma_istream {
116N/A struct istream_private istream;
0N/A
0N/A lzma_stream strm;
0N/A uoff_t eof_offset, stream_size;
115N/A size_t high_pos;
115N/A struct stat last_parent_statbuf;
115N/A
13N/A bool log_errors:1;
0N/A bool marked:1;
0N/A bool strm_closed:1;
0N/A};
0N/A
0N/Astatic void i_stream_lzma_close(struct iostream_private *stream,
0N/A bool close_parent)
0N/A{
0N/A struct lzma_istream *zstream = (struct lzma_istream *)stream;
0N/A
0N/A if (!zstream->strm_closed) {
0N/A lzma_end(&zstream->strm);
0N/A zstream->strm_closed = TRUE;
115N/A }
115N/A if (close_parent)
115N/A i_stream_close(zstream->istream.parent);
115N/A}
115N/A
0N/Astatic void lzma_read_error(struct lzma_istream *zstream, const char *error)
0N/A{
0N/A io_stream_set_error(&zstream->istream.iostream,
0N/A "lzma.read(%s): %s at %"PRIuUOFF_T,
0N/A i_stream_get_name(&zstream->istream.istream), error,
0N/A i_stream_get_absolute_offset(&zstream->istream.istream));
0N/A if (zstream->log_errors)
0N/A i_error("%s", zstream->istream.iostream.error);
299N/A}
0N/A
299N/Astatic void lzma_stream_end(struct lzma_istream *zstream)
299N/A{
0N/A zstream->eof_offset = zstream->istream.istream.v_offset +
0N/A (zstream->istream.pos - zstream->istream.skip);
0N/A zstream->stream_size = zstream->eof_offset;
0N/A}
0N/A
205N/Astatic ssize_t i_stream_lzma_read(struct istream_private *stream)
0N/A{
0N/A struct lzma_istream *zstream = (struct lzma_istream *)stream;
0N/A const unsigned char *data;
0N/A uoff_t high_offset;
0N/A size_t size, out_size;
0N/A lzma_ret ret;
0N/A
0N/A high_offset = stream->istream.v_offset + (stream->pos - stream->skip);
0N/A if (zstream->eof_offset == high_offset) {
0N/A i_assert(zstream->high_pos == 0 ||
0N/A zstream->high_pos == stream->pos);
0N/A stream->istream.eof = TRUE;
0N/A return -1;
0N/A }
0N/A
0N/A if (stream->pos < zstream->high_pos) {
0N/A /* we're here because we seeked back within the read buffer. */
0N/A ret = zstream->high_pos - stream->pos;
0N/A stream->pos = zstream->high_pos;
0N/A zstream->high_pos = 0;
0N/A
0N/A if (zstream->eof_offset != (uoff_t)-1) {
115N/A high_offset = stream->istream.v_offset +
0N/A (stream->pos - stream->skip);
116N/A i_assert(zstream->eof_offset == high_offset);
0N/A stream->istream.eof = TRUE;
0N/A }
0N/A return ret;
0N/A }
0N/A zstream->high_pos = 0;
0N/A
0N/A if (stream->pos + CHUNK_SIZE > stream->buffer_size) {
0N/A /* try to keep at least CHUNK_SIZE available */
0N/A if (!zstream->marked && stream->skip > 0) {
0N/A /* don't try to keep anything cached if we don't
0N/A have a seek mark. */
0N/A i_stream_compress(stream);
0N/A }
0N/A if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
0N/A i_stream_grow_buffer(stream, CHUNK_SIZE);
116N/A
116N/A if (stream->pos == stream->buffer_size) {
116N/A if (stream->skip > 0) {
116N/A /* lose our buffer cache */
116N/A i_stream_compress(stream);
116N/A }
116N/A
116N/A if (stream->pos == stream->buffer_size)
116N/A return -2; /* buffer full */
116N/A }
116N/A }
116N/A
116N/A if (i_stream_read_more(stream->parent, &data, &size) < 0) {
116N/A if (stream->parent->stream_errno != 0) {
116N/A stream->istream.stream_errno =
164N/A stream->parent->stream_errno;
116N/A } else {
116N/A i_assert(stream->parent->eof);
116N/A lzma_stream_end(zstream);
116N/A stream->istream.eof = TRUE;
116N/A }
116N/A return -1;
116N/A }
116N/A if (size == 0) {
116N/A /* no more input */
116N/A i_assert(!stream->istream.blocking);
116N/A return 0;
116N/A }
116N/A
116N/A zstream->strm.next_in = data;
120N/A zstream->strm.avail_in = size;
120N/A
177N/A out_size = stream->buffer_size - stream->pos;
177N/A zstream->strm.next_out = stream->w_buffer + stream->pos;
120N/A zstream->strm.avail_out = out_size;
177N/A ret = lzma_code(&zstream->strm, LZMA_RUN);
177N/A
177N/A out_size -= zstream->strm.avail_out;
177N/A stream->pos += out_size;
177N/A
177N/A i_stream_skip(stream->parent, size - zstream->strm.avail_in);
177N/A
177N/A switch (ret) {
177N/A case LZMA_OK:
177N/A break;
177N/A case LZMA_DATA_ERROR:
177N/A case LZMA_BUF_ERROR:
177N/A lzma_read_error(zstream, "corrupted data");
177N/A stream->istream.stream_errno = EINVAL;
177N/A return -1;
177N/A case LZMA_FORMAT_ERROR:
120N/A lzma_read_error(zstream, "wrong magic in header (not xz file?)");
0N/A stream->istream.stream_errno = EINVAL;
0N/A return -1;
0N/A case LZMA_OPTIONS_ERROR:
0N/A lzma_read_error(zstream, "Unsupported xz options");
0N/A stream->istream.stream_errno = EINVAL;
0N/A return -1;
0N/A case LZMA_MEM_ERROR:
0N/A i_fatal_status(FATAL_OUTOFMEM, "lzma.read(%s): Out of memory",
0N/A i_stream_get_name(&stream->istream));
0N/A case LZMA_STREAM_END:
0N/A lzma_stream_end(zstream);
0N/A if (out_size == 0) {
0N/A stream->istream.eof = TRUE;
0N/A return -1;
0N/A }
0N/A break;
295N/A default:
295N/A lzma_read_error(zstream, t_strdup_printf(
295N/A "lzma_code() failed with %d", ret));
295N/A stream->istream.stream_errno = EINVAL;
295N/A return -1;
295N/A }
295N/A if (out_size == 0) {
295N/A /* read more input */
295N/A return i_stream_lzma_read(stream);
295N/A }
295N/A return out_size;
295N/A}
295N/A
295N/Astatic void i_stream_lzma_init(struct lzma_istream *zstream)
295N/A{
295N/A lzma_ret ret;
295N/A
295N/A ret = lzma_stream_decoder(&zstream->strm, LZMA_MEMORY_LIMIT,
295N/A LZMA_CONCATENATED);
295N/A switch (ret) {
295N/A case LZMA_OK:
295N/A break;
295N/A case LZMA_MEM_ERROR:
295N/A i_fatal_status(FATAL_OUTOFMEM, "lzma: Out of memory");
295N/A default:
301N/A i_fatal("lzma_stream_decoder() failed with ret=%d", ret);
301N/A }
301N/A}
301N/A
301N/Astatic void i_stream_lzma_reset(struct lzma_istream *zstream)
336N/A{
301N/A struct istream_private *stream = &zstream->istream;
301N/A
301N/A i_stream_seek(stream->parent, stream->parent_start_offset);
301N/A zstream->eof_offset = (uoff_t)-1;
301N/A zstream->strm.next_in = NULL;
301N/A zstream->strm.avail_in = 0;
301N/A
301N/A stream->parent_expected_offset = stream->parent_start_offset;
301N/A stream->skip = stream->pos = 0;
301N/A stream->istream.v_offset = 0;
301N/A zstream->high_pos = 0;
301N/A
301N/A lzma_end(&zstream->strm);
301N/A i_stream_lzma_init(zstream);
301N/A}
301N/A
301N/Astatic void
301N/Ai_stream_lzma_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
301N/A{
301N/A struct lzma_istream *zstream = (struct lzma_istream *) stream;
301N/A uoff_t start_offset = stream->istream.v_offset - stream->skip;
301N/A
301N/A if (v_offset < start_offset) {
301N/A /* have to seek backwards */
301N/A i_stream_lzma_reset(zstream);
301N/A start_offset = 0;
301N/A } else if (zstream->high_pos != 0) {
301N/A stream->pos = zstream->high_pos;
301N/A zstream->high_pos = 0;
301N/A }
301N/A
301N/A if (v_offset <= start_offset + stream->pos) {
301N/A /* seeking backwards within what's already cached */
301N/A stream->skip = v_offset - start_offset;
301N/A stream->istream.v_offset = v_offset;
301N/A zstream->high_pos = stream->pos;
301N/A stream->pos = stream->skip;
301N/A } else {
301N/A /* read and cache forward */
301N/A do {
301N/A size_t avail = stream->pos - stream->skip;
301N/A
301N/A if (stream->istream.v_offset + avail >= v_offset) {
0N/A i_stream_skip(&stream->istream,
0N/A v_offset -
0N/A stream->istream.v_offset);
0N/A break;
0N/A }
13N/A
136N/A i_stream_skip(&stream->istream, avail);
} while (i_stream_read(&stream->istream) >= 0);
if (stream->istream.v_offset != v_offset) {
/* some failure, we've broken it */
if (stream->istream.stream_errno != 0) {
i_error("lzma_istream.seek(%s) failed: %s",
i_stream_get_name(&stream->istream),
strerror(stream->istream.stream_errno));
i_stream_close(&stream->istream);
} else {
/* unexpected EOF. allow it since we may just
want to check if there's anything.. */
i_assert(stream->istream.eof);
}
}
}
if (mark)
zstream->marked = TRUE;
}
static int
i_stream_lzma_stat(struct istream_private *stream, bool exact)
{
struct lzma_istream *zstream = (struct lzma_istream *) stream;
const struct stat *st;
size_t size;
if (i_stream_stat(stream->parent, exact, &st) < 0) {
stream->istream.stream_errno = stream->parent->stream_errno;
return -1;
}
stream->statbuf = *st;
/* when exact=FALSE always return the parent stat's size, even if we
know the exact value. this is necessary because otherwise e.g. mbox
code can see two different values and think that a compressed mbox
file keeps changing. */
if (!exact)
return 0;
if (zstream->stream_size == (uoff_t)-1) {
uoff_t old_offset = stream->istream.v_offset;
do {
size = i_stream_get_data_size(&stream->istream);
i_stream_skip(&stream->istream, size);
} while (i_stream_read(&stream->istream) > 0);
i_stream_seek(&stream->istream, old_offset);
if (zstream->stream_size == (uoff_t)-1)
return -1;
}
stream->statbuf.st_size = zstream->stream_size;
return 0;
}
static void i_stream_lzma_sync(struct istream_private *stream)
{
struct lzma_istream *zstream = (struct lzma_istream *) stream;
const struct stat *st;
if (i_stream_stat(stream->parent, FALSE, &st) < 0) {
if (memcmp(&zstream->last_parent_statbuf,
st, sizeof(*st)) == 0) {
/* a compressed file doesn't change unexpectedly,
don't clear our caches unnecessarily */
return;
}
zstream->last_parent_statbuf = *st;
}
i_stream_lzma_reset(zstream);
}
struct istream *i_stream_create_lzma(struct istream *input, bool log_errors)
{
struct lzma_istream *zstream;
zstream = i_new(struct lzma_istream, 1);
zstream->eof_offset = (uoff_t)-1;
zstream->stream_size = (uoff_t)-1;
zstream->log_errors = log_errors;
i_stream_lzma_init(zstream);
zstream->istream.iostream.close = i_stream_lzma_close;
zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
zstream->istream.read = i_stream_lzma_read;
zstream->istream.seek = i_stream_lzma_seek;
zstream->istream.stat = i_stream_lzma_stat;
zstream->istream.sync = i_stream_lzma_sync;
zstream->istream.istream.readable_fd = FALSE;
zstream->istream.istream.blocking = input->blocking;
zstream->istream.istream.seekable = input->seekable;
return i_stream_create(&zstream->istream, input,
i_stream_get_fd(input));
}
#endif