istream-lz4.c revision d868a04630bd7bfe9c1543a7c3f68703b3e276e4
294N/A/* Copyright (c) 2013-2016 Dovecot authors, see the included COPYING file */
294N/A
787N/A#include "lib.h"
789N/A
789N/A#ifdef HAVE_LZ4
789N/A
1336N/A#include "buffer.h"
789N/A#include "istream-private.h"
789N/A#include "istream-zlib.h"
873N/A#include "iostream-lz4.h"
789N/A#include <lz4.h>
877N/A
789N/Astruct lz4_istream {
294N/A struct istream_private istream;
873N/A
873N/A uoff_t stream_size;
789N/A struct stat last_parent_statbuf;
789N/A
789N/A buffer_t *chunk_buf;
873N/A uint32_t chunk_size, chunk_left, max_uncompressed_chunk_size;
789N/A
789N/A unsigned int log_errors:1;
789N/A unsigned int marked:1;
1393N/A unsigned int header_read:1;
1393N/A};
1393N/A
789N/Astatic void i_stream_lz4_close(struct iostream_private *stream,
1342N/A bool close_parent)
1342N/A{
789N/A struct lz4_istream *zstream = (struct lz4_istream *)stream;
789N/A
789N/A if (zstream->chunk_buf != NULL)
789N/A buffer_free(&zstream->chunk_buf);
789N/A if (close_parent)
1336N/A i_stream_close(zstream->istream.parent);
1336N/A}
789N/A
873N/Astatic void lz4_read_error(struct lz4_istream *zstream, const char *error)
1336N/A{
1336N/A io_stream_set_error(&zstream->istream.iostream,
1336N/A "lz4.read(%s): %s at %"PRIuUOFF_T,
1336N/A i_stream_get_name(&zstream->istream.istream), error,
1392N/A zstream->istream.abs_start_offset +
1336N/A zstream->istream.istream.v_offset);
1336N/A if (zstream->log_errors)
1336N/A i_error("%s", zstream->istream.iostream.error);
1336N/A}
1336N/A
1336N/Astatic int i_stream_lz4_read_header(struct lz4_istream *zstream)
1336N/A{
1336N/A const struct iostream_lz4_header *hdr;
1336N/A const unsigned char *data;
1336N/A size_t size;
1336N/A int ret;
1336N/A
1336N/A ret = i_stream_read_bytes(zstream->istream.parent, &data, &size,
873N/A sizeof(*hdr));
873N/A if (ret < 0) {
873N/A zstream->istream.istream.stream_errno =
789N/A zstream->istream.parent->stream_errno;
787N/A return ret;
294N/A }
1336N/A if (ret == 0 && !zstream->istream.istream.eof)
1336N/A return 0;
1336N/A hdr = (const void *)data;
1336N/A if (ret == 0 || memcmp(hdr->magic, IOSTREAM_LZ4_MAGIC,
1337N/A IOSTREAM_LZ4_MAGIC_LEN) != 0) {
294N/A lz4_read_error(zstream, "wrong magic in header (not lz4 file?)");
294N/A zstream->istream.istream.stream_errno = EINVAL;
294N/A return -1;
1336N/A }
1336N/A zstream->max_uncompressed_chunk_size =
1337N/A ((uint32_t)hdr->max_uncompressed_chunk_size[0] << 24) |
1337N/A (hdr->max_uncompressed_chunk_size[1] << 16) |
1431N/A (hdr->max_uncompressed_chunk_size[2] << 8) |
1337N/A hdr->max_uncompressed_chunk_size[3];
1337N/A if (zstream->max_uncompressed_chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
1337N/A lz4_read_error(zstream, t_strdup_printf(
1337N/A "lz4 max chunk size too large (%u > %u)",
1337N/A zstream->max_uncompressed_chunk_size,
1337N/A ISTREAM_LZ4_CHUNK_SIZE));
1336N/A zstream->istream.istream.stream_errno = EINVAL;
1337N/A return -1;
1337N/A }
1337N/A i_stream_skip(zstream->istream.parent, sizeof(*hdr));
294N/A return 1;
294N/A}
294N/A
294N/Astatic ssize_t i_stream_lz4_read(struct istream_private *stream)
789N/A{
294N/A struct lz4_istream *zstream = (struct lz4_istream *)stream;
1109N/A const unsigned char *data;
1109N/A size_t size, max_size;
1337N/A int ret;
1337N/A
1109N/A if (!zstream->header_read) {
1337N/A if ((ret = i_stream_lz4_read_header(zstream)) <= 0)
1337N/A return ret;
1109N/A zstream->header_read = TRUE;
787N/A }
869N/A
869N/A if (zstream->chunk_left == 0) {
869N/A ret = i_stream_read_bytes(stream->parent, &data, &size,
869N/A IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
789N/A if (ret < 0) {
789N/A stream->istream.stream_errno =
789N/A stream->parent->stream_errno;
789N/A if (stream->istream.stream_errno == 0) {
789N/A stream->istream.eof = TRUE;
789N/A zstream->stream_size = stream->istream.v_offset +
294N/A stream->pos - stream->skip;
789N/A }
294N/A return ret;
294N/A }
789N/A if (ret == 0 && !stream->istream.eof)
294N/A return 0;
294N/A zstream->chunk_size = zstream->chunk_left =
294N/A ((uint32_t)data[0] << 24) |
294N/A (data[1] << 16) | (data[2] << 8) | data[3];
294N/A if (zstream->chunk_size == 0 ||
294N/A zstream->chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
294N/A lz4_read_error(zstream, t_strdup_printf(
789N/A "invalid lz4 chunk size: %u", zstream->chunk_size));
789N/A stream->istream.stream_errno = EINVAL;
789N/A return -1;
787N/A }
789N/A i_stream_skip(stream->parent, IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
294N/A buffer_set_used_size(zstream->chunk_buf, 0);
869N/A }
868N/A
789N/A /* read the whole compressed chunk into memory */
789N/A while (zstream->chunk_left > 0 &&
869N/A (ret = i_stream_read_more(zstream->istream.parent, &data, &size)) > 0) {
869N/A if (size > zstream->chunk_left)
869N/A size = zstream->chunk_left;
869N/A buffer_append(zstream->chunk_buf, data, size);
789N/A i_stream_skip(zstream->istream.parent, size);
789N/A zstream->chunk_left -= size;
869N/A }
949N/A if (zstream->chunk_left > 0) {
789N/A if (ret == -1 && zstream->istream.parent->stream_errno == 0) {
789N/A lz4_read_error(zstream, "truncated lz4 chunk");
979N/A stream->istream.stream_errno = EINVAL;
1340N/A return -1;
1340N/A }
789N/A zstream->istream.istream.stream_errno =
787N/A zstream->istream.parent->stream_errno;
787N/A return ret;
294N/A }
294N/A /* if we already have max_buffer_size amount of data, fail here */
789N/A i_stream_compress(stream);
294N/A if (stream->pos >= stream->max_buffer_size)
294N/A return -2;
294N/A /* allocate enough space for the old data and the new
294N/A decompressed chunk. we don't know the original compressed size,
789N/A so just allocate the max amount of memory. */
294N/A max_size = stream->pos + zstream->max_uncompressed_chunk_size;
294N/A if (stream->buffer_size < max_size) {
294N/A stream->w_buffer = i_realloc(stream->w_buffer,
789N/A stream->buffer_size, max_size);
294N/A stream->buffer_size = max_size;
789N/A stream->buffer = stream->w_buffer;
294N/A }
789N/A ret = LZ4_decompress_safe(zstream->chunk_buf->data,
789N/A (void *)(stream->w_buffer + stream->pos),
911N/A zstream->chunk_buf->used,
911N/A stream->buffer_size - stream->pos);
911N/A i_assert(ret <= (int)zstream->max_uncompressed_chunk_size);
915N/A if (ret < 0) {
911N/A lz4_read_error(zstream, "corrupted lz4 chunk");
911N/A stream->istream.stream_errno = EINVAL;
911N/A return -1;
869N/A }
869N/A i_assert(ret > 0);
869N/A stream->pos += ret;
869N/A i_assert(stream->pos <= stream->buffer_size);
789N/A return ret;
857N/A}
789N/A
789N/Astatic void i_stream_lz4_reset(struct lz4_istream *zstream)
789N/A{
789N/A struct istream_private *stream = &zstream->istream;
789N/A
294N/A i_stream_seek(stream->parent, stream->parent_start_offset);
294N/A zstream->header_read = FALSE;
789N/A zstream->chunk_size = zstream->chunk_left = 0;
294N/A
1393N/A stream->parent_expected_offset = stream->parent_start_offset;
1393N/A stream->skip = stream->pos = 0;
1393N/A stream->istream.v_offset = 0;
1393N/A}
1393N/A
1393N/Astatic void
1393N/Ai_stream_lz4_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
294N/A{
789N/A struct lz4_istream *zstream = (struct lz4_istream *) stream;
294N/A uoff_t start_offset = stream->istream.v_offset - stream->skip;
294N/A
294N/A if (v_offset < start_offset) {
789N/A /* have to seek backwards */
294N/A i_stream_lz4_reset(zstream);
1339N/A start_offset = 0;
1339N/A }
1340N/A
1340N/A if (v_offset <= start_offset + stream->pos) {
877N/A /* seeking backwards within what's already cached */
869N/A stream->skip = v_offset - start_offset;
869N/A stream->istream.v_offset = v_offset;
868N/A stream->pos = stream->skip;
869N/A } else {
869N/A /* read and cache forward */
869N/A do {
868N/A size_t avail = stream->pos - stream->skip;
789N/A
941N/A if (stream->istream.v_offset + avail >= v_offset) {
1086N/A i_stream_skip(&stream->istream,
911N/A v_offset -
941N/A stream->istream.v_offset);
911N/A break;
294N/A }
1392N/A
941N/A i_stream_skip(&stream->istream, avail);
1392N/A } while (i_stream_read(&stream->istream) >= 0);
872N/A
941N/A if (stream->istream.v_offset != v_offset) {
872N/A /* some failure, we've broken it */
869N/A if (stream->istream.stream_errno != 0) {
994N/A i_error("lz4_istream.seek(%s) failed: %s",
1086N/A i_stream_get_name(&stream->istream),
994N/A strerror(stream->istream.stream_errno));
994N/A i_stream_close(&stream->istream);
994N/A } else {
994N/A /* unexpected EOF. allow it since we may just
869N/A want to check if there's anything.. */
1109N/A i_assert(stream->istream.eof);
1109N/A }
1109N/A }
1109N/A }
1109N/A
1109N/A if (mark)
868N/A zstream->marked = TRUE;
869N/A}
869N/A
869N/Astatic int
1428N/Ai_stream_lz4_stat(struct istream_private *stream, bool exact)
868N/A{
870N/A struct lz4_istream *zstream = (struct lz4_istream *) stream;
870N/A const struct stat *st;
870N/A size_t size;
870N/A
870N/A if (i_stream_stat(stream->parent, exact, &st) < 0) {
869N/A stream->istream.stream_errno = stream->parent->stream_errno;
869N/A return -1;
869N/A }
869N/A stream->statbuf = *st;
1336N/A
1336N/A /* when exact=FALSE always return the parent stat's size, even if we
1336N/A know the exact value. this is necessary because otherwise e.g. mbox
1336N/A code can see two different values and think that a compressed mbox
1336N/A file keeps changing. */
1336N/A if (!exact)
1336N/A return 0;
1336N/A
1336N/A if (zstream->stream_size == (uoff_t)-1) {
1336N/A uoff_t old_offset = stream->istream.v_offset;
1336N/A
1336N/A do {
1336N/A size = i_stream_get_data_size(&stream->istream);
869N/A i_stream_skip(&stream->istream, size);
869N/A } while (i_stream_read(&stream->istream) > 0);
1342N/A
868N/A i_stream_seek(&stream->istream, old_offset);
789N/A if (zstream->stream_size == (uoff_t)-1)
789N/A return -1;
789N/A }
789N/A stream->statbuf.st_size = zstream->stream_size;
789N/A return 0;
877N/A}
877N/A
877N/Astatic void i_stream_lz4_sync(struct istream_private *stream)
877N/A{
877N/A struct lz4_istream *zstream = (struct lz4_istream *) stream;
787N/A const struct stat *st;
294N/A
789N/A if (i_stream_stat(stream->parent, FALSE, &st) < 0) {
789N/A if (memcmp(&zstream->last_parent_statbuf,
789N/A st, sizeof(*st)) == 0) {
789N/A /* a compressed file doesn't change unexpectedly,
789N/A don't clear our caches unnecessarily */
789N/A return;
877N/A }
877N/A zstream->last_parent_statbuf = *st;
877N/A }
877N/A i_stream_lz4_reset(zstream);
877N/A}
789N/A
294N/Astruct istream *i_stream_create_lz4(struct istream *input, bool log_errors)
869N/A{
869N/A struct lz4_istream *zstream;
1091N/A
869N/A zstream = i_new(struct lz4_istream, 1);
869N/A zstream->stream_size = (uoff_t)-1;
869N/A zstream->log_errors = log_errors;
869N/A
1086N/A zstream->istream.iostream.close = i_stream_lz4_close;
1086N/A zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
1086N/A zstream->istream.read = i_stream_lz4_read;
1086N/A zstream->istream.seek = i_stream_lz4_seek;
294N/A zstream->istream.stat = i_stream_lz4_stat;
294N/A zstream->istream.sync = i_stream_lz4_sync;
294N/A
294N/A zstream->istream.istream.readable_fd = FALSE;
294N/A zstream->istream.istream.blocking = input->blocking;
294N/A zstream->istream.istream.seekable = input->seekable;
294N/A zstream->chunk_buf = buffer_create_dynamic(default_pool, 1024);
294N/A
294N/A return i_stream_create(&zstream->istream, input,
294N/A i_stream_get_fd(input));
869N/A}
294N/A#endif
294N/A