istream-lz4.c revision 7f74811b78f8915e73dffc88bb49009e98b6846d
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen/* Copyright (c) 2013-2016 Dovecot authors, see the included COPYING file */
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#include "lib.h"
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#ifdef HAVE_LZ4
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch#include "buffer.h"
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#include "istream-private.h"
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#include "istream-zlib.h"
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#include "iostream-lz4.h"
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#include <lz4.h>
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainenstruct lz4_istream {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen struct istream_private istream;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch uoff_t stream_size;
e0cf44fb802a5e7aa4cbeee5e80ef8f3f6aecdbeStephan Bosch struct stat last_parent_statbuf;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen buffer_t *chunk_buf;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen uint32_t chunk_size, chunk_left, max_uncompressed_chunk_size;
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch bool log_errors:1;
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch bool marked:1;
e0cf44fb802a5e7aa4cbeee5e80ef8f3f6aecdbeStephan Bosch bool header_read:1;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen};
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainenstatic void i_stream_lz4_close(struct iostream_private *stream,
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen bool close_parent)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen{
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen struct lz4_istream *zstream = (struct lz4_istream *)stream;
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch if (zstream->chunk_buf != NULL)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen buffer_free(&zstream->chunk_buf);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (close_parent)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_stream_close(zstream->istream.parent);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen}
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Boschstatic void lz4_read_error(struct lz4_istream *zstream, const char *error)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen{
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen io_stream_set_error(&zstream->istream.iostream,
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen "lz4.read(%s): %s at %"PRIuUOFF_T,
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch i_stream_get_name(&zstream->istream.istream), error,
e0cf44fb802a5e7aa4cbeee5e80ef8f3f6aecdbeStephan Bosch i_stream_get_absolute_offset(&zstream->istream.istream));
e0cf44fb802a5e7aa4cbeee5e80ef8f3f6aecdbeStephan Bosch if (zstream->log_errors)
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch i_error("%s", zstream->istream.iostream.error);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen}
feba5e502b2131c9a1c766b7ef9ff041dbf71d1dStephan Bosch
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainenstatic int i_stream_lz4_read_header(struct lz4_istream *zstream)
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch{
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch const struct iostream_lz4_header *hdr;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen const unsigned char *data;
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch size_t size;
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch int ret;
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch ret = i_stream_read_bytes(zstream->istream.parent, &data, &size,
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch sizeof(*hdr));
6dad0888fcec8372f230941c70d8940b8c203b32Stephan Bosch if (ret < 0) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->istream.istream.stream_errno =
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch zstream->istream.parent->stream_errno;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return ret;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (ret == 0 && !zstream->istream.istream.eof)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return 0;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen hdr = (const void *)data;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (ret == 0 || memcmp(hdr->magic, IOSTREAM_LZ4_MAGIC,
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen IOSTREAM_LZ4_MAGIC_LEN) != 0) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen lz4_read_error(zstream, "wrong magic in header (not lz4 file?)");
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->istream.istream.stream_errno = EINVAL;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch return -1;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->max_uncompressed_chunk_size =
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen ((uint32_t)hdr->max_uncompressed_chunk_size[0] << 24) |
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen (hdr->max_uncompressed_chunk_size[1] << 16) |
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen (hdr->max_uncompressed_chunk_size[2] << 8) |
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen hdr->max_uncompressed_chunk_size[3];
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (zstream->max_uncompressed_chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch lz4_read_error(zstream, t_strdup_printf(
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch "lz4 max chunk size too large (%u > %u)",
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->max_uncompressed_chunk_size,
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen ISTREAM_LZ4_CHUNK_SIZE));
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->istream.istream.stream_errno = EINVAL;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return -1;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_stream_skip(zstream->istream.parent, sizeof(*hdr));
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return 1;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen}
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Boschstatic ssize_t i_stream_lz4_read(struct istream_private *stream)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen{
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen struct lz4_istream *zstream = (struct lz4_istream *)stream;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen const unsigned char *data;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen size_t size, max_size;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen int ret;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch if (!zstream->header_read) {
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch if ((ret = i_stream_lz4_read_header(zstream)) <= 0)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return ret;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->header_read = TRUE;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (zstream->chunk_left == 0) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen ret = i_stream_read_bytes(stream->parent, &data, &size,
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch if (ret < 0) {
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch stream->istream.stream_errno =
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch stream->parent->stream_errno;
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch if (stream->istream.stream_errno == 0) {
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch stream->istream.eof = TRUE;
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch zstream->stream_size = stream->istream.v_offset +
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch stream->pos - stream->skip;
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch }
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch return ret;
1175415b88ff168e367c77df23901eada13225b9Stephan Bosch }
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch if (ret == 0 && !stream->istream.eof)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return 0;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->chunk_size = zstream->chunk_left =
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen ((uint32_t)data[0] << 24) |
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen (data[1] << 16) | (data[2] << 8) | data[3];
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (zstream->chunk_size == 0 ||
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->chunk_size > ISTREAM_LZ4_CHUNK_SIZE) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen lz4_read_error(zstream, t_strdup_printf(
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch "invalid lz4 chunk size: %u", zstream->chunk_size));
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch stream->istream.stream_errno = EINVAL;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return -1;
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_stream_skip(stream->parent, IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen buffer_set_used_size(zstream->chunk_buf, 0);
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch /* read the whole compressed chunk into memory */
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch while (zstream->chunk_left > 0 &&
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch (ret = i_stream_read_more(zstream->istream.parent, &data, &size)) > 0) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (size > zstream->chunk_left)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch size = zstream->chunk_left;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch buffer_append(zstream->chunk_buf, data, size);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_skip(zstream->istream.parent, size);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch zstream->chunk_left -= size;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (zstream->chunk_left > 0) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (ret == -1 && zstream->istream.parent->stream_errno == 0) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch lz4_read_error(zstream, "truncated lz4 chunk");
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->istream.stream_errno = EINVAL;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return -1;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch zstream->istream.istream.stream_errno =
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch zstream->istream.parent->stream_errno;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return ret;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen /* if we already have max_buffer_size amount of data, fail here */
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_stream_compress(stream);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (stream->pos >= i_stream_get_max_buffer_size(&stream->istream))
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen return -2;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen /* allocate enough space for the old data and the new
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen decompressed chunk. we don't know the original compressed size,
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen so just allocate the max amount of memory. */
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch max_size = stream->pos + zstream->max_uncompressed_chunk_size;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (stream->buffer_size < max_size) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->w_buffer = i_realloc(stream->w_buffer,
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->buffer_size, max_size);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->buffer_size = max_size;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->buffer = stream->w_buffer;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch ret = LZ4_decompress_safe(zstream->chunk_buf->data,
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch (void *)(stream->w_buffer + stream->pos),
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->chunk_buf->used,
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->buffer_size - stream->pos);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_assert(ret <= (int)zstream->max_uncompressed_chunk_size);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (ret < 0) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen lz4_read_error(zstream, "corrupted lz4 chunk");
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->istream.stream_errno = EINVAL;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return -1;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_assert(ret > 0);
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch stream->pos += ret;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch i_assert(stream->pos <= stream->buffer_size);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return ret;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch}
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainenstatic void i_stream_lz4_reset(struct lz4_istream *zstream)
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch{
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch struct istream_private *stream = &zstream->istream;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_stream_seek(stream->parent, stream->parent_start_offset);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch zstream->header_read = FALSE;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->chunk_size = zstream->chunk_left = 0;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->parent_expected_offset = stream->parent_start_offset;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->skip = stream->pos = 0;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->istream.v_offset = 0;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch}
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Boschstatic void
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Boschi_stream_lz4_seek(struct istream_private *stream, uoff_t v_offset, bool mark)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch{
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch struct lz4_istream *zstream = (struct lz4_istream *) stream;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen uoff_t start_offset = stream->istream.v_offset - stream->skip;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (v_offset < start_offset) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen /* have to seek backwards */
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen i_stream_lz4_reset(zstream);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen start_offset = 0;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen if (v_offset <= start_offset + stream->pos) {
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen /* seeking backwards within what's already cached */
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->skip = v_offset - start_offset;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->istream.v_offset = v_offset;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen stream->pos = stream->skip;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen } else {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch /* read and cache forward */
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen ssize_t ret;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch do {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch size_t avail = stream->pos - stream->skip;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (stream->istream.v_offset + avail >= v_offset) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_skip(&stream->istream,
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch v_offset -
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->istream.v_offset);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch break;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_skip(&stream->istream, avail);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch } while ((ret = i_stream_read(&stream->istream)) > 0);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_assert(ret == -1);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (stream->istream.v_offset != v_offset) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch /* some failure, we've broken it */
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (stream->istream.stream_errno != 0) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_error("lz4_istream.seek(%s) failed: %s",
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_get_name(&stream->istream),
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch strerror(stream->istream.stream_errno));
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_close(&stream->istream);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch } else {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch /* unexpected EOF. allow it since we may just
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch want to check if there's anything.. */
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_assert(stream->istream.eof);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (mark)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch zstream->marked = TRUE;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch}
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Boschstatic int
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Boschi_stream_lz4_stat(struct istream_private *stream, bool exact)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch{
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch struct lz4_istream *zstream = (struct lz4_istream *) stream;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch const struct stat *st;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch size_t size;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (i_stream_stat(stream->parent, exact, &st) < 0) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->istream.stream_errno = stream->parent->stream_errno;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return -1;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->statbuf = *st;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch /* when exact=FALSE always return the parent stat's size, even if we
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch know the exact value. this is necessary because otherwise e.g. mbox
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch code can see two different values and think that a compressed mbox
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch file keeps changing. */
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (!exact)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return 0;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (zstream->stream_size == (uoff_t)-1) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch uoff_t old_offset = stream->istream.v_offset;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch ssize_t ret;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch do {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch size = i_stream_get_data_size(&stream->istream);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_skip(&stream->istream, size);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch } while ((ret = i_stream_read(&stream->istream)) > 0);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_assert(ret == -1);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_seek(&stream->istream, old_offset);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (zstream->stream_size == (uoff_t)-1)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return -1;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch stream->statbuf.st_size = zstream->stream_size;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return 0;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch}
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Boschstatic void i_stream_lz4_sync(struct istream_private *stream)
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch{
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch struct lz4_istream *zstream = (struct lz4_istream *) stream;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch const struct stat *st;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (i_stream_stat(stream->parent, FALSE, &st) < 0) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch if (memcmp(&zstream->last_parent_statbuf,
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch st, sizeof(*st)) == 0) {
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch /* a compressed file doesn't change unexpectedly,
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch don't clear our caches unnecessarily */
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch return;
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch }
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->last_parent_statbuf = *st;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch }
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch i_stream_lz4_reset(zstream);
9e7bf91667639a2390207ab4d90bf88e2afcec2aStephan Bosch}
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainenstruct istream *i_stream_create_lz4(struct istream *input, bool log_errors)
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen{
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen struct lz4_istream *zstream;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream = i_new(struct lz4_istream, 1);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->stream_size = (uoff_t)-1;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->log_errors = log_errors;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->istream.iostream.close = i_stream_lz4_close;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->istream.read = i_stream_lz4_read;
81498cffac5fb96fcbab6a30418017b04ffb59d1Stephan Bosch zstream->istream.seek = i_stream_lz4_seek;
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch zstream->istream.stat = i_stream_lz4_stat;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->istream.sync = i_stream_lz4_sync;
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen zstream->istream.istream.readable_fd = FALSE;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->istream.istream.blocking = input->blocking;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->istream.istream.seekable = input->seekable;
208dcaf62332b80b220c8c66e776f7cc0c39253bStephan Bosch zstream->chunk_buf = buffer_create_dynamic(default_pool, 1024);
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch return i_stream_create(&zstream->istream, input,
e8f1e510df3ab051a816715c2056f0d10aee929eStephan Bosch i_stream_get_fd(input));
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen}
b72c3363092b73cab1da2de4a9d75592e7d8fd6bTimo Sirainen#endif
81498cffac5fb96fcbab6a30418017b04ffb59d1Stephan Bosch