src/lib-index/mail-cache-compress.c

	mail-cache-compress.c revision 6157a322f2ac1ea1332d9003ecb0b11466aa8fe7
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "lib.h"
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen#include "array.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "ostream.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "nfs-workarounds.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "read-full.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "close-keep-errno.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "file-dotlock.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include "file-cache.h"
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen#include "file-set-size.h"
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen#include "mail-cache-private.h"
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen#include <sys/stat.h>
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenstruct mail_cache_copy_context {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    struct mail_cache *cache;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    buffer_t *buffer, *field_seen;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    ARRAY_DEFINE(bitmask_pos, unsigned int);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    uint32_t *field_file_map;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    uint8_t field_seen_value;
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen    bool new_msg;
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen};
072f06b60d69fe9456c3fffe20e72a7d09f2825dTimo Sirainen
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainenstatic void
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainenmail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen             const struct mail_cache_iterate_field *field)
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen{
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    unsigned char *dest;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    unsigned int i, *pos;
395682d473b161c86165e7b9323ce4e45afb94bdTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    if (*pos == 0) {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        /* we decided to drop this field */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        return;
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen    }
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    for (i = 0; i < field->size; i++)
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        dest[i] |= ((const unsigned char*)field->data)[i];
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen}
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainenstatic void
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainenmail_cache_compress_field(struct mail_cache_copy_context *ctx,
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen              const struct mail_cache_iterate_field *field)
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen{
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        struct mail_cache_field *cache_field;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    enum mail_cache_decision_type dec;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    uint32_t file_field_idx, size32;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    uint8_t *field_seen;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    file_field_idx = ctx->field_file_map[field->field_idx];
51327f2489a4e0e615eb9f7d921473cf8512bb79Timo Sirainen    if (file_field_idx == (uint32_t)-1)
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        return;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    cache_field = &ctx->cache->fields[field->field_idx].field;
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    field_seen = buffer_get_space_unsafe(ctx->field_seen,
31a574fda352ef4f71dbff9c30e15e4744e132c0Timo Sirainen                         field->field_idx, 1);
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    if (*field_seen == ctx->field_seen_value) {
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        /* duplicate */
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen            mail_cache_merge_bitmask(ctx, field);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        return;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    }
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    *field_seen = ctx->field_seen_value;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen    if (ctx->new_msg) {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        if (dec == MAIL_CACHE_DECISION_NO)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen            return;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    } else {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        if (dec != MAIL_CACHE_DECISION_YES)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen            return;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    }
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    if (cache_field->field_size == (unsigned int)-1) {
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        size32 = (uint32_t)field->size;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen        buffer_append(ctx->buffer, &size32, sizeof(size32));
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    }
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen    if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen        /* remember the position in case we need to update it */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        unsigned int pos = ctx->buffer->used;
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen        array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    }
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen    buffer_append(ctx->buffer, field->data, field->size);
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen    if ((field->size & 3) != 0)
99695d99930b35c2bac85d52e976b44cf8485d83Timo Sirainen        buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen}
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenstatic uint32_t
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenget_next_file_seq(struct mail_cache *cache, struct mail_index_view *view)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen{
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    const struct mail_index_ext *ext;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    uint32_t file_seq;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        file_seq = cache->hdr->file_seq + 1;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    return file_seq != 0 ? file_seq : 1;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen}
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenstatic void
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainenmail_cache_compress_get_fields(struct mail_cache_copy_context *ctx,
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen                   unsigned int used_fields_count)
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen{
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    struct mail_cache *cache = ctx->cache;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    struct mail_cache_field *field;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    unsigned int i, j, idx;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    /* Make mail_cache_header_fields_get() return the fields in
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen       the same order as we saved them. */
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    memcpy(cache->field_file_map, ctx->field_file_map,
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen           sizeof(uint32_t) * cache->fields_count);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    /* reverse mapping */
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen    cache->file_fields_count = used_fields_count;
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen    i_free(cache->file_field_map);
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen    cache->file_field_map = used_fields_count == 0 ? NULL :
eb0816090cf5a549280ad783b9aa6fec199d36baTimo Sirainen        i_new(unsigned int, used_fields_count);
363929157786b549c80630bda3c3575f5115c6c5Timo Sirainen    for (i = j = 0; i < cache->fields_count; i++) {
4182d8cd818e76856a5a1e25b343fe5ddf69fd8eTimo Sirainen        idx = cache->field_file_map[i];
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen        if (idx != (uint32_t)-1) {
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen            i_assert(idx < used_fields_count &&
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen                 cache->file_field_map[idx] == 0);
c115c742f730e312d6b6ab5064595cd0d8b4e26eTimo Sirainen            cache->file_field_map[idx] = i;
            j++;
        }

        /* change permanent decisions to temporary decisions.
           if they're still permanent they'll get updated later. */
        field = &cache->fields[i].field;
        if (field->decision == MAIL_CACHE_DECISION_YES)
            field->decision = MAIL_CACHE_DECISION_TEMP;
    }
    i_assert(j == used_fields_count);

    buffer_set_used_size(ctx->buffer, 0);
    mail_cache_header_fields_get(cache, ctx->buffer);
}

static int
mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
        int fd, uint32_t *file_seq_r,
        ARRAY_TYPE(uint32_t) *ext_offsets)
{
        struct mail_cache_copy_context ctx;
    struct mail_cache_lookup_iterate_ctx iter;
    struct mail_cache_iterate_field field;
    struct mail_index_view *view;
    struct mail_cache_view *cache_view;
    const struct mail_index_header *idx_hdr;
    struct mail_cache_header hdr;
    struct mail_cache_record cache_rec;
    struct ostream *output;
    uint32_t message_count, seq, first_new_seq, ext_offset;
    unsigned int i, used_fields_count, orig_fields_count;
    time_t max_drop_time;

    view = mail_index_transaction_get_view(trans);

    /* get sequence of first message which doesn't need its temp fields
       removed. */
    idx_hdr = mail_index_get_header(view);
    if (idx_hdr->day_first_uid[7] == 0) {
        first_new_seq = 1;
        message_count = mail_index_view_get_messages_count(view);
    } else {
        if (!mail_index_lookup_seq_range(view,
                         idx_hdr->day_first_uid[7],
                         (uint32_t)-1, &first_new_seq,
                         &message_count))
            first_new_seq = message_count+1;
    }

    cache_view = mail_cache_view_open(cache, view);
    output = o_stream_create_fd_file(fd, 0, FALSE);

    memset(&hdr, 0, sizeof(hdr));
    hdr.version = MAIL_CACHE_VERSION;
    hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
    hdr.indexid = cache->index->indexid;
    hdr.file_seq = get_next_file_seq(cache, view);
    o_stream_send(output, &hdr, sizeof(hdr));

    memset(&ctx, 0, sizeof(ctx));
    ctx.cache = cache;
    ctx.buffer = buffer_create_dynamic(default_pool, 4096);
    ctx.field_seen = buffer_create_dynamic(default_pool, 64);
    ctx.field_seen_value = 0;
    ctx.field_file_map = t_new(uint32_t, cache->fields_count);
    t_array_init(&ctx.bitmask_pos, 32);

    /* @UNSAFE: drop unused fields and create a field mapping for
       used fields */
    max_drop_time = idx_hdr->day_stamp == 0 ? 0 :
        idx_hdr->day_stamp - MAIL_CACHE_FIELD_DROP_SECS;

    /* if some fields' "last used" time is zero, they were probably just
       added by us. change them to the current time. */
    for (i = 0; i < cache->fields_count; i++) {
        if (cache->fields[i].last_used == 0)
            cache->fields[i].last_used = ioloop_time;
    }

    orig_fields_count = cache->fields_count;
    if (cache->file_fields_count == 0) {
        /* creating the initial cache file. add all fields. */
        for (i = 0; i < orig_fields_count; i++)
            ctx.field_file_map[i] = i;
        used_fields_count = i;
    } else {
        for (i = used_fields_count = 0; i < orig_fields_count; i++) {
            if (cache->fields[i].last_used < max_drop_time)
                cache->fields[i].used = FALSE;

            ctx.field_file_map[i] = !cache->fields[i].used ?
                (uint32_t)-1 : used_fields_count++;
        }
    }

    i_array_init(ext_offsets, message_count);
    for (seq = 1; seq <= message_count; seq++) {
        if (mail_index_transaction_is_expunged(trans, seq)) {
            (void)array_append_space(ext_offsets);
            continue;
        }

        ctx.new_msg = seq >= first_new_seq;
        buffer_set_used_size(ctx.buffer, 0);

        if (++ctx.field_seen_value == 0) {
            memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
                   0, buffer_get_size(ctx.field_seen));
            ctx.field_seen_value++;
        }

        memset(&cache_rec, 0, sizeof(cache_rec));
        buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));

        mail_cache_lookup_iter_init(cache_view, seq, &iter);
        while (mail_cache_lookup_iter_next(&iter, &field) > 0)
            mail_cache_compress_field(&ctx, &field);

        cache_rec.size = buffer_get_used_size(ctx.buffer);
        if (cache_rec.size == sizeof(cache_rec)) {
            /* nothing cached */
            ext_offset = 0;
        } else {
            ext_offset = output->offset;
            buffer_write(ctx.buffer, 0, &cache_rec,
                     sizeof(cache_rec));
            o_stream_send(output, ctx.buffer->data, cache_rec.size);
        }

        array_append(ext_offsets, &ext_offset, 1);
    }
    i_assert(array_count(ext_offsets) == message_count);
    i_assert(orig_fields_count == cache->fields_count);

    hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
    mail_cache_compress_get_fields(&ctx, used_fields_count);
    o_stream_send(output, ctx.buffer->data, ctx.buffer->used);

    hdr.used_file_size = output->offset;
    buffer_free(&ctx.buffer);
    buffer_free(&ctx.field_seen);

    o_stream_seek(output, 0);
    o_stream_send(output, &hdr, sizeof(hdr));

    mail_cache_view_close(cache_view);

    if (o_stream_flush(output) < 0) {
        errno = output->stream_errno;
        mail_cache_set_syscall_error(cache, "o_stream_flush()");
        o_stream_destroy(&output);
        array_free(ext_offsets);
        return -1;
    }

    if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) {
        /* grow the file some more. doesn't matter if it fails */
        (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE);
    }

    o_stream_destroy(&output);

    if (!cache->index->fsync_disable) {
        if (fdatasync(fd) < 0) {
            mail_cache_set_syscall_error(cache, "fdatasync()");
            array_free(ext_offsets);
            return -1;
        }
    }

    *file_seq_r = hdr.file_seq;
    return 0;
}

static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
{
    struct mail_cache_header hdr;
    unsigned int i;
    int fd, ret;

    for (i = 0;; i++) {
        fd = nfs_safe_open(cache->filepath, O_RDONLY);
        if (fd == -1) {
            if (errno == ENOENT)
                return 0;

            mail_cache_set_syscall_error(cache, "open()");
            return -1;
        }

        ret = read_full(fd, &hdr, sizeof(hdr));
        close_keep_errno(fd);

        if (ret >= 0) {
            if (ret == 0)
                return 0;
            if (cache->need_compress_file_seq == 0) {
                /* previously it didn't exist */
                return 1;
            }
            return hdr.file_seq != cache->need_compress_file_seq;
        } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
            mail_cache_set_syscall_error(cache, "read()");
            return -1;
        }
    }
}

static int mail_cache_compress_locked(struct mail_cache *cache,
                      struct mail_index_transaction *trans,
                      bool *unlock)
{
    struct dotlock *dotlock;
        mode_t old_mask;
    uint32_t file_seq, old_offset;
    ARRAY_TYPE(uint32_t) ext_offsets;
    const uint32_t *offsets;
    unsigned int i, count;
    int fd, ret;

    if (!MAIL_CACHE_IS_UNUSABLE(cache))
        mail_cache_flush_read_cache(cache, TRUE);

    /* get the latest info on fields */
    if (mail_cache_header_fields_read(cache) < 0)
        return -1;

    old_mask = umask(cache->index->mode ^ 0666);
    fd = file_dotlock_open(&cache->dotlock_settings, cache->filepath,
                   0, &dotlock);
    umask(old_mask);

    if (fd == -1) {
        mail_cache_set_syscall_error(cache, "file_dotlock_open()");
        return -1;
    }

    if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
        if (ret < 0)
            return -1;

        /* was just compressed, forget this */
        cache->need_compress_file_seq = 0;
        file_dotlock_delete(&dotlock);

        if (*unlock) {
            (void)mail_cache_unlock(cache);
            *unlock = FALSE;
        }

        return mail_cache_reopen(cache);
    }

    if (cache->index->gid != (gid_t)-1 &&
        fchown(fd, (uid_t)-1, cache->index->gid) < 0) {
        mail_cache_set_syscall_error(cache, "fchown()");
        file_dotlock_delete(&dotlock);
        return -1;
    }

    if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets) < 0) {
        (void)file_dotlock_delete(&dotlock);
        return -1;
    }

    if (file_dotlock_replace(&dotlock,
                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
        mail_cache_set_syscall_error(cache,
                         "file_dotlock_replace()");
        (void)close(fd);
        array_free(&ext_offsets);
        return -1;
    }

    /* once we're sure that the compression was successful,
       update the offsets */
    mail_index_ext_reset(trans, cache->ext_id, file_seq);
    offsets = array_get(&ext_offsets, &count);
    for (i = 0; i < count; i++) {
        if (offsets[i] != 0) {
            mail_index_update_ext(trans, i + 1, cache->ext_id,
                          &offsets[i], &old_offset);
        }
    }
    array_free(&ext_offsets);

    if (*unlock) {
        (void)mail_cache_unlock(cache);
        *unlock = FALSE;
    }

    mail_cache_file_close(cache);
    cache->fd = fd;

    if (cache->file_cache != NULL)
        file_cache_set_fd(cache->file_cache, cache->fd);

    if (mail_cache_map(cache, 0, 0) < 0)
        return -1;
    if (mail_cache_header_fields_read(cache) < 0)
        return -1;

    cache->need_compress_file_seq = 0;
    return 0;
}

int mail_cache_compress(struct mail_cache *cache,
            struct mail_index_transaction *trans)
{
    bool unlock = FALSE;
    int ret;

    if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
        return 0;

    if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
        /* we're using dotlocking, cache file creation itself creates
           the dotlock file we need. */
        return mail_cache_compress_locked(cache, trans, &unlock);
    }

    switch (mail_cache_lock(cache, FALSE)) {
    case -1:
        return -1;
    case 0:
        /* couldn't lock, either it's broken or doesn't exist.
           just start creating it. */
        return mail_cache_compress_locked(cache, trans, &unlock);
    default:
        /* locking succeeded. */
        unlock = TRUE;
        ret = mail_cache_compress_locked(cache, trans, &unlock);
        if (unlock) {
            if (mail_cache_unlock(cache) < 0)
                ret = -1;
        }
        return ret;
    }
}

bool mail_cache_need_compress(struct mail_cache *cache)
{
    return cache->need_compress_file_seq != 0 &&
        !cache->index->readonly;
}