src/lib-index/mail-cache-compress.c

	mail-cache-compress.c revision 6237f743bbaf74de5a2d2051672baed87023657b
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (C) 2003-2004 Timo Sirainen */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "lib.h"
2423da95ee20fd4b3c260c1389cf2952d25f099cTimo Sirainen#include "array.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "ostream.h"
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen#include "nfs-workarounds.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "read-full.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "close-keep-errno.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "file-dotlock.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "file-cache.h"
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen#include "file-set-size.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "mail-cache-private.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include <sys/stat.h>
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainenstruct mail_cache_copy_context {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_cache *cache;
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen    buffer_t *buffer, *field_seen;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    ARRAY_DEFINE(bitmask_pos, unsigned int);
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    uint8_t field_seen_value;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    bool new_msg;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen};
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainenstatic void
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainenmail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen             const struct mail_cache_iterate_field *field)
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen{
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    unsigned char *dest;
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    unsigned int i, *pos;
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen    if (*pos == 0) {
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen        /* we decided to drop this field */
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen        return;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    for (i = 0; i < field->size; i++)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        dest[i] |= ((const unsigned char*)field->data)[i];
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainenstatic void
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenmail_cache_compress_field(struct mail_cache_copy_context *ctx,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen              const struct mail_cache_iterate_field *field)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    uint32_t field_idx = field->field_idx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        struct mail_cache_field *cache_field;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    enum mail_cache_decision_type dec;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    uint8_t *field_seen;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    uint32_t size32;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    cache_field = &ctx->cache->fields[field_idx].field;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    field_seen = buffer_get_space_unsafe(ctx->field_seen, field_idx, 1);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (*field_seen == ctx->field_seen_value) {
c040ee67d0ac0fb7375bb543965bf67dcae6affaTimo Sirainen        /* duplicate */
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen        if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
533bfba437e4120aa29dd45bca2aa87e30ee28a2Timo Sirainen            mail_cache_merge_bitmask(ctx, field);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        return;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen    *field_seen = ctx->field_seen_value;
d92f33f13830ba23d814342bf3ea8db721a15bb1Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
61e84692827b6a64912343f515c984853021483aTimo Sirainen    if (ctx->new_msg) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        if (dec == MAIL_CACHE_DECISION_NO)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            return;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    } else {
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen        if (dec != MAIL_CACHE_DECISION_YES)
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen            return;
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    }
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    buffer_append(ctx->buffer, &field_idx, sizeof(field_idx));
61b0637759146621cbb7edcbd0b03a71cfd66dfeTimo Sirainen
dca6d617a23e3f93af3b8df59acb46478179fe55Timo Sirainen    if (cache_field->field_size == (unsigned int)-1) {
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen        size32 = (uint32_t)field->size;
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen        buffer_append(ctx->buffer, &size32, sizeof(size32));
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen    }
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen
4ba962c3e78f140facdcfb1e093c4c46de75ae24Timo Sirainen    if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        /* remember the position in case we need to update it */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        unsigned int pos = ctx->buffer->used;
3c493c276f599d9b9cd10764876d648003046954Timo Sirainen
a91bd6256b33729531c33ff8bc66ee1ae95840f9Timo Sirainen        array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen    }
2f30b72d49fbff0c4096125c139e4bdfef45669cTimo Sirainen    buffer_append(ctx->buffer, field->data, field->size);
0b25846ba794ce19536a24d4065beaf2a0bd9464Timo Sirainen    if ((field->size & 3) != 0)
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen        buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen}
91b203fd2132510a47a4b34252c0ae0efd688a19Timo Sirainen
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainenstatic uint32_t
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainenget_next_file_seq(struct mail_cache *cache, struct mail_index_view *view)
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen{
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    const struct mail_index_ext *ext;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    uint32_t file_seq;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen    file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
8854395cdd21ca521b37ce669f3acb8445792b20Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        file_seq = cache->hdr->file_seq + 1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    return file_seq != 0 ? file_seq : 1;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen}
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenstatic int
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainenmail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen        int fd, uint32_t *file_seq_r,
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen        ARRAY_TYPE(uint32_t) *ext_offsets)
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        struct mail_cache_copy_context ctx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_cache_lookup_iterate_ctx iter;
bbd0a870f8639767e4e4011d2aedadac08d5c66fTimo Sirainen    struct mail_cache_iterate_field field;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_index_view *view;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_cache_view *cache_view;
539977f9257bd8985be5a8093658da266ae9cd19Timo Sirainen    const struct mail_index_header *idx_hdr;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_cache_header hdr;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct mail_cache_record cache_rec;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    struct ostream *output;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen    buffer_t *buffer;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    uint32_t i, message_count, seq, first_new_seq, ext_offset;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    view = mail_index_transaction_get_view(trans);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    /* get sequence of first message which doesn't need its temp fields
106b804c819443791f1324f8bbe34429eeea6a13Timo Sirainen       removed. */
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen    idx_hdr = mail_index_get_header(view);
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen    if (idx_hdr->day_first_uid[7] == 0) {
e156adefc1260d31a145df2f5e9b3c82050d4163Timo Sirainen        first_new_seq = 1;
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen        message_count = mail_index_view_get_messages_count(view);
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen    } else {
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen        if (mail_index_lookup_uid_range(view, idx_hdr->day_first_uid[7],
b13f738e8eb3f24dc2abf2c804f954b4d864ac6fTimo Sirainen                        (uint32_t)-1, &first_new_seq,
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen                        &message_count) < 0)
3fe67ec75ccae1230bb9eb9f16affc48377f6441Timo Sirainen            return -1;
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen        if (first_new_seq == 0)
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen            first_new_seq = message_count+1;
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen    }
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen    cache_view = mail_cache_view_open(cache, view);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    output = o_stream_create_file(fd, default_pool, 0, FALSE);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    memset(&hdr, 0, sizeof(hdr));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    hdr.version = MAIL_CACHE_VERSION;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen    hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen    hdr.indexid = cache->index->indexid;
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen    hdr.file_seq = get_next_file_seq(cache, view);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    o_stream_send(output, &hdr, sizeof(hdr));
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    memset(&ctx, 0, sizeof(ctx));
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    ctx.cache = cache;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    ctx.buffer = buffer_create_dynamic(default_pool, 4096);
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen    ctx.field_seen = buffer_create_dynamic(default_pool, 64);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    ctx.field_seen_value = 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    t_array_init(&ctx.bitmask_pos, 32);
7ff6268cc35102675d73d44d680bed13d0709f7bTimo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    t_array_init(ext_offsets, message_count);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    for (seq = 1; seq <= message_count; seq++) {
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen        if (mail_index_transaction_is_expunged(trans, seq)) {
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen            (void)array_append_space(ext_offsets);
eb1572d7c44ebc7b0b039d085c3dbab2ef7043ddTimo Sirainen            continue;
ed354926406e28254b581f821bb052f38d9c14e8Timo Sirainen        }
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen        ctx.new_msg = seq >= first_new_seq;
9f46aa48a9982567a37bb08dd95af8bee5100c7eTimo Sirainen        buffer_set_used_size(ctx.buffer, 0);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        if (++ctx.field_seen_value == 0) {
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen            memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen                   0, buffer_get_size(ctx.field_seen));
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen            ctx.field_seen_value++;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        }
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        memset(&cache_rec, 0, sizeof(cache_rec));
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        mail_cache_lookup_iter_init(cache_view, seq, &iter);
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        while (mail_cache_lookup_iter_next(&iter, &field) > 0)
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen            mail_cache_compress_field(&ctx, &field);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        cache_rec.size = buffer_get_used_size(ctx.buffer);
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen        if (cache_rec.size == sizeof(cache_rec)) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen            /* nothing cached */
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen            ext_offset = 0;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen        } else {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen            ext_offset = output->offset;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen            buffer_write(ctx.buffer, 0, &cache_rec,
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen                     sizeof(cache_rec));
4645cc6c911a95991d7af43b40f88e99506ea5e9Timo Sirainen            o_stream_send(output, ctx.buffer->data, cache_rec.size);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen        array_append(ext_offsets, &ext_offset, 1);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen    }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen    i_assert(array_count(ext_offsets) == message_count);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
e0ca8f2484847b57e20798a9f9c7040708696a90Timo Sirainen    if (cache->fields_count != 0) {
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen        hdr.field_header_offset =
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen            mail_index_uint32_to_offset(output->offset);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen        /* we wrote everything using our internal field ids. so we want
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen           mail_cache_header_fields_get() to use them and ignore any
b039dabf4c53f72454e795930e7643b6e0e625f9Timo Sirainen           existing id mappings in the old cache file. */
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        cache->file_fields_count = 0;
4366a21968093172d9b757fe6894b1ee8916434eTimo Sirainen        for (i = 0; i < cache->fields_count; i++)
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen                        cache->field_file_map[i] = (uint32_t)-1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        t_push();
96f2533c48ce5def0004931606a2fdf275578880Timo Sirainen        buffer = buffer_create_dynamic(pool_datastack_create(), 256);
96f2533c48ce5def0004931606a2fdf275578880Timo Sirainen        mail_cache_header_fields_get(cache, buffer);
4d84348ffcbb60de566108562c95ad64629e7a53Timo Sirainen        o_stream_send(output, buffer_get_data(buffer, NULL),
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen                  buffer_get_used_size(buffer));
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen        t_pop();
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    hdr.used_file_size = output->offset;
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen    buffer_free(ctx.buffer);
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen    buffer_free(ctx.field_seen);
0185427dd52fddec6fc76d6e99c7659620d4366eTimo Sirainen
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen    o_stream_seek(output, 0);
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen    o_stream_send(output, &hdr, sizeof(hdr));
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    mail_cache_view_close(cache_view);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen    if (o_stream_flush(output) < 0) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen        errno = output->stream_errno;
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen        mail_cache_set_syscall_error(cache, "o_stream_flush()");
5fb3bff645380804c9db2510940c41db6b8fdb01Timo Sirainen        o_stream_destroy(&output);
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen        return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen    }
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
    if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) {
        /* grow the file some more. doesn't matter if it fails */
        (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE);
    }

    o_stream_destroy(&output);

    if (!cache->index->fsync_disable) {
        if (fdatasync(fd) < 0) {
            mail_cache_set_syscall_error(cache, "fdatasync()");
            return -1;
        }
    }

    *file_seq_r = hdr.file_seq;
    return 0;
}

static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
{
    struct mail_cache_header hdr;
    unsigned int i;
    int fd, ret;

    for (i = 0;; i++) {
        fd = nfs_safe_open(cache->filepath, O_RDONLY);
        if (fd == -1) {
            if (errno == ENOENT)
                return 0;

            mail_cache_set_syscall_error(cache, "open()");
            return -1;
        }

        mail_cache_flush_read_cache(cache, FALSE);
        ret = read_full(fd, &hdr, sizeof(hdr));
        close_keep_errno(fd);

        if (ret >= 0) {
            if (ret == 0)
                return 0;
            if (cache->need_compress_file_seq == 0) {
                /* previously it didn't exist */
                return 1;
            }
            return hdr.file_seq != cache->need_compress_file_seq;
        } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
            mail_cache_set_syscall_error(cache, "read()");
            return -1;
        }
    }
}

static int mail_cache_compress_locked(struct mail_cache *cache,
                      struct mail_index_transaction *trans,
                      bool *unlock)
{
    struct dotlock *dotlock;
        mode_t old_mask;
    uint32_t file_seq, old_offset;
    ARRAY_TYPE(uint32_t) ext_offsets;
    const uint32_t *offsets;
    unsigned int i, count;
    int fd, ret;

    if (cache->fd != -1)
        mail_cache_flush_read_cache(cache, TRUE);

    /* get the latest info on fields */
    if (mail_cache_header_fields_read(cache) < 0)
        return -1;

    old_mask = umask(cache->index->mode ^ 0666);
    fd = file_dotlock_open(&cache->dotlock_settings, cache->filepath,
                   0, &dotlock);
    umask(old_mask);

    if (fd == -1) {
        mail_cache_set_syscall_error(cache, "file_dotlock_open()");
        return -1;
    }

    if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
        if (ret < 0)
            return -1;

        /* was just compressed, forget this */
        cache->need_compress_file_seq = 0;
        file_dotlock_delete(&dotlock);

        if (*unlock) {
            (void)mail_cache_unlock(cache);
            *unlock = FALSE;
        }

        return mail_cache_reopen(cache);
    }

    if (cache->index->gid != (gid_t)-1 &&
        fchown(fd, (uid_t)-1, cache->index->gid) < 0) {
        mail_cache_set_syscall_error(cache, "fchown()");
        file_dotlock_delete(&dotlock);
        return -1;
    }

    t_push();
    if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets) < 0) {
        (void)file_dotlock_delete(&dotlock);
        t_pop();
        return -1;
    }

    if (file_dotlock_replace(&dotlock,
                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
        mail_cache_set_syscall_error(cache,
                         "file_dotlock_replace()");
        (void)close(fd);
        t_pop();
        return -1;
    }

    /* once we're sure that the compression was successful,
       update the offsets */
    mail_index_ext_reset(trans, cache->ext_id, file_seq);
    offsets = array_get(&ext_offsets, &count);
    for (i = 0; i < count; i++) {
        if (offsets[i] != 0) {
            mail_index_update_ext(trans, i + 1, cache->ext_id,
                          &offsets[i], &old_offset);
        }
    }
    t_pop();

    if (*unlock) {
        (void)mail_cache_unlock(cache);
        *unlock = FALSE;
    }

    mail_cache_file_close(cache);
    cache->fd = fd;

    if (cache->file_cache != NULL)
        file_cache_set_fd(cache->file_cache, cache->fd);

    if (mail_cache_map(cache, 0, 0) < 0)
        return -1;
    if (mail_cache_header_fields_read(cache) < 0)
        return -1;

    cache->need_compress_file_seq = 0;
    return 0;
}

int mail_cache_compress(struct mail_cache *cache,
            struct mail_index_transaction *trans)
{
    bool unlock = FALSE;
    int ret;

    if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
        return 0;

    if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
        /* we're using dotlocking, cache file creation itself creates
           the dotlock file we need. */
        return mail_cache_compress_locked(cache, trans, &unlock);
    }

    switch (mail_cache_lock(cache, FALSE)) {
    case -1:
        return -1;
    case 0:
        /* couldn't lock, either it's broken or doesn't exist.
           just start creating it. */
        return mail_cache_compress_locked(cache, trans, &unlock);
    default:
        /* locking succeeded. */
        unlock = TRUE;
        ret = mail_cache_compress_locked(cache, trans, &unlock);
        if (unlock) {
            if (mail_cache_unlock(cache) < 0)
                ret = -1;
        }
        return ret;
    }
}

bool mail_cache_need_compress(struct mail_cache *cache)
{
    return cache->need_compress_file_seq != 0;
}