src/lib-index/mail-cache-compress.c

	mail-cache-compress.c revision 6157a322f2ac1ea1332d9003ecb0b11466aa8fe7
89a126810703c666309310d0f3189e9834d70b5bTimo Sirainen/* Copyright (c) 2003-2007 Dovecot authors, see the included COPYING file */
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "lib.h"
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen#include "array.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "ostream.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "nfs-workarounds.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "read-full.h"
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen#include "close-keep-errno.h"
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen#include "file-dotlock.h"
1098fc409a45e7603701dc94635927a673bee0c1Timo Sirainen#include "file-cache.h"
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen#include "file-set-size.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen#include "mail-cache-private.h"
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen#include <sys/stat.h>
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainenstruct mail_cache_copy_context {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache *cache;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    buffer_t *buffer, *field_seen;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    ARRAY_DEFINE(bitmask_pos, unsigned int);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    uint32_t *field_file_map;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint8_t field_seen_value;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    bool new_msg;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen};
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainenstatic void
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainenmail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen             const struct mail_cache_iterate_field *field)
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen{
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    unsigned char *dest;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    unsigned int i, *pos;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    if (*pos == 0) {
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        /* we decided to drop this field */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        return;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    }
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    for (i = 0; i < field->size; i++)
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        dest[i] |= ((const unsigned char*)field->data)[i];
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen}
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenstatic void
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainenmail_cache_compress_field(struct mail_cache_copy_context *ctx,
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen              const struct mail_cache_iterate_field *field)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen        struct mail_cache_field *cache_field;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    enum mail_cache_decision_type dec;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    uint32_t file_field_idx, size32;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    uint8_t *field_seen;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    file_field_idx = ctx->field_file_map[field->field_idx];
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    if (file_field_idx == (uint32_t)-1)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen        return;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    cache_field = &ctx->cache->fields[field->field_idx].field;
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    field_seen = buffer_get_space_unsafe(ctx->field_seen,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen                         field->field_idx, 1);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    if (*field_seen == ctx->field_seen_value) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        /* duplicate */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen            mail_cache_merge_bitmask(ctx, field);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        return;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    }
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    *field_seen = ctx->field_seen_value;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    if (ctx->new_msg) {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (dec == MAIL_CACHE_DECISION_NO)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            return;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    } else {
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (dec != MAIL_CACHE_DECISION_YES)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            return;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    }
1171f0abf442638bac1827bb24a0b6b8eb682a82Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
dd4b5f14b71b01a84af942e720a2d6e5f15ee1a7Timo Sirainen    if (cache_field->field_size == (unsigned int)-1) {
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        size32 = (uint32_t)field->size;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        buffer_append(ctx->buffer, &size32, sizeof(size32));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        /* remember the position in case we need to update it */
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        unsigned int pos = ctx->buffer->used;
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen        array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    }
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    buffer_append(ctx->buffer, field->data, field->size);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    if ((field->size & 3) != 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenstatic uint32_t
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainenget_next_file_seq(struct mail_cache *cache, struct mail_index_view *view)
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen{
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    const struct mail_index_ext *ext;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    uint32_t file_seq;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen    if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen        file_seq = cache->hdr->file_seq + 1;
6237f743bbaf74de5a2d2051672baed87023657bTimo Sirainen
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    return file_seq != 0 ? file_seq : 1;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen}
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainenstatic void
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainenmail_cache_compress_get_fields(struct mail_cache_copy_context *ctx,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen                   unsigned int used_fields_count)
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen{
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    struct mail_cache *cache = ctx->cache;
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen    struct mail_cache_field *field;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    unsigned int i, j, idx;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    /* Make mail_cache_header_fields_get() return the fields in
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen       the same order as we saved them. */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    memcpy(cache->field_file_map, ctx->field_file_map,
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen           sizeof(uint32_t) * cache->fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    /* reverse mapping */
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    cache->file_fields_count = used_fields_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    i_free(cache->file_field_map);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    cache->file_field_map = used_fields_count == 0 ? NULL :
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen        i_new(unsigned int, used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    for (i = j = 0; i < cache->fields_count; i++) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen        idx = cache->field_file_map[i];
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen        if (idx != (uint32_t)-1) {
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen            i_assert(idx < used_fields_count &&
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen                 cache->file_field_map[idx] == 0);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen            cache->file_field_map[idx] = i;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen            j++;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen        }
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen        /* change permanent decisions to temporary decisions.
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen           if they're still permanent they'll get updated later. */
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen        field = &cache->fields[i].field;
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen        if (field->decision == MAIL_CACHE_DECISION_YES)
fd44baee6e92f13d42ff711895d6510067f70955Timo Sirainen            field->decision = MAIL_CACHE_DECISION_TEMP;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    i_assert(j == used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    buffer_set_used_size(ctx->buffer, 0);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    mail_cache_header_fields_get(cache, ctx->buffer);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen}
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainenstatic int
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainenmail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        int fd, uint32_t *file_seq_r,
c740209eb3db031022c6fd436fa2ef1aa0ec46b8Timo Sirainen        ARRAY_TYPE(uint32_t) *ext_offsets)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        struct mail_cache_copy_context ctx;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_lookup_iterate_ctx iter;
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    struct mail_cache_iterate_field field;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    struct mail_index_view *view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    struct mail_cache_view *cache_view;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    const struct mail_index_header *idx_hdr;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    struct mail_cache_header hdr;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen    struct mail_cache_record cache_rec;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    struct ostream *output;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    uint32_t message_count, seq, first_new_seq, ext_offset;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    unsigned int i, used_fields_count, orig_fields_count;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    time_t max_drop_time;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    view = mail_index_transaction_get_view(trans);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
325d4ad220bd13f6d176391d962a0e33c856a7f6Timo Sirainen    /* get sequence of first message which doesn't need its temp fields
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen       removed. */
b20fb5b1df9d604a7541f5118fc5b4b466d211efTimo Sirainen    idx_hdr = mail_index_get_header(view);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    if (idx_hdr->day_first_uid[7] == 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        first_new_seq = 1;
df4018ae2f0a95be602f724ca70df7e0e3bd6a7dTimo Sirainen        message_count = mail_index_view_get_messages_count(view);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    } else {
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen        if (!mail_index_lookup_seq_range(view,
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen                         idx_hdr->day_first_uid[7],
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen                         (uint32_t)-1, &first_new_seq,
3da614c39dd29f536c485089e67839b4cf89fed3Timo Sirainen                         &message_count))
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen            first_new_seq = message_count+1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    }
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    cache_view = mail_cache_view_open(cache, view);
93fa87cf1a96c4f279ec4f5c311820313ba12c34Timo Sirainen    output = o_stream_create_fd_file(fd, 0, FALSE);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    memset(&hdr, 0, sizeof(hdr));
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    hdr.version = MAIL_CACHE_VERSION;
811f2e26d9782d9cb99fdf82e18ffa0a77564fe2Timo Sirainen    hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen    hdr.indexid = cache->index->indexid;
6a19e109ee8c5a6f688da83a86a7f6abeb71abddTimo Sirainen    hdr.file_seq = get_next_file_seq(cache, view);
4bbee99b3aef449a9a2a11a5b5cf1ca486915c49Timo Sirainen    o_stream_send(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    memset(&ctx, 0, sizeof(ctx));
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen    ctx.cache = cache;
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    ctx.buffer = buffer_create_dynamic(default_pool, 4096);
4b058f90f9e8a2c6b2eed275de4eb8cc5195a71dTimo Sirainen    ctx.field_seen = buffer_create_dynamic(default_pool, 64);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    ctx.field_seen_value = 0;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    ctx.field_file_map = t_new(uint32_t, cache->fields_count);
7d75b95c260e58ee5039fa3afd79e9fda3bc8002Timo Sirainen    t_array_init(&ctx.bitmask_pos, 32);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    /* @UNSAFE: drop unused fields and create a field mapping for
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen       used fields */
8d77ab87a57ae98532c62e5d1548bd131dee3074Timo Sirainen    max_drop_time = idx_hdr->day_stamp == 0 ? 0 :
8d77ab87a57ae98532c62e5d1548bd131dee3074Timo Sirainen        idx_hdr->day_stamp - MAIL_CACHE_FIELD_DROP_SECS;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen    /* if some fields' "last used" time is zero, they were probably just
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen       added by us. change them to the current time. */
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen    for (i = 0; i < cache->fields_count; i++) {
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen        if (cache->fields[i].last_used == 0)
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen            cache->fields[i].last_used = ioloop_time;
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen    }
6157a322f2ac1ea1332d9003ecb0b11466aa8fe7Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    orig_fields_count = cache->fields_count;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen    if (cache->file_fields_count == 0) {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen        /* creating the initial cache file. add all fields. */
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen        for (i = 0; i < orig_fields_count; i++)
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen            ctx.field_file_map[i] = i;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen        used_fields_count = i;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen    } else {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen        for (i = used_fields_count = 0; i < orig_fields_count; i++) {
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen            if (cache->fields[i].last_used < max_drop_time)
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen                cache->fields[i].used = FALSE;
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen            ctx.field_file_map[i] = !cache->fields[i].used ?
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen                (uint32_t)-1 : used_fields_count++;
fe6cf42464c36ab281c0e0740f0255be77453670Timo Sirainen        }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    }
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen    i_array_init(ext_offsets, message_count);
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    for (seq = 1; seq <= message_count; seq++) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        if (mail_index_transaction_is_expunged(trans, seq)) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            (void)array_append_space(ext_offsets);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            continue;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        }
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        ctx.new_msg = seq >= first_new_seq;
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        buffer_set_used_size(ctx.buffer, 0);
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        if (++ctx.field_seen_value == 0) {
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen            memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen                   0, buffer_get_size(ctx.field_seen));
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen            ctx.field_seen_value++;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        }
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        memset(&cache_rec, 0, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen        buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
992a13add4eea0810e4db0f042a595dddf85536aTimo Sirainen
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        mail_cache_lookup_iter_init(cache_view, seq, &iter);
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen        while (mail_cache_lookup_iter_next(&iter, &field) > 0)
6b85bc4b03e552cfaeeae872d63c2d8ac5fcb7c4Timo Sirainen            mail_cache_compress_field(&ctx, &field);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen        cache_rec.size = buffer_get_used_size(ctx.buffer);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        if (cache_rec.size == sizeof(cache_rec)) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            /* nothing cached */
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            ext_offset = 0;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        } else {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            ext_offset = output->offset;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            buffer_write(ctx.buffer, 0, &cache_rec,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen                     sizeof(cache_rec));
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            o_stream_send(output, ctx.buffer->data, cache_rec.size);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        }
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        array_append(ext_offsets, &ext_offset, 1);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    i_assert(array_count(ext_offsets) == message_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    i_assert(orig_fields_count == cache->fields_count);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    mail_cache_compress_get_fields(&ctx, used_fields_count);
9a583c7a827f7a4d89ee43774f2d51ea6a214543Timo Sirainen    o_stream_send(output, ctx.buffer->data, ctx.buffer->used);
4edf90751f075cc6ab3d6f53fc78b656efa80922Timo Sirainen
a40d26f83af808a0ea1e212c001d682a96d870b0Timo Sirainen    hdr.used_file_size = output->offset;
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen    buffer_free(&ctx.buffer);
0f66f12eb4cdbf47670975044c88d8f388bf92dfTimo Sirainen    buffer_free(&ctx.field_seen);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    o_stream_seek(output, 0);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    o_stream_send(output, &hdr, sizeof(hdr));
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    mail_cache_view_close(cache_view);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    if (o_stream_flush(output) < 0) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        errno = output->stream_errno;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        mail_cache_set_syscall_error(cache, "o_stream_flush()");
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen        o_stream_destroy(&output);
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen        array_free(ext_offsets);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        return -1;
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
aff7542e1d2f48b030560a4f01096a2cc3f671ceTimo Sirainen    if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) {
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        /* grow the file some more. doesn't matter if it fails */
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen        (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen    }
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
cd56a23e21f1df3f79648cf07e2f4385e2fadebbTimo Sirainen    o_stream_destroy(&output);
fd2f5fbc1f07aa93e2214a28cdf02437fb7d06c8Timo Sirainen
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen    if (!cache->index->fsync_disable) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen        if (fdatasync(fd) < 0) {
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen            mail_cache_set_syscall_error(cache, "fdatasync()");
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen            array_free(ext_offsets);
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen            return -1;
369a1084c500a9df7448ffa9409ce32e42060bc2Timo Sirainen        }
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    *file_seq_r = hdr.file_seq;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    return 0;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainenstatic int mail_cache_compress_has_file_changed(struct mail_cache *cache)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen{
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    struct mail_cache_header hdr;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    unsigned int i;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    int fd, ret;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    for (i = 0;; i++) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        fd = nfs_safe_open(cache->filepath, O_RDONLY);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        if (fd == -1) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            if (errno == ENOENT)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen                return 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            mail_cache_set_syscall_error(cache, "open()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        ret = read_full(fd, &hdr, sizeof(hdr));
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        close_keep_errno(fd);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        if (ret >= 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            if (ret == 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen                return 0;
c18e2301f4c81f2819711661890db7796d4450c3Timo Sirainen            if (cache->need_compress_file_seq == 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen                /* previously it didn't exist */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen                return 1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            return hdr.file_seq != cache->need_compress_file_seq;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            mail_cache_set_syscall_error(cache, "read()");
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen}
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainenstatic int mail_cache_compress_locked(struct mail_cache *cache,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen                      struct mail_index_transaction *trans,
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen                      bool *unlock)
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen{
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen    struct dotlock *dotlock;
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen        mode_t old_mask;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    uint32_t file_seq, old_offset;
c740209eb3db031022c6fd436fa2ef1aa0ec46b8Timo Sirainen    ARRAY_TYPE(uint32_t) ext_offsets;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    const uint32_t *offsets;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    unsigned int i, count;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    int fd, ret;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
f8523f81a8f5fafbb30bb09c0fbeb4d4625be5d5Timo Sirainen    if (!MAIL_CACHE_IS_UNUSABLE(cache))
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen        mail_cache_flush_read_cache(cache, TRUE);
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen    /* get the latest info on fields */
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    if (mail_cache_header_fields_read(cache) < 0)
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen        return -1;
e015e2f7e7f48874495f9df8b0dd192b7ffcb5ccTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen    old_mask = umask(cache->index->mode ^ 0666);
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen    fd = file_dotlock_open(&cache->dotlock_settings, cache->filepath,
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen                   0, &dotlock);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen    umask(old_mask);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    if (fd == -1) {
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen        mail_cache_set_syscall_error(cache, "file_dotlock_open()");
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        if (ret < 0)
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen            return -1;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        /* was just compressed, forget this */
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        cache->need_compress_file_seq = 0;
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        file_dotlock_delete(&dotlock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen            (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen            *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen        return mail_cache_reopen(cache);
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    }
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen    if (cache->index->gid != (gid_t)-1 &&
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen        fchown(fd, (uid_t)-1, cache->index->gid) < 0) {
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen        mail_cache_set_syscall_error(cache, "fchown()");
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen        file_dotlock_delete(&dotlock);
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen        return -1;
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen    }
d22390f33eedbd2413debabc0662dde5241b1aa6Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets) < 0) {
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen        (void)file_dotlock_delete(&dotlock);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        return -1;
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen    if (file_dotlock_replace(&dotlock,
13c6532dc104d23061e6901783ceb1ff8872c206Timo Sirainen                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        mail_cache_set_syscall_error(cache,
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen                         "file_dotlock_replace()");
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        (void)close(fd);
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen        array_free(&ext_offsets);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    /* once we're sure that the compression was successful,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen       update the offsets */
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    mail_index_ext_reset(trans, cache->ext_id, file_seq);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    offsets = array_get(&ext_offsets, &count);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    for (i = 0; i < count; i++) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        if (offsets[i] != 0) {
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            mail_index_update_ext(trans, i + 1, cache->ext_id,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen                          &offsets[i], &old_offset);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        }
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen    }
3140537477eecdc5ea0a99f77df233c297dae58dTimo Sirainen    array_free(&ext_offsets);
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    if (*unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        (void)mail_cache_unlock(cache);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        *unlock = FALSE;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    }
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    mail_cache_file_close(cache);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    cache->fd = fd;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    if (cache->file_cache != NULL)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        file_cache_set_fd(cache->file_cache, cache->fd);
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    if (mail_cache_map(cache, 0, 0) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    if (mail_cache_header_fields_read(cache) < 0)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
58be9d6bcc3800f5b3d76a064ee767fbe31a5a8aTimo Sirainen    cache->need_compress_file_seq = 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    return 0;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen}
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainenint mail_cache_compress(struct mail_cache *cache,
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen            struct mail_index_transaction *trans)
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen{
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    bool unlock = FALSE;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    int ret;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen    if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen        return 0;
4b231ca0bbe3b536acbd350101e183441ce0247aTimo Sirainen
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen    if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen        /* we're using dotlocking, cache file creation itself creates
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen           the dotlock file we need. */
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        return mail_cache_compress_locked(cache, trans, &unlock);
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen    }
365435901c22df7e5838f574c950b0e32e77f78aTimo Sirainen
df6478c4cf605bd81b3891c148b84c14eb6c4035Timo Sirainen    switch (mail_cache_lock(cache, FALSE)) {
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    case -1:
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        return -1;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    case 0:
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        /* couldn't lock, either it's broken or doesn't exist.
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen           just start creating it. */
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        return mail_cache_compress_locked(cache, trans, &unlock);
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    default:
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        /* locking succeeded. */
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        unlock = TRUE;
7ca9da144f79317f9a0d3b0b5cc1fe21e44a5bf4Timo Sirainen        ret = mail_cache_compress_locked(cache, trans, &unlock);
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        if (unlock) {
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen            if (mail_cache_unlock(cache) < 0)
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen                ret = -1;
d9fdacd5fb3e07997e5c389739d2054f0c8441d8Timo Sirainen        }
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen        return ret;
6fdf8b5e4e71a69f5974f59eec2b8c19bc421fe2Timo Sirainen    }
0cb2e8eb55e70f8ebe1e8349bdf49e4cbe5d8834Timo Sirainen}
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen
6ef7e31619edfaa17ed044b45861d106a86191efTimo Sirainenbool mail_cache_need_compress(struct mail_cache *cache)
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen{
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen    return cache->need_compress_file_seq != 0 &&
c53e8ee216904ffe6de4f6518d9f9f5107b7610eTimo Sirainen        !cache->index->readonly;
8aacc9e7c84f8376822823ec98c2f551d4919b2eTimo Sirainen}