src/lib-index/mail-cache-compress.c

	mail-cache-compress.c revision b780aa272b742a43579cdb523cc79cc8d4521306
7cb128dc4cae2a03a742f63ba7afee23c78e3af0Phil Carmody/* Copyright (c) 2003-2010 Dovecot authors, see the included COPYING file */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
08d6658a4e2ec8104cd1307f6baa75fdb07a24f8Mark Washenberger#include "lib.h"
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen#include "array.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "ostream.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "nfs-workarounds.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "read-full.h"
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen#include "close-keep-errno.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "file-dotlock.h"
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen#include "file-cache.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "file-set-size.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-cache-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include <sys/stat.h>
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b321df9603081896b70ec44635af96d674a9839aTimo Sirainenstruct mail_cache_copy_context {
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    struct mail_cache *cache;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    buffer_t *buffer, *field_seen;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    ARRAY_DEFINE(bitmask_pos, unsigned int);
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    uint32_t *field_file_map;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    uint8_t field_seen_value;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    bool new_msg;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen};
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainenstatic void
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenmail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen             const struct mail_cache_iterate_field *field)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    unsigned char *dest;
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    unsigned int i, *pos;
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen
9f431ccfb6932746db56245c8a3d3415717ef545Timo Sirainen    pos = array_idx_modifiable(&ctx->bitmask_pos, field->field_idx);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    if (*pos == 0) {
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        /* we decided to drop this field */
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen        return;
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen    }
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen    for (i = 0; i < field->size; i++)
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen        dest[i] |= ((const unsigned char*)field->data)[i];
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen}
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen
b4f35fa953a95e4d06fdae54d394095073fcfea0Timo Sirainenstatic void
b4f35fa953a95e4d06fdae54d394095073fcfea0Timo Sirainenmail_cache_compress_field(struct mail_cache_copy_context *ctx,
b4f35fa953a95e4d06fdae54d394095073fcfea0Timo Sirainen              const struct mail_cache_iterate_field *field)
b4f35fa953a95e4d06fdae54d394095073fcfea0Timo Sirainen{
b4f35fa953a95e4d06fdae54d394095073fcfea0Timo Sirainen        struct mail_cache_field *cache_field;
31e7be5e1d41a77f08d26cef46aba1df24b3f1baTimo Sirainen    enum mail_cache_decision_type dec;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    uint32_t file_field_idx, size32;
31e7be5e1d41a77f08d26cef46aba1df24b3f1baTimo Sirainen    uint8_t *field_seen;
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen    file_field_idx = ctx->field_file_map[field->field_idx];
c02a056b724abd6578fb8c4e439de0e94eaea6feTimo Sirainen    if (file_field_idx == (uint32_t)-1)
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen        return;
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    cache_field = &ctx->cache->fields[field->field_idx].field;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen    field_seen = buffer_get_space_unsafe(ctx->field_seen,
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen                         field->field_idx, 1);
517d1e7142d57299c733b30423e35e7e1f8d01d6Timo Sirainen    if (*field_seen == ctx->field_seen_value) {
a9c8c1f74e5d2911d3c15657727a30b649d3bbc4Timo Sirainen        /* duplicate */
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen        if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            mail_cache_merge_bitmask(ctx, field);
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen        return;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    *field_seen = ctx->field_seen_value;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen    dec = cache_field->decision & ~MAIL_CACHE_DECISION_FORCED;
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen    if (ctx->new_msg) {
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        if (dec == MAIL_CACHE_DECISION_NO)
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen            return;
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen    } else {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen        if (dec != MAIL_CACHE_DECISION_YES)
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen            return;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen    }
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen    buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen    if (cache_field->field_size == (unsigned int)-1) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen        size32 = (uint32_t)field->size;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen        buffer_append(ctx->buffer, &size32, sizeof(size32));
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    }
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        /* remember the position in case we need to update it */
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        unsigned int pos = ctx->buffer->used;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    }
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    buffer_append(ctx->buffer, field->data, field->size);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    if ((field->size & 3) != 0)
dc8ba4890b74bde400e0feef96114940129ab181Timo Sirainen        buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
dc8ba4890b74bde400e0feef96114940129ab181Timo Sirainen}
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainenstatic uint32_t
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainenget_next_file_seq(struct mail_cache *cache, struct mail_index_view *view)
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen{
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen    const struct mail_index_ext *ext;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    uint32_t file_seq;
1e7252421b9c85b898fef7e75bd6422ef1f046e4Timo Sirainen
31633d676642b83305b8d46da495d9bb4e2d1ff8Timo Sirainen    ext = mail_index_view_get_ext(view, cache->ext_id);
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen    file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
5af8ea0a24c5930a8e310ebc4f33fba1d084217cTimo Sirainen
1e7252421b9c85b898fef7e75bd6422ef1f046e4Timo Sirainen    if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        file_seq = cache->hdr->file_seq + 1;
17e09bf093bf968f383a90c559399656dffafe73Timo Sirainen
17e09bf093bf968f383a90c559399656dffafe73Timo Sirainen    return file_seq != 0 ? file_seq : 1;
17e09bf093bf968f383a90c559399656dffafe73Timo Sirainen}
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic void
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainenmail_cache_compress_get_fields(struct mail_cache_copy_context *ctx,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen                   unsigned int used_fields_count)
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen{
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    struct mail_cache *cache = ctx->cache;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    struct mail_cache_field *field;
1108376e39a19912e8394e64e19b1bc6f6691cf6Timo Sirainen    unsigned int i, j, idx;
a8e132559a7ebe54c8269d79ce29fa3338c76199Timo Sirainen
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    /* Make mail_cache_header_fields_get() return the fields in
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen       the same order as we saved them. */
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    memcpy(cache->field_file_map, ctx->field_file_map,
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen           sizeof(uint32_t) * cache->fields_count);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    /* reverse mapping */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    cache->file_fields_count = used_fields_count;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    i_free(cache->file_field_map);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    cache->file_field_map = used_fields_count == 0 ? NULL :
1108376e39a19912e8394e64e19b1bc6f6691cf6Timo Sirainen        i_new(unsigned int, used_fields_count);
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen    for (i = j = 0; i < cache->fields_count; i++) {
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        idx = cache->field_file_map[i];
1108376e39a19912e8394e64e19b1bc6f6691cf6Timo Sirainen        if (idx != (uint32_t)-1) {
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen            i_assert(idx < used_fields_count &&
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen                 cache->file_field_map != NULL &&
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen                 cache->file_field_map[idx] == 0);
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen            cache->file_field_map[idx] = i;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen            j++;
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        }
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen
f968e62caa52a8924bd05ebf76ff515b5c18e17bTimo Sirainen        /* change permanent decisions to temporary decisions.
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen           if they're still permanent they'll get updated later. */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        field = &cache->fields[i].field;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        if (field->decision == MAIL_CACHE_DECISION_YES)
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen            field->decision = MAIL_CACHE_DECISION_TEMP;
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen    }
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen    i_assert(j == used_fields_count);
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen    buffer_set_used_size(ctx->buffer, 0);
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen    mail_cache_header_fields_get(cache, ctx->buffer);
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen}
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainen
3c9783956dea385b322cd7fa6bf8c98c17a907a0Timo Sirainenstatic int
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenmail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        int fd, uint32_t *file_seq_r,
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        ARRAY_TYPE(uint32_t) *ext_offsets)
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen{
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        struct mail_cache_copy_context ctx;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    struct mail_cache_lookup_iterate_ctx iter;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    struct mail_cache_iterate_field field;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    struct mail_index_view *view;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    struct mail_cache_view *cache_view;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    const struct mail_index_header *idx_hdr;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    struct mail_cache_header hdr;
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen    struct mail_cache_record cache_rec;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    struct ostream *output;
a7bee3930831a9261fa6180d02af29c484d862e9Timo Sirainen    uint32_t message_count, seq, first_new_seq, ext_offset;
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    unsigned int i, used_fields_count, orig_fields_count;
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    time_t max_drop_time;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    view = mail_index_transaction_get_view(trans);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    cache_view = mail_cache_view_open(cache, view);
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    output = o_stream_create_fd_file(fd, 0, FALSE);
43d32cbe60fdaef2699d99f1ca259053e9350411Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    memset(&hdr, 0, sizeof(hdr));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    hdr.version = MAIL_CACHE_VERSION;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    hdr.indexid = cache->index->indexid;
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen    hdr.file_seq = get_next_file_seq(cache, view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    o_stream_send(output, &hdr, sizeof(hdr));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    memset(&ctx, 0, sizeof(ctx));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    ctx.cache = cache;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    ctx.buffer = buffer_create_dynamic(default_pool, 4096);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    ctx.field_seen = buffer_create_dynamic(default_pool, 64);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    ctx.field_seen_value = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    ctx.field_file_map = t_new(uint32_t, cache->fields_count + 1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    t_array_init(&ctx.bitmask_pos, 32);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    /* @UNSAFE: drop unused fields and create a field mapping for
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen       used fields */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    idx_hdr = mail_index_get_header(view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    max_drop_time = idx_hdr->day_stamp == 0 ? 0 :
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        idx_hdr->day_stamp - MAIL_CACHE_FIELD_DROP_SECS;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    orig_fields_count = cache->fields_count;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    if (cache->file_fields_count == 0) {
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen        /* creating the initial cache file. add all fields. */
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen        for (i = 0; i < orig_fields_count; i++)
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            ctx.field_file_map[i] = i;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen        used_fields_count = i;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen    } else {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen        for (i = used_fields_count = 0; i < orig_fields_count; i++) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen            struct mail_cache_field_private *priv =
484e12acec34f16e5a8adc001e23ae48f1dda8c7Timo Sirainen                &cache->fields[i];
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen            enum mail_cache_decision_type dec =
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen                priv->field.decision;
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen            /* if the decision isn't forced and this field hasn't
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen               been accessed for a while, drop it */
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen            if ((dec & MAIL_CACHE_DECISION_FORCED) == 0 &&
6135260095e1704ed6edff9d00bdfc043c11429cTimo Sirainen                (time_t)priv->last_used < max_drop_time &&
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen                !priv->adding) {
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen                dec = MAIL_CACHE_DECISION_NO;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen                priv->field.decision = dec;
48010d123abfac8cb19f33f1fe12f33a7090089eTimo Sirainen            }
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
484e12acec34f16e5a8adc001e23ae48f1dda8c7Timo Sirainen            /* drop all fields we don't want */
484e12acec34f16e5a8adc001e23ae48f1dda8c7Timo Sirainen            if ((dec & ~MAIL_CACHE_DECISION_FORCED) ==
484e12acec34f16e5a8adc001e23ae48f1dda8c7Timo Sirainen                MAIL_CACHE_DECISION_NO && !priv->adding) {
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen                priv->used = FALSE;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen                priv->last_used = 0;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            }
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            ctx.field_file_map[i] = !priv->used ?
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen                (uint32_t)-1 : used_fields_count++;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        }
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen    }
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen    /* get sequence of first message which doesn't need its temp fields
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen       removed. */
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    first_new_seq = mail_cache_get_first_new_seq(view);
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen    message_count = mail_index_view_get_messages_count(view);
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen    i_array_init(ext_offsets, message_count);
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen    for (seq = 1; seq <= message_count; seq++) {
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen        if (mail_index_transaction_is_expunged(trans, seq)) {
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            (void)array_append_space(ext_offsets);
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            continue;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        }
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        ctx.new_msg = seq >= first_new_seq;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        buffer_set_used_size(ctx.buffer, 0);
38ceb710e2bc957a66a75c68957cb87746682a75Timo Sirainen
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        if (++ctx.field_seen_value == 0) {
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen                   0, buffer_get_size(ctx.field_seen));
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen            ctx.field_seen_value++;
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        }
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen        memset(&cache_rec, 0, sizeof(cache_rec));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        mail_cache_lookup_iter_init(cache_view, seq, &iter);
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen        while (mail_cache_lookup_iter_next(&iter, &field) > 0)
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen            mail_cache_compress_field(&ctx, &field);
e665999b757e60bfb98e5a84a78b05f061453140Timo Sirainen
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        cache_rec.size = buffer_get_used_size(ctx.buffer);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        if (cache_rec.size == sizeof(cache_rec)) {
849969f639a00eab26791db3cb1b66430420c0cdTimo Sirainen            /* nothing cached */
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen            ext_offset = 0;
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        } else {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            ext_offset = output->offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            buffer_write(ctx.buffer, 0, &cache_rec,
f3d506e525a720f214020ca0f989a1966b30edaeTimo Sirainen                     sizeof(cache_rec));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen            o_stream_send(output, ctx.buffer->data, cache_rec.size);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        }
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen        array_append(ext_offsets, &ext_offset, 1);
66d2db642fe24d555d113ba463e446b038d476efTimo Sirainen    }
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    i_assert(orig_fields_count == cache->fields_count);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
f8a86fdfb0048f9c87bf223373b35416ceb5856bTimo Sirainen    hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    mail_cache_compress_get_fields(&ctx, used_fields_count);
f93c833d644ecff0b0f80bee4f1cdde3e697b5c8Timo Sirainen    o_stream_send(output, ctx.buffer->data, ctx.buffer->used);
e665999b757e60bfb98e5a84a78b05f061453140Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    hdr.used_file_size = output->offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    buffer_free(&ctx.buffer);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    buffer_free(&ctx.field_seen);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    o_stream_seek(output, 0);
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen    o_stream_send(output, &hdr, sizeof(hdr));
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen
d5cebe7f98e63d4e2822863ef2faa4971e8b3a5dTimo Sirainen    mail_cache_view_close(cache_view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    if (o_stream_flush(output) < 0) {
b321df9603081896b70ec44635af96d674a9839aTimo Sirainen        errno = output->stream_errno;
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen        mail_cache_set_syscall_error(cache, "o_stream_flush()");
1f1e81aab38d833d1c9cdc244c91fd762e0080d4Timo Sirainen        o_stream_destroy(&output);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        array_free(ext_offsets);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    }
e4d34f2fbee451219599d71505594df704093ce3Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) {
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        /* grow the file some more. doesn't matter if it fails */
35136dd2baf8dc30e4e754294ed81ff48e8c1e64Timo Sirainen        (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen    }
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen    o_stream_destroy(&output);
7bafda1813454621e03615e83d55bccfa7cc56bdTimo Sirainen
d5abbb932a0a598f002da39a8b3326643b1b5efcTimo Sirainen    if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen        if (fdatasync(fd) < 0) {
            mail_cache_set_syscall_error(cache, "fdatasync()");
            array_free(ext_offsets);
            return -1;
        }
    }

    *file_seq_r = hdr.file_seq;
    return 0;
}

static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
{
    struct mail_cache_header hdr;
    unsigned int i;
    int fd, ret;

    for (i = 0;; i++) {
        fd = nfs_safe_open(cache->filepath, O_RDONLY);
        if (fd == -1) {
            if (errno == ENOENT)
                return 0;

            mail_cache_set_syscall_error(cache, "open()");
            return -1;
        }

        ret = read_full(fd, &hdr, sizeof(hdr));
        close_keep_errno(fd);

        if (ret >= 0) {
            if (ret == 0)
                return 0;
            if (cache->need_compress_file_seq == 0) {
                /* previously it didn't exist */
                return 1;
            }
            return hdr.file_seq != cache->need_compress_file_seq;
        } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
            mail_cache_set_syscall_error(cache, "read()");
            return -1;
        }
    }
}

static int mail_cache_compress_locked(struct mail_cache *cache,
                      struct mail_index_transaction *trans,
                      bool *unlock)
{
    struct dotlock *dotlock;
    struct stat st;
    mode_t old_mask;
    uint32_t file_seq, old_offset;
    ARRAY_TYPE(uint32_t) ext_offsets;
    const uint32_t *offsets;
    unsigned int i, count;
    int fd, ret;

    /* get the latest info on fields */
    if (mail_cache_header_fields_read(cache) < 0)
        return -1;

    old_mask = umask(cache->index->mode ^ 0666);
    fd = file_dotlock_open(&cache->dotlock_settings, cache->filepath,
                   DOTLOCK_CREATE_FLAG_NONBLOCK, &dotlock);
    umask(old_mask);

    if (fd == -1) {
        if (errno != EAGAIN)
            mail_cache_set_syscall_error(cache, "file_dotlock_open()");
        return -1;
    }

    if ((ret = mail_cache_compress_has_file_changed(cache)) != 0) {
        if (ret < 0)
            return -1;

        /* was just compressed, forget this */
        cache->need_compress_file_seq = 0;
        file_dotlock_delete(&dotlock);

        if (*unlock) {
            (void)mail_cache_unlock(cache);
            *unlock = FALSE;
        }

        return mail_cache_reopen(cache);
    }

    mail_index_fchown(cache->index, fd,
              file_dotlock_get_lock_path(dotlock));

    if (mail_cache_copy(cache, trans, fd, &file_seq, &ext_offsets) < 0) {
        /* the fields may have been updated in memory already.
           reverse those changes by re-reading them from file. */
        if (mail_cache_header_fields_read(cache) < 0)
            return -1;
        (void)file_dotlock_delete(&dotlock);
        return -1;
    }

    if (fstat(fd, &st) < 0) {
        mail_cache_set_syscall_error(cache, "fstat()");
        (void)file_dotlock_delete(&dotlock);
        return -1;
    }

    if (file_dotlock_replace(&dotlock,
                 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
        mail_cache_set_syscall_error(cache,
                         "file_dotlock_replace()");
        (void)close(fd);
        array_free(&ext_offsets);
        return -1;
    }

    /* once we're sure that the compression was successful,
       update the offsets */
    mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE);
    offsets = array_get(&ext_offsets, &count);
    for (i = 0; i < count; i++) {
        if (offsets[i] != 0) {
            mail_index_update_ext(trans, i + 1, cache->ext_id,
                          &offsets[i], &old_offset);
        }
    }
    array_free(&ext_offsets);

    if (*unlock) {
        (void)mail_cache_unlock(cache);
        *unlock = FALSE;
    }

    mail_cache_file_close(cache);
    cache->fd = fd;
    cache->st_ino = st.st_ino;
    cache->st_dev = st.st_dev;
    cache->field_header_write_pending = FALSE;

    if (cache->file_cache != NULL)
        file_cache_set_fd(cache->file_cache, cache->fd);

    if (mail_cache_map(cache, 0, 0) < 0)
        return -1;
    if (mail_cache_header_fields_read(cache) < 0)
        return -1;

    cache->need_compress_file_seq = 0;
    return 0;
}

int mail_cache_compress(struct mail_cache *cache,
            struct mail_index_transaction *trans)
{
    bool unlock = FALSE;
    int ret;

    i_assert(!cache->compressing);

    if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
        return 0;

    if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
        /* we're using dotlocking, cache file creation itself creates
           the dotlock file we need. */
        if (!MAIL_CACHE_IS_UNUSABLE(cache)) {
            mail_index_flush_read_cache(cache->index,
                            cache->filepath, cache->fd,
                            FALSE);
        }
    } else {
        switch (mail_cache_try_lock(cache)) {
        case -1:
            return -1;
        case 0:
            /* couldn't lock, either it's broken or doesn't exist.
               just start creating it. */
            break;
        default:
            /* locking succeeded. */
            unlock = TRUE;
        }
    }
    cache->compressing = TRUE;
    ret = mail_cache_compress_locked(cache, trans, &unlock);
    cache->compressing = FALSE;
    if (unlock) {
        if (mail_cache_unlock(cache) < 0)
            ret = -1;
    }
    return ret;
}

bool mail_cache_need_compress(struct mail_cache *cache)
{
    return cache->need_compress_file_seq != 0 &&
        !cache->index->readonly;
}