src/lib-index/mail-cache-transaction.c

	mail-cache-transaction.c revision c1b9c4531186c6a7cd92d2c353273a834f8ee66f
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek/* Copyright (c) 2003-2012 Dovecot authors, see the included COPYING file */
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "lib.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "ioloop.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "array.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "buffer.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "module-context.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "file-cache.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "file-set-size.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "read-full.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "write-full.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "mail-cache-private.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include "mail-index-transaction-private.h"
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include <stddef.h>
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#include <sys/stat.h>
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#define MAIL_CACHE_INIT_WRITE_BUFFER (1024*16)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#define MAIL_CACHE_MAX_WRITE_BUFFER (1024*256)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek#define CACHE_TRANS_CONTEXT(obj) \
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    MODULE_CONTEXT(obj, cache_mail_index_transaction_module)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekstruct mail_cache_transaction_ctx {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    union mail_index_transaction_module_context module_ctx;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_index_transaction_vfuncs super;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_cache *cache;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_cache_view *view;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_index_transaction *trans;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
07e941c1bbdc752142bbd3b838c540bc7ecd0ed7Stef Walter    uint32_t cache_file_seq;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    uint32_t first_new_seq;
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walter    buffer_t *cache_data;
769347ad4d35d43488eb98f980143495b0db415dStef Walter    ARRAY(uint32_t) cache_data_seq;
769347ad4d35d43488eb98f980143495b0db415dStef Walter    uint32_t prev_seq;
769347ad4d35d43488eb98f980143495b0db415dStef Walter    size_t last_rec_pos;
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walter    uoff_t bytes_written;
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walter    unsigned int tried_compression:1;
769347ad4d35d43488eb98f980143495b0db415dStef Walter    unsigned int changes:1;
b76419cf8830440b46c20a15585562343c7b1924Jakub Hrozek};
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekstatic MODULE_CONTEXT_DEFINE_INIT(cache_mail_index_transaction_module,
769347ad4d35d43488eb98f980143495b0db415dStef Walter                  &mail_index_module_register);
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walterstatic int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx);
769347ad4d35d43488eb98f980143495b0db415dStef Walterstatic int mail_cache_link_locked(struct mail_cache *cache,
769347ad4d35d43488eb98f980143495b0db415dStef Walter                  uint32_t old_offset, uint32_t new_offset);
769347ad4d35d43488eb98f980143495b0db415dStef Walter
769347ad4d35d43488eb98f980143495b0db415dStef Walterstatic void mail_index_transaction_cache_reset(struct mail_index_transaction *t)
769347ad4d35d43488eb98f980143495b0db415dStef Walter{
769347ad4d35d43488eb98f980143495b0db415dStef Walter    struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT(t);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_index_transaction_vfuncs super = ctx->super;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    mail_cache_transaction_reset(ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    super.reset(t);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekstatic int
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekmail_index_transaction_cache_commit(struct mail_index_transaction *t,
3bea01f01d76e1e95a8239c0d3f67073992136a1Jan Zeleny                    struct mail_index_transaction_commit_result *result_r)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek{
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT(t);
c90fdc85cfb552911e29b4d284fd4c40ea1bb4e9Petr Cech    struct mail_index_transaction_vfuncs super = ctx->super;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    /* a failed cache commit isn't important enough to fail the entire
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek       index transaction, so we'll just ignore it */
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    (void)mail_cache_transaction_commit(&ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    return super.commit(t, result_r);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekstatic void
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekmail_index_transaction_cache_rollback(struct mail_index_transaction *t)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek{
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT(t);
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    struct mail_index_transaction_vfuncs super = ctx->super;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    mail_cache_transaction_rollback(&ctx);
83a79d93035c2d75a1941f3b54426119174044a0Pavel Březina    super.rollback(t);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
65976ea5e9767bfaced81dfb97dc87d59f50b57eSimo Sorcestruct mail_cache_transaction_ctx *
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekmail_cache_get_transaction(struct mail_cache_view *view,
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek               struct mail_index_transaction *t)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek{
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_cache_transaction_ctx *ctx;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    ctx = !cache_mail_index_transaction_module.id.module_id_set ? NULL :
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov        CACHE_TRANS_CONTEXT(t);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if (ctx != NULL)
07e941c1bbdc752142bbd3b838c540bc7ecd0ed7Stef Walter        return ctx;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek
07e941c1bbdc752142bbd3b838c540bc7ecd0ed7Stef Walter    ctx = i_new(struct mail_cache_transaction_ctx, 1);
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    ctx->cache = view->cache;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    ctx->view = view;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    ctx->trans = t;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    i_assert(view->transaction == NULL);
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    view->transaction = ctx;
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    view->trans_view = mail_index_transaction_open_updated_view(t);
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    ctx->super = t->v;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    t->v.reset = mail_index_transaction_cache_reset;
d9577dbd92555b0755881e37724019ef9c578404Stef Walter    t->v.commit = mail_index_transaction_cache_commit;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek    t->v.rollback = mail_index_transaction_cache_rollback;
ccb2c1f30b04bf1f7a33f47748664dedb7ddd0e3Jakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    MODULE_CONTEXT_SET(t, cache_mail_index_transaction_module, ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    return ctx;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekvoid mail_cache_transaction_reset(struct mail_cache_transaction_ctx *ctx)
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina{
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    ctx->cache_file_seq = MAIL_CACHE_IS_UNUSABLE(ctx->cache) ? 0 :
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        ctx->cache->hdr->file_seq;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    mail_index_ext_set_reset_id(ctx->trans, ctx->cache->ext_id,
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                    ctx->cache_file_seq);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if (ctx->cache_data != NULL)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        buffer_set_used_size(ctx->cache_data, 0);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if (array_is_created(&ctx->cache_data_seq))
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina        array_clear(&ctx->cache_data_seq);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    ctx->prev_seq = 0;
8bccd95e275fae760a991da394235e4e70e57bbdMichal Zidek    ctx->last_rec_pos = 0;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    ctx->changes = FALSE;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
07e941c1bbdc752142bbd3b838c540bc7ecd0ed7Stef Walter
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekvoid mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **_ctx)
07e941c1bbdc752142bbd3b838c540bc7ecd0ed7Stef Walter{
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    struct mail_cache_transaction_ctx *ctx = *_ctx;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    *_ctx = NULL;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    if (ctx->bytes_written > 0) {
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina        /* we already wrote to the cache file. we can't (or don't want
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina           to) delete that data, so just mark it as deleted space */
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina        if (mail_cache_transaction_lock(ctx) > 0) {
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov            ctx->cache->hdr_copy.deleted_space +=
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina                ctx->bytes_written;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina            (void)mail_cache_unlock(ctx->cache);
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina        }
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    }
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    MODULE_CONTEXT_UNSET(ctx->trans, cache_mail_index_transaction_module);
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina    ctx->view->transaction = NULL;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    ctx->view->trans_seq1 = ctx->view->trans_seq2 = 0;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    mail_index_view_close(&ctx->view->trans_view);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if (ctx->cache_data != NULL)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        buffer_free(&ctx->cache_data);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if (array_is_created(&ctx->cache_data_seq))
3bea01f01d76e1e95a8239c0d3f67073992136a1Jan Zeleny        array_free(&ctx->cache_data_seq);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    i_free(ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekstatic int
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekmail_cache_transaction_compress(struct mail_cache_transaction_ctx *ctx)
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov{
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina    struct mail_cache *cache = ctx->cache;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_index_view *view;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    struct mail_index_transaction *trans;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    int ret;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    ctx->tried_compression = TRUE;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    cache->need_compress_file_seq =
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        MAIL_CACHE_IS_UNUSABLE(cache) ? 0 : cache->hdr->file_seq;
08c72b84d85d482f030a30cf74786695f097e91cJakub Hrozek
08c72b84d85d482f030a30cf74786695f097e91cJakub Hrozek    view = mail_index_view_open(cache->index);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    trans = mail_index_transaction_begin(view,
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                    MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    if (mail_cache_compress(cache, trans) < 0) {
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina        mail_index_transaction_rollback(&trans);
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina        ret = -1;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    } else {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        ret = mail_index_transaction_commit(&trans);
0528fdec17d0031996e919fcd852459e86592c35Jakub Hrozek    }
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose    mail_index_view_close(&view);
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    mail_cache_transaction_reset(ctx);
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose    return ret;
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose}
909a86af4eb99f5d311d7136cab78dca535ae304Sumit Bose
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashovstatic void
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekmail_cache_transaction_open_if_needed(struct mail_cache_transaction_ctx *ctx)
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina{
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina    struct mail_cache *cache = ctx->cache;
7c9fe57ad82747a32721ca0a08c5569282f3e0c4Pavel Březina    const struct mail_index_ext *ext;
6f8ae17869f4f8a1496e3f171ae6b5c11af1845cPavel Březina    uint32_t idx;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    int i;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if (!cache->opened) {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        (void)mail_cache_open_and_verify(cache);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        return;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    }
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    /* see if we should try to reopen the cache file */
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek    for (i = 0;; i++) {
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek        if (MAIL_CACHE_IS_UNUSABLE(cache))
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            return;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        if (!mail_index_map_get_ext_idx(cache->index->map,
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                        cache->ext_id, &idx)) {
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek            /* index doesn't have a cache extension, but the cache
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek               file exists (corrupted indexes fixed?). fix it. */
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            if (i == 2)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                break;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        } else {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            ext = array_idx(&cache->index->map->extensions, idx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            if (ext->reset_id == cache->hdr->file_seq || i == 2)
458f5245dd5130d12666cce6faf8ef1ec7f80169Pavel Reichl                break;
458f5245dd5130d12666cce6faf8ef1ec7f80169Pavel Reichl
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            /* index offsets don't match the cache file */
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            if (ext->reset_id > cache->hdr->file_seq) {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                /* the cache file appears to be too old.
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                   reopening should help. */
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                if (mail_cache_reopen(cache) != 0)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                    break;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            }
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        }
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        /* cache file sequence might be broken. it's also possible
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek           that it was just compressed and we just haven't yet seen
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek           the changes in index. try if refreshing index helps.
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek           if not, compress the cache file. */
9e2c64c6d4f5560e27207193efea6536a566865eMichal Zidek        if (i == 0) {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            if (ctx->tried_compression)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                break;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            /* get the latest reset ID */
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            if (mail_index_refresh(ctx->cache->index) < 0)
287cc55b9086dd3c4e2a5fb84784e09767860142Jakub Hrozek                return;
ac40d2f2b2b2fc35c95389f5e28febd580bd2b7aJakub Hrozek        } else {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            i_assert(i == 1);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            (void)mail_cache_transaction_compress(ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        }
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    }
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek}
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozekstatic int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx)
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov{
a3c8390d19593b1e5277d95bfb4ab206d4785150Nikolai Kondrashov    struct mail_cache *cache = ctx->cache;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    int ret;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    mail_cache_transaction_open_if_needed(ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek    if ((ret = mail_cache_lock(cache, FALSE)) <= 0) {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        if (ret < 0)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            return -1;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        if (!ctx->tried_compression && MAIL_CACHE_IS_UNUSABLE(cache)) {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            if (mail_cache_transaction_compress(ctx) < 0)
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek                return -1;
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            return mail_cache_transaction_lock(ctx);
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek        } else {
9e80079370ff3b943832adc3c5ef430e64be0a0cJakub Hrozek            return 0;
        }
    }
    i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));

    if (ctx->cache_file_seq == 0) {
        i_assert(ctx->cache_data == NULL ||
             ctx->cache_data->used == 0);
        ctx->cache_file_seq = cache->hdr->file_seq;
    } else if (ctx->cache_file_seq != cache->hdr->file_seq) {
        if (mail_cache_unlock(cache) < 0)
            return -1;
        mail_cache_transaction_reset(ctx);
        return 0;
    }
    return 1;
}

static int
mail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx,
                    uint32_t write_offset)
{
    struct mail_cache *cache = ctx->cache;
    const struct mail_cache_record *rec = ctx->cache_data->data;
    const uint32_t *seqs;
    uint32_t i, seq_count, old_offset;

    mail_index_ext_using_reset_id(ctx->trans, ctx->cache->ext_id,
                      ctx->cache_file_seq);

    /* write the cache_offsets to index file. records' prev_offset
       is updated to point to old cache record when index is being
       synced. */
    seqs = array_get(&ctx->cache_data_seq, &seq_count);
    for (i = 0; i < seq_count; i++) {
        mail_index_update_ext(ctx->trans, seqs[i], cache->ext_id,
                      &write_offset, &old_offset);
        if (old_offset != 0) {
            /* we added records for this message multiple
               times in this same uncommitted transaction.
               only the new one will be written to
               transaction log, we need to do the linking
               ourself here. */
            if (old_offset > write_offset) {
                if (mail_cache_link_locked(cache, old_offset,
                               write_offset) < 0)
                    return -1;
            } else {
                /* if we're combining multiple transactions,
                   make sure the one with the smallest offset
                   is written into index. this is required for
                   non-file-mmaped cache to work properly. */
                mail_index_update_ext(ctx->trans, seqs[i],
                              cache->ext_id,
                              &old_offset, NULL);
                if (mail_cache_link_locked(cache, write_offset,
                               old_offset) < 0)
                    return -1;
            }
        }

        write_offset += rec->size;
        rec = CONST_PTR_OFFSET(rec, rec->size);
    }
    return 0;
}

static int
mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
{
    uint32_t write_offset;
    int ret;

    i_assert(!ctx->cache->locked);

    if (mail_cache_transaction_lock(ctx) <= 0)
        return -1;

    /* first write the actual data to cache file */
    i_assert(ctx->cache_data != NULL);
    i_assert(ctx->last_rec_pos <= ctx->cache_data->used);
    if (mail_cache_append(ctx->cache, ctx->cache_data->data,
                  ctx->last_rec_pos, &write_offset) < 0)
        ret = -1;
    else {
        /* update records' cache offsets to index */
        ctx->bytes_written += ctx->last_rec_pos;
        ret = mail_cache_transaction_update_index(ctx, write_offset);
    }
    if (mail_cache_unlock(ctx->cache) < 0)
        ret = -1;

    /* drop the written data from buffer */
    buffer_copy(ctx->cache_data, 0,
            ctx->cache_data, ctx->last_rec_pos, (size_t)-1);
    buffer_set_used_size(ctx->cache_data,
                 ctx->cache_data->used - ctx->last_rec_pos);
    ctx->last_rec_pos = 0;

    array_clear(&ctx->cache_data_seq);
    return ret;
}

static void
mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache_record *rec;
    void *data;
    size_t size;

    data = buffer_get_modifiable_data(ctx->cache_data, &size);
    rec = PTR_OFFSET(data, ctx->last_rec_pos);
    rec->size = size - ctx->last_rec_pos;
    i_assert(rec->size > sizeof(*rec));

    /* FIXME: here would be a good place to set prev_offset to
       avoid doing it later, but avoid circular prev_offsets
       when cache is updated multiple times within the same
       transaction */

    array_append(&ctx->cache_data_seq, &ctx->prev_seq, 1);
    ctx->last_rec_pos = size;
}

static void
mail_cache_transaction_switch_seq(struct mail_cache_transaction_ctx *ctx)
{
    struct mail_cache_record new_rec;

    if (ctx->prev_seq != 0) {
        /* update previously added cache record's size */
        mail_cache_transaction_update_last_rec(ctx);
    } else if (ctx->cache_data == NULL) {
        ctx->cache_data =
            buffer_create_dynamic(default_pool,
                          MAIL_CACHE_INIT_WRITE_BUFFER);
        i_array_init(&ctx->cache_data_seq, 64);
    }

    memset(&new_rec, 0, sizeof(new_rec));
    buffer_append(ctx->cache_data, &new_rec, sizeof(new_rec));

    ctx->prev_seq = 0;
    ctx->changes = TRUE;
}

int mail_cache_transaction_commit(struct mail_cache_transaction_ctx **_ctx)
{
    struct mail_cache_transaction_ctx *ctx = *_ctx;
    int ret = 0;

    if (ctx->changes) {
        if (ctx->prev_seq != 0)
            mail_cache_transaction_update_last_rec(ctx);
        if (mail_cache_transaction_flush(ctx) < 0)
            ret = -1;
        else {
            /* successfully wrote everything */
            ctx->bytes_written = 0;
        }
        /* Here would be a good place to do fdatasync() to make sure
           everything is written before offsets are updated to index.
           However it slows down I/O unneededly and we're pretty good
           at catching and fixing cache corruption, so we no longer do
           it. */
    }
    mail_cache_transaction_rollback(_ctx);
    return ret;
}

static int
mail_cache_header_fields_write(struct mail_cache_transaction_ctx *ctx,
                   const buffer_t *buffer)
{
    struct mail_cache *cache = ctx->cache;
    uint32_t offset, hdr_offset;

    i_assert(cache->locked);

    if (mail_cache_append(cache, buffer->data, buffer->used, &offset) < 0)
        return -1;

    if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
        if (fdatasync(cache->fd) < 0) {
            mail_cache_set_syscall_error(cache, "fdatasync()");
            return -1;
        }
    }
    /* find offset to the previous header's "next_offset" field */
    if (mail_cache_header_fields_get_next_offset(cache, &hdr_offset) < 0)
        return -1;

    /* update the next_offset offset, so our new header will be found */
    offset = mail_index_uint32_to_offset(offset);
    if (mail_cache_write(cache, &offset, sizeof(offset), hdr_offset) < 0)
        return -1;

    if (hdr_offset == offsetof(struct mail_cache_header,
                   field_header_offset)) {
        /* we're adding the first field. hdr_copy needs to be kept
           in sync so unlocking won't overwrite it. */
        cache->hdr_copy.field_header_offset = hdr_offset;
        cache->hdr_ro_copy.field_header_offset = hdr_offset;
    }
    return 0;
}

static void mail_cache_mark_adding(struct mail_cache *cache, bool set)
{
    unsigned int i;

    /* we want to avoid adding all the fields one by one to the cache file,
       so just add all of them at once in here. the unused ones get dropped
       later when compressing. */
    for (i = 0; i < cache->fields_count; i++) {
        if (set)
            cache->fields[i].used = TRUE;
        cache->fields[i].adding = set;
    }
}

static int mail_cache_header_add_field(struct mail_cache_transaction_ctx *ctx,
                       unsigned int field_idx)
{
    struct mail_cache *cache = ctx->cache;
    int ret;

    if (mail_cache_transaction_lock(ctx) <= 0) {
        if (MAIL_CACHE_IS_UNUSABLE(cache))
            return -1;

        /* if we compressed the cache, the field should be there now.
           it's however possible that someone else just compressed it
           and we only reopened the cache file. */
        if (cache->field_file_map[field_idx] != (uint32_t)-1)
            return 0;

        /* need to add it */
        if (mail_cache_transaction_lock(ctx) <= 0)
            return -1;
    }

    /* re-read header to make sure we don't lose any fields. */
    if (mail_cache_header_fields_read(cache) < 0) {
        (void)mail_cache_unlock(cache);
        return -1;
    }

    if (cache->field_file_map[field_idx] != (uint32_t)-1) {
        /* it was already added */
        if (mail_cache_unlock(cache) < 0)
            return -1;
        return 0;
    }

    T_BEGIN {
        buffer_t *buffer;

        buffer = buffer_create_dynamic(pool_datastack_create(), 256);
        mail_cache_header_fields_get(cache, buffer);
        ret = mail_cache_header_fields_write(ctx, buffer);
    } T_END;

    if (ret == 0) {
        /* we wrote all the headers, so there are no pending changes */
        cache->field_header_write_pending = FALSE;
        ret = mail_cache_header_fields_read(cache);
    }
    if (ret == 0 && cache->field_file_map[field_idx] == (uint32_t)-1) {
        mail_index_set_error(cache->index,
                     "Cache file %s: Newly added field got "
                     "lost unexpectedly", cache->filepath);
        ret = -1;
    }

    if (mail_cache_unlock(cache) < 0)
        ret = -1;
    return ret;
}

void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
            unsigned int field_idx, const void *data, size_t data_size)
{
    uint32_t file_field, data_size32;
    unsigned int fixed_size;
    size_t full_size;
    int ret;

    i_assert(field_idx < ctx->cache->fields_count);
    i_assert(data_size < (uint32_t)-1);

    if (ctx->cache->fields[field_idx].field.decision ==
        (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED))
        return;

    if (ctx->cache_file_seq == 0) {
        mail_cache_transaction_open_if_needed(ctx);
        if (!MAIL_CACHE_IS_UNUSABLE(ctx->cache))
            ctx->cache_file_seq = ctx->cache->hdr->file_seq;
    } else if (!MAIL_CACHE_IS_UNUSABLE(ctx->cache) &&
           ctx->cache_file_seq != ctx->cache->hdr->file_seq) {
        /* cache was compressed within this transaction */
        mail_cache_transaction_reset(ctx);
    }

    file_field = ctx->cache->field_file_map[field_idx];
    if (MAIL_CACHE_IS_UNUSABLE(ctx->cache) || file_field == (uint32_t)-1) {
        /* we'll have to add this field to headers */
        mail_cache_mark_adding(ctx->cache, TRUE);
        ret = mail_cache_header_add_field(ctx, field_idx);
        mail_cache_mark_adding(ctx->cache, FALSE);
        if (ret < 0)
            return;

        if (ctx->cache_file_seq == 0)
            ctx->cache_file_seq = ctx->cache->hdr->file_seq;

        file_field = ctx->cache->field_file_map[field_idx];
        i_assert(file_field != (uint32_t)-1);
    }
    i_assert(ctx->cache_file_seq != 0);

    mail_cache_decision_add(ctx->view, seq, field_idx);

    fixed_size = ctx->cache->fields[field_idx].field.field_size;
    i_assert(fixed_size == (unsigned int)-1 || fixed_size == data_size);

    data_size32 = (uint32_t)data_size;

    if (ctx->prev_seq != seq) {
        mail_cache_transaction_switch_seq(ctx);
        ctx->prev_seq = seq;

        /* remember roughly what we have modified, so cache lookups can
           look into transactions to see changes. */
        if (seq < ctx->view->trans_seq1 || ctx->view->trans_seq1 == 0)
            ctx->view->trans_seq1 = seq;
        if (seq > ctx->view->trans_seq2)
            ctx->view->trans_seq2 = seq;
    }

    /* remember that this value exists, in case we try to look it up */
    buffer_write(ctx->view->cached_exists_buf, field_idx,
             &ctx->view->cached_exists_value, 1);

    full_size = (data_size + 3) & ~3;
    if (fixed_size == (unsigned int)-1)
        full_size += sizeof(data_size32);

    if (ctx->cache_data->used + full_size > MAIL_CACHE_MAX_WRITE_BUFFER &&
        ctx->last_rec_pos > 0) {
        /* time to flush our buffer. if flushing fails because the
           cache file had been compressed and was reopened, return
           without adding the cached data since cache_data buffer
           doesn't contain the cache_rec anymore. */
        if (mail_cache_transaction_flush(ctx) < 0) {
            /* make sure the transaction is reset, so we don't
               constantly try to flush for each call to this
               function */
            mail_cache_transaction_reset(ctx);
            return;
        }
    }

    buffer_append(ctx->cache_data, &file_field, sizeof(file_field));
    if (fixed_size == -1U) {
        buffer_append(ctx->cache_data, &data_size32,
                  sizeof(data_size32));
    }

    buffer_append(ctx->cache_data, data, data_size);
    if ((data_size & 3) != 0)
                buffer_append_zero(ctx->cache_data, 4 - (data_size & 3));
}

bool mail_cache_field_want_add(struct mail_cache_transaction_ctx *ctx,
                   uint32_t seq, unsigned int field_idx)
{
    enum mail_cache_decision_type decision;

    mail_cache_transaction_open_if_needed(ctx);

    decision = mail_cache_field_get_decision(ctx->view->cache, field_idx);
    decision &= ~MAIL_CACHE_DECISION_FORCED;
    switch (decision) {
    case MAIL_CACHE_DECISION_NO:
        return FALSE;
    case MAIL_CACHE_DECISION_TEMP:
        /* add it only if it's newer than what we would drop when
           compressing */
        if (ctx->first_new_seq == 0) {
            ctx->first_new_seq =
                mail_cache_get_first_new_seq(ctx->view->view);
        }
        if (seq < ctx->first_new_seq)
            return FALSE;
        break;
    default:
        break;
    }

    return mail_cache_field_exists(ctx->view, seq, field_idx) == 0;
}

bool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx,
                  uint32_t seq, unsigned int field_idx)
{
    enum mail_cache_decision_type decision;

    mail_cache_transaction_open_if_needed(ctx);

    decision = mail_cache_field_get_decision(ctx->view->cache, field_idx);
    if (decision == (MAIL_CACHE_DECISION_FORCED | MAIL_CACHE_DECISION_NO))
        return FALSE;

    return mail_cache_field_exists(ctx->view, seq, field_idx) == 0;
}

static int mail_cache_link_locked(struct mail_cache *cache,
                  uint32_t old_offset, uint32_t new_offset)
{
    new_offset += offsetof(struct mail_cache_record, prev_offset);
    return mail_cache_write(cache, &old_offset, sizeof(old_offset),
                new_offset);
}

int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
            uint32_t new_offset)
{
    const struct mail_cache_record *rec;
    const void *data;
    int ret;

    i_assert(cache->locked);

    if (MAIL_CACHE_IS_UNUSABLE(cache))
        return -1;

    /* this function is called for each added cache record (or cache
       extension record update actually) with new_offset pointing to the
       new record and old_offset pointing to the previous record.

       we want to keep the old and new records linked so both old and new
       cached data is found. normally they are already linked correctly.
       the problem only comes when multiple processes are adding cache
       records at the same time. we'd rather not lose those additions, so
       force the linking order to be new_offset -> old_offset if it isn't
       already. */
    ret = mail_cache_map(cache, new_offset, sizeof(*rec), &data);
    if (ret <= 0) {
        if (ret == 0) {
            mail_cache_set_corrupted(cache,
                "Cache record offset %u points outside file",
                new_offset);
        }
        return -1;
    }
    rec = data;
    if (rec->prev_offset == old_offset) {
        /* link is already correct */
        return 0;
    }

    if (mail_cache_link_locked(cache, old_offset, new_offset) < 0)
        return -1;

    cache->hdr_copy.continued_record_count++;
    cache->hdr_modified = TRUE;
    return 0;
}

static int mail_cache_delete_real(struct mail_cache *cache, uint32_t offset)
{
    const struct mail_cache_record *rec;
    struct mail_cache_loop_track loop_track;
    int ret = 0;

    i_assert(cache->locked);

    /* we'll only update the deleted_space in header. we can't really
       do any actual deleting as other processes might still be using
       the data. also it's actually useful as some index views are still
       able to ask cached data from messages that have already been
       expunged. */
    memset(&loop_track, 0, sizeof(loop_track));
    while (offset != 0 &&
           (ret = mail_cache_get_record(cache, offset, &rec)) == 0) {
        if (mail_cache_track_loops(&loop_track, offset, rec->size)) {
            mail_cache_set_corrupted(cache,
                         "record list is circular");
            return -1;
        }

        cache->hdr_copy.deleted_space += rec->size;
        offset = rec->prev_offset;
    }
    return ret;
}

int mail_cache_delete(struct mail_cache *cache, uint32_t offset)
{
    int ret;

    i_assert(cache->locked);
    T_BEGIN {
        ret = mail_cache_delete_real(cache, offset);
    } T_END;
    cache->hdr_modified = TRUE;
    return ret;
}