mail-cache-transaction.c revision e6d4f540bf5c3b7ef5d6e154b217a2422210048c
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2003-2017 Dovecot authors, see the included COPYING file */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "lib.h"
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen#include "ioloop.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "array.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "buffer.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "module-context.h"
0371406d952fe51367c7be91703e5634b7d9d225Timo Sirainen#include "file-cache.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "file-set-size.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "read-full.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "write-full.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-cache-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include "mail-index-transaction-private.h"
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#include <stddef.h>
22627da0fb77c1d0d9a8e8bc485ef5540b6f2e69Timo Sirainen#include <sys/stat.h>
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#define MAIL_CACHE_INIT_WRITE_BUFFER (1024*16)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen#define MAIL_CACHE_MAX_WRITE_BUFFER (1024*256)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen#define CACHE_TRANS_CONTEXT(obj) \
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen MODULE_CONTEXT(obj, cache_mail_index_transaction_module)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstruct mail_cache_transaction_rec {
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen uint32_t seq;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen uint32_t cache_data_pos;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen};
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstruct mail_cache_transaction_ctx {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen union mail_index_transaction_module_context module_ctx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index_transaction_vfuncs super;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen struct mail_cache *cache;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen struct mail_cache_view *view;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen struct mail_index_transaction *trans;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen uint32_t cache_file_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t first_new_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_t *cache_data;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ARRAY(struct mail_cache_transaction_rec) cache_data_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t prev_seq, min_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen size_t last_rec_pos;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int records_written;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen bool tried_compression:1;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen bool changes:1;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen};
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic MODULE_CONTEXT_DEFINE_INIT(cache_mail_index_transaction_module,
68f0dfb4b2815ecbc1bd8d8a68adcfd577ec55aeTimo Sirainen &mail_index_module_register);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic size_t mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenstatic void mail_index_transaction_cache_reset(struct mail_index_transaction *t)
9c2b0eb659540b9db8dd3a8a6a2515921fbe8eebTimo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT(t);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen struct mail_index_transaction_vfuncs super = ctx->super;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen mail_cache_transaction_reset(ctx);
892b3cbf0eba9ba455448adcf71864a409345c6dTimo Sirainen super.reset(t);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen}
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
f4c0b1874b0533bcf2df1d28d584ff02cfdae3faTimo Sirainenstatic int
f4c0b1874b0533bcf2df1d28d584ff02cfdae3faTimo Sirainenmail_index_transaction_cache_commit(struct mail_index_transaction *t,
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen struct mail_index_transaction_commit_result *result_r)
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT(t);
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen struct mail_index_transaction_vfuncs super = ctx->super;
0db5b158a00c08955bdacc99b1e2cd1ec07f4311Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* a failed cache commit isn't important enough to fail the entire
d16b506f5540e3407d256bda35624b38a5ecf88fTimo Sirainen index transaction, so we'll just ignore it */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen (void)mail_cache_transaction_commit(&ctx);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen return super.commit(t, result_r);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen}
554c1c792dc6fce1e25c74555c2da786bffde75fTimo Sirainen
554c1c792dc6fce1e25c74555c2da786bffde75fTimo Sirainenstatic void
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainenmail_index_transaction_cache_rollback(struct mail_index_transaction *t)
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen{
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen struct mail_cache_transaction_ctx *ctx = CACHE_TRANS_CONTEXT(t);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen struct mail_index_transaction_vfuncs super = ctx->super;
6df2db16b3920346ed07cefb86e8bdcb7e1faec5Timo Sirainen
6df2db16b3920346ed07cefb86e8bdcb7e1faec5Timo Sirainen mail_cache_transaction_rollback(&ctx);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen super.rollback(t);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen}
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainenstruct mail_cache_transaction_ctx *
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainenmail_cache_get_transaction(struct mail_cache_view *view,
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen struct mail_index_transaction *t)
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen{
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen struct mail_cache_transaction_ctx *ctx;
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen ctx = !cache_mail_index_transaction_module.id.module_id_set ? NULL :
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen CACHE_TRANS_CONTEXT(t);
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen if (ctx != NULL)
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen return ctx;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen ctx = i_new(struct mail_cache_transaction_ctx, 1);
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen ctx->cache = view->cache;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->view = view;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->trans = t;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen i_assert(view->transaction == NULL);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen view->transaction = ctx;
407caeb5d0c8a6b158e2caef48dd909011d40340Timo Sirainen view->trans_view = mail_index_transaction_open_updated_view(t);
407caeb5d0c8a6b158e2caef48dd909011d40340Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->super = t->v;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen t->v.reset = mail_index_transaction_cache_reset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen t->v.commit = mail_index_transaction_cache_commit;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen t->v.rollback = mail_index_transaction_cache_rollback;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen MODULE_CONTEXT_SET(t, cache_mail_index_transaction_module, ctx);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen return ctx;
a54fa00087ba926a3d966a8449d8d7579e89911cTimo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenvoid mail_cache_transaction_reset(struct mail_cache_transaction_ctx *ctx)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache_file_seq = MAIL_CACHE_IS_UNUSABLE(ctx->cache) ? 0 :
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache->hdr->file_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_ext_set_reset_id(ctx->trans, ctx->cache->ext_id,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_file_seq);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->cache_data != NULL)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_set_used_size(ctx->cache_data, 0);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (array_is_created(&ctx->cache_data_seq))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_clear(&ctx->cache_data_seq);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->prev_seq = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->last_rec_pos = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->changes = FALSE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenvoid mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **_ctx)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache_transaction_ctx *ctx = *_ctx;
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen *_ctx = NULL;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->records_written > 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we already wrote to the cache file. we can't (or don't want
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen to) delete that data, so just mark it as deleted space */
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen if (mail_cache_transaction_lock(ctx) > 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache->hdr_copy.deleted_record_count +=
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->records_written;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen (void)mail_cache_unlock(ctx->cache);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen MODULE_CONTEXT_UNSET(ctx->trans, cache_mail_index_transaction_module);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->view->transaction = NULL;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->view->trans_seq1 = ctx->view->trans_seq2 = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_view_close(&ctx->view->trans_view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->cache_data != NULL)
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen buffer_free(&ctx->cache_data);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen if (array_is_created(&ctx->cache_data_seq))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_free(&ctx->cache_data_seq);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_free(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainenstatic int
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainenmail_cache_transaction_compress(struct mail_cache_transaction_ctx *ctx)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache *cache = ctx->cache;
d16b506f5540e3407d256bda35624b38a5ecf88fTimo Sirainen struct mail_index_view *view;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen struct mail_index_transaction *trans;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen struct mail_cache_compress_lock *lock;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen int ret;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->tried_compression = TRUE;
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen cache->need_compress_file_seq =
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen MAIL_CACHE_IS_UNUSABLE(cache) ? 0 : cache->hdr->file_seq;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen view = mail_index_view_open(cache->index);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen trans = mail_index_transaction_begin(view,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (mail_cache_compress(cache, trans, &lock) < 0) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen mail_index_transaction_rollback(&trans);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ret = -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen } else {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ret = mail_index_transaction_commit(&trans);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (lock != NULL)
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen mail_cache_compress_unlock(&lock);
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen mail_index_view_close(&view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_reset(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return ret;
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainenstatic void
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenmail_cache_transaction_open_if_needed(struct mail_cache_transaction_ctx *ctx)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache *cache = ctx->cache;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const struct mail_index_ext *ext;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen uint32_t idx;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen int i;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen if (!cache->opened) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen (void)mail_cache_open_and_verify(cache);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen }
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* see if we should try to reopen the cache file */
a0b89f3b1df99b3a32f44623f13ad1893118825bTimo Sirainen for (i = 0;; i++) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (MAIL_CACHE_IS_UNUSABLE(cache))
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (!mail_index_map_get_ext_idx(cache->index->map,
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen cache->ext_id, &idx)) {
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen /* index doesn't have a cache extension, but the cache
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen file exists (corrupted indexes fixed?). fix it. */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (i == 2)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen break;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen } else {
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen ext = array_idx(&cache->index->map->extensions, idx);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ext->reset_id == cache->hdr->file_seq || i == 2)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen break;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* index offsets don't match the cache file */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ext->reset_id > cache->hdr->file_seq) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* the cache file appears to be too old.
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen reopening should help. */
9349a0afffad990e45d3ad33081e1d2d9e68a753Timo Sirainen if (mail_cache_reopen(cache) != 0)
9349a0afffad990e45d3ad33081e1d2d9e68a753Timo Sirainen break;
9349a0afffad990e45d3ad33081e1d2d9e68a753Timo Sirainen }
4c9c55e15f35474f53f11659e796c63b1c34e884Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
4c9c55e15f35474f53f11659e796c63b1c34e884Timo Sirainen /* cache file sequence might be broken. it's also possible
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen that it was just compressed and we just haven't yet seen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen the changes in index. try if refreshing index helps.
a94936bafd127680184da114c6a177b37ff656e5Timo Sirainen if not, compress the cache file. */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (i == 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->tried_compression)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen break;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* get the latest reset ID */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (mail_index_refresh(ctx->cache->index) < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return;
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen } else {
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen i_assert(i == 1);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen (void)mail_cache_transaction_compress(ctx);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen }
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen }
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen}
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainenstatic int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx)
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache *cache = ctx->cache;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen int ret;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
a94936bafd127680184da114c6a177b37ff656e5Timo Sirainen mail_cache_transaction_open_if_needed(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if ((ret = mail_cache_lock(cache)) <= 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ret < 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
d16b506f5540e3407d256bda35624b38a5ecf88fTimo Sirainen if (!ctx->tried_compression && MAIL_CACHE_IS_UNUSABLE(cache)) {
4c9c55e15f35474f53f11659e796c63b1c34e884Timo Sirainen if (mail_cache_transaction_compress(ctx) < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen return mail_cache_transaction_lock(ctx);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen } else {
22627da0fb77c1d0d9a8e8bc485ef5540b6f2e69Timo Sirainen return 0;
3e28b527dd6048a40684afd29cff0ee008fc0014Timo Sirainen }
3e28b527dd6048a40684afd29cff0ee008fc0014Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(!MAIL_CACHE_IS_UNUSABLE(cache));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ctx->cache_file_seq == 0) {
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen i_assert(ctx->cache_data == NULL ||
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache_data->used == 0);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache_file_seq = cache->hdr->file_seq;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen } else if (ctx->cache_file_seq != cache->hdr->file_seq) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (mail_cache_unlock(cache) < 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return -1;
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen mail_cache_transaction_reset(ctx);
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen return 0;
ccffb125d94adff0ad776de5a96e22f864d6fb0aTimo Sirainen }
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen return 1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenconst struct mail_cache_record *
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainenmail_cache_transaction_lookup_rec(struct mail_cache_transaction_ctx *ctx,
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen unsigned int seq,
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen unsigned int *trans_next_idx)
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen{
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen const struct mail_cache_transaction_rec *recs;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen unsigned int i, count;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (MAIL_CACHE_IS_UNUSABLE(ctx->cache) ||
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen ctx->cache_file_seq != ctx->cache->hdr->file_seq) {
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen /* Cache was compressed during this transaction. We can't
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen safely use the data anymore, since its fields won't match
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen cache->file_fields_map. */
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen return NULL;
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen }
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen recs = array_get(&ctx->cache_data_seq, &count);
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen for (i = *trans_next_idx; i < count; i++) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (recs[i].seq == seq) {
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen *trans_next_idx = i + 1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return CONST_PTR_OFFSET(ctx->cache_data->data,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen recs[i].cache_data_pos);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen *trans_next_idx = i + 1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (seq == ctx->prev_seq && i == count) {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* update the unfinished record's (temporary) size and
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return it */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen mail_cache_transaction_update_last_rec_size(ctx);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return CONST_PTR_OFFSET(ctx->cache_data->data,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->last_rec_pos);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return NULL;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstatic int
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenmail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen uint32_t write_offset)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
a1aaf11831cab8346d6d0dc702e37b3f1d95eb43Timo Sirainen struct mail_cache *cache = ctx->cache;
a1aaf11831cab8346d6d0dc702e37b3f1d95eb43Timo Sirainen const struct mail_cache_record *rec = ctx->cache_data->data;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen const struct mail_cache_transaction_rec *recs;
a1aaf11831cab8346d6d0dc702e37b3f1d95eb43Timo Sirainen uint32_t i, seq_count;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_ext_using_reset_id(ctx->trans, ctx->cache->ext_id,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_file_seq);
c44f402f17f9a58ead24ac0083945cae86fb172bTimo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* write the cache_offsets to index file. records' prev_offset
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen is updated to point to old cache record when index is being
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen synced. */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen recs = array_get(&ctx->cache_data_seq, &seq_count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen for (i = 0; i < seq_count; i++) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_update_ext(ctx->trans, recs[i].seq, cache->ext_id,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen &write_offset, NULL);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen write_offset += rec->size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen rec = CONST_PTR_OFFSET(rec, rec->size);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen }
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen return 0;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen}
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainenstatic int
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainenmail_cache_link_records(struct mail_cache_transaction_ctx *ctx,
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen uint32_t write_offset)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_index_map *map;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache_record *rec;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const struct mail_cache_transaction_rec *recs;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen const uint32_t *prev_offsetp;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ARRAY_TYPE(uint32_t) seq_offsets;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen uint32_t i, seq_count, reset_id, prev_offset, *offsetp;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen const void *data;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen i_assert(ctx->min_seq != 0);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen i_array_init(&seq_offsets, 64);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen recs = array_get(&ctx->cache_data_seq, &seq_count);
68f0dfb4b2815ecbc1bd8d8a68adcfd577ec55aeTimo Sirainen rec = buffer_get_modifiable_data(ctx->cache_data, NULL);
e958a3c4573058f17999f0083a34080ca35e34d8Timo Sirainen for (i = 0; i < seq_count; i++) {
e958a3c4573058f17999f0083a34080ca35e34d8Timo Sirainen offsetp = array_idx_modifiable(&seq_offsets,
e958a3c4573058f17999f0083a34080ca35e34d8Timo Sirainen recs[i].seq - ctx->min_seq);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (*offsetp != 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen prev_offset = *offsetp;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen else {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen mail_index_lookup_ext_full(ctx->view->trans_view, recs[i].seq,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache->ext_id, &map,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen &data, NULL);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen prev_offsetp = data;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (prev_offsetp == NULL || *prev_offsetp == 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen prev_offset = 0;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen else if (mail_index_ext_get_reset_id(ctx->view->trans_view, map,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache->ext_id,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen &reset_id) &&
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen reset_id == ctx->cache_file_seq)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen prev_offset = *prev_offsetp;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen else
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen prev_offset = 0;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (prev_offset >= write_offset) {
68f0dfb4b2815ecbc1bd8d8a68adcfd577ec55aeTimo Sirainen mail_cache_set_corrupted(ctx->cache,
68f0dfb4b2815ecbc1bd8d8a68adcfd577ec55aeTimo Sirainen "Cache record offset points outside existing file");
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen array_free(&seq_offsets);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen return -1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (prev_offset != 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* link this record to previous one */
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen rec->prev_offset = prev_offset;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen ctx->cache->hdr_copy.continued_record_count++;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen } else {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache->hdr_copy.record_count++;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen *offsetp = write_offset;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen write_offset += rec->size;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen rec = PTR_OFFSET(rec, rec->size);
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen }
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen array_free(&seq_offsets);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache->hdr_modified = TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic int
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenmail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
2cfe9983ce7a6280636ee12beccc2e865111967bTimo Sirainen struct stat st;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t write_offset = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen int ret = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(!ctx->cache->locked);
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen if (array_count(&ctx->cache_data_seq) == 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we had done some changes, but they were aborted. */
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen i_assert(ctx->last_rec_pos == 0);
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen ctx->min_seq = 0;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen return 0;
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen if (mail_cache_transaction_lock(ctx) <= 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(ctx->cache_data != NULL);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(ctx->last_rec_pos <= ctx->cache_data->used);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we need to get the final write offset for linking records */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (fstat(ctx->cache->fd, &st) < 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (!ESTALE_FSTAT(errno))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_set_syscall_error(ctx->cache, "fstat()");
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ret = -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen } else if ((uint32_t)-1 < st.st_size + ctx->last_rec_pos) {
f119596e34bc4a7ce374f4aa5f4f1eb12061a372Timo Sirainen mail_cache_set_corrupted(ctx->cache, "Cache file too large");
f119596e34bc4a7ce374f4aa5f4f1eb12061a372Timo Sirainen ret = -1;
f119596e34bc4a7ce374f4aa5f4f1eb12061a372Timo Sirainen } else {
f119596e34bc4a7ce374f4aa5f4f1eb12061a372Timo Sirainen write_offset = st.st_size;
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen if (mail_cache_link_records(ctx, write_offset) < 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ret = -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* write to cache file */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ret < 0 ||
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_append(ctx->cache, ctx->cache_data->data,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->last_rec_pos, &write_offset) < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen else {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* update records' cache offsets to index */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->records_written++;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = mail_cache_transaction_update_index(ctx, write_offset);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
294c71436de227178c709e4d498e7be9b5d8d7feTimo Sirainen if (mail_cache_unlock(ctx->cache) < 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ret = -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* drop the written data from buffer */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_copy(ctx->cache_data, 0,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache_data, ctx->last_rec_pos, (size_t)-1);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen buffer_set_used_size(ctx->cache_data,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->cache_data->used - ctx->last_rec_pos);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->last_rec_pos = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->min_seq = 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen array_clear(&ctx->cache_data_seq);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return ret;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenstatic size_t
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenmail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache_record *rec;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen void *data;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen size_t size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
13a8c553f293349248b161ff851743498916e26eTimo Sirainen data = buffer_get_modifiable_data(ctx->cache_data, &size);
13a8c553f293349248b161ff851743498916e26eTimo Sirainen rec = PTR_OFFSET(data, ctx->last_rec_pos);
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen rec->size = size - ctx->last_rec_pos;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen i_assert(rec->size > sizeof(*rec));
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen return rec->size;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen}
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainenstatic void
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenmail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx)
9fcf7b79236b0045f7709718f7b65ada516565e7Timo Sirainen{
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen struct mail_cache_transaction_rec *trans_rec;
9fcf7b79236b0045f7709718f7b65ada516565e7Timo Sirainen size_t size;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen size = mail_cache_transaction_update_last_rec_size(ctx);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (size > MAIL_CACHE_RECORD_MAX_SIZE) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos);
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen return;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ctx->min_seq > ctx->prev_seq || ctx->min_seq == 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->min_seq = ctx->prev_seq;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen trans_rec = array_append_space(&ctx->cache_data_seq);
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen trans_rec->seq = ctx->prev_seq;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen trans_rec->cache_data_pos = ctx->last_rec_pos;
24ce0c343cefe54af841871fa39dbc3464028b06Timo Sirainen ctx->last_rec_pos = ctx->cache_data->used;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstatic void
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenmail_cache_transaction_switch_seq(struct mail_cache_transaction_ctx *ctx)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
5fbf8719b9ef072295c16bc4492f9f0ece92117dTimo Sirainen struct mail_cache_record new_rec;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ctx->prev_seq != 0) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* update previously added cache record's size */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_update_last_rec(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen } else if (ctx->cache_data == NULL) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_data =
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_create_dynamic(default_pool,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen MAIL_CACHE_INIT_WRITE_BUFFER);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_array_init(&ctx->cache_data_seq, 64);
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_zero(&new_rec);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_append(ctx->cache_data, &new_rec, sizeof(new_rec));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->prev_seq = 0;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ctx->changes = TRUE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenint mail_cache_transaction_commit(struct mail_cache_transaction_ctx **_ctx)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen struct mail_cache_transaction_ctx *ctx = *_ctx;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen int ret = 0;
3656c91dcb8336814bebd4500e81c3dde25233e6Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->changes) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->prev_seq != 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_update_last_rec(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (mail_cache_transaction_flush(ctx) < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = -1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen else {
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* successfully wrote everything */
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen ctx->records_written = 0;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* Here would be a good place to do fdatasync() to make sure
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen everything is written before offsets are updated to index.
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen However it slows down I/O unneededly and we're pretty good
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen at catching and fixing cache corruption, so we no longer do
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen it. */
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen }
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen mail_cache_transaction_rollback(_ctx);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen return ret;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstatic int
5fe06fea9fee0f5e4e9cb49f6866877223f78b85Timo Sirainenmail_cache_header_fields_write(struct mail_cache_transaction_ctx *ctx,
5fe06fea9fee0f5e4e9cb49f6866877223f78b85Timo Sirainen const buffer_t *buffer)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen struct mail_cache *cache = ctx->cache;
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen uint32_t offset, hdr_offset;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen i_assert(cache->locked);
ab1b9a793d57a60c230a41f65f1a25d52c026233Timo Sirainen
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen offset = 0;
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen if (mail_cache_append(cache, buffer->data, buffer->used, &offset) < 0)
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen return -1;
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (fdatasync(cache->fd) < 0) {
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen mail_cache_set_syscall_error(cache, "fdatasync()");
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen return -1;
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen }
36c4702131e5a04984ad5d07cf5d8d5c633d43c3Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* find offset to the previous header's "next_offset" field */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (mail_cache_header_fields_get_next_offset(cache, &hdr_offset) < 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen /* update the next_offset offset, so our new header will be found */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen offset = mail_index_uint32_to_offset(offset);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (mail_cache_write(cache, &offset, sizeof(offset), hdr_offset) < 0)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen if (hdr_offset == offsetof(struct mail_cache_header,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen field_header_offset)) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* we're adding the first field. hdr_copy needs to be kept
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen in sync so unlocking won't overwrite it. */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen cache->hdr_copy.field_header_offset = hdr_offset;
840a3701b7a0f7fadd17738998c33790a8dfad2dTimo Sirainen cache->hdr_ro_copy.field_header_offset = hdr_offset;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return 0;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenstatic void mail_cache_mark_adding(struct mail_cache *cache, bool set)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen{
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen unsigned int i;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* we want to avoid adding all the fields one by one to the cache file,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen so just add all of them at once in here. the unused ones get dropped
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen later when compressing. */
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen for (i = 0; i < cache->fields_count; i++) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (set)
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen cache->fields[i].used = TRUE;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen cache->fields[i].adding = set;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen}
f6e301cb2060c4367d8145e2bf5d553ba87ceb34Timo Sirainen
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainenstatic int mail_cache_header_add_field(struct mail_cache_transaction_ctx *ctx,
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen unsigned int field_idx)
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen{
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen struct mail_cache *cache = ctx->cache;
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen int ret;
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen
87b8a4b6805d0b13af6a417dc70bdc74027bf1d3Timo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) {
87b8a4b6805d0b13af6a417dc70bdc74027bf1d3Timo Sirainen if (cache->file_fields_count <= field_idx) {
87b8a4b6805d0b13af6a417dc70bdc74027bf1d3Timo Sirainen cache->file_field_map =
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen i_realloc_type(cache->file_field_map,
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen unsigned int,
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen cache->file_fields_count,
06f537a8e0b399222cc2a7755015ef3963525fd2Timo Sirainen field_idx+1);
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen cache->file_fields_count = field_idx+1;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen cache->file_field_map[field_idx] = field_idx;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen cache->field_file_map[field_idx] = field_idx;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return 0;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (mail_cache_transaction_lock(ctx) <= 0) {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (MAIL_CACHE_IS_UNUSABLE(cache))
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen return -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen /* if we compressed the cache, the field should be there now.
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen it's however possible that someone else just compressed it
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen and we only reopened the cache file. */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (cache->field_file_map[field_idx] != (uint32_t)-1)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return 0;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* need to add it */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (mail_cache_transaction_lock(ctx) <= 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen /* re-read header to make sure we don't lose any fields. */
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen if (mail_cache_header_fields_read(cache) < 0) {
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen (void)mail_cache_unlock(cache);
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen return -1;
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen }
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen if (cache->field_file_map[field_idx] != (uint32_t)-1) {
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen /* it was already added */
26cdaf7097427fa90343260fa236af12ab93cca3Timo Sirainen if (mail_cache_unlock(cache) < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return 0;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen T_BEGIN {
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen buffer_t *buffer;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer = buffer_create_dynamic(pool_datastack_create(), 256);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_header_fields_get(cache, buffer);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = mail_cache_header_fields_write(ctx, buffer);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen } T_END;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ret == 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we wrote all the headers, so there are no pending changes */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen cache->field_header_write_pending = FALSE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = mail_cache_header_fields_read(cache);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen if (ret == 0 && cache->field_file_map[field_idx] == (uint32_t)-1) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_index_set_error(cache->index,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "Cache file %s: Newly added field got "
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen "lost unexpectedly", cache->filepath);
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen ret = -1;
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (mail_cache_unlock(cache) < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = -1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return ret;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenvoid mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int field_idx, const void *data, size_t data_size)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t file_field, data_size32;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen unsigned int fixed_size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen size_t full_size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen int ret;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(field_idx < ctx->cache->fields_count);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(data_size < (uint32_t)-1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->cache->fields[field_idx].field.decision ==
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->cache_file_seq == 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_open_if_needed(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (!MAIL_CACHE_IS_UNUSABLE(ctx->cache))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_file_seq = ctx->cache->hdr->file_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen } else if (!MAIL_CACHE_IS_UNUSABLE(ctx->cache) &&
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_file_seq != ctx->cache->hdr->file_seq) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* cache was compressed within this transaction */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_reset(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen file_field = ctx->cache->field_file_map[field_idx];
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (MAIL_CACHE_IS_UNUSABLE(ctx->cache) || file_field == (uint32_t)-1) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* we'll have to add this field to headers */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_mark_adding(ctx->cache, TRUE);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ret = mail_cache_header_add_field(ctx, field_idx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_mark_adding(ctx->cache, FALSE);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ret < 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->cache_file_seq == 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (MAIL_INDEX_IS_IN_MEMORY(ctx->cache->index))
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_file_seq = 1;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen else
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->cache_file_seq = ctx->cache->hdr->file_seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen file_field = ctx->cache->field_file_map[field_idx];
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(file_field != (uint32_t)-1);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(ctx->cache_file_seq != 0);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_decision_add(ctx->view, seq, field_idx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen fixed_size = ctx->cache->fields[field_idx].field.field_size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen i_assert(fixed_size == UINT_MAX || fixed_size == data_size);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen data_size32 = (uint32_t)data_size;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->prev_seq != seq) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_switch_seq(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->prev_seq = seq;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen /* remember roughly what we have modified, so cache lookups can
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen look into transactions to see changes. */
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen if (seq < ctx->view->trans_seq1 || ctx->view->trans_seq1 == 0)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen ctx->view->trans_seq1 = seq;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen if (seq > ctx->view->trans_seq2)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen ctx->view->trans_seq2 = seq;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen }
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen /* remember that this value exists, in case we try to look it up */
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen buffer_write(ctx->view->cached_exists_buf, field_idx,
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen &ctx->view->cached_exists_value, 1);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen full_size = (data_size + 3) & ~3;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen if (fixed_size == UINT_MAX)
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen full_size += sizeof(data_size32);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen if (ctx->cache_data->used + full_size > MAIL_CACHE_MAX_WRITE_BUFFER &&
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen ctx->last_rec_pos > 0) {
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen /* time to flush our buffer. if flushing fails because the
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen cache file had been compressed and was reopened, return
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen without adding the cached data since cache_data buffer
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen doesn't contain the cache_rec anymore. */
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen if (mail_cache_transaction_flush(ctx) < 0) {
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen /* make sure the transaction is reset, so we don't
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen constantly try to flush for each call to this
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen function */
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen mail_cache_transaction_reset(ctx);
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen return;
1ac19c5c2b66a12f5598792aad15114ee3eb62e2Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_append(ctx->cache_data, &file_field, sizeof(file_field));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (fixed_size == UINT_MAX) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_append(ctx->cache_data, &data_size32,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen sizeof(data_size32));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_append(ctx->cache_data, data, data_size);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if ((data_size & 3) != 0)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen buffer_append_zero(ctx->cache_data, 4 - (data_size & 3));
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen}
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainenbool mail_cache_field_want_add(struct mail_cache_transaction_ctx *ctx,
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen uint32_t seq, unsigned int field_idx)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen{
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen enum mail_cache_decision_type decision;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_transaction_open_if_needed(ctx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen decision = mail_cache_field_get_decision(ctx->view->cache, field_idx);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen decision &= ~MAIL_CACHE_DECISION_FORCED;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen switch (decision) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen case MAIL_CACHE_DECISION_NO:
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return FALSE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen case MAIL_CACHE_DECISION_TEMP:
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen /* add it only if it's newer than what we would drop when
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen compressing */
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (ctx->first_new_seq == 0) {
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen ctx->first_new_seq =
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen mail_cache_get_first_new_seq(ctx->view->view);
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen if (seq < ctx->first_new_seq)
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen return FALSE;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen break;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen default:
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen break;
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen }
6c2c5f20760b06bfb4a40b0ee2ef5ab016bc41f0Timo Sirainen
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen return mail_cache_field_exists(ctx->view, seq, field_idx) == 0;
dc9de21d4375faeedbe5b7e941502ac578650da9Timo Sirainen}
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainenbool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx,
6b2738c39a868ff9291867138c55029fc40cf105Timo Sirainen uint32_t seq, unsigned int field_idx)
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen{
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen enum mail_cache_decision_type decision;
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen mail_cache_transaction_open_if_needed(ctx);
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen decision = mail_cache_field_get_decision(ctx->view->cache, field_idx);
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen if (decision == (MAIL_CACHE_DECISION_FORCED | MAIL_CACHE_DECISION_NO))
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen return FALSE;
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen return mail_cache_field_exists(ctx->view, seq, field_idx) == 0;
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen}
9672bb2a11c37c275d695451accd824da5c9e485Timo Sirainen