index/dbox-multi/mdbox-sync.c

bcb4e51a409d94ae670de96afb8483a4f7855294Stephan Bosch/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen/*
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen   Expunging works like:
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen   1. Lock map index by beginning a map sync.
65b94e73c305dcb209cf958f938b93ec061c67a9Timo Sirainen   2. Write map UID refcount changes to map index (=> tail != head).
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen   3. Expunge messages from mailbox index.
65b94e73c305dcb209cf958f938b93ec061c67a9Timo Sirainen   4. Finish map sync, which updates tail=head and unlocks map index.
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen
65b94e73c305dcb209cf958f938b93ec061c67a9Timo Sirainen   If something crashes after 2 but before 4 is finished, tail != head and
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen   reader can do a full resync to figure out what got broken.
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen*/
c977374bd4651cafc1626ebe308aa66dfd8b30e0Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "lib.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "array.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "ioloop.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "str.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "mdbox-storage.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "mdbox-storage-rebuild.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "mdbox-map.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "mdbox-file.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen#include "mdbox-sync.h"
436adac819e7cbeef04af08dcc6a4f3ecd4a1d9eMartti Rannanjärvi#include "mailbox-recent-flags.h"
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
9955f6cba7652469b1d600a3674e8d27dd4e61bdTimo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenstatic int
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainendbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq,
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen                  const guid_128_t guid_128)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    const void *data;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    uint32_t uid;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    mail_index_lookup_uid(ctx->sync_view, seq, &uid);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    mail_index_lookup_ext(ctx->sync_view, seq,
03010dbaa74ec70f062994dfe3cd39bedc99a28bTimo Sirainen                  ctx->mbox->guid_ext_id, &data, NULL);
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen    if (guid_128_is_empty(guid_128) ||
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen        memcmp(data, guid_128, GUID_128_SIZE) == 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi    mailbox_set_critical(&ctx->mbox->box,
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi        "Expunged GUID mismatch for UID %u: %s vs %s",
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi        uid, guid_128_to_string(data), guid_128_to_string(guid_128));
5b62dea2f88165f3f4d87bba9011343f3ff415ffTimo Sirainen    mdbox_storage_set_corrupted(ctx->mbox->storage);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenstatic int mdbox_sync_expunge(struct mdbox_sync_context *ctx, uint32_t seq,
de62ce819d59a529530da4b57be1b8d6dad13d6bTimo Sirainen                  const guid_128_t guid_128)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    uint32_t map_uid;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen    if (seq_range_array_add(&ctx->expunged_seqs, seq)) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        /* already marked as expunged in this sync */
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (dbox_sync_verify_expunge_guid(ctx, seq, guid_128) < 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return -1;
c18ff860dc22960fd37c272d929f889c7939a2c8Timo Sirainen    if (mdbox_map_update_refcount(ctx->map_trans, map_uid, -1) < 0)
73a87c2ff65c6116cde6fb158dfddb8ef7346901Timo Sirainen        return -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
73a87c2ff65c6116cde6fb158dfddb8ef7346901Timo Sirainenstatic int mdbox_sync_rec(struct mdbox_sync_context *ctx,
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen              const struct mail_index_sync_rec *sync_rec)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    uint32_t seq, seq1, seq2;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        /* not interested */
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (!mail_index_lookup_seq_range(ctx->sync_view,
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen                     sync_rec->uid1, sync_rec->uid2,
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen                     &seq1, &seq2)) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        /* already expunged everything. nothing to do. */
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    for (seq = seq1; seq <= seq2; seq++) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        if (mdbox_sync_expunge(ctx, seq, sync_rec->guid_128) < 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            return -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainenstatic int dbox_sync_mark_expunges(struct mdbox_sync_context *ctx)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    enum mail_index_transaction_flags flags =
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
d22301419109ed4a38351715e6760011421dadecTimo Sirainen    struct mailbox *box = &ctx->mbox->box;
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    struct mail_index_transaction *trans;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    struct seq_range_iter iter;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    unsigned int n;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    const void *data;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    uint32_t seq, uid;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    /* use a separate transaction here so that we can commit the changes
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen       during map transaction */
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    trans = mail_index_transaction_begin(ctx->sync_view, flags);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    while (seq_range_array_iter_nth(&iter, n++, &seq)) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        mail_index_lookup_uid(ctx->sync_view, seq, &uid);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        mail_index_lookup_ext(ctx->sync_view, seq,
03010dbaa74ec70f062994dfe3cd39bedc99a28bTimo Sirainen                      ctx->mbox->guid_ext_id, &data, NULL);
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        mail_index_expunge_guid(trans, seq, data);
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    }
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    if (mail_index_transaction_commit(&trans) < 0)
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        return -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    if (box->v.sync_notify != NULL) {
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        /* do notifications after commit finished successfully */
6ae329de09afb7214c906d762320847e05469d53Timo Sirainen        box->tmp_sync_view = ctx->sync_view;
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0;
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        while (seq_range_array_iter_nth(&iter, n++, &seq)) {
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen            mail_index_lookup_uid(ctx->sync_view, seq, &uid);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen        }
6ae329de09afb7214c906d762320847e05469d53Timo Sirainen        box->tmp_sync_view = NULL;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
637455ebee0453f860c9bce0626c485e35fb83deTimo Sirainen    return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenstatic int mdbox_sync_index(struct mdbox_sync_context *ctx)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
d22301419109ed4a38351715e6760011421dadecTimo Sirainen    struct mailbox *box = &ctx->mbox->box;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    const struct mail_index_header *hdr;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    struct mail_index_sync_rec sync_rec;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    uint32_t seq1, seq2;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    int ret = 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    hdr = mail_index_get_header(ctx->sync_view);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (hdr->uid_validity == 0) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        /* newly created index file */
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen        if (hdr->next_uid == 1) {
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen            /* could be just a race condition where we opened the
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen               mailbox between mkdir and index creation. fix this
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen               silently. */
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen            if (mdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0)
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen                return -1;
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen            return 1;
f605df8a4c15cc7a11e16fdde994d51473700890Timo Sirainen        }
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi        mailbox_set_critical(box, "Broken index: missing UIDVALIDITY");
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    /* mark the newly seen messages as recent */
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen                    hdr->next_uid, &seq1, &seq2)) {
817d027593510c3ba70ad542ce0011f5f6916d1eTimo Sirainen        mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view,
817d027593510c3ba70ad542ce0011f5f6916d1eTimo Sirainen                          seq1, seq2);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
65b94e73c305dcb209cf958f938b93ec061c67a9Timo Sirainen    /* handle syncing records without map being locked. */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if (mdbox_map_atomic_is_locked(ctx->atomic)) {
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        ctx->map_trans = mdbox_map_transaction_begin(ctx->atomic, FALSE);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        i_array_init(&ctx->expunged_seqs, 64);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
73a87c2ff65c6116cde6fb158dfddb8ef7346901Timo Sirainen        if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            break;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
73a87c2ff65c6116cde6fb158dfddb8ef7346901Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    /* write refcount changes to map index. transaction commit updates the
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen       log head, while tail is left behind. */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if (mdbox_map_atomic_is_locked(ctx->atomic)) {
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        if (ret == 0)
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen            ret = mdbox_map_transaction_commit(ctx->map_trans, "mdbox syncing");
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        /* write changes to mailbox index */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        if (ret == 0)
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen            ret = dbox_sync_mark_expunges(ctx);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        /* finish the map changes and unlock the map. this also updates
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen           map's tail -> head. */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        if (ret < 0)
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen            mdbox_map_atomic_set_failed(ctx->atomic);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        mdbox_map_transaction_free(&ctx->map_trans);
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        ctx->expunged_count = seq_range_count(&ctx->expunged_seqs);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        array_free(&ctx->expunged_seqs);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (box->v.sync_notify != NULL)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        box->v.sync_notify(box, 0, 0);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return ret == 0 ? 1 :
5b62dea2f88165f3f4d87bba9011343f3ff415ffTimo Sirainen        (ctx->mbox->storage->corrupted ? 0 : -1);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainenstatic int mdbox_sync_try_begin(struct mdbox_sync_context *ctx,
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen                enum mail_index_sync_flags sync_flags)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    struct mdbox_mailbox *mbox = ctx->mbox;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    int ret;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen    ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx,
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen                        &ctx->sync_view, &ctx->trans, sync_flags);
d0b2bd9e2246eb68ed952c7f2e13d1969d657c8fTimo Sirainen    if (mail_index_reset_fscked(mbox->box.index))
d0b2bd9e2246eb68ed952c7f2e13d1969d657c8fTimo Sirainen        mdbox_storage_set_corrupted(mbox->storage);
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen    if (ret <= 0)
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen        return ret; /* error / nothing to do */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if (!mdbox_map_atomic_is_locked(ctx->atomic) &&
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        mail_index_sync_has_expunges(ctx->index_sync_ctx)) {
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        /* we have expunges, so we need to write to map.
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen           it needs to be locked before mailbox index. */
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        mail_index_sync_set_reason(ctx->index_sync_ctx, "mdbox expunge check");
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        mail_index_sync_rollback(&ctx->index_sync_ctx);
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen        index_storage_expunging_deinit(&ctx->mbox->box);
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        if (mdbox_map_atomic_lock(ctx->atomic, "mdbox syncing with expunges") < 0)
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen            return -1;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        return mdbox_sync_try_begin(ctx, sync_flags);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    }
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    return 1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenint mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen             struct mdbox_map_atomic_context *atomic,
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen             struct mdbox_sync_context **ctx_r)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
96088ba9ef3b6c113a0a41f3778cd38f437ddc3dTimo Sirainen    const struct mail_index_header *hdr =
96088ba9ef3b6c113a0a41f3778cd38f437ddc3dTimo Sirainen        mail_index_get_header(mbox->box.view);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    struct mdbox_sync_context *ctx;
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen    const char *reason;
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen    enum mail_index_sync_flags sync_flags;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    int ret;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    bool rebuild, storage_rebuilt = FALSE;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    *ctx_r = NULL;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen
dd4f30895ebbddd77e000472fbadcb3128ae2883Timo Sirainen    /* avoid race conditions with mailbox creation, don't check for dbox
dd4f30895ebbddd77e000472fbadcb3128ae2883Timo Sirainen       headers until syncing has locked the mailbox */
dd4f30895ebbddd77e000472fbadcb3128ae2883Timo Sirainen    rebuild = mbox->storage->corrupted ||
96088ba9ef3b6c113a0a41f3778cd38f437ddc3dTimo Sirainen        (hdr->flags & MAIL_INDEX_HDR_FLAG_FSCKD) != 0 ||
96088ba9ef3b6c113a0a41f3778cd38f437ddc3dTimo Sirainen        mdbox_map_is_fscked(mbox->storage->map) ||
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        (flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
f71c2d4e6b802bf8e622bcd5df29286262d05d5aTimo Sirainen    if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) {
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            return -1;
817d027593510c3ba70ad542ce0011f5f6916d1eTimo Sirainen        mailbox_recent_flags_reset(&mbox->box);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        storage_rebuilt = TRUE;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    ctx = i_new(struct mdbox_sync_context, 1);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    ctx->mbox = mbox;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    ctx->flags = flags;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    ctx->atomic = atomic;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
3f91e60401495a4046c73992fabaa5e77200a451Timo Sirainen    sync_flags = index_storage_get_sync_flags(&mbox->box);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    /* don't write unnecessary dirty flag updates */
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    ret = mdbox_sync_try_begin(ctx, sync_flags);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if (ret <= 0) {
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        /* failed / nothing to do */
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen        index_storage_expunging_deinit(&mbox->box);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        i_free(ctx);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        return ret;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if ((ret = mdbox_sync_index(ctx)) <= 0) {
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        mail_index_sync_set_reason(ctx->index_sync_ctx,
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen            ret < 0 ? "mdbox syncing failed" :
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen            "mdbox syncing found corruption");
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        mail_index_sync_rollback(&ctx->index_sync_ctx);
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen        index_storage_expunging_deinit(&mbox->box);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        i_free_and_null(ctx);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        if (ret < 0)
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen            return -1;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        /* corrupted */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        if (storage_rebuilt) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi                "mdbox: Storage keeps breaking");
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            return -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        }
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        /* we'll need to rebuild storage.
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen           try again from the beginning. */
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        mdbox_storage_set_corrupted(mbox->storage);
37ee89f3cba22cd975912a882f0d3097fa5031e1Timo Sirainen        if ((flags & MDBOX_SYNC_FLAG_NO_REBUILD) != 0) {
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi            mailbox_set_critical(&mbox->box,
d4002fe1f64d25a792f76fb102ef7dc519cd4e24Martti Rannanjärvi                "mdbox: Can't rebuild storage");
37ee89f3cba22cd975912a882f0d3097fa5031e1Timo Sirainen            return -1;
37ee89f3cba22cd975912a882f0d3097fa5031e1Timo Sirainen        }
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        return mdbox_sync_begin(mbox, flags, atomic, ctx_r);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
9963bef626fd9ea227fb606e8b1694cdb1ab39aaTimo Sirainen    index_storage_expunging_deinit(&mbox->box);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen    if (!mdbox_map_atomic_is_locked(ctx->atomic))
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        reason = "mdbox synced";
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen    else {
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        /* may be 0 msgs, but that still informs that the map
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen           was locked */
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen        reason = t_strdup_printf("mdbox synced - %u msgs expunged",
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen                     ctx->expunged_count);
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen    }
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen    mail_index_sync_set_reason(ctx->index_sync_ctx, reason);
62958c5eefcd7dd84717b487ca36ec3a86949eb9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    *ctx_r = ctx;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenint mdbox_sync_finish(struct mdbox_sync_context **_ctx, bool success)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    struct mdbox_sync_context *ctx = *_ctx;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    int ret = success ? 0 : -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    *_ctx = NULL;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    if (success) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
1f1ee8db68d9ae1604350801cd8dc33ebe29fe8aTimo Sirainen            mailbox_set_index_error(&ctx->mbox->box);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            ret = -1;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    } else {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        mail_index_sync_rollback(&ctx->index_sync_ctx);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    i_free(ctx);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return ret;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenint mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    struct mdbox_sync_context *sync_ctx;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    struct mdbox_map_atomic_context *atomic;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    int ret;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    atomic = mdbox_map_atomic_begin(mbox->storage->map);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    ret = mdbox_sync_begin(mbox, flags, atomic, &sync_ctx);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if (ret == 0 && sync_ctx != NULL)
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        ret = mdbox_sync_finish(&sync_ctx, TRUE);
4fded1eec06aba9ce37887ac30619768760cd0d0Timo Sirainen    if (ret == 0)
4fded1eec06aba9ce37887ac30619768760cd0d0Timo Sirainen        mdbox_map_atomic_set_success(atomic);
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    if (mdbox_map_atomic_finish(&atomic) < 0)
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen        ret = -1;
08a8b3de61139ba02371afc8240ac85be0e8b17cTimo Sirainen    return ret;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenstruct mailbox_sync_context *
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainenmdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen{
b67974c4b89ab6950c2694cce8dfb1b6561cc084Josef 'Jeff' Sipek    struct mdbox_mailbox *mbox = MDBOX_MAILBOX(box);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    enum mdbox_sync_flags mdbox_sync_flags = 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    int ret = 0;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen    if (mail_index_reset_fscked(box->index))
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen        mdbox_storage_set_corrupted(mbox->storage);
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen    if (index_mailbox_want_full_sync(&mbox->box, flags) ||
e098ac1cd4ff0afb33b22cc7fd288014f4fdb0bcTimo Sirainen        mbox->storage->corrupted) {
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen            mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD;
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen        ret = mdbox_sync(mbox, mdbox_sync_flags);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    }
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen    return index_mailbox_sync_init(box, flags, ret < 0);
1ac7c8e9040e0d0b7e9f849e45b94bfe919595a9Timo Sirainen}