index/dbox-single/sdbox-sync.c

	sdbox-sync.c revision d0b2bd9e2246eb68ed952c7f2e13d1969d657c8f
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi/* Copyright (c) 2007-2011 Dovecot authors, see the included COPYING file */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "lib.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "dbox-attachment.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "sdbox-storage.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "sdbox-file.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#include "sdbox-sync.h"
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi#define SDBOX_REBUILD_COUNT 3
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic void
1d980e5489836e977ba59b419e27b0ec875c4bd3takashidbox_sync_file_move_if_needed(struct dbox_file *file,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                  enum sdbox_sync_entry_type type)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    bool move_to_alt = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT;
3f08db06526d6901aa08c110b5bc7dde6bc39905nd    bool deleted;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (move_to_alt != dbox_file_is_in_alt(file)) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        /* move the file. if it fails, nothing broke so
3f08db06526d6901aa08c110b5bc7dde6bc39905nd           don't worry about it. */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (dbox_file_open(file, &deleted) > 0 && !deleted)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            (void)sdbox_file_move(file, move_to_alt);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
11495c9f0bd33e51a25b4d532beadfbcf9b944a3nilgun
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic void sdbox_sync_file(struct sdbox_sync_context *ctx,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                uint32_t seq, uint32_t uid,
f086b4b402fa9a2fefc7dda85de2a3cc1cd0a654rjung                enum sdbox_sync_entry_type type)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct dbox_file *file;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    switch (type) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    case SDBOX_SYNC_ENTRY_TYPE_EXPUNGE:
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if (!mail_index_transaction_is_expunged(ctx->trans, seq)) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            mail_index_expunge(ctx->trans, seq);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            array_append(&ctx->expunged_uids, &uid, 1);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        break;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    case SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT:
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    case SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT:
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        file = sdbox_file_init(ctx->mbox, uid);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        dbox_sync_file_move_if_needed(file, type);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        dbox_file_unref(&file);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        break;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic void sdbox_sync_add(struct sdbox_sync_context *ctx,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi               const struct mail_index_sync_rec *sync_rec)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t uid;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    enum sdbox_sync_entry_type type;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t seq, seq1, seq2;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        /* we're interested */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        type = SDBOX_SYNC_ENTRY_TYPE_EXPUNGE;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        /* we care only about alt flag changes */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun            type = SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        else if ((sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) != 0)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            type = SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        else
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            return;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    } else {
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        /* not interested */
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        return;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    }
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (!mail_index_lookup_seq_range(ctx->sync_view,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                     sync_rec->uid1, sync_rec->uid2,
b9f522ae1c0ed2bf3fc4444245bf28b2e2449a65nd                     &seq1, &seq2)) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        /* already expunged everything. nothing to do. */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    for (seq = seq1; seq <= seq2; seq++) {
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        mail_index_lookup_uid(ctx->sync_view, seq, &uid);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        sdbox_sync_file(ctx, seq, uid, type);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic int sdbox_sync_index(struct sdbox_sync_context *ctx)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    struct mailbox *box = &ctx->mbox->box;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    const struct mail_index_header *hdr;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_index_sync_rec sync_rec;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    uint32_t seq1, seq2;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    hdr = mail_index_get_header(ctx->sync_view);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (hdr->uid_validity == 0) {
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        /* newly created index file */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        mail_storage_set_critical(box->storage,
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun            "sdbox %s: Broken index: missing UIDVALIDITY",
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi            mailbox_get_path(box));
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        return 0;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    }
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    /* mark the newly seen messages as recent */
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                    hdr->next_uid, &seq1, &seq2))
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        index_mailbox_set_recent_seq(box, ctx->sync_view, seq1, seq2);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        sdbox_sync_add(ctx, &sync_rec);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    return 1;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic void dbox_sync_file_expunge(struct sdbox_sync_context *ctx,
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi                   uint32_t uid)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mailbox *box = &ctx->mbox->box;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct dbox_file *file;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct sdbox_file *sfile;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    int ret;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    file = sdbox_file_init(ctx->mbox, uid);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    sfile = (struct sdbox_file *)file;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    if (file->storage->attachment_dir != NULL)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        ret = sdbox_file_unlink_with_attachments(sfile);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    else
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        ret = dbox_file_unlink(file);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    /* do sync_notify only when the file was unlinked by us */
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    if (ret > 0 && box->v.sync_notify != NULL)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    dbox_file_unref(&file);
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi}
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi
1d980e5489836e977ba59b419e27b0ec875c4bd3takashistatic void dbox_sync_expunge_files(struct sdbox_sync_context *ctx)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    const uint32_t *uidp;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    /* NOTE: Index is no longer locked. Multiple processes may be unlinking
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun       the files at the same time. */
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    array_foreach(&ctx->expunged_uids, uidp)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        dbox_sync_file_expunge(ctx, *uidp);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    if (ctx->mbox->box.v.sync_notify != NULL)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        ctx->mbox->box.v.sync_notify(&ctx->mbox->box, 0, 0);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun}
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunstatic int
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunsdbox_refresh_header(struct sdbox_mailbox *mbox, bool retry, bool log_error)
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun{
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    struct mail_index_view *view;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    struct sdbox_index_header hdr;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    int ret;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    view = mail_index_view_open(mbox->box.index);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    ret = sdbox_read_header(mbox, &hdr, log_error);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    mail_index_view_close(&view);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    if (ret < 0 && retry) {
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        (void)mail_index_refresh(mbox->box.index);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun        return sdbox_refresh_header(mbox, FALSE, log_error);
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    }
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun    return ret;
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun}
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgunint sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags,
b2a930a0c94e9fd25f8d2b3a2c53573235db3f06nilgun             struct sdbox_sync_context **ctx_r)
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi{
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct mail_storage *storage = mbox->box.storage;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    struct sdbox_sync_context *ctx;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    enum mail_index_sync_flags sync_flags;
11495c9f0bd33e51a25b4d532beadfbcf9b944a3nilgun    unsigned int i;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    int ret;
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    bool rebuild, force_rebuild;
f086b4b402fa9a2fefc7dda85de2a3cc1cd0a654rjung
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi    force_rebuild = (flags & SDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
5effc8b39fae5cd169d17f342bfc265705840014rbowen    rebuild = force_rebuild ||
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        mbox->corrupted_rebuild_count != 0 ||
1d980e5489836e977ba59b419e27b0ec875c4bd3takashi        sdbox_refresh_header(mbox, TRUE, FALSE) < 0;

    ctx = i_new(struct sdbox_sync_context, 1);
    ctx->mbox = mbox;
    ctx->flags = flags;
    i_array_init(&ctx->expunged_uids, 32);

    sync_flags = index_storage_get_sync_flags(&mbox->box);
    if (!rebuild && (flags & SDBOX_SYNC_FLAG_FORCE) == 0)
        sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
    if ((flags & SDBOX_SYNC_FLAG_FSYNC) != 0)
        sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
    /* don't write unnecessary dirty flag updates */
    sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;

    for (i = 0;; i++) {
        ret = mail_index_sync_begin(mbox->box.index,
                        &ctx->index_sync_ctx,
                        &ctx->sync_view, &ctx->trans,
                        sync_flags);
        if (mail_index_reset_fscked(mbox->box.index))
            sdbox_set_mailbox_corrupted(&mbox->box);
        if (ret <= 0) {
            if (ret < 0)
                mail_storage_set_index_error(&mbox->box);
            array_free(&ctx->expunged_uids);
            i_free(ctx);
            *ctx_r = NULL;
            return ret;
        }

        if (rebuild)
            ret = 0;
        else {
            if ((ret = sdbox_sync_index(ctx)) > 0)
                break;
        }

        /* failure. keep the index locked while we're doing a
           rebuild. */
        if (ret == 0) {
            if (i >= SDBOX_REBUILD_COUNT) {
                mail_storage_set_critical(storage,
                    "sdbox %s: Index keeps breaking",
                    mailbox_get_path(&ctx->mbox->box));
                ret = -1;
            } else {
                /* do a full resync and try again. */
                i_warning("sdbox %s: Rebuilding index",
                      mailbox_get_path(&ctx->mbox->box));
                rebuild = FALSE;
                ret = sdbox_sync_index_rebuild(mbox,
                                   force_rebuild);
            }
        }
        mail_index_sync_rollback(&ctx->index_sync_ctx);
        if (ret < 0) {
            array_free(&ctx->expunged_uids);
            i_free(ctx);
            return -1;
        }
    }

    *ctx_r = ctx;
    return 0;
}

int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success)
{
    struct sdbox_sync_context *ctx = *_ctx;
    int ret = success ? 0 : -1;

    *_ctx = NULL;

    if (success) {
        if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
            mail_storage_set_index_error(&ctx->mbox->box);
            ret = -1;
        } else {
            dbox_sync_expunge_files(ctx);
        }
    } else {
        mail_index_sync_rollback(&ctx->index_sync_ctx);
    }

    array_free(&ctx->expunged_uids);
    i_free(ctx);
    return ret;
}

int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags)
{
    struct sdbox_sync_context *sync_ctx;

    if (sdbox_sync_begin(mbox, flags, &sync_ctx) < 0)
        return -1;

    if (sync_ctx == NULL)
        return 0;
    return sdbox_sync_finish(&sync_ctx, TRUE);
}

struct mailbox_sync_context *
sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
    struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box;
    enum sdbox_sync_flags sdbox_sync_flags = 0;
    int ret = 0;

    if (!box->opened) {
        if (mailbox_open(box) < 0)
            ret = -1;
    }

    if (mail_index_reset_fscked(box->index))
        sdbox_set_mailbox_corrupted(box);
    if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) ||
             mbox->corrupted_rebuild_count != 0)) {
        if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
            sdbox_sync_flags |= SDBOX_SYNC_FLAG_FORCE_REBUILD;
        ret = sdbox_sync(mbox, sdbox_sync_flags);
    }

    return index_mailbox_sync_init(box, flags, ret < 0);
}