index/dbox-single/sdbox-sync.c

	sdbox-sync.c revision ecd54acb3f5be767cfe3e4a759f3edeb24abf6d3
0N/A/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */
2362N/A
0N/A#include "lib.h"
0N/A#include "dbox-attachment.h"
0N/A#include "sdbox-storage.h"
0N/A#include "sdbox-file.h"
2362N/A#include "sdbox-sync.h"
0N/A#include "mailbox-recent-flags.h"
2362N/A
0N/A#define SDBOX_REBUILD_COUNT 3
0N/A
0N/Astatic void
0N/Adbox_sync_file_move_if_needed(struct dbox_file *file,
0N/A                  enum sdbox_sync_entry_type type)
0N/A{
0N/A    struct stat st;
0N/A    bool move_to_alt = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT;
0N/A    bool deleted;
0N/A
0N/A    if (move_to_alt == dbox_file_is_in_alt(file) &&
2362N/A        !move_to_alt) {
2362N/A        /* unopened dbox files default to primary dir.
2362N/A           stat the file to update its location. */
0N/A        (void)dbox_file_stat(file, &st);
0N/A
0N/A    }
0N/A    if (move_to_alt != dbox_file_is_in_alt(file)) {
0N/A        /* move the file. if it fails, nothing broke so
0N/A           don't worry about it. */
0N/A        if (dbox_file_open(file, &deleted) > 0 && !deleted)
0N/A            (void)sdbox_file_move(file, move_to_alt);
0N/A    }
0N/A}
0N/A
0N/Astatic void sdbox_sync_file(struct sdbox_sync_context *ctx,
0N/A                uint32_t seq, uint32_t uid,
0N/A                enum sdbox_sync_entry_type type)
0N/A{
0N/A    struct dbox_file *file;
0N/A    enum modify_type modify_type;
0N/A
0N/A    switch (type) {
0N/A    case SDBOX_SYNC_ENTRY_TYPE_EXPUNGE:
0N/A        if (!mail_index_transaction_is_expunged(ctx->trans, seq)) {
0N/A            mail_index_expunge(ctx->trans, seq);
0N/A            array_append(&ctx->expunged_uids, &uid, 1);
0N/A        }
0N/A        break;
0N/A    case SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT:
0N/A    case SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT:
0N/A        /* update flags in the sync transaction, mainly to make
0N/A           sure that these alt changes get marked as synced
0N/A           and won't be retried */
0N/A        modify_type = type == SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT ?
0N/A            MODIFY_ADD : MODIFY_REMOVE;
0N/A        mail_index_update_flags(ctx->trans, seq, modify_type,
0N/A                    (enum mail_flags)DBOX_INDEX_FLAG_ALT);
0N/A        file = sdbox_file_init(ctx->mbox, uid);
0N/A        dbox_sync_file_move_if_needed(file, type);
0N/A        dbox_file_unref(&file);
0N/A        break;
0N/A    }
0N/A}
0N/A
0N/Astatic void sdbox_sync_add(struct sdbox_sync_context *ctx,
0N/A               const struct mail_index_sync_rec *sync_rec)
0N/A{
0N/A    uint32_t uid;
0N/A    enum sdbox_sync_entry_type type;
0N/A    uint32_t seq, seq1, seq2;
0N/A
0N/A    if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
0N/A        /* we're interested */
0N/A        type = SDBOX_SYNC_ENTRY_TYPE_EXPUNGE;
0N/A    } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) {
0N/A        /* we care only about alt flag changes */
0N/A        if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0)
0N/A            type = SDBOX_SYNC_ENTRY_TYPE_MOVE_TO_ALT;
0N/A        else if ((sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) != 0)
0N/A            type = SDBOX_SYNC_ENTRY_TYPE_MOVE_FROM_ALT;
0N/A        else
0N/A            return;
0N/A    } else {
0N/A        /* not interested */
0N/A        return;
0N/A    }
0N/A
0N/A    if (!mail_index_lookup_seq_range(ctx->sync_view,
0N/A                     sync_rec->uid1, sync_rec->uid2,
0N/A                     &seq1, &seq2)) {
0N/A        /* already expunged everything. nothing to do. */
0N/A        return;
0N/A    }
0N/A
0N/A    for (seq = seq1; seq <= seq2; seq++) {
0N/A        mail_index_lookup_uid(ctx->sync_view, seq, &uid);
0N/A        sdbox_sync_file(ctx, seq, uid, type);
0N/A    }
0N/A}
0N/A
0N/Astatic int sdbox_sync_index(struct sdbox_sync_context *ctx)
0N/A{
0N/A    struct mailbox *box = &ctx->mbox->box;
0N/A    const struct mail_index_header *hdr;
0N/A    struct mail_index_sync_rec sync_rec;
0N/A    uint32_t seq1, seq2;
0N/A
0N/A    hdr = mail_index_get_header(ctx->sync_view);
0N/A    if (hdr->uid_validity == 0) {
0N/A        /* newly created index file */
0N/A        if (hdr->next_uid == 1) {
0N/A            /* could be just a race condition where we opened the
0N/A               mailbox between mkdir and index creation. fix this
0N/A               silently. */
0N/A            if (sdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0)
0N/A                return -1;
0N/A            return 1;
0N/A        }
0N/A        mail_storage_set_critical(box->storage,
0N/A            "sdbox %s: Broken index: missing UIDVALIDITY",
0N/A            mailbox_get_path(box));
0N/A        sdbox_set_mailbox_corrupted(box);
0N/A        return 0;
0N/A    }
0N/A
0N/A    /* mark the newly seen messages as recent */
0N/A    if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
0N/A                    hdr->next_uid, &seq1, &seq2))
0N/A        mailbox_recent_flags_set_seqs(box, ctx->sync_view, seq1, seq2);
0N/A
0N/A    while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))
0N/A        sdbox_sync_add(ctx, &sync_rec);
0N/A    return 1;
0N/A}
0N/A
0N/Astatic void dbox_sync_file_expunge(struct sdbox_sync_context *ctx,
0N/A                   uint32_t uid)
{
    struct mailbox *box = &ctx->mbox->box;
    struct dbox_file *file;
    struct sdbox_file *sfile;
    int ret;

    file = sdbox_file_init(ctx->mbox, uid);
    sfile = (struct sdbox_file *)file;
    if (file->storage->attachment_dir != NULL)
        ret = sdbox_file_unlink_with_attachments(sfile);
    else
        ret = dbox_file_unlink(file);

    /* do sync_notify only when the file was unlinked by us */
    if (ret > 0 && box->v.sync_notify != NULL)
        box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
    dbox_file_unref(&file);
}

static void dbox_sync_expunge_files(struct sdbox_sync_context *ctx)
{
    const uint32_t *uidp;

    /* NOTE: Index is no longer locked. Multiple processes may be unlinking
       the files at the same time. */
    ctx->mbox->box.tmp_sync_view = ctx->sync_view;
    array_foreach(&ctx->expunged_uids, uidp) T_BEGIN {
        dbox_sync_file_expunge(ctx, *uidp);
    } T_END;
    if (ctx->mbox->box.v.sync_notify != NULL)
        ctx->mbox->box.v.sync_notify(&ctx->mbox->box, 0, 0);
    ctx->mbox->box.tmp_sync_view = NULL;
}

static int
sdbox_refresh_header(struct sdbox_mailbox *mbox, bool retry, bool log_error)
{
    struct mail_index_view *view;
    struct sdbox_index_header hdr;
    bool need_resize;
    int ret;

    view = mail_index_view_open(mbox->box.index);
    ret = sdbox_read_header(mbox, &hdr, log_error, &need_resize);
    mail_index_view_close(&view);

    if (ret < 0 && retry) {
        mail_index_refresh(mbox->box.index);
        return sdbox_refresh_header(mbox, FALSE, log_error);
    }
    return ret;
}

int sdbox_sync_begin(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags,
             struct sdbox_sync_context **ctx_r)
{
    struct mail_storage *storage = mbox->box.storage;
    struct sdbox_sync_context *ctx;
    enum mail_index_sync_flags sync_flags;
    unsigned int i;
    int ret;
    bool rebuild, force_rebuild;

    force_rebuild = (flags & SDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
    rebuild = force_rebuild ||
        mbox->corrupted_rebuild_count != 0 ||
        sdbox_refresh_header(mbox, TRUE, FALSE) < 0;

    ctx = i_new(struct sdbox_sync_context, 1);
    ctx->mbox = mbox;
    ctx->flags = flags;
    i_array_init(&ctx->expunged_uids, 32);

    sync_flags = index_storage_get_sync_flags(&mbox->box);
    if (!rebuild && (flags & SDBOX_SYNC_FLAG_FORCE) == 0)
        sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
    if ((flags & SDBOX_SYNC_FLAG_FSYNC) != 0)
        sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
    /* don't write unnecessary dirty flag updates */
    sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;

    for (i = 0;; i++) {
        ret = index_storage_expunged_sync_begin(&mbox->box,
                &ctx->index_sync_ctx, &ctx->sync_view,
                &ctx->trans, sync_flags);
        if (mail_index_reset_fscked(mbox->box.index))
            sdbox_set_mailbox_corrupted(&mbox->box);
        if (ret <= 0) {
            array_free(&ctx->expunged_uids);
            i_free(ctx);
            *ctx_r = NULL;
            return ret;
        }

        if (rebuild)
            ret = 0;
        else {
            if ((ret = sdbox_sync_index(ctx)) > 0)
                break;
        }

        /* failure. keep the index locked while we're doing a
           rebuild. */
        if (ret == 0) {
            if (i >= SDBOX_REBUILD_COUNT) {
                mail_storage_set_critical(storage,
                    "sdbox %s: Index keeps breaking",
                    mailbox_get_path(&ctx->mbox->box));
                ret = -1;
            } else {
                /* do a full resync and try again. */
                rebuild = FALSE;
                ret = sdbox_sync_index_rebuild(mbox,
                                   force_rebuild);
            }
        }
        mail_index_sync_rollback(&ctx->index_sync_ctx);
        if (ret < 0) {
            index_storage_expunging_deinit(&ctx->mbox->box);
            array_free(&ctx->expunged_uids);
            i_free(ctx);
            return -1;
        }
    }

    *ctx_r = ctx;
    return 0;
}

int sdbox_sync_finish(struct sdbox_sync_context **_ctx, bool success)
{
    struct sdbox_sync_context *ctx = *_ctx;
    int ret = success ? 0 : -1;

    *_ctx = NULL;

    if (success) {
        mail_index_view_ref(ctx->sync_view);

        if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
            mailbox_set_index_error(&ctx->mbox->box);
            ret = -1;
        } else {
            dbox_sync_expunge_files(ctx);
            mail_index_view_close(&ctx->sync_view);
        }
    } else {
        mail_index_sync_rollback(&ctx->index_sync_ctx);
    }

    index_storage_expunging_deinit(&ctx->mbox->box);
    array_free(&ctx->expunged_uids);
    i_free(ctx);
    return ret;
}

int sdbox_sync(struct sdbox_mailbox *mbox, enum sdbox_sync_flags flags)
{
    struct sdbox_sync_context *sync_ctx;

    if (sdbox_sync_begin(mbox, flags, &sync_ctx) < 0)
        return -1;

    if (sync_ctx == NULL)
        return 0;
    return sdbox_sync_finish(&sync_ctx, TRUE);
}

struct mailbox_sync_context *
sdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
    struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box;
    enum sdbox_sync_flags sdbox_sync_flags = 0;
    int ret = 0;

    if (!box->opened) {
        if (mailbox_open(box) < 0)
            ret = -1;
    }

    if (ret == 0 && mail_index_reset_fscked(box->index))
        sdbox_set_mailbox_corrupted(box);
    if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) ||
             mbox->corrupted_rebuild_count != 0)) {
        if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
            sdbox_sync_flags |= SDBOX_SYNC_FLAG_FORCE_REBUILD;
        ret = sdbox_sync(mbox, sdbox_sync_flags);
    }

    return index_mailbox_sync_init(box, flags, ret < 0);
}