index/dbox-multi/mdbox-sync.c

	mdbox-sync.c revision 3f91e60401495a4046c73992fabaa5e77200a451
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
c25356d5978632df6203437e1953bcb29e0c736fTimo Sirainen
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "lib.h"
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "array.h"
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "ioloop.h"
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "hex-binary.h"
8e6e668126491f248d5305b5f7b15f46f03f07b1Timo Sirainen#include "str.h"
8e6e668126491f248d5305b5f7b15f46f03f07b1Timo Sirainen#include "mdbox-storage.h"
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "mdbox-storage-rebuild.h"
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "mdbox-map.h"
a128456c8530046dd9a998386f383160e70b532fTimo Sirainen#include "mdbox-file.h"
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen#include "mdbox-sync.h"
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen#include <stdlib.h>
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen#include <dirent.h>
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen#define DBOX_REBUILD_COUNT 3
9e95ca63385f300c7625beba9174cbf0e019d406Timo Sirainen
a128456c8530046dd9a998386f383160e70b532fTimo Sirainenstatic int
dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq,
                  const uint8_t guid_128[MAIL_GUID_128_SIZE])
{
    const void *data;
    uint32_t uid;

    mail_index_lookup_uid(ctx->sync_view, seq, &uid);
    mail_index_lookup_ext(ctx->sync_view, seq,
                  ctx->mbox->guid_ext_id, &data, NULL);
    if (mail_guid_128_is_empty(guid_128) ||
        memcmp(data, guid_128, MAIL_GUID_128_SIZE) == 0)
        return 0;

    mail_storage_set_critical(&ctx->mbox->storage->storage.storage,
        "Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s",
        ctx->mbox->box.vname, uid,
        binary_to_hex(data, MAIL_GUID_128_SIZE),
        binary_to_hex(guid_128, MAIL_GUID_128_SIZE));
    ctx->mbox->storage->storage.files_corrupted = TRUE;
    return -1;
}

static int mdbox_sync_expunge(struct mdbox_sync_context *ctx, uint32_t seq,
                  const uint8_t guid_128[MAIL_GUID_128_SIZE])
{
    uint32_t map_uid;

    if (seq_range_exists(&ctx->expunged_seqs, seq)) {
        /* already marked as expunged in this sync */
        return 0;
    }

    if (dbox_sync_verify_expunge_guid(ctx, seq, guid_128) < 0)
        return -1;

    if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0)
        return -1;

    seq_range_array_add(&ctx->expunged_seqs, 0, seq);
    array_append(&ctx->expunged_map_uids, &map_uid, 1);
    return 0;
}

static int mdbox_sync_add(struct mdbox_sync_context *ctx,
              const struct mail_index_sync_rec *sync_rec)
{
    uint32_t seq, seq1, seq2;

    if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
        /* not interested */
        return 0;
    }

    if (!mail_index_lookup_seq_range(ctx->sync_view,
                     sync_rec->uid1, sync_rec->uid2,
                     &seq1, &seq2)) {
        /* already expunged everything. nothing to do. */
        return 0;
    }

    for (seq = seq1; seq <= seq2; seq++) {
        if (mdbox_sync_expunge(ctx, seq, sync_rec->guid_128) < 0)
            return -1;
    }
    return 0;
}

static void dbox_sync_mark_expunges(struct mdbox_sync_context *ctx)
{
    struct mailbox *box = &ctx->mbox->box;
    struct seq_range_iter iter;
    unsigned int n;
    const void *data;
    uint32_t seq, uid;

    seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0;
    while (seq_range_array_iter_nth(&iter, n++, &seq)) {
        mail_index_lookup_uid(ctx->sync_view, seq, &uid);
        mail_index_lookup_ext(ctx->sync_view, seq,
                      ctx->mbox->guid_ext_id, &data, NULL);
        mail_index_expunge_guid(ctx->trans, seq, data);

        if (box->v.sync_notify != NULL)
            box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
    }
}

static int mdbox_sync_index_finish_expunges(struct mdbox_sync_context *ctx)
{
    struct dbox_map_transaction_context *map_trans;
    int ret;

    map_trans = dbox_map_transaction_begin(ctx->mbox->storage->map, FALSE);
    ret = dbox_map_update_refcounts(map_trans, &ctx->expunged_map_uids, -1);
    if (ret == 0) {
        ret = dbox_map_transaction_commit(map_trans);
        if (ret == 0)
            dbox_sync_mark_expunges(ctx);
    }

    dbox_map_transaction_free(&map_trans);
    return ret;
}

static int mdbox_sync_index(struct mdbox_sync_context *ctx)
{
    struct mailbox *box = &ctx->mbox->box;
    const struct mail_index_header *hdr;
    struct mail_index_sync_rec sync_rec;
    uint32_t seq1, seq2;
    int ret = 0;

    hdr = mail_index_get_header(ctx->sync_view);
    if (hdr->uid_validity == 0) {
        /* newly created index file */
        return 0;
    }

    /* mark the newly seen messages as recent */
    if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid,
                    hdr->next_uid, &seq1, &seq2)) {
        index_mailbox_set_recent_seq(&ctx->mbox->box, ctx->sync_view,
                         seq1, seq2);
    }

    /* read all changes and group changes to same file_id together */
    i_array_init(&ctx->expunged_seqs, 64);
    i_array_init(&ctx->expunged_map_uids, 64);
    while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) {
        if ((ret = mdbox_sync_add(ctx, &sync_rec)) < 0)
            break;
    }
    if (ret == 0 && array_count(&ctx->expunged_seqs) > 0)
        ret = mdbox_sync_index_finish_expunges(ctx);

    if (box->v.sync_notify != NULL)
        box->v.sync_notify(box, 0, 0);

    array_free(&ctx->expunged_seqs);
    array_free(&ctx->expunged_map_uids);
    return ret == 0 ? 1 :
        (ctx->mbox->storage->storage.files_corrupted ? 0 : -1);
}

static int mdbox_refresh_header(struct mdbox_mailbox *mbox, bool retry)
{
    struct mail_index_view *view;
    struct mdbox_index_header hdr;
    int ret;

    view = mail_index_view_open(mbox->box.index);
    ret = mdbox_read_header(mbox, &hdr);
    mail_index_view_close(&view);

    if (ret == 0) {
        ret = mbox->storage->storage.files_corrupted ? -1 : 0;
    } else if (retry) {
        (void)mail_index_refresh(mbox->box.index);
        return mdbox_refresh_header(mbox, FALSE);
    }
    return ret;
}

int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
             struct mdbox_sync_context **ctx_r)
{
    struct mail_storage *storage = mbox->box.storage;
    struct mdbox_sync_context *ctx;
    enum mail_index_sync_flags sync_flags;
    unsigned int i;
    int ret;
    bool rebuild, storage_rebuilt = FALSE;

    rebuild = mdbox_refresh_header(mbox, TRUE) < 0 ||
        (flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
    if (rebuild) {
        if (mdbox_storage_rebuild(mbox->storage) < 0)
            return -1;
        index_mailbox_reset_uidvalidity(&mbox->box);
        storage_rebuilt = TRUE;
    }

    ctx = i_new(struct mdbox_sync_context, 1);
    ctx->mbox = mbox;
    ctx->flags = flags;

    sync_flags = index_storage_get_sync_flags(&mbox->box);
    if (!rebuild && (flags & MDBOX_SYNC_FLAG_FORCE) == 0)
        sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
    if ((flags & MDBOX_SYNC_FLAG_FSYNC) != 0)
        sync_flags |= MAIL_INDEX_SYNC_FLAG_FSYNC;
    /* don't write unnecessary dirty flag updates */
    sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;

    for (i = 0;; i++) {
        ret = mail_index_sync_begin(mbox->box.index,
                        &ctx->index_sync_ctx,
                        &ctx->sync_view, &ctx->trans,
                        sync_flags);
        if (ret <= 0) {
            if (ret < 0)
                mail_storage_set_index_error(&mbox->box);
            i_free(ctx);
            *ctx_r = NULL;
            return ret;
        }

        /* now that we're locked, check again if we want to rebuild */
        if (mdbox_refresh_header(mbox, FALSE) < 0)
            ret = 0;
        else {
            if ((ret = mdbox_sync_index(ctx)) > 0)
                break;
        }

        /* failure. keep the index locked while we're doing a
           rebuild. */
        if (ret == 0) {
            if (!storage_rebuilt) {
                /* we'll need to rebuild storage too.
                   try again from the beginning. */
                mbox->storage->storage.files_corrupted = TRUE;
                mail_index_sync_rollback(&ctx->index_sync_ctx);
                i_free(ctx);
                return mdbox_sync_begin(mbox, flags, ctx_r);
            }
            mail_storage_set_critical(storage,
                "dbox %s: Storage keeps breaking",
                ctx->mbox->box.path);
            ret = -1;
        }
        mail_index_sync_rollback(&ctx->index_sync_ctx);
        if (ret < 0) {
            i_free(ctx);
            return -1;
        }
    }

    *ctx_r = ctx;
    return 0;
}

int mdbox_sync_finish(struct mdbox_sync_context **_ctx, bool success)
{
    struct mdbox_sync_context *ctx = *_ctx;
    int ret = success ? 0 : -1;

    *_ctx = NULL;

    if (success) {
        if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
            mail_storage_set_index_error(&ctx->mbox->box);
            ret = -1;
        }
    } else {
        mail_index_sync_rollback(&ctx->index_sync_ctx);
    }

    i_free(ctx);
    return ret;
}

int mdbox_sync(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags)
{
    struct mdbox_sync_context *sync_ctx;

    if (mdbox_sync_begin(mbox, flags, &sync_ctx) < 0)
        return -1;

    if (sync_ctx == NULL)
        return 0;
    return mdbox_sync_finish(&sync_ctx, TRUE);
}

struct mailbox_sync_context *
mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
    struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box;
    enum mdbox_sync_flags mdbox_sync_flags = 0;
    int ret = 0;

    if (!box->opened) {
        if (mailbox_open(box) < 0)
            ret = -1;
    }

    if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) ||
             mbox->storage->storage.files_corrupted)) {
        if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)
            mdbox_sync_flags |= MDBOX_SYNC_FLAG_FORCE_REBUILD;
        ret = mdbox_sync(mbox, mdbox_sync_flags);
    }

    return index_mailbox_sync_init(box, flags, ret < 0);
}

static int mdbox_sync_altmove_add_files(struct mdbox_storage *dstorage,
                    ARRAY_TYPE(seq_range) *file_ids)
{
    struct mail_storage *storage = &dstorage->storage.storage;
    DIR *dir;
    struct dirent *d;
    struct stat st;
    time_t altmove_mtime;
    string_t *path;
    unsigned int file_id, dir_len;
    int ret = 0;

    if (dstorage->set->mdbox_altmove == 0 ||
        dstorage->alt_storage_dir == NULL)
        return 0;

    altmove_mtime = ioloop_time - dstorage->set->mdbox_altmove;

    /* we want to quickly find the latest alt file, but we also want to
       avoid accessing the alt storage as much as possible. so we'll do
       this by finding the lowest numbered file (n) from primary storage.
       hopefully one of n-[1..m] is appendable in alt storage. */
    dir = opendir(dstorage->storage_dir);
    if (dir == NULL) {
        mail_storage_set_critical(storage,
            "opendir(%s) failed: %m", dstorage->storage_dir);
        return -1;
    }

    path = t_str_new(256);
    str_append(path, dstorage->storage_dir);
    str_append_c(path, '/');
    dir_len = str_len(path);

    for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
        if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
                strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
            continue;

        str_truncate(path, dir_len);
        str_append(path, d->d_name);

        file_id = strtoul(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
                  NULL, 10);

        if (stat(str_c(path), &st) < 0) {
            mail_storage_set_critical(storage,
                "stat(%s) failed: %m", str_c(path));
        } else if (st.st_mtime < altmove_mtime) {
            seq_range_array_add(file_ids, 0, file_id);
        }
    }
    if (errno != 0) {
        mail_storage_set_critical(storage,
            "readdir(%s) failed: %m", dstorage->storage_dir);
        ret = -1;
    }
    if (closedir(dir) < 0) {
        mail_storage_set_critical(storage,
            "closedir(%s) failed: %m", dstorage->storage_dir);
        ret = -1;
    }
    return ret;
}

int mdbox_sync_purge(struct mail_storage *_storage)
{
    struct mdbox_storage *storage = (struct mdbox_storage *)_storage;
    ARRAY_TYPE(seq_range) ref0_file_ids;
    struct dbox_file *file;
    struct seq_range_iter iter;
    unsigned int i = 0;
    uint32_t file_id;
    bool deleted;
    int ret = 0;

    i_array_init(&ref0_file_ids, 64);
    if (dbox_map_get_zero_ref_files(storage->map, &ref0_file_ids) < 0)
        ret = -1;

    /* add also files that can be altmoved */
    if (mdbox_sync_altmove_add_files(storage, &ref0_file_ids) < 0)
        ret = -1;

    seq_range_array_iter_init(&iter, &ref0_file_ids); i = 0;
    while (seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN {
        file = mdbox_file_init(storage, file_id);
        if (dbox_file_open(file, &deleted) > 0 && !deleted) {
            if (mdbox_file_purge(file) < 0)
                ret = -1;
        } else {
            dbox_map_remove_file_id(storage->map, file_id);
        }
        dbox_file_unref(&file);
    } T_END;
    array_free(&ref0_file_ids);
    return ret;
}