mdbox-storage-rebuild.c revision b58aafbd21b365117538f73f306d22f75acd91f1
493N/A/* Copyright (c) 2009-2010 Dovecot authors, see the included COPYING file */
493N/A
493N/A#include "lib.h"
970N/A#include "array.h"
493N/A#include "ioloop.h"
493N/A#include "istream.h"
919N/A#include "hash.h"
919N/A#include "str.h"
919N/A#include "dbox-sync-rebuild.h"
919N/A#include "mdbox-storage.h"
919N/A#include "mdbox-file.h"
919N/A#include "mdbox-map-private.h"
919N/A#include "mdbox-sync.h"
919N/A#include "mdbox-storage-rebuild.h"
919N/A
919N/A#include <stdlib.h>
919N/A#include <dirent.h>
919N/A#include <unistd.h>
919N/A
919N/Astruct mdbox_rebuild_msg {
919N/A uint8_t guid_128[MAIL_GUID_128_SIZE];
919N/A uint32_t file_id;
919N/A uint32_t offset;
493N/A uint32_t size;
493N/A uint32_t map_uid;
493N/A
493N/A uint16_t refcount;
493N/A unsigned int seen_zero_ref_in_map:1;
493N/A};
493N/A
493N/Astruct rebuild_msg_mailbox {
493N/A struct mailbox *box;
493N/A struct mail_index_sync_ctx *sync_ctx;
493N/A struct mail_index_view *view;
970N/A struct mail_index_transaction *trans;
970N/A uint32_t next_uid;
970N/A};
493N/A
970N/Astruct mdbox_storage_rebuild_context {
493N/A struct mdbox_storage *storage;
493N/A pool_t pool;
970N/A
493N/A struct hash_table *guid_hash;
970N/A ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *);
970N/A
970N/A uint32_t prev_file_id;
970N/A uint32_t highest_seen_map_uid;
970N/A
970N/A struct mailbox_list *default_list;
493N/A struct mail_index_sync_ctx *sync_ctx;
493N/A struct mail_index_view *sync_view;
493N/A struct mail_index_transaction *trans;
493N/A
970N/A struct rebuild_msg_mailbox prev_msg;
970N/A
970N/A unsigned int msgs_unsorted:1;
970N/A};
970N/A
970N/Astatic unsigned int guid_hash(const void *p)
493N/A{
493N/A const uint8_t *s = p;
970N/A unsigned int i, g, h = 0;
493N/A
493N/A for (i = 0; i < MAIL_GUID_128_SIZE; i++) {
493N/A h = (h << 4) + s[i];
493N/A if ((g = h & 0xf0000000UL)) {
493N/A h = h ^ (g >> 24);
493N/A h = h ^ g;
493N/A }
493N/A }
493N/A return h;
493N/A}
493N/A
493N/Astatic int guid_cmp(const void *p1, const void *p2)
493N/A{
493N/A return memcmp(p1, p2, MAIL_GUID_128_SIZE);
493N/A}
493N/A
493N/Astatic struct mdbox_storage_rebuild_context *
493N/Amdbox_storage_rebuild_init(struct mdbox_storage *storage)
493N/A{
493N/A struct mdbox_storage_rebuild_context *ctx;
493N/A
493N/A ctx = i_new(struct mdbox_storage_rebuild_context, 1);
493N/A ctx->storage = storage;
493N/A ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256);
493N/A ctx->guid_hash = hash_table_create(default_pool, ctx->pool, 0,
970N/A guid_hash, guid_cmp);
970N/A i_array_init(&ctx->msgs, 512);
970N/A return ctx;
970N/A}
970N/A
970N/Astatic void
970N/Amdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx)
970N/A{
970N/A if (ctx->sync_ctx != NULL)
970N/A mail_index_sync_rollback(&ctx->sync_ctx);
970N/A
970N/A hash_table_destroy(&ctx->guid_hash);
970N/A pool_unref(&ctx->pool);
970N/A array_free(&ctx->msgs);
970N/A i_free(ctx);
970N/A}
970N/A
970N/Astatic int mdbox_rebuild_msg_offset_cmp(const void *p1, const void *p2)
970N/A{
970N/A const struct mdbox_rebuild_msg *const *m1 = p1, *const *m2 = p2;
970N/A
970N/A if ((*m1)->file_id < (*m2)->file_id)
970N/A return -1;
970N/A if ((*m1)->file_id > (*m2)->file_id)
970N/A return 1;
970N/A
970N/A if ((*m1)->offset < (*m2)->offset)
970N/A return -1;
970N/A if ((*m1)->offset > (*m2)->offset)
970N/A return 1;
970N/A return 0;
970N/A}
970N/A
970N/Astatic int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1,
970N/A struct mdbox_rebuild_msg *const *m2)
970N/A{
970N/A if ((*m1)->map_uid < (*m2)->map_uid)
970N/A return -1;
970N/A if ((*m1)->map_uid > (*m2)->map_uid)
970N/A return 1;
970N/A return 0;
970N/A}
970N/A
970N/Astatic int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
970N/A struct dbox_file *file)
970N/A{
970N/A const char *guid;
970N/A struct mdbox_rebuild_msg *rec;
970N/A uoff_t offset, prev_offset;
970N/A bool last, first, fixed = FALSE;
493N/A int ret;
493N/A
493N/A dbox_file_seek_rewind(file);
493N/A prev_offset = 0;
493N/A while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) {
493N/A if (ret > 0) {
493N/A if ((ret = dbox_file_metadata_read(file)) < 0)
970N/A break;
493N/A }
493N/A
493N/A if (ret == 0) {
493N/A /* file is corrupted. fix it and retry. */
493N/A if (fixed || last)
493N/A break;
493N/A first = prev_offset == 0;
810N/A if (prev_offset == 0) {
970N/A /* use existing file header if it was ok */
493N/A prev_offset = offset;
493N/A }
493N/A if (dbox_file_fix(file, prev_offset) < 0) {
493N/A ret = -1;
493N/A break;
493N/A }
493N/A fixed = TRUE;
493N/A if (!first) {
493N/A /* seek to the offset where we last left off */
493N/A ret = dbox_file_get_mail_stream(file,
493N/A prev_offset, NULL);
493N/A if (ret <= 0)
493N/A break;
493N/A }
493N/A continue;
493N/A }
493N/A prev_offset = offset;
493N/A
493N/A guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID);
493N/A if (guid == NULL) {
970N/A dbox_file_set_corrupted(file,
493N/A "Message is missing GUID");
970N/A ret = 0;
970N/A break;
970N/A }
493N/A
493N/A rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1);
493N/A rec->file_id = ctx->prev_file_id;
493N/A rec->offset = offset;
493N/A rec->size = file->input->v_offset - offset;
493N/A mail_generate_guid_128_hash(guid, rec->guid_128);
493N/A array_append(&ctx->msgs, &rec, 1);
493N/A
493N/A if (hash_table_lookup(ctx->guid_hash, rec->guid_128) != NULL) {
493N/A /* duplicate. save this as a refcount=0 to map,
493N/A so it will eventually be deleted. */
493N/A rec->seen_zero_ref_in_map = TRUE;
493N/A } else {
810N/A hash_table_insert(ctx->guid_hash, rec->guid_128, rec);
493N/A }
493N/A }
493N/A if (ret < 0)
493N/A return -1;
493N/A else if (ret == 0 && !last)
810N/A return 0;
493N/A else
493N/A return 1;
493N/A}
493N/A
493N/Astatic int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx,
493N/A const char *path)
493N/A{
493N/A struct dbox_file *file;
493N/A uint32_t file_id;
493N/A const char *fname;
493N/A unsigned int len;
493N/A bool deleted;
493N/A int ret = 0;
493N/A
493N/A fname = strrchr(path, '/');
493N/A i_assert(fname != NULL);
493N/A fname += strlen(MDBOX_MAIL_FILE_PREFIX) + 1;
493N/A
493N/A file_id = strtoul(fname, NULL, 10);
493N/A if (!is_numeric(fname, '\0') || file_id == 0) {
493N/A len = strlen(fname);
493N/A if (len > 7 && strcmp(fname + len - 7, ".broken") != 0) {
493N/A i_warning("dbox rebuild: File name is missing ID: %s",
493N/A path);
493N/A }
493N/A return 0;
970N/A }
970N/A
493N/A /* small optimization: typically files are returned sorted. in that
493N/A case we don't need to sort them ourself. */
970N/A if (file_id < ctx->prev_file_id)
493N/A ctx->msgs_unsorted = TRUE;
493N/A ctx->prev_file_id = file_id;
493N/A
493N/A file = mdbox_file_init(ctx->storage, file_id);
493N/A if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted)
493N/A ret = rebuild_file_mails(ctx, file);
493N/A if (ret == 0)
493N/A i_error("dbox rebuild: Failed to fix file %s", path);
493N/A dbox_file_unref(&file);
970N/A return ret < 0 ? -1 : 0;
493N/A}
493N/A
493N/Astatic void
493N/Arebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx,
493N/A uint32_t next_uid)
493N/A{
493N/A struct mdbox_rebuild_msg **msgs;
493N/A struct dbox_map_mail_index_record rec;
493N/A unsigned int i, count;
493N/A uint32_t seq;
493N/A
493N/A memset(&rec, 0, sizeof(rec));
493N/A msgs = array_get_modifiable(&ctx->msgs, &count);
970N/A for (i = 0; i < count; i++) {
493N/A if (msgs[i]->map_uid != 0)
493N/A continue;
493N/A
970N/A rec.file_id = msgs[i]->file_id;
970N/A rec.offset = msgs[i]->offset;
970N/A rec.size = msgs[i]->size;
970N/A
970N/A msgs[i]->map_uid = next_uid++;
493N/A mail_index_append(ctx->trans, msgs[i]->map_uid, &seq);
970N/A mail_index_update_ext(ctx->trans, seq,
493N/A ctx->storage->map->map_ext_id,
493N/A &rec, NULL);
970N/A }
970N/A}
970N/A
970N/Astatic int rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx)
970N/A{
970N/A struct dbox_map *map = ctx->storage->map;
493N/A const struct mail_index_header *hdr;
493N/A struct mdbox_rebuild_msg *const *msgs, **pos;
493N/A struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
970N/A struct dbox_mail_lookup_rec rec;
493N/A uint32_t seq;
970N/A unsigned int count;
970N/A
970N/A if (ctx->msgs_unsorted)
970N/A array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp);
970N/A
970N/A msgs = array_get_modifiable(&ctx->msgs, &count);
970N/A hdr = mail_index_get_header(ctx->sync_view);
970N/A for (seq = 1; seq <= hdr->messages_count; seq++) {
970N/A if (dbox_map_view_lookup_rec(map, ctx->sync_view,
970N/A seq, &rec) < 0)
970N/A return -1;
970N/A
970N/A /* look up the rebuild msg record for this message */
970N/A search_msg.file_id = rec.rec.file_id;
970N/A search_msg.offset = rec.rec.offset;
970N/A pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs),
970N/A mdbox_rebuild_msg_offset_cmp);
493N/A if (pos == NULL) {
970N/A /* map record points to non-existing message. */
970N/A mail_index_expunge(ctx->trans, seq);
970N/A } else {
970N/A (*pos)->map_uid = rec.map_uid;
970N/A if (rec.refcount == 0)
970N/A (*pos)->seen_zero_ref_in_map = TRUE;
970N/A }
970N/A }
970N/A rebuild_add_missing_map_uids(ctx, hdr->next_uid);
970N/A
493N/A /* afterwards we're interested in looking up map_uids.
970N/A re-sort the messages to make it easier. */
970N/A array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp);
970N/A return 0;
970N/A}
970N/A
970N/Astatic struct mdbox_rebuild_msg *
970N/Arebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx,
970N/A uint32_t map_uid)
493N/A{
493N/A struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
970N/A struct mdbox_rebuild_msg **pos;
970N/A
970N/A search_msg.map_uid = map_uid;
970N/A pos = array_bsearch(&ctx->msgs, &search_msgp,
970N/A mdbox_rebuild_msg_uid_cmp);
970N/A return pos == NULL ? NULL : *pos;
970N/A}
970N/A
970N/Astatic void
970N/Arebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx,
970N/A struct dbox_sync_rebuild_context *rebuild_ctx,
970N/A struct mdbox_mailbox *mbox,
970N/A struct mail_index_view *view,
970N/A struct mail_index_transaction *trans)
970N/A{
970N/A const struct mdbox_mail_index_record *dbox_rec;
970N/A struct mdbox_mail_index_record new_dbox_rec;
970N/A const struct mail_index_header *hdr;
970N/A struct mdbox_rebuild_msg *rec;
970N/A const void *data;
702N/A bool expunged;
970N/A uint32_t seq, uid, new_seq, map_uid;
970N/A
970N/A memset(&new_dbox_rec, 0, sizeof(new_dbox_rec));
970N/A hdr = mail_index_get_header(view);
970N/A for (seq = 1; seq <= hdr->messages_count; seq++) {
970N/A mail_index_lookup_ext(view, seq, mbox->ext_id,
970N/A &data, &expunged);
970N/A dbox_rec = data;
970N/A map_uid = dbox_rec == NULL ? 0 : dbox_rec->map_uid;
970N/A
970N/A mail_index_lookup_ext(view, seq, mbox->guid_ext_id,
970N/A &data, &expunged);
970N/A
970N/A /* see if we can find this message based on
970N/A 1) GUID, 2) map_uid */
970N/A rec = data == NULL ? NULL :
970N/A hash_table_lookup(ctx->guid_hash, data);
970N/A if (rec == NULL) {
970N/A if (map_uid == 0) {
970N/A /* not a multi-dbox message, ignore. */
970N/A continue;
970N/A }
702N/A /* multi-dbox message that wasn't found with GUID.
970N/A either it's lost or GUID has been corrupted. we can
970N/A still try to look it up using map_uid. */
970N/A rec = rebuild_lookup_map_uid(ctx, map_uid);
970N/A if (rec != NULL) {
970N/A mail_index_update_ext(trans, seq,
970N/A mbox->guid_ext_id,
970N/A rec->guid_128, NULL);
970N/A }
970N/A } else if (map_uid != rec->map_uid) {
970N/A /* map_uid is wrong, update it */
970N/A i_assert(rec->map_uid != 0);
970N/A new_dbox_rec.map_uid = rec->map_uid;
970N/A mail_index_update_ext(trans, seq, mbox->ext_id,
970N/A &new_dbox_rec, NULL);
970N/A } else {
970N/A /* everything was ok */
970N/A }
970N/A
970N/A if (rec != NULL) T_BEGIN {
970N/A /* keep this message */
970N/A rec->refcount++;
970N/A
970N/A mail_index_lookup_uid(view, seq, &uid);
970N/A mail_index_append(trans, uid, &new_seq);
970N/A dbox_sync_rebuild_index_metadata(rebuild_ctx,
970N/A new_seq, uid);
970N/A
970N/A new_dbox_rec.map_uid = rec->map_uid;
970N/A mail_index_update_ext(trans, new_seq, mbox->ext_id,
970N/A &new_dbox_rec, NULL);
970N/A } T_END;
970N/A }
970N/A}
970N/A
970N/Astatic int
970N/Arebuild_mailbox(struct mdbox_storage_rebuild_context *ctx,
970N/A struct mail_namespace *ns, const char *name)
970N/A{
970N/A struct mailbox *box;
970N/A struct mdbox_mailbox *mbox;
970N/A struct mail_index_sync_ctx *sync_ctx;
970N/A struct mail_index_view *view;
493N/A struct mail_index_transaction *trans;
970N/A struct dbox_sync_rebuild_context *rebuild_ctx;
493N/A enum mail_error error;
493N/A int ret;
box = mdbox_mailbox_alloc(&ctx->storage->storage.storage,
ns->list, name, MAILBOX_FLAG_READONLY |
MAILBOX_FLAG_KEEP_RECENT |
MAILBOX_FLAG_IGNORE_ACLS);
if (dbox_mailbox_open(box) < 0) {
(void)mail_storage_get_last_error(box->storage, &error);
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
/* non-temporary error, ignore */
return 0;
}
mbox = (struct mdbox_mailbox *)box;
ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans,
MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES);
if (ret <= 0) {
i_assert(ret != 0);
mail_storage_set_index_error(box);
mailbox_free(&box);
return -1;
}
rebuild_ctx = dbox_sync_index_rebuild_init(&mbox->box, view, trans);
rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
dbox_sync_index_rebuild_deinit(&rebuild_ctx);
if (mail_index_sync_commit(&sync_ctx) < 0) {
mail_storage_set_index_error(box);
ret = -1;
}
mailbox_free(&box);
return ret < 0 ? -1 : 0;
}
static int
rebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx,
struct mail_namespace *ns)
{
struct mailbox_list_iterate_context *iter;
const struct mailbox_info *info;
int ret = 0;
if (ctx->default_list == NULL ||
(ns->flags & NAMESPACE_FLAG_INBOX) != 0)
ctx->default_list = ns->list;
iter = mailbox_list_iter_init(ns->list, "*",
MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
while ((info = mailbox_list_iter_next(iter)) != NULL) {
if ((info->flags & (MAILBOX_NONEXISTENT |
MAILBOX_NOSELECT)) == 0) {
T_BEGIN {
ret = rebuild_mailbox(ctx, ns, info->name);
} T_END;
if (ret < 0) {
ret = -1;
break;
}
}
}
if (mailbox_list_iter_deinit(&iter) < 0)
ret = -1;
return ret;
}
static int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx)
{
struct mail_user *user = ctx->storage->storage.storage.user;
struct mail_namespace *ns;
for (ns = user->namespaces; ns != NULL; ns = ns->next) {
if (ns->storage == &ctx->storage->storage.storage &&
ns->alias_for == NULL) {
if (rebuild_namespace_mailboxes(ctx, ns) < 0)
return -1;
}
}
return 0;
}
static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg)
{
if (mail_index_sync_commit(&msg->sync_ctx) < 0)
return -1;
mailbox_free(&msg->box);
memset(msg, 0, sizeof(*msg));
return 0;
}
static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx,
struct mdbox_rebuild_msg *msg)
{
struct mail_storage *storage = &ctx->storage->storage.storage;
struct dbox_file *file;
const struct mail_index_header *hdr;
struct mdbox_mail_index_record dbox_rec;
const char *mailbox = NULL;
struct mailbox *box;
struct mdbox_mailbox *mbox;
enum mail_error error;
bool deleted, created;
int ret;
uint32_t seq;
/* first see if message contains the mailbox it was originally
saved to */
file = mdbox_file_init(ctx->storage, msg->file_id);
ret = dbox_file_open(file, &deleted);
if (ret > 0)
ret = dbox_file_get_mail_stream(file, msg->offset, NULL);
if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) {
mailbox = dbox_file_metadata_get(file,
DBOX_METADATA_ORIG_MAILBOX);
}
dbox_file_unref(&file);
if (ret <= 0 || deleted) {
if (ret < 0)
return -1;
/* we shouldn't get here, so apparently we couldn't fix
something. just ignore the mail.. */
return 0;
}
if (mailbox == NULL)
mailbox = "INBOX";
/* we have the destination mailbox. now open it and add the message
there. */
created = FALSE;
box = ctx->prev_msg.box != NULL &&
strcmp(mailbox, ctx->prev_msg.box->name) == 0 ?
ctx->prev_msg.box : NULL;
while (box == NULL) {
box = mdbox_mailbox_alloc(storage, ctx->default_list,
mailbox, MAILBOX_FLAG_READONLY |
MAILBOX_FLAG_KEEP_RECENT |
MAILBOX_FLAG_IGNORE_ACLS);
if (dbox_mailbox_open(box) == 0)
break;
(void)mail_storage_get_last_error(box->storage, &error);
if (error == MAIL_ERROR_NOTFOUND && !created) {
/* mailbox doesn't exist currently? see if creating
it helps. */
created = TRUE;
(void)mailbox_create(box, NULL, FALSE);
mailbox_free(&box);
continue;
}
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
if (strcmp(mailbox, "INBOX") != 0) {
/* see if we can save to INBOX instead. */
mailbox = "INBOX";
} else {
/* this shouldn't happen */
return -1;
}
}
mbox = (struct mdbox_mailbox *)box;
/* switch the mailbox cache if necessary */
if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) {
if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
return -1;
}
if (ctx->prev_msg.box == NULL) {
ret = mail_index_sync_begin(box->index,
&ctx->prev_msg.sync_ctx,
&ctx->prev_msg.view,
&ctx->prev_msg.trans, 0);
if (ret <= 0) {
i_assert(ret != 0);
mail_storage_set_index_error(box);
mailbox_free(&box);
return -1;
}
ctx->prev_msg.box = box;
hdr = mail_index_get_header(ctx->prev_msg.view);
ctx->prev_msg.next_uid = hdr->next_uid;
}
/* add the new message */
memset(&dbox_rec, 0, sizeof(dbox_rec));
dbox_rec.map_uid = msg->map_uid;
dbox_rec.save_date = ioloop_time;
mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id,
&dbox_rec, NULL);
mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id,
msg->guid_128, NULL);
msg->refcount++;
return 0;
}
static int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx)
{
struct mdbox_rebuild_msg **msgs;
unsigned int i, count;
/* if we have messages at this point which have refcount=0, they're
either already expunged or they were somehow lost for some reason.
we'll need to figure out what to do about them. */
msgs = array_get_modifiable(&ctx->msgs, &count);
for (i = 0; i < count; i++) {
if (msgs[i]->refcount != 0)
continue;
if (msgs[i]->seen_zero_ref_in_map) {
/* we've seen the map record, trust it. */
continue;
}
/* either map record was lost for this message or the message
was lost from its mailbox. safest way to handle this is to
restore the message. */
if (rebuild_restore_msg(ctx, msgs[i]) < 0)
return -1;
}
if (ctx->prev_msg.box != NULL) {
if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
return -1;
}
return 0;
}
static void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx)
{
const struct mail_index_header *hdr;
const void *data;
struct mdbox_rebuild_msg **msgs;
const uint16_t *ref16_p;
bool expunged;
uint32_t seq, map_uid;
unsigned int i, count;
/* update refcounts for existing map records */
msgs = array_get_modifiable(&ctx->msgs, &count);
hdr = mail_index_get_header(ctx->sync_view);
for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) {
mail_index_lookup_uid(ctx->sync_view, seq, &map_uid);
if (map_uid != msgs[i]->map_uid) {
/* we've already expunged this map record */
i_assert(map_uid < msgs[i]->map_uid);
continue;
}
mail_index_lookup_ext(ctx->sync_view, seq,
ctx->storage->map->ref_ext_id,
&data, &expunged);
ref16_p = data;
if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) {
mail_index_update_ext(ctx->trans, seq,
ctx->storage->map->ref_ext_id,
&msgs[i]->refcount, NULL);
}
i++;
}
/* update refcounts for newly created map records */
for (; i < count; i++, seq++) {
mail_index_update_ext(ctx->trans, seq,
ctx->storage->map->ref_ext_id,
&msgs[i]->refcount, NULL);
}
}
static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx)
{
if (rebuild_handle_zero_refs(ctx) < 0)
return -1;
rebuild_update_refcounts(ctx);
return 0;
}
static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx)
{
const struct mail_index_header *hdr;
DIR *dir;
struct dirent *d;
string_t *path;
unsigned int dir_len;
uint32_t uid_validity;
int ret = 0;
if (dbox_map_open(ctx->storage->map, TRUE) < 0)
return -1;
/* begin by locking the map, so that other processes can't try to
rebuild at the same time. */
ret = mail_index_sync_begin(ctx->storage->map->index, &ctx->sync_ctx,
&ctx->sync_view, &ctx->trans, 0);
if (ret <= 0) {
i_assert(ret != 0);
mail_storage_set_internal_error(&ctx->storage->storage.storage);
mail_index_reset_error(ctx->storage->map->index);
return -1;
}
uid_validity = dbox_map_get_uid_validity(ctx->storage->map);
hdr = mail_index_get_header(ctx->sync_view);
if (hdr->uid_validity != uid_validity) {
mail_index_update_header(ctx->trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
}
dir = opendir(ctx->storage->storage_dir);
if (dir == NULL) {
mail_storage_set_critical(&ctx->storage->storage.storage,
"opendir(%s) failed: %m", ctx->storage->storage_dir);
return -1;
}
path = t_str_new(256);
str_append(path, ctx->storage->storage_dir);
str_append_c(path, '/');
dir_len = str_len(path);
for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) {
str_truncate(path, dir_len);
str_append(path, d->d_name);
T_BEGIN {
ret = rebuild_add_file(ctx, str_c(path));
} T_END;
if (ret < 0) {
ret = -1;
break;
}
}
}
if (ret == 0 && errno != 0) {
mail_storage_set_critical(&ctx->storage->storage.storage,
"readdir(%s) failed: %m", ctx->storage->storage_dir);
ret = -1;
}
if (closedir(dir) < 0) {
mail_storage_set_critical(&ctx->storage->storage.storage,
"closedir(%s) failed: %m", ctx->storage->storage_dir);
ret = -1;
}
if (ret < 0 ||
rebuild_apply_map(ctx) < 0 ||
rebuild_mailboxes(ctx) < 0 ||
rebuild_finish(ctx) < 0 ||
mail_index_sync_commit(&ctx->sync_ctx) < 0)
return -1;
return 0;
}
int mdbox_storage_rebuild(struct mdbox_storage *storage)
{
struct mdbox_storage_rebuild_context *ctx;
struct stat st;
int ret;
if (stat(storage->storage_dir, &st) < 0) {
if (errno == ENOENT) {
/* no multi-dbox files */
return 0;
}
mail_storage_set_critical(&storage->storage.storage,
"stat(%s) failed: %m", storage->storage_dir);
return -1;
}
i_warning("dbox %s: rebuilding indexes", storage->storage_dir);
ctx = mdbox_storage_rebuild_init(storage);
ret = mdbox_storage_rebuild_scan(ctx);
mdbox_storage_rebuild_deinit(ctx);
if (ret == 0)
storage->storage.files_corrupted = FALSE;
return ret;
}