mdbox-storage-rebuild.c revision 47ede56f4e6eebfe631a1f0febf74d7adcdbcd00
5a580c3a38ced62d4bcc95b8ac7c4f2935b5d294Timo Sirainen/* Copyright (c) 2009-2013 Dovecot authors, see the included COPYING file */
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "lib.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "array.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "ioloop.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "istream.h"
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen#include "hash.h"
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen#include "str.h"
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen#include "mail-cache.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "index-rebuild.h"
eb5ea3f4513ff2999892b8d904551f58b74f65f9Timo Sirainen#include "mail-namespace.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "mailbox-list-private.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "mdbox-storage.h"
eb5ea3f4513ff2999892b8d904551f58b74f65f9Timo Sirainen#include "mdbox-file.h"
25480af2e21cf136e461ec802177f52b43154485Timo Sirainen#include "mdbox-map-private.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "mdbox-sync.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include "mdbox-storage-rebuild.h"
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include <stdlib.h>
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include <dirent.h>
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen#include <unistd.h>
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainenstruct mdbox_rebuild_msg {
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mdbox_rebuild_msg *guid_hash_next;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen guid_128_t guid_128;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen uint32_t file_id;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen uint32_t offset;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen uint32_t rec_size;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen uoff_t mail_size;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen uint32_t map_uid;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen uint16_t refcount;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen unsigned int seen_zero_ref_in_map:1;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen};
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainenstruct rebuild_msg_mailbox {
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen struct mailbox *box;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mail_index_sync_ctx *sync_ctx;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mail_index_view *view;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mail_index_transaction *trans;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen uint32_t next_uid;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen};
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainenstruct mdbox_storage_rebuild_context {
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mdbox_storage *storage;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mdbox_map_atomic_context *atomic;
c991d8c2c0d5d6c025e24fc00cb06dd61c42456dTimo Sirainen pool_t pool;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mdbox_map_mail_index_header orig_map_hdr;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen HASH_TABLE(uint8_t *, struct mdbox_rebuild_msg *) guid_hash;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen ARRAY(struct mdbox_rebuild_msg *) msgs;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen ARRAY_TYPE(seq_range) seen_file_ids;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen uint32_t rebuild_count;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen uint32_t highest_file_id;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen struct mailbox_list *default_list;
c991d8c2c0d5d6c025e24fc00cb06dd61c42456dTimo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct rebuild_msg_mailbox prev_msg;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen unsigned int have_pop3_uidls:1;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen unsigned int have_pop3_orders:1;
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen};
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainen
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainenstatic struct mdbox_storage_rebuild_context *
4082d5b171d1c3a00ba705093d62b8afc9cf17aeTimo Sirainenmdbox_storage_rebuild_init(struct mdbox_storage *storage,
c991d8c2c0d5d6c025e24fc00cb06dd61c42456dTimo Sirainen struct mdbox_map_atomic_context *atomic)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen{
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mdbox_storage_rebuild_context *ctx;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen i_assert(!storage->rebuilding_storage);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
c991d8c2c0d5d6c025e24fc00cb06dd61c42456dTimo Sirainen ctx = i_new(struct mdbox_storage_rebuild_context, 1);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen ctx->storage = storage;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen ctx->atomic = atomic;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen hash_table_create(&ctx->guid_hash, ctx->pool, 0,
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen guid_128_hash, guid_128_cmp);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen i_array_init(&ctx->msgs, 512);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen i_array_init(&ctx->seen_file_ids, 128);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen ctx->storage->rebuilding_storage = TRUE;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen return ctx;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen}
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainenstatic void
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainenmdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx)
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen{
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen i_assert(ctx->storage->rebuilding_storage);
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen ctx->storage->rebuilding_storage = FALSE;
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen hash_table_destroy(&ctx->guid_hash);
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen pool_unref(&ctx->pool);
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen array_free(&ctx->seen_file_ids);
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen array_free(&ctx->msgs);
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen i_free(ctx);
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen}
d06d6667bac64aabe1efb216af56ca45108d63b0Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainenstatic int
c991d8c2c0d5d6c025e24fc00cb06dd61c42456dTimo Sirainenmdbox_rebuild_msg_offset_cmp(struct mdbox_rebuild_msg *const *m1,
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen struct mdbox_rebuild_msg *const *m2)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen{
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen if ((*m1)->file_id < (*m2)->file_id)
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen return -1;
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen if ((*m1)->file_id > (*m2)->file_id)
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen return 1;
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if ((*m1)->offset < (*m2)->offset)
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen return -1;
86bde2c1838d1ce967fa2b394bb952004a4adcb7Timo Sirainen if ((*m1)->offset > (*m2)->offset)
605eca549c08af753e05c25937bcccd66079c321Timo Sirainen return 1;
605eca549c08af753e05c25937bcccd66079c321Timo Sirainen
605eca549c08af753e05c25937bcccd66079c321Timo Sirainen if ((*m1)->rec_size < (*m2)->rec_size)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen return -1;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen if ((*m1)->rec_size > (*m2)->rec_size)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen return 1;
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen return 0;
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen}
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainenstatic int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1,
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen struct mdbox_rebuild_msg *const *m2)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen{
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen if ((*m1)->map_uid < (*m2)->map_uid)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen return -1;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen if ((*m1)->map_uid > (*m2)->map_uid)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen return 1;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen return 0;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen}
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainenstatic void rebuild_scan_metadata(struct mdbox_storage_rebuild_context *ctx,
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen struct dbox_file *file)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen{
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL) != NULL)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen ctx->have_pop3_uidls = TRUE;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen if (dbox_file_metadata_get(file, DBOX_METADATA_POP3_ORDER) != NULL)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen ctx->have_pop3_orders = TRUE;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen}
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainenstatic int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen struct dbox_file *file, uint32_t file_id)
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen{
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen const char *guid;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen uint8_t *guid_p;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen struct mdbox_rebuild_msg *rec, *old_rec;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen uoff_t offset, prev_offset;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen bool last, first, fixed = FALSE;
355fe8b5d02904df39e793f66da5432d86649d4aTimo Sirainen int ret;
605eca549c08af753e05c25937bcccd66079c321Timo Sirainen
605eca549c08af753e05c25937bcccd66079c321Timo Sirainen dbox_file_seek_rewind(file);
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen prev_offset = 0;
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) {
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen if (ret > 0) {
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen if ((ret = dbox_file_metadata_read(file)) < 0)
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen break;
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen }
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen if (ret == 0) {
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen /* file is corrupted. fix it and retry. */
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen if (fixed || last)
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen break;
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen first = prev_offset == 0;
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen if (prev_offset == 0) {
6646bd844c85d5b27451199d8868b6d2357cd293Timo Sirainen /* use existing file header if it was ok */
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen prev_offset = offset;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen if ((ret = dbox_file_fix(file, prev_offset)) < 0)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen break;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen if (ret == 0) {
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen /* file was deleted */
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen return 1;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
8d80659e504ffb34bb0c6a633184fece35751b18Timo Sirainen fixed = TRUE;
e8ecd8f24ffc612f5d0be10f7931ac619f1eab88Timo Sirainen if (!first) {
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen /* seek to the offset where we last left off */
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen ret = dbox_file_seek(file, prev_offset);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen if (ret <= 0)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen break;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen continue;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen prev_offset = offset;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID);
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen if (guid == NULL || *guid == '\0') {
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen dbox_file_set_corrupted(file,
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen "Message is missing GUID");
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen ret = 0;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen break;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen rebuild_scan_metadata(ctx, file);
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1);
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen rec->file_id = file_id;
e262f3aa3429dbc74f668bc8bd501cf08b955778Timo Sirainen rec->offset = offset;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen rec->rec_size = file->input->v_offset - offset;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen rec->mail_size = dbox_file_get_plaintext_size(file);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen mail_generate_guid_128_hash(guid, rec->guid_128);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen i_assert(!guid_128_is_empty(rec->guid_128));
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen array_append(&ctx->msgs, &rec, 1);
25480af2e21cf136e461ec802177f52b43154485Timo Sirainen
8bb360f9e5de1c25e4f875205bb06e8bf15dae14Timo Sirainen guid_p = rec->guid_128;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen old_rec = hash_table_lookup(ctx->guid_hash, guid_p);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen if (old_rec == NULL)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen hash_table_insert(ctx->guid_hash, guid_p, rec);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen else if (rec->mail_size == old_rec->mail_size) {
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen /* two mails' GUID and size are the same, which quite
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen likely means that their contents are the same as
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen well. we'll compare the mail sizes instead of the
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen record sizes, because the records' metadata may
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen differ.
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen save this duplicate mail with refcount=0 to the map,
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen so it will eventually be purged. */
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen rec->seen_zero_ref_in_map = TRUE;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen } else {
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen /* duplicate GUID, but not a duplicate message. */
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen i_error("mdbox %s: Duplicate GUID %s in "
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen "m.%u:%u (size=%"PRIuUOFF_T") and m.%u:%u "
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen "(size=%"PRIuUOFF_T")",
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen ctx->storage->storage_dir, guid,
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen old_rec->file_id, old_rec->offset, old_rec->mail_size,
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen rec->file_id, rec->offset, rec->mail_size);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen rec->guid_hash_next = old_rec->guid_hash_next;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen old_rec->guid_hash_next = rec;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen }
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen if (ret < 0)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen return -1;
4c9a72e0988d462df49810984dc93b3fd4a24c23Timo Sirainen else if (ret == 0 && !last)
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen return 0;
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen else
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen return 1;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen}
1d3f7c1278168d5b1cbfa9a2cc9929a0909056b4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainenstatic int
503a863a317acba125a4e46435694e35fad769e4Timo Sirainenrebuild_rename_file(struct mdbox_storage_rebuild_context *ctx,
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen const char *dir, const char **fname_p, uint32_t *file_id_r)
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen{
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen const char *old_path, *new_path, *fname = *fname_p;
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen old_path = t_strconcat(dir, "/", fname, NULL);
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen do {
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen new_path = t_strdup_printf("%s/"MDBOX_MAIL_FILE_FORMAT,
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen dir, ++ctx->highest_file_id);
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen /* use link()+unlink() instead of rename() to make sure we
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen don't overwrite any files. */
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen if (link(old_path, new_path) == 0) {
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if (unlink(old_path) < 0)
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen i_error("unlink(%s) failed: %m", old_path);
749a9676d265a517c7a731f5b9336c524a49e6a6Timo Sirainen *fname_p = strrchr(new_path, '/') + 1;
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen *file_id_r = ctx->highest_file_id;
749a9676d265a517c7a731f5b9336c524a49e6a6Timo Sirainen return 0;
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen }
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen } while (errno == EEXIST);
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen
5e85a6a1349177c613dea55aabb20d857b8240a5Timo Sirainen i_error("link(%s, %s) failed: %m", old_path, new_path);
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen return -1;
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen}
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainenstatic int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx,
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen const char *dir, const char *fname)
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen{
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen struct dbox_file *file;
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen uint32_t file_id;
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen const char *id_str, *ext;
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen bool deleted;
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen int ret = 0;
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen id_str = fname + strlen(MDBOX_MAIL_FILE_PREFIX);
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen if (str_to_uint32(id_str, &file_id) < 0 || file_id == 0) {
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen /* m.*.broken files are created by file fixing
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen m.*.lock files are created if flock() isn't available */
ef174bf5299348e8c0662d235341869f319cfe54Timo Sirainen ext = strrchr(id_str, '.');
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen if (ext == NULL || (strcmp(ext, ".broken") != 0 &&
ba14267101444b8f144091cefd437e1ea44d3e32Timo Sirainen strcmp(ext, ".lock") != 0)) {
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen i_warning("mdbox rebuild: "
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen "Skipping file with missing ID: %s/%s",
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen dir, fname);
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen }
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen return 0;
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen }
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen if (!seq_range_exists(&ctx->seen_file_ids, file_id)) {
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if (ctx->highest_file_id < file_id)
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen ctx->highest_file_id = file_id;
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen } else {
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen /* duplicate file. either readdir() returned it twice
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen (unlikely) or it exists in both alt and primary storage.
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen to make sure we don't lose any mails from either of the
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen files, give this file a new ID and rename it. */
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen if (rebuild_rename_file(ctx, dir, &fname, &file_id) < 0)
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen return -1;
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen }
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen seq_range_array_add(&ctx->seen_file_ids, file_id);
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen file = mdbox_file_init(ctx->storage, file_id);
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted)
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen ret = rebuild_file_mails(ctx, file, file_id);
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen if (ret == 0)
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen i_error("mdbox rebuild: Failed to fix file %s/%s", dir, fname);
e34d170f8f0e084bd94bfbc1a7085ece67e508dfTimo Sirainen dbox_file_unref(&file);
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen return ret < 0 ? -1 : 0;
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen}
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainenstatic void
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainenrebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx,
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen uint32_t next_uid)
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen{
461ffead9720d1e516b959d5e41f049c73d38c7cTimo Sirainen struct mdbox_rebuild_msg **msgs;
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen struct mdbox_map_mail_index_record rec;
c47e837a127c533e67debafde8ccf9691041be16Timo Sirainen unsigned int i, count;
eb5ea3f4513ff2999892b8d904551f58b74f65f9Timo Sirainen uint32_t seq;
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen memset(&rec, 0, sizeof(rec));
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen msgs = array_get_modifiable(&ctx->msgs, &count);
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen for (i = 0; i < count; i++) {
503a863a317acba125a4e46435694e35fad769e4Timo Sirainen if (msgs[i]->map_uid != 0)
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen continue;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen rec.file_id = msgs[i]->file_id;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen rec.offset = msgs[i]->offset;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen rec.size = msgs[i]->rec_size;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen msgs[i]->map_uid = next_uid++;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen mail_index_append(ctx->atomic->sync_trans,
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen msgs[i]->map_uid, &seq);
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen mail_index_update_ext(ctx->atomic->sync_trans, seq,
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen ctx->storage->map->map_ext_id,
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen &rec, NULL);
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen }
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen}
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainenstatic int rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx)
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen{
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen struct mdbox_map *map = ctx->storage->map;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen const struct mail_index_header *hdr;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen struct mdbox_rebuild_msg **pos;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
e4cebacdec9c9e5b685dde5f7cbf7a5cf7e1d248Timo Sirainen struct dbox_mail_lookup_rec rec;
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen uint32_t seq;
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp);
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen /* msgs now contains a list of all messages that exists in m.* files,
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen sorted by file_id,offset */
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen
e20e638805c4bd54e039891a3e92760b1dfa189aTimo Sirainen hdr = mail_index_get_header(ctx->atomic->sync_view);
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen if (mdbox_map_view_lookup_rec(map, ctx->atomic->sync_view,
9905ec03fb2011419caeac4cd5a1b6c28ab50a73Timo Sirainen seq, &rec) < 0)
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen return -1;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen /* look up the rebuild msg record for this message based on
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen the (file_id, offset, size) triplet */
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen search_msg.file_id = rec.rec.file_id;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen search_msg.offset = rec.rec.offset;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen search_msg.rec_size = rec.rec.size;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen pos = array_bsearch(&ctx->msgs, &search_msgp,
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen mdbox_rebuild_msg_offset_cmp);
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen if (pos == NULL || (*pos)->map_uid != 0) {
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen /* map record points to nonexistent or
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen a duplicate message. */
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen mail_index_expunge(ctx->atomic->sync_trans, seq);
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen } else {
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen /* remember this message's map_uid */
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen (*pos)->map_uid = rec.map_uid;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen if (rec.refcount == 0)
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen (*pos)->seen_zero_ref_in_map = TRUE;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen }
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen }
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen rebuild_add_missing_map_uids(ctx, hdr->next_uid);
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen /* afterwards we're interested in looking up map_uids.
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen re-sort the messages to make it easier. */
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp);
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen return 0;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen}
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainenstatic struct mdbox_rebuild_msg *
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainenrebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx,
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen uint32_t map_uid)
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen{
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen struct mdbox_rebuild_msg **pos;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen search_msg.map_uid = map_uid;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen pos = array_bsearch(&ctx->msgs, &search_msgp,
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen mdbox_rebuild_msg_uid_cmp);
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen return pos == NULL ? NULL : *pos;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen}
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainenstatic bool
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainenguid_hash_have_map_uid(struct mdbox_rebuild_msg **recp, uint32_t map_uid)
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen{
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen struct mdbox_rebuild_msg *rec;
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen for (rec = *recp; rec != NULL; rec = rec->guid_hash_next) {
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen if (rec->map_uid == map_uid) {
90b50df264b57e0f63cd8cc6aea1ce3bb7cf5f64Timo Sirainen *recp = rec;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen return TRUE;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen }
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen }
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen return FALSE;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen}
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainenstatic void
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainenrebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx,
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen struct index_rebuild_context *rebuild_ctx,
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen struct mdbox_mailbox *mbox,
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen struct mail_index_view *view,
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen struct mail_index_transaction *trans)
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen{
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen struct mdbox_mail_index_record new_dbox_rec;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen const struct mail_index_header *hdr;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen struct mdbox_rebuild_msg *rec;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen const void *data;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen const uint8_t *guid_p;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen uint32_t old_seq, new_seq, uid, map_uid;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen /* Rebuild the mailbox's index. Note that index is reset at this point,
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen so although we can still access the old messages, we'll need to
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen append anything we want to keep as new messages. */
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen hdr = mail_index_get_header(view);
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen for (old_seq = 1; old_seq <= hdr->messages_count; old_seq++) {
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen mail_index_lookup_ext(view, old_seq, mbox->ext_id,
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen &data, NULL);
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen if (data == NULL) {
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen memset(&new_dbox_rec, 0, sizeof(new_dbox_rec));
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen map_uid = 0;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen } else {
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen memcpy(&new_dbox_rec, data, sizeof(new_dbox_rec));
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen map_uid = new_dbox_rec.map_uid;
5fdeff082e329e4a85bb7e74aaec2c35e2288557Timo Sirainen }
mail_index_lookup_ext(view, old_seq, mbox->guid_ext_id,
&data, NULL);
guid_p = data;
/* see if we can find this message based on
1) GUID, 2) map_uid */
rec = guid_p == NULL ? NULL :
hash_table_lookup(ctx->guid_hash, guid_p);
if (rec == NULL) {
/* multi-dbox message that wasn't found with GUID.
either it's lost or GUID has been corrupted. we can
still try to look it up using map_uid. */
rec = map_uid == 0 ? NULL :
rebuild_lookup_map_uid(ctx, map_uid);
map_uid = rec == NULL ? 0 : rec->map_uid;
} else if (!guid_hash_have_map_uid(&rec, map_uid)) {
/* message's GUID and map_uid point to different
physical messages. assume that GUID is correct and
map_uid is wrong. */
map_uid = rec->map_uid;
} else {
/* everything was ok. use this specific record's
map_uid to avoid duplicating mails in case the same
GUID exists multiple times */
}
if (rec != NULL) T_BEGIN {
/* keep this message. add it to mailbox index. */
i_assert(map_uid != 0);
rec->refcount++;
mail_index_lookup_uid(view, old_seq, &uid);
mail_index_append(trans, uid, &new_seq);
index_rebuild_index_metadata(rebuild_ctx,
new_seq, uid);
new_dbox_rec.map_uid = map_uid;
mail_index_update_ext(trans, new_seq, mbox->ext_id,
&new_dbox_rec, NULL);
mail_index_update_ext(trans, new_seq, mbox->guid_ext_id,
rec->guid_128, NULL);
} T_END;
}
}
static void
mdbox_rebuild_get_header(struct mail_index_view *view, uint32_t hdr_ext_id,
struct mdbox_index_header *hdr_r)
{
const void *data;
size_t data_size;
mail_index_get_header_ext(view, hdr_ext_id, &data, &data_size);
memset(hdr_r, 0, sizeof(*hdr_r));
memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
}
static void mdbox_header_update(struct mdbox_storage_rebuild_context *ctx,
struct index_rebuild_context *rebuild_ctx,
struct mdbox_mailbox *mbox)
{
struct mdbox_index_header hdr, backup_hdr;
mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id, &hdr);
if (rebuild_ctx->backup_view == NULL)
memset(&backup_hdr, 0, sizeof(backup_hdr));
else {
mdbox_rebuild_get_header(rebuild_ctx->backup_view,
mbox->hdr_ext_id, &backup_hdr);
}
/* make sure we have valid mailbox guid */
if (guid_128_is_empty(hdr.mailbox_guid)) {
if (!guid_128_is_empty(backup_hdr.mailbox_guid)) {
memcpy(hdr.mailbox_guid, backup_hdr.mailbox_guid,
sizeof(hdr.mailbox_guid));
} else {
guid_128_generate(hdr.mailbox_guid);
}
}
/* update map's uid-validity */
hdr.map_uid_validity = mdbox_map_get_uid_validity(mbox->storage->map);
if (ctx->have_pop3_uidls)
hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_UIDLS;
if (ctx->have_pop3_orders)
hdr.flags |= DBOX_INDEX_HEADER_FLAG_HAVE_POP3_ORDERS;
/* and write changes */
mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0,
&hdr, sizeof(hdr));
}
static int
rebuild_mailbox(struct mdbox_storage_rebuild_context *ctx,
struct mail_namespace *ns, const char *vname)
{
struct mailbox *box;
struct mdbox_mailbox *mbox;
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_view *view;
struct mail_index_transaction *trans;
struct index_rebuild_context *rebuild_ctx;
enum mail_error error;
int ret;
box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY |
MAILBOX_FLAG_IGNORE_ACLS);
if (box->storage != &ctx->storage->storage.storage) {
/* the namespace has multiple storages. */
mailbox_free(&box);
return 0;
}
if (mailbox_open(box) < 0) {
error = mailbox_get_last_mail_error(box);
i_error("Couldn't open mailbox '%s': %s",
vname, mailbox_get_last_error(box, NULL));
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
/* non-temporary error, ignore */
return 0;
}
mbox = (struct mdbox_mailbox *)box;
ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans,
MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES);
if (ret <= 0) {
i_assert(ret != 0);
mailbox_set_index_error(box);
mailbox_free(&box);
return -1;
}
rebuild_ctx = index_index_rebuild_init(&mbox->box, view, trans);
mdbox_header_update(ctx, rebuild_ctx, mbox);
rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
index_index_rebuild_deinit(&rebuild_ctx, dbox_get_uidvalidity_next);
if (mail_index_sync_commit(&sync_ctx) < 0) {
mailbox_set_index_error(box);
ret = -1;
}
mailbox_free(&box);
return ret < 0 ? -1 : 0;
}
static int
rebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx,
struct mail_namespace *ns)
{
struct mailbox_list_iterate_context *iter;
const struct mailbox_info *info;
int ret = 0;
if (ctx->default_list == NULL ||
(ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0)
ctx->default_list = ns->list;
iter = mailbox_list_iter_init(ns->list, "*",
MAILBOX_LIST_ITER_RAW_LIST |
MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
while ((info = mailbox_list_iter_next(iter)) != NULL) {
if ((info->flags & (MAILBOX_NONEXISTENT |
MAILBOX_NOSELECT)) == 0) {
T_BEGIN {
ret = rebuild_mailbox(ctx, ns, info->vname);
} T_END;
if (ret < 0) {
ret = -1;
break;
}
}
}
if (mailbox_list_iter_deinit(&iter) < 0)
ret = -1;
return ret;
}
static int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx)
{
struct mail_storage *storage = &ctx->storage->storage.storage;
struct mail_namespace *ns;
for (ns = storage->user->namespaces; ns != NULL; ns = ns->next) {
if (ns->storage == storage && ns->alias_for == NULL) {
if (rebuild_namespace_mailboxes(ctx, ns) < 0)
return -1;
}
}
if (ctx->default_list == NULL)
i_panic("No namespace found for storage=%s", storage->name);
return 0;
}
static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg)
{
if (mail_index_sync_commit(&msg->sync_ctx) < 0)
return -1;
mailbox_free(&msg->box);
memset(msg, 0, sizeof(*msg));
return 0;
}
static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx,
struct mdbox_rebuild_msg *msg)
{
struct mail_storage *storage = &ctx->storage->storage.storage;
struct dbox_file *file;
const struct mail_index_header *hdr;
struct mdbox_mail_index_record dbox_rec;
const char *mailbox = NULL;
struct mailbox *box;
struct mdbox_mailbox *mbox;
enum mail_error error;
bool deleted, created;
int ret;
uint32_t seq;
/* first see if message contains the mailbox it was originally
saved to */
file = mdbox_file_init(ctx->storage, msg->file_id);
ret = dbox_file_open(file, &deleted);
if (ret > 0 && !deleted)
ret = dbox_file_seek(file, msg->offset);
if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) {
mailbox = dbox_file_metadata_get(file,
DBOX_METADATA_ORIG_MAILBOX);
if (mailbox != NULL) {
mailbox = mailbox_list_get_vname(ctx->default_list, mailbox);
mailbox = t_strdup(mailbox);
}
rebuild_scan_metadata(ctx, file);
}
dbox_file_unref(&file);
if (ret <= 0 || deleted) {
if (ret < 0)
return -1;
/* we shouldn't get here, so apparently we couldn't fix
something. just ignore the mail.. */
return 0;
}
if (mailbox == NULL)
mailbox = "INBOX";
/* we have the destination mailbox. now open it and add the message
there. */
created = FALSE;
box = ctx->prev_msg.box != NULL &&
strcmp(mailbox, ctx->prev_msg.box->vname) == 0 ?
ctx->prev_msg.box : NULL;
while (box == NULL) {
box = mailbox_alloc(ctx->default_list, mailbox,
MAILBOX_FLAG_READONLY |
MAILBOX_FLAG_IGNORE_ACLS);
i_assert(box->storage == storage);
if (mailbox_open(box) == 0)
break;
error = mailbox_get_last_mail_error(box);
if (error == MAIL_ERROR_NOTFOUND && !created) {
/* mailbox doesn't exist currently? see if creating
it helps. */
created = TRUE;
(void)mailbox_create(box, NULL, FALSE);
mailbox_free(&box);
continue;
}
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
if (strcmp(mailbox, "INBOX") != 0) {
/* see if we can save to INBOX instead. */
mailbox = "INBOX";
} else {
/* this shouldn't happen */
return -1;
}
}
mbox = (struct mdbox_mailbox *)box;
/* switch the mailbox cache if necessary */
if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) {
if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
return -1;
}
if (ctx->prev_msg.box == NULL) {
ret = mail_index_sync_begin(box->index,
&ctx->prev_msg.sync_ctx,
&ctx->prev_msg.view,
&ctx->prev_msg.trans, 0);
if (ret <= 0) {
i_assert(ret != 0);
mailbox_set_index_error(box);
mailbox_free(&box);
return -1;
}
ctx->prev_msg.box = box;
hdr = mail_index_get_header(ctx->prev_msg.view);
ctx->prev_msg.next_uid = hdr->next_uid;
}
/* add the new message */
memset(&dbox_rec, 0, sizeof(dbox_rec));
dbox_rec.map_uid = msg->map_uid;
dbox_rec.save_date = ioloop_time;
mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id,
&dbox_rec, NULL);
mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id,
msg->guid_128, NULL);
msg->refcount++;
return 0;
}
static int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx)
{
struct mdbox_rebuild_msg **msgs;
unsigned int i, count;
/* if we have messages at this point which have refcount=0, they're
either already expunged or they were somehow lost for some reason.
we'll need to figure out what to do about them. */
msgs = array_get_modifiable(&ctx->msgs, &count);
for (i = 0; i < count; i++) {
if (msgs[i]->refcount != 0)
continue;
if (msgs[i]->seen_zero_ref_in_map) {
/* we've seen the map record, trust it. */
continue;
}
/* either map record was lost for this message or the message
was lost from its mailbox. safest way to handle this is to
restore the message. */
if (rebuild_restore_msg(ctx, msgs[i]) < 0)
return -1;
}
if (ctx->prev_msg.box != NULL) {
if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
return -1;
}
return 0;
}
static void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx)
{
const struct mail_index_header *hdr;
const void *data;
struct mdbox_rebuild_msg **msgs;
const uint16_t *ref16_p;
uint32_t seq, map_uid;
unsigned int i, count;
/* update refcounts for existing map records */
msgs = array_get_modifiable(&ctx->msgs, &count);
hdr = mail_index_get_header(ctx->atomic->sync_view);
for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) {
mail_index_lookup_uid(ctx->atomic->sync_view, seq, &map_uid);
if (map_uid != msgs[i]->map_uid) {
/* we've already expunged this map record */
i_assert(map_uid < msgs[i]->map_uid);
continue;
}
mail_index_lookup_ext(ctx->atomic->sync_view, seq,
ctx->storage->map->ref_ext_id,
&data, NULL);
ref16_p = data;
if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) {
mail_index_update_ext(ctx->atomic->sync_trans, seq,
ctx->storage->map->ref_ext_id,
&msgs[i]->refcount, NULL);
}
i++;
}
/* update refcounts for newly created map records */
for (; i < count; i++, seq++) {
mail_index_update_ext(ctx->atomic->sync_trans, seq,
ctx->storage->map->ref_ext_id,
&msgs[i]->refcount, NULL);
}
}
static int rebuild_finish(struct mdbox_storage_rebuild_context *ctx)
{
struct mdbox_map_mail_index_header map_hdr;
i_assert(ctx->default_list != NULL);
if (rebuild_handle_zero_refs(ctx) < 0)
return -1;
rebuild_update_refcounts(ctx);
/* update map header */
map_hdr = ctx->orig_map_hdr;
map_hdr.highest_file_id = ctx->highest_file_id;
map_hdr.rebuild_count = ++ctx->rebuild_count;
mail_index_update_header_ext(ctx->atomic->sync_trans,
ctx->storage->map->map_ext_id,
0, &map_hdr, sizeof(map_hdr));
return 0;
}
static int
mdbox_storage_rebuild_scan_dir(struct mdbox_storage_rebuild_context *ctx,
const char *storage_dir, bool alt)
{
DIR *dir;
struct dirent *d;
int ret = 0;
dir = opendir(storage_dir);
if (dir == NULL) {
if (alt && errno == ENOENT)
return 0;
mail_storage_set_critical(&ctx->storage->storage.storage,
"opendir(%s) failed: %m", storage_dir);
return -1;
}
for (errno = 0; (d = readdir(dir)) != NULL && ret == 0; errno = 0) {
if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) T_BEGIN {
ret = rebuild_add_file(ctx, storage_dir, d->d_name);
} T_END;
}
if (ret == 0 && errno != 0) {
mail_storage_set_critical(&ctx->storage->storage.storage,
"readdir(%s) failed: %m", storage_dir);
ret = -1;
}
if (closedir(dir) < 0) {
mail_storage_set_critical(&ctx->storage->storage.storage,
"closedir(%s) failed: %m", storage_dir);
ret = -1;
}
return ret;
}
static int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx)
{
const void *data;
size_t data_size;
if (mdbox_map_open_or_create(ctx->storage->map) < 0)
return -1;
/* begin by locking the map, so that other processes can't try to
rebuild at the same time. */
if (mdbox_map_atomic_lock(ctx->atomic) < 0)
return -1;
/* fsck the map just in case its UIDs are broken */
if (mail_index_fsck(ctx->storage->map->index) < 0) {
mail_storage_set_internal_error(&ctx->storage->storage.storage);
return -1;
}
/* get old map header */
mail_index_get_header_ext(ctx->atomic->sync_view,
ctx->storage->map->map_ext_id,
&data, &data_size);
memset(&ctx->orig_map_hdr, 0, sizeof(ctx->orig_map_hdr));
memcpy(&ctx->orig_map_hdr, data,
I_MIN(data_size, sizeof(ctx->orig_map_hdr)));
ctx->highest_file_id = ctx->orig_map_hdr.highest_file_id;
/* get storage rebuild counter after locking */
ctx->rebuild_count = mdbox_map_get_rebuild_count(ctx->storage->map);
if (ctx->rebuild_count != ctx->storage->corrupted_rebuild_count &&
ctx->storage->corrupted) {
/* storage was already rebuilt by someone else */
return 0;
}
i_warning("mdbox %s: rebuilding indexes", ctx->storage->storage_dir);
if (mdbox_storage_rebuild_scan_dir(ctx, ctx->storage->storage_dir,
FALSE) < 0)
return -1;
if (ctx->storage->alt_storage_dir != NULL) {
if (mdbox_storage_rebuild_scan_dir(ctx,
ctx->storage->alt_storage_dir, TRUE) < 0)
return -1;
}
if (rebuild_apply_map(ctx) < 0 ||
rebuild_mailboxes(ctx) < 0 ||
rebuild_finish(ctx) < 0) {
mdbox_map_atomic_set_failed(ctx->atomic);
return -1;
}
return 0;
}
int mdbox_storage_rebuild_in_context(struct mdbox_storage *storage,
struct mdbox_map_atomic_context *atomic)
{
struct mdbox_storage_rebuild_context *ctx;
int ret;
if (dbox_verify_alt_storage(storage->map->root_list) < 0) {
mail_storage_set_critical(&storage->storage.storage,
"mdbox rebuild: Alt storage %s not mounted, aborting",
storage->alt_storage_dir);
mdbox_map_atomic_set_failed(atomic);
return -1;
}
ctx = mdbox_storage_rebuild_init(storage, atomic);
ret = mdbox_storage_rebuild_scan(ctx);
mdbox_storage_rebuild_deinit(ctx);
if (ret == 0) {
storage->corrupted = FALSE;
storage->corrupted_rebuild_count = 0;
}
return ret;
}
int mdbox_storage_rebuild(struct mdbox_storage *storage)
{
struct mdbox_map_atomic_context *atomic;
int ret;
atomic = mdbox_map_atomic_begin(storage->map);
ret = mdbox_storage_rebuild_in_context(storage, atomic);
mdbox_map_atomic_set_success(atomic);
if (mdbox_map_atomic_finish(&atomic) < 0)
ret = -1;
return ret;
}