mdbox-storage-rebuild.c revision cd89cbcf9ef73a0855ffcbe799b4fc2442b28cbb
5f5870385cff47efd2f58e7892f251cf13761528Timo Sirainen/* Copyright (c) 2009-2010 Dovecot authors, see the included COPYING file */
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "lib.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "array.h"
65cca8364f483126b396aeb2036dc879ad45ab8dTimo Sirainen#include "ioloop.h"
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen#include "istream.h"
472369cba85d9f7c995dda60e7cd01d78b4a960aTimo Sirainen#include "hash.h"
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen#include "str.h"
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include "dbox-sync-rebuild.h"
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen#include "mdbox-storage.h"
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen#include "mdbox-file.h"
b1f37113a5760bee842c5a7678bb5fa6f5bd8b60Timo Sirainen#include "mdbox-map-private.h"
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen#include "mdbox-sync.h"
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen#include "mdbox-storage-rebuild.h"
e28fa207d1a097fa6e4a867f74ee0761472ef1ceTimo Sirainen
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen#include <stdlib.h>
ef50336eefcb9ba99f73c6af37420eaf8857a39bTimo Sirainen#include <dirent.h>
13d98ffa534f2e7d04a832c9d0153fc9c568b878Timo Sirainen#include <unistd.h>
13d98ffa534f2e7d04a832c9d0153fc9c568b878Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstruct mdbox_rebuild_msg {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen uint8_t guid_128[MAIL_GUID_128_SIZE];
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen uint32_t file_id;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen uint32_t offset;
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen uint32_t size;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen uint32_t map_uid;
5694eeb99b69dea8033ca77ad69743c6b4871370Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen uint16_t refcount;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen unsigned int seen_zero_ref_in_map:1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen};
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstruct rebuild_msg_mailbox {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mailbox *box;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_sync_ctx *sync_ctx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_view *view;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_transaction *trans;
8e371a3ce32bd64288786855b8ce0cb63f19f7d1Timo Sirainen uint32_t next_uid;
220e21750948941dc6e33b8f11b552fa21d7f81eTimo Sirainen};
4ee00532a265bdfb38539d811fcd12d51210ac35Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstruct mdbox_storage_rebuild_context {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen struct mdbox_storage *storage;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen pool_t pool;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen struct hash_table *guid_hash;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen ARRAY_DEFINE(msgs, struct mdbox_rebuild_msg *);
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen uint32_t prev_file_id;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen uint32_t highest_seen_map_uid;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen struct mailbox_list *default_list;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_sync_ctx *sync_ctx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_view *sync_view;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mail_index_transaction *trans;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct rebuild_msg_mailbox prev_msg;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen unsigned int msgs_unsorted:1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen};
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic unsigned int guid_hash(const void *p)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen const uint8_t *s = p;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen unsigned int i, g, h = 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen for (i = 0; i < MAIL_GUID_128_SIZE; i++) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen h = (h << 4) + s[i];
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if ((g = h & 0xf0000000UL)) {
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen h = h ^ (g >> 24);
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen h = h ^ g;
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen }
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen }
8a0ad174adb1eb5108511b90e97f4e5f9089b0eeTimo Sirainen return h;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic int guid_cmp(const void *p1, const void *p2)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return memcmp(p1, p2, MAIL_GUID_128_SIZE);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic struct mdbox_storage_rebuild_context *
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenmdbox_storage_rebuild_init(struct mdbox_storage *storage)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen struct mdbox_storage_rebuild_context *ctx;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ctx = i_new(struct mdbox_storage_rebuild_context, 1);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ctx->storage = storage;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ctx->guid_hash = hash_table_create(default_pool, ctx->pool, 0,
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen guid_hash, guid_cmp);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen i_array_init(&ctx->msgs, 512);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen return ctx;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen}
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainenstatic void
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainenmdbox_storage_rebuild_deinit(struct mdbox_storage_rebuild_context *ctx)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen{
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen if (ctx->sync_ctx != NULL)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen mail_index_sync_rollback(&ctx->sync_ctx);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen hash_table_destroy(&ctx->guid_hash);
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen pool_unref(&ctx->pool);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen array_free(&ctx->msgs);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen i_free(ctx);
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen}
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic int mdbox_rebuild_msg_offset_cmp(const void *p1, const void *p2)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen const struct mdbox_rebuild_msg *const *m1 = p1, *const *m2 = p2;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if ((*m1)->file_id < (*m2)->file_id)
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen return -1;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen if ((*m1)->file_id > (*m2)->file_id)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return 1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if ((*m1)->offset < (*m2)->offset)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen if ((*m1)->offset > (*m2)->offset)
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen return 1;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen return 0;
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainenstatic int mdbox_rebuild_msg_uid_cmp(struct mdbox_rebuild_msg *const *m1,
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen struct mdbox_rebuild_msg *const *m2)
63e207529879438e9f4412d97cdc34bdc82a3702Timo Sirainen{
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen if ((*m1)->map_uid < (*m2)->map_uid)
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen return -1;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen if ((*m1)->map_uid > (*m2)->map_uid)
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen return 1;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen return 0;
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen}
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainenstatic int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
2649b237dd4690575e75a30b2bf3b39ebd37b835Timo Sirainen struct dbox_file *file)
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen{
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen const char *guid;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen struct mdbox_rebuild_msg *rec;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uoff_t offset, prev_offset;
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen bool last, first, fixed = FALSE;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen int ret;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen dbox_file_seek_rewind(file);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen prev_offset = 0;
76b91bac787101e6b0075122ab6478dd98c8a884Timo Sirainen while ((ret = dbox_file_seek_next(file, &offset, &last)) >= 0) {
76b91bac787101e6b0075122ab6478dd98c8a884Timo Sirainen if (ret > 0) {
76b91bac787101e6b0075122ab6478dd98c8a884Timo Sirainen if ((ret = dbox_file_metadata_read(file)) < 0)
3fe67ec75ccae1230bb9eb9f16affc48377f6441Timo Sirainen break;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen if (ret == 0) {
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen /* file is corrupted. fix it and retry. */
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen if (fixed || last)
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen break;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen first = prev_offset == 0;
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen if (prev_offset == 0) {
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen /* use existing file header if it was ok */
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen prev_offset = offset;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen }
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen if (dbox_file_fix(file, prev_offset) < 0) {
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen ret = -1;
2a6dcd984104fed84bed8795ccdfabb20e41ce52Timo Sirainen break;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen fixed = TRUE;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (!first) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* seek to the offset where we last left off */
5214b67a7dabab87da74e04bb8b227f94b95bce4Timo Sirainen ret = dbox_file_get_mail_stream(file,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen prev_offset, NULL);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (ret <= 0)
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen break;
27586e4785d56aeb76e1fd96af8db799688dc64aTimo Sirainen }
360123b1b41b7aa8af6c4a91c39046be646cd349Timo Sirainen continue;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
5214b67a7dabab87da74e04bb8b227f94b95bce4Timo Sirainen prev_offset = offset;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (guid == NULL) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen dbox_file_set_corrupted(file,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen "Message is missing GUID");
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ret = 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen break;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen rec = p_new(ctx->pool, struct mdbox_rebuild_msg, 1);
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen rec->file_id = ctx->prev_file_id;
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen rec->offset = offset;
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen rec->size = file->input->v_offset - offset;
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen mail_generate_guid_128_hash(guid, rec->guid_128);
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen array_append(&ctx->msgs, &rec, 1);
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen
917498e6f84969d2b93410c1e479735abe8e0ed7Timo Sirainen if (hash_table_lookup(ctx->guid_hash, rec->guid_128) != NULL) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen /* duplicate. save this as a refcount=0 to map,
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen so it will eventually be deleted. */
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk rec->seen_zero_ref_in_map = TRUE;
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk } else {
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk hash_table_insert(ctx->guid_hash, rec->guid_128, rec);
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen }
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk }
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk if (ret < 0)
e5acc283bf030b0b5c79ca4e52d315c516a299faPascal Volk return -1;
7920a47321690c932ffd4d286cd16b4048d22d41Timo Sirainen else if (ret == 0 && !last)
7920a47321690c932ffd4d286cd16b4048d22d41Timo Sirainen return 0;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen else
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen return 1;
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen}
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainen
db8b0a3f74a20528d66a3c4be7df920e5c4554c2Timo Sirainenstatic int rebuild_add_file(struct mdbox_storage_rebuild_context *ctx,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen const char *path)
5666a3d6a7ea89362b8d9e8b39b15424cd9d6388Timo Sirainen{
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen struct dbox_file *file;
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen uint32_t file_id;
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen const char *fname;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen unsigned int len;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen bool deleted;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen int ret = 0;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen fname = strrchr(path, '/');
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen i_assert(fname != NULL);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen fname += strlen(MDBOX_MAIL_FILE_PREFIX) + 1;
9abf5be0962538e1f6f5c73c838ff677341da0c9Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen file_id = strtoul(fname, NULL, 10);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen if (!is_numeric(fname, '\0') || file_id == 0) {
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen len = strlen(fname);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen if (len > 7 && strcmp(fname + len - 7, ".broken") != 0) {
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen i_warning("dbox rebuild: File name is missing ID: %s",
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen path);
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen }
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen return 0;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen }
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen /* small optimization: typically files are returned sorted. in that
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen case we don't need to sort them ourself. */
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen if (file_id < ctx->prev_file_id)
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen ctx->msgs_unsorted = TRUE;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen ctx->prev_file_id = file_id;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen file = mdbox_file_init(ctx->storage, file_id);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if ((ret = dbox_file_open(file, &deleted)) > 0 && !deleted)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen ret = rebuild_file_mails(ctx, file);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (ret == 0)
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen i_error("dbox rebuild: Failed to fix file %s", path);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen dbox_file_unref(&file);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen return ret < 0 ? -1 : 0;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen}
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainenstatic void
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainenrebuild_add_missing_map_uids(struct mdbox_storage_rebuild_context *ctx,
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen uint32_t next_uid)
5c597df6aa8d81de4053c6986fab7739f3b44b20Timo Sirainen{
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen struct mdbox_rebuild_msg **msgs;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen struct dbox_map_mail_index_record rec;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen unsigned int i, count;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen uint32_t seq;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen memset(&rec, 0, sizeof(rec));
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen msgs = array_get_modifiable(&ctx->msgs, &count);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen for (i = 0; i < count; i++) {
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (msgs[i]->map_uid != 0)
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen continue;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen rec.file_id = msgs[i]->file_id;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen rec.offset = msgs[i]->offset;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen rec.size = msgs[i]->size;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen msgs[i]->map_uid = next_uid++;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_append(ctx->trans, msgs[i]->map_uid, &seq);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen mail_index_update_ext(ctx->trans, seq,
5214b67a7dabab87da74e04bb8b227f94b95bce4Timo Sirainen ctx->storage->map->map_ext_id,
5214b67a7dabab87da74e04bb8b227f94b95bce4Timo Sirainen &rec, NULL);
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen}
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainenstatic int rebuild_apply_map(struct mdbox_storage_rebuild_context *ctx)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen{
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen struct dbox_map *map = ctx->storage->map;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen const struct mail_index_header *hdr;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen struct mdbox_rebuild_msg *const *msgs, **pos;
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen struct dbox_mail_lookup_rec rec;
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen uint32_t seq;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen unsigned int count;
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen if (ctx->msgs_unsorted)
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen array_sort(&ctx->msgs, mdbox_rebuild_msg_offset_cmp);
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen
c58c12049c883b281c088d47a2a7278c21c390e1Timo Sirainen msgs = array_get_modifiable(&ctx->msgs, &count);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen hdr = mail_index_get_header(ctx->sync_view);
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen if (dbox_map_view_lookup_rec(map, ctx->sync_view,
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen seq, &rec) < 0)
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen return -1;
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen /* look up the rebuild msg record for this message */
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen search_msg.file_id = rec.rec.file_id;
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen search_msg.offset = rec.rec.offset;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs),
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen mdbox_rebuild_msg_offset_cmp);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (pos == NULL) {
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen /* map record points to non-existing message. */
c1d19144dd7b1de6822df6ed1d10af0c9cb38840Timo Sirainen mail_index_expunge(ctx->trans, seq);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen } else {
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen (*pos)->map_uid = rec.map_uid;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen if (rec.refcount == 0)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen (*pos)->seen_zero_ref_in_map = TRUE;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen }
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen }
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen rebuild_add_missing_map_uids(ctx, hdr->next_uid);
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen
5da1aa5197a43d83f0fb3eeb83125c7cd73d1b62Timo Sirainen /* afterwards we're interested in looking up map_uids.
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen re-sort the messages to make it easier. */
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen array_sort(&ctx->msgs, mdbox_rebuild_msg_uid_cmp);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return 0;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen}
360123b1b41b7aa8af6c4a91c39046be646cd349Timo Sirainen
360123b1b41b7aa8af6c4a91c39046be646cd349Timo Sirainenstatic struct mdbox_rebuild_msg *
360123b1b41b7aa8af6c4a91c39046be646cd349Timo Sirainenrebuild_lookup_map_uid(struct mdbox_storage_rebuild_context *ctx,
360123b1b41b7aa8af6c4a91c39046be646cd349Timo Sirainen uint32_t map_uid)
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen{
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen struct mdbox_rebuild_msg search_msg, *search_msgp = &search_msg;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen struct mdbox_rebuild_msg **pos;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen search_msg.map_uid = map_uid;
5494a6bc149da8f02fd25c0434a9d612ac33f659Timo Sirainen pos = array_bsearch(&ctx->msgs, &search_msgp,
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen mdbox_rebuild_msg_uid_cmp);
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen return pos == NULL ? NULL : *pos;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen}
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainenstatic void
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainenrebuild_mailbox_multi(struct mdbox_storage_rebuild_context *ctx,
94b0ff77495c3ed14bdd4b5d7ae1eb37e8c9efb5Timo Sirainen struct dbox_sync_rebuild_context *rebuild_ctx,
02e61e13a8360a9d3ec92c5fa5ae60c0f0181b71Timo Sirainen struct mdbox_mailbox *mbox,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen struct mail_index_view *view,
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen struct mail_index_transaction *trans)
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen{
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen const struct mdbox_mail_index_record *dbox_rec;
b365bd121cdc87f63e1dd47c5085a27091118e00Timo Sirainen struct mdbox_mail_index_record new_dbox_rec;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen const struct mail_index_header *hdr;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen struct mdbox_rebuild_msg *rec;
adb6413686e52e00dded4932babcc08ff041876bTimo Sirainen const void *data;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen bool expunged;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen uint32_t seq, uid, new_seq, map_uid;
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen memset(&new_dbox_rec, 0, sizeof(new_dbox_rec));
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen hdr = mail_index_get_header(view);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen for (seq = 1; seq <= hdr->messages_count; seq++) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen mail_index_lookup_ext(view, seq, mbox->ext_id,
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen &data, &expunged);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen dbox_rec = data;
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen map_uid = dbox_rec == NULL ? 0 : dbox_rec->map_uid;
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen mail_index_lookup_ext(view, seq, mbox->guid_ext_id,
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen &data, &expunged);
1433bf361ddb0bba8878c8ada5726d0284edad57Timo Sirainen
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen /* see if we can find this message based on
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen 1) GUID, 2) map_uid */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen rec = data == NULL ? NULL :
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen hash_table_lookup(ctx->guid_hash, data);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (rec == NULL) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (map_uid == 0) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* not a multi-dbox message, ignore. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen continue;
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen }
4a514fb20e04df397842cde11cc9ea92abfe9728Timo Sirainen /* multi-dbox message that wasn't found with GUID.
94d8e51119003d2bc5a100c663f90141f297385dTimo Sirainen either it's lost or GUID has been corrupted. we can
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen still try to look it up using map_uid. */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen rec = rebuild_lookup_map_uid(ctx, map_uid);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen if (rec != NULL) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen mail_index_update_ext(trans, seq,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen mbox->guid_ext_id,
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen rec->guid_128, NULL);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen }
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen } else if (map_uid != rec->map_uid) {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* map_uid is wrong, update it */
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen i_assert(rec->map_uid != 0);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen new_dbox_rec.map_uid = rec->map_uid;
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen mail_index_update_ext(trans, seq, mbox->ext_id,
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen &new_dbox_rec, NULL);
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen } else {
0b4e1043e596bfb36d999dacbf1d4d63ee96d75fTimo Sirainen /* everything was ok */
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen }
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen if (rec != NULL) T_BEGIN {
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen /* keep this message */
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen rec->refcount++;
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen mail_index_lookup_uid(view, seq, &uid);
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen mail_index_append(trans, uid, &new_seq);
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen dbox_sync_rebuild_index_metadata(rebuild_ctx,
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen new_seq, uid);
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen new_dbox_rec.map_uid = rec->map_uid;
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen mail_index_update_ext(trans, new_seq, mbox->ext_id,
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen &new_dbox_rec, NULL);
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen } T_END;
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen }
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen}
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainenstatic void
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenmdbox_rebuild_get_header(struct mail_index_view *view, uint32_t hdr_ext_id,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct mdbox_index_header *hdr_r)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen const void *data;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen size_t data_size;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen mail_index_get_header_ext(view, hdr_ext_id, &data, &data_size);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen memset(hdr_r, 0, sizeof(*hdr_r));
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen memcpy(hdr_r, data, I_MIN(data_size, sizeof(*hdr_r)));
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen}
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenstatic void mdbox_header_update(struct dbox_sync_rebuild_context *rebuild_ctx,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct mdbox_mailbox *mbox)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct mdbox_index_header hdr, backup_hdr;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mdbox_rebuild_get_header(rebuild_ctx->view, mbox->hdr_ext_id, &hdr);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (rebuild_ctx->backup_view == NULL)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen memset(&backup_hdr, 0, sizeof(backup_hdr));
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen else {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mdbox_rebuild_get_header(rebuild_ctx->backup_view,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mbox->hdr_ext_id, &backup_hdr);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* make sure we have valid mailbox guid */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (mail_guid_128_is_empty(hdr.mailbox_guid)) {
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen if (!mail_guid_128_is_empty(backup_hdr.mailbox_guid)) {
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen memcpy(hdr.mailbox_guid, backup_hdr.mailbox_guid,
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen sizeof(hdr.mailbox_guid));
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen } else {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_generate_guid_128(hdr.mailbox_guid);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
d508ab8db2b0f74b5e225d199b4aaa5293342746Timo Sirainen /* update map's uid-validity */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen hdr.map_uid_validity = dbox_map_get_uid_validity(mbox->storage->map);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* and write changes */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mail_index_update_header_ext(rebuild_ctx->trans, mbox->hdr_ext_id, 0,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen &hdr, sizeof(hdr));
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen}
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainenstatic int
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainenrebuild_mailbox(struct mdbox_storage_rebuild_context *ctx,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mail_namespace *ns, const char *name)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen{
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mailbox *box;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mdbox_mailbox *mbox;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mail_index_sync_ctx *sync_ctx;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mail_index_view *view;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mail_index_transaction *trans;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct dbox_sync_rebuild_context *rebuild_ctx;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen enum mail_error error;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen int ret;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen box = mdbox_mailbox_alloc(&ctx->storage->storage.storage,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen ns->list, name, MAILBOX_FLAG_READONLY |
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen MAILBOX_FLAG_KEEP_RECENT |
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen MAILBOX_FLAG_IGNORE_ACLS);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (dbox_mailbox_open(box) < 0) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen (void)mail_storage_get_last_error(box->storage, &error);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox_free(&box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (error == MAIL_ERROR_TEMP)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen /* non-temporary error, ignore */
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return 0;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mbox = (struct mdbox_mailbox *)box;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (ret <= 0) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen i_assert(ret != 0);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mail_storage_set_index_error(box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox_free(&box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen rebuild_ctx = dbox_sync_index_rebuild_init(&mbox->box, view, trans);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mdbox_header_update(rebuild_ctx, mbox);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen dbox_sync_index_rebuild_deinit(&rebuild_ctx);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (mail_index_sync_commit(&sync_ctx) < 0) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mail_storage_set_index_error(box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen ret = -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen mailbox_free(&box);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return ret < 0 ? -1 : 0;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen}
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainenstatic int
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainenrebuild_namespace_mailboxes(struct mdbox_storage_rebuild_context *ctx,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct mail_namespace *ns)
e58f291e777b25c1286965b80e142ada6fdacb03Timo Sirainen{
e58f291e777b25c1286965b80e142ada6fdacb03Timo Sirainen struct mailbox_list_iterate_context *iter;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen const struct mailbox_info *info;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen int ret = 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
494a5de15db3b2806ab31d5ecc3e1c306ae14d06Timo Sirainen if (ctx->default_list == NULL ||
e58f291e777b25c1286965b80e142ada6fdacb03Timo Sirainen (ns->flags & NAMESPACE_FLAG_INBOX) != 0)
e58f291e777b25c1286965b80e142ada6fdacb03Timo Sirainen ctx->default_list = ns->list;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen iter = mailbox_list_iter_init(ns->list, "*",
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen while ((info = mailbox_list_iter_next(iter)) != NULL) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if ((info->flags & (MAILBOX_NONEXISTENT |
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen MAILBOX_NOSELECT)) == 0) {
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen T_BEGIN {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ret = rebuild_mailbox(ctx, ns, info->name);
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen } T_END;
08ed4ab71fd2a4e800d9025a736f0f46b771ea90Timo Sirainen if (ret < 0) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ret = -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen break;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen }
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen }
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen if (mailbox_list_iter_deinit(&iter) < 0)
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen ret = -1;
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen return ret;
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen}
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainenstatic int rebuild_mailboxes(struct mdbox_storage_rebuild_context *ctx)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct mail_user *user = ctx->storage->storage.storage.user;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct mail_namespace *ns;
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen
0bd259973f98837cf0e41fdee3e2a578e51ad09eTimo Sirainen for (ns = user->namespaces; ns != NULL; ns = ns->next) {
0bd259973f98837cf0e41fdee3e2a578e51ad09eTimo Sirainen if (ns->storage == &ctx->storage->storage.storage &&
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen ns->alias_for == NULL) {
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen if (rebuild_namespace_mailboxes(ctx, ns) < 0)
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen return -1;
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return 0;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen}
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainenstatic int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen{
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (mail_index_sync_commit(&msg->sync_ctx) < 0)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox_free(&msg->box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen memset(msg, 0, sizeof(*msg));
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return 0;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen}
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainenstatic int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mdbox_rebuild_msg *msg)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen{
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct mail_storage *storage = &ctx->storage->storage.storage;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen struct dbox_file *file;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen const struct mail_index_header *hdr;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mdbox_mail_index_record dbox_rec;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen const char *mailbox = NULL;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mailbox *box;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen struct mdbox_mailbox *mbox;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen enum mail_error error;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen bool deleted, created;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen int ret;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen uint32_t seq;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* first see if message contains the mailbox it was originally
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen saved to */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen file = mdbox_file_init(ctx->storage, msg->file_id);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ret = dbox_file_open(file, &deleted);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (ret > 0)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen ret = dbox_file_get_mail_stream(file, msg->offset, NULL);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mailbox = dbox_file_metadata_get(file,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen DBOX_METADATA_ORIG_MAILBOX);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen dbox_file_unref(&file);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (ret <= 0 || deleted) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (ret < 0)
8f70c97f7ab7b7e1683ed5cfcd96721a899c2520Timo Sirainen return -1;
8f70c97f7ab7b7e1683ed5cfcd96721a899c2520Timo Sirainen /* we shouldn't get here, so apparently we couldn't fix
8f70c97f7ab7b7e1683ed5cfcd96721a899c2520Timo Sirainen something. just ignore the mail.. */
8f70c97f7ab7b7e1683ed5cfcd96721a899c2520Timo Sirainen return 0;
8f70c97f7ab7b7e1683ed5cfcd96721a899c2520Timo Sirainen }
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (mailbox == NULL)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mailbox = "INBOX";
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* we have the destination mailbox. now open it and add the message
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen there. */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen created = FALSE;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen box = ctx->prev_msg.box != NULL &&
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen strcmp(mailbox, ctx->prev_msg.box->name) == 0 ?
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen ctx->prev_msg.box : NULL;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen while (box == NULL) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen box = mdbox_mailbox_alloc(storage, ctx->default_list,
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen mailbox, MAILBOX_FLAG_READONLY |
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen MAILBOX_FLAG_KEEP_RECENT |
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen MAILBOX_FLAG_IGNORE_ACLS);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (dbox_mailbox_open(box) == 0)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen break;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen (void)mail_storage_get_last_error(box->storage, &error);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen if (error == MAIL_ERROR_NOTFOUND && !created) {
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* mailbox doesn't exist currently? see if creating
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen it helps. */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen created = TRUE;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen (void)mailbox_create(box, NULL, FALSE);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox_free(&box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen continue;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox_free(&box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (error == MAIL_ERROR_TEMP)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (strcmp(mailbox, "INBOX") != 0) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen /* see if we can save to INBOX instead. */
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox = "INBOX";
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen } else {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen /* this shouldn't happen */
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mbox = (struct mdbox_mailbox *)box;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen /* switch the mailbox cache if necessary */
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (ctx->prev_msg.box == NULL) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen ret = mail_index_sync_begin(box->index,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen &ctx->prev_msg.sync_ctx,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen &ctx->prev_msg.view,
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen &ctx->prev_msg.trans, 0);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (ret <= 0) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen i_assert(ret != 0);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mail_storage_set_index_error(box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen mailbox_free(&box);
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ctx->prev_msg.box = box;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen hdr = mail_index_get_header(ctx->prev_msg.view);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen ctx->prev_msg.next_uid = hdr->next_uid;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* add the new message */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen memset(&dbox_rec, 0, sizeof(dbox_rec));
9b00ecffbe74fd864d0d72e6112ec53b86f619baTimo Sirainen dbox_rec.map_uid = msg->map_uid;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen dbox_rec.save_date = ioloop_time;
9b00ecffbe74fd864d0d72e6112ec53b86f619baTimo Sirainen mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
9b00ecffbe74fd864d0d72e6112ec53b86f619baTimo Sirainen mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->ext_id,
9b00ecffbe74fd864d0d72e6112ec53b86f619baTimo Sirainen &dbox_rec, NULL);
9b00ecffbe74fd864d0d72e6112ec53b86f619baTimo Sirainen mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id,
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen msg->guid_128, NULL);
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen
3e0bae44b65f5c46989fcef3d1e07203f496327eTimo Sirainen msg->refcount++;
0bd259973f98837cf0e41fdee3e2a578e51ad09eTimo Sirainen return 0;
0bd259973f98837cf0e41fdee3e2a578e51ad09eTimo Sirainen}
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainenstatic int rebuild_handle_zero_refs(struct mdbox_storage_rebuild_context *ctx)
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen{
e4cb3bfcd42f1f2c9e676ece6f7f53803f5c6a16Timo Sirainen struct mdbox_rebuild_msg **msgs;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen unsigned int i, count;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* if we have messages at this point which have refcount=0, they're
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen either already expunged or they were somehow lost for some reason.
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen we'll need to figure out what to do about them. */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen msgs = array_get_modifiable(&ctx->msgs, &count);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen for (i = 0; i < count; i++) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (msgs[i]->refcount != 0)
1c1cecd3dfaf71b0c9499b044023e631841e88aaTimo Sirainen continue;
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen if (msgs[i]->seen_zero_ref_in_map) {
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen /* we've seen the map record, trust it. */
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen continue;
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen }
636f017be100bce67d66fd3ae1544a47681efd33Timo Sirainen /* either map record was lost for this message or the message
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen was lost from its mailbox. safest way to handle this is to
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen restore the message. */
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen if (rebuild_restore_msg(ctx, msgs[i]) < 0)
69af83d4e6c2c5c825a17edd7a41a4fb014caa8fTimo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (ctx->prev_msg.box != NULL) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return 0;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen}
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainenstatic void rebuild_update_refcounts(struct mdbox_storage_rebuild_context *ctx)
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen{
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen const struct mail_index_header *hdr;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen const void *data;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen struct mdbox_rebuild_msg **msgs;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen const uint16_t *ref16_p;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen bool expunged;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen uint32_t seq, map_uid;
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen unsigned int i, count;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen /* update refcounts for existing map records */
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen msgs = array_get_modifiable(&ctx->msgs, &count);
dc5606fb66d30a659459446b6ca1a8b4f1146052Timo Sirainen hdr = mail_index_get_header(ctx->sync_view);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen mail_index_lookup_uid(ctx->sync_view, seq, &map_uid);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen if (map_uid != msgs[i]->map_uid) {
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen /* we've already expunged this map record */
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen i_assert(map_uid < msgs[i]->map_uid);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen continue;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
1701e3f91107051b1704721bf1dc1e32491faaf9Timo Sirainen mail_index_lookup_ext(ctx->sync_view, seq,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ctx->storage->map->ref_ext_id,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen &data, &expunged);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen ref16_p = data;
ff7056842f14fd3b30a2d327dfab165b9d15dd30Timo Sirainen if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) {
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen mail_index_update_ext(ctx->trans, seq,
678d0463849ba777106eb7875f27db07a5d8e3dfTimo Sirainen ctx->storage->map->ref_ext_id,
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen &msgs[i]->refcount, NULL);
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen }
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen i++;
bd4e36a8cd7257cca7d1434c49a1e343ed7c5100Timo Sirainen }
cb933f0a570a9cef5c975eadb818aa6b1002a269Timo Sirainen
cb933f0a570a9cef5c975eadb818aa6b1002a269Timo Sirainen /* update refcounts for newly created map records */
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen for (; i < count; i++, seq++) {
37847ec8eaec9ad55c9df10ae109efe7b37ac573Timo Sirainen mail_index_update_ext(ctx->trans, seq,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen ctx->storage->map->ref_ext_id,
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen &msgs[i]->refcount, NULL);
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen }
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen}
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainenstatic int rebuild_finish(struct mdbox_storage_rebuild_context *ctx)
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen{
a27e065f1a1f91c7fbdf7c2ea1c387441af0cbb3Timo Sirainen i_assert(ctx->default_list != NULL);
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen if (rebuild_handle_zero_refs(ctx) < 0)
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen return -1;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen rebuild_update_refcounts(ctx);
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen return 0;
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen}
68a4946b12583b88fa802e52ebee45cd96056772Timo Sirainen
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainenstatic int mdbox_storage_rebuild_scan(struct mdbox_storage_rebuild_context *ctx)
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen{
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen const struct mail_index_header *hdr;
c58906589cafc32df4c04ffbef933baadd3f2276Timo Sirainen DIR *dir;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen struct dirent *d;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen string_t *path;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen unsigned int dir_len;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen uint32_t uid_validity;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen int ret = 0;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen if (dbox_map_open(ctx->storage->map, TRUE) < 0)
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen return -1;
6dd77763f5451269ace733579cf58f2f3b18bca4Timo Sirainen
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen /* begin by locking the map, so that other processes can't try to
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen rebuild at the same time. */
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen ret = mail_index_sync_begin(ctx->storage->map->index, &ctx->sync_ctx,
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen &ctx->sync_view, &ctx->trans, 0);
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen if (ret <= 0) {
0df9428baed48afaff90b4d4f03792d2fd756a43Timo Sirainen i_assert(ret != 0);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen mail_storage_set_internal_error(&ctx->storage->storage.storage);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_reset_error(ctx->storage->map->index);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen return -1;
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen uid_validity = dbox_map_get_uid_validity(ctx->storage->map);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen hdr = mail_index_get_header(ctx->sync_view);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (hdr->uid_validity != uid_validity) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_index_update_header(ctx->trans,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen offsetof(struct mail_index_header, uid_validity),
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen &uid_validity, sizeof(uid_validity), TRUE);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen }
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen dir = opendir(ctx->storage->storage_dir);
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen if (dir == NULL) {
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen mail_storage_set_critical(&ctx->storage->storage.storage,
2615df45a8027948a474abe5e817b34b0499c171Timo Sirainen "opendir(%s) failed: %m", ctx->storage->storage_dir);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen return -1;
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen }
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen path = t_str_new(256);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen str_append(path, ctx->storage->storage_dir);
c0a87e5f3316a57e6f915882fa1951d0fbb74a61Timo Sirainen str_append_c(path, '/');
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen dir_len = str_len(path);
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
a10ed8c47534b4c6b6bf2711ccfe577e720a47b4Timo Sirainen if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen strlen(MDBOX_MAIL_FILE_PREFIX)) == 0) {
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen str_truncate(path, dir_len);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen str_append(path, d->d_name);
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen T_BEGIN {
3f2dd5867a85b15555eb43a33ea5029393a45103Timo Sirainen ret = rebuild_add_file(ctx, str_c(path));
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen } T_END;
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen if (ret < 0) {
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen ret = -1;
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen break;
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen }
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen }
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen }
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen if (ret == 0 && errno != 0) {
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen mail_storage_set_critical(&ctx->storage->storage.storage,
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen "readdir(%s) failed: %m", ctx->storage->storage_dir);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen ret = -1;
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen }
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen if (closedir(dir) < 0) {
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen mail_storage_set_critical(&ctx->storage->storage.storage,
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen "closedir(%s) failed: %m", ctx->storage->storage_dir);
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen ret = -1;
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen }
9fc97c8aa8190df87624d214bcc5d0b5362bec93Timo Sirainen
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen if (ret < 0 ||
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen rebuild_apply_map(ctx) < 0 ||
a24519c36d5f8fa22f58b2c693ba547e8d175a54Timo Sirainen rebuild_mailboxes(ctx) < 0 ||
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen rebuild_finish(ctx) < 0 ||
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen mail_index_sync_commit(&ctx->sync_ctx) < 0)
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen return -1;
a64adf62fa33f2463a86f990217b0c9078531a40Timo Sirainen return 0;
e200d1ba38eeebfb0b9e60150d93753ec6d823c8Timo Sirainen}
e200d1ba38eeebfb0b9e60150d93753ec6d823c8Timo Sirainen
e200d1ba38eeebfb0b9e60150d93753ec6d823c8Timo Sirainenint mdbox_storage_rebuild(struct mdbox_storage *storage)
d02d34e138e32b4266f5a403d6c51d7803bf322fTimo Sirainen{
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen struct mdbox_storage_rebuild_context *ctx;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen struct stat st;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen int ret;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen if (stat(storage->storage_dir, &st) < 0) {
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen if (errno == ENOENT) {
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen /* no multi-dbox files */
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen return 0;
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen }
d06c46087e9e6e66bbbbb9df1d5b33154349515cTimo Sirainen
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen mail_storage_set_critical(&storage->storage.storage,
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen "stat(%s) failed: %m", storage->storage_dir);
1d2b188f0eedc3cab6e27ceac5425a037f38042eTimo Sirainen return -1;
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen }
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen i_warning("dbox %s: rebuilding indexes", storage->storage_dir);
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen ctx = mdbox_storage_rebuild_init(storage);
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen ret = mdbox_storage_rebuild_scan(ctx);
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen mdbox_storage_rebuild_deinit(ctx);
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen if (ret == 0)
47d5cc09738defd0020c797866e2a25a344aa65aTimo Sirainen storage->storage.files_corrupted = FALSE;
94f84d1c3f786d1b92dd2a1507f83a2dad887c56Timo Sirainen return ret;
e91543761d0b7b97a1dc28e036e44d76405545c2Timo Sirainen}
490f66d6476d51cc02333d6eb398a5cd94b67f48Timo Sirainen