mdbox-storage-rebuild.c revision 18122b1fb9aa58945ed12d7020e91fbf4f04a0cc
/* Copyright (c) 2009-2017 Dovecot authors, see the included COPYING file */
#include "lib.h"
#include "array.h"
#include "ioloop.h"
#include "istream.h"
#include "hash.h"
#include "str.h"
#include "mail-cache.h"
#include "index-rebuild.h"
#include "mail-namespace.h"
#include "mailbox-list-private.h"
#include "mdbox-storage.h"
#include "mdbox-file.h"
#include "mdbox-map-private.h"
#include "mdbox-sync.h"
#include "mdbox-storage-rebuild.h"
#include <dirent.h>
#include <unistd.h>
#define REBUILD_MAX_REFCOUNT 32768
struct mdbox_rebuild_msg {
struct mdbox_rebuild_msg *guid_hash_next;
bool seen_zero_ref_in_map:1;
};
struct rebuild_msg_mailbox {
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_view *view;
struct mail_index_transaction *trans;
};
struct mdbox_storage_rebuild_context {
struct mdbox_storage *storage;
struct mdbox_map_atomic_context *atomic;
struct mailbox_list *default_list;
struct rebuild_msg_mailbox prev_msg;
bool have_pop3_uidls:1;
bool have_pop3_orders:1;
};
static struct mdbox_storage_rebuild_context *
struct mdbox_map_atomic_context *atomic)
{
struct mdbox_storage_rebuild_context *ctx;
return ctx;
}
static void
{
}
static int
struct mdbox_rebuild_msg *const *m2)
{
return -1;
return 1;
return -1;
return 1;
return -1;
return 1;
return 0;
}
struct mdbox_rebuild_msg *const *m2)
{
return -1;
return 1;
return 0;
}
{
}
{
const char *guid;
int ret;
prev_offset = 0;
if (ret > 0) {
break;
}
if (ret == 0) {
/* file is corrupted. fix it and retry. */
break;
first = prev_offset == 0;
if (prev_offset == 0) {
/* use existing file header if it was ok */
}
break;
if (ret == 0) {
/* file was deleted */
return 1;
}
if (!first) {
/* seek to the offset where we last left off */
if (ret <= 0)
break;
}
continue;
}
"Message is missing GUID");
ret = 0;
break;
}
/* two mails' GUID and size are the same, which quite
likely means that their contents are the same as
well. we'll compare the mail sizes instead of the
record sizes, because the records' metadata may
differ.
save this duplicate mail with refcount=0 to the map,
so it will eventually be purged. */
} else {
/* duplicate GUID, but not a duplicate message. */
i_error("mdbox %s: Duplicate GUID %s in "
}
}
if (ret < 0)
return -1;
return 0;
else
return 1;
}
static int
{
do {
/* use link()+unlink() instead of rename() to make sure we
don't overwrite any files. */
return 0;
}
return -1;
}
{
bool deleted;
int ret = 0;
/* m.*.broken files are created by file fixing
m.*.lock files are created if flock() isn't available */
i_warning("mdbox rebuild: "
"Skipping file with missing ID: %s/%s",
}
return 0;
}
} else {
/* duplicate file. either readdir() returned it twice
(unlikely) or it exists in both alt and primary storage.
to make sure we don't lose any mails from either of the
files, give this file a new ID and rename it. */
return -1;
}
if (ret == 0)
return ret < 0 ? -1 : 0;
}
static void
{
struct mdbox_rebuild_msg **msgs;
struct mdbox_map_mail_index_record rec;
unsigned int i, count;
for (i = 0; i < count; i++) {
continue;
}
}
{
const struct mail_index_header *hdr;
struct mdbox_rebuild_msg **pos;
struct dbox_mail_lookup_rec rec;
/* msgs now contains a list of all messages that exists in m.* files,
sorted by file_id,offset */
return -1;
/* look up the rebuild msg record for this message based on
the (file_id, offset, size) triplet */
/* map record points to nonexistent or
a duplicate message. */
} else {
/* remember this message's map_uid */
}
}
/* afterwards we're interested in looking up map_uids.
re-sort the messages to make it easier. */
return 0;
}
static struct mdbox_rebuild_msg *
{
struct mdbox_rebuild_msg **pos;
}
static bool
{
struct mdbox_rebuild_msg *rec;
return TRUE;
}
}
return FALSE;
}
static void
struct index_rebuild_context *rebuild_ctx,
struct mdbox_mailbox *mbox,
struct mail_index_view *view,
struct mail_index_transaction *trans)
{
struct mdbox_mail_index_record new_dbox_rec;
const struct mail_index_header *hdr;
struct mdbox_rebuild_msg *rec;
const void *data;
/* Rebuild the mailbox's index. Note that index is reset at this point,
so although we can still access the old messages, we'll need to
append anything we want to keep as new messages. */
map_uid = 0;
} else {
}
/* see if we can find this message based on
1) GUID, 2) map_uid */
/* multi-dbox message that wasn't found with GUID.
either it's lost or GUID has been corrupted. we can
still try to look it up using map_uid. */
/* message's GUID and map_uid point to different
physical messages. assume that GUID is correct and
map_uid is wrong. */
} else {
/* everything was ok. use this specific record's
map_uid to avoid duplicating mails in case the same
GUID exists multiple times */
}
/* keep this message. add it to mailbox index. */
&new_dbox_rec, NULL);
} T_END;
}
}
static void
{
const void *data;
}
struct index_rebuild_context *rebuild_ctx,
struct mdbox_mailbox *mbox)
{
bool need_resize, need_resize_backup;
&hdr, &need_resize);
i_zero(&backup_hdr);
need_resize = TRUE;
} else {
}
/* make sure we have valid mailbox guid */
sizeof(hdr.mailbox_guid));
} else {
}
}
/* update map's uid-validity */
if (ctx->have_pop3_uidls)
if (ctx->have_pop3_orders)
/* and write changes */
if (need_resize) {
sizeof(hdr));
}
}
static int
{
struct mdbox_mailbox *mbox;
struct mail_index_sync_ctx *sync_ctx;
struct mail_index_view *view;
struct mail_index_transaction *trans;
struct index_rebuild_context *rebuild_ctx;
enum mail_error error;
int ret;
/* the namespace has multiple storages. */
mailbox_free(&box);
return 0;
}
if (mailbox_open(box) < 0) {
i_error("Couldn't open mailbox '%s': %s",
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
/* non-temporary error, ignore */
return 0;
}
if (ret <= 0) {
mailbox_free(&box);
return -1;
}
if (mail_index_sync_commit(&sync_ctx) < 0) {
ret = -1;
}
mailbox_free(&box);
return ret < 0 ? -1 : 0;
}
static int
struct mail_namespace *ns)
{
struct mailbox_list_iterate_context *iter;
const struct mailbox_info *info;
int ret = 0;
MAILBOX_NOSELECT)) == 0) {
T_BEGIN {
} T_END;
if (ret < 0) {
ret = -1;
break;
}
}
}
if (mailbox_list_iter_deinit(&iter) < 0)
ret = -1;
return ret;
}
{
struct mail_namespace *ns;
return -1;
}
}
return 0;
}
{
return -1;
return 0;
}
struct mdbox_rebuild_msg *msg)
{
const struct mail_index_header *hdr;
struct mdbox_mail_index_record dbox_rec;
struct mdbox_mailbox *mbox;
enum mail_error error;
int ret;
/* first see if message contains the mailbox it was originally
saved to */
}
}
if (ret < 0)
return -1;
/* we shouldn't get here, so apparently we couldn't fix
something. just ignore the mail.. */
return 0;
}
mailbox = "INBOX";
/* we have the destination mailbox. now open it and add the message
there. */
if (mailbox_open(box) == 0)
break;
/* mailbox doesn't exist currently? see if creating
it helps. */
mailbox_free(&box);
continue;
}
mailbox_free(&box);
if (error == MAIL_ERROR_TEMP)
return -1;
/* see if we can save to INBOX instead. */
mailbox = "INBOX";
} else {
/* this shouldn't happen */
return -1;
}
}
/* switch the mailbox cache if necessary */
return -1;
}
if (ret <= 0) {
mailbox_free(&box);
return -1;
}
}
/* add the new message */
return 0;
}
{
struct mdbox_rebuild_msg **msgs;
unsigned int i, count;
/* if we have messages at this point which have refcount=0, they're
either already expunged or they were somehow lost for some reason.
we'll need to figure out what to do about them. */
for (i = 0; i < count; i++) {
continue;
if (msgs[i]->seen_zero_ref_in_map) {
/* we've seen the map record, trust it. */
continue;
}
/* either map record was lost for this message or the message
was lost from its mailbox. safest way to handle this is to
restore the message. */
return -1;
}
return -1;
}
return 0;
}
{
const struct mail_index_header *hdr;
const void *data;
struct mdbox_rebuild_msg **msgs;
unsigned int i, count;
/* update refcounts for existing map records */
/* we've already expunged this map record */
continue;
}
}
i++;
}
/* update refcounts for newly created map records */
}
}
{
struct mdbox_map_mail_index_header map_hdr;
if (rebuild_handle_zero_refs(ctx) < 0)
return -1;
/* update map header */
return 0;
}
static int
const char *storage_dir, bool alt)
{
struct dirent *d;
int ret = 0;
return 0;
"opendir(%s) failed: %m", storage_dir);
return -1;
}
} T_END;
}
"readdir(%s) failed: %m", storage_dir);
ret = -1;
}
"closedir(%s) failed: %m", storage_dir);
ret = -1;
}
return ret;
}
{
const void *data;
return -1;
/* begin by locking the map, so that other processes can't try to
rebuild at the same time. */
return -1;
/* fsck the map just in case its UIDs are broken */
return -1;
}
/* get old map header */
/* get storage rebuild counter after locking */
/* storage was already rebuilt by someone else */
return 0;
}
FALSE) < 0)
return -1;
return -1;
}
if (rebuild_apply_map(ctx) < 0 ||
rebuild_mailboxes(ctx) < 0 ||
rebuild_finish(ctx) < 0) {
return -1;
}
return 0;
}
struct mdbox_map_atomic_context *atomic)
{
struct mdbox_storage_rebuild_context *ctx;
int ret;
"mdbox rebuild: Alt storage %s not mounted, aborting",
return -1;
}
if (ret == 0) {
}
return ret;
}
{
struct mdbox_map_atomic_context *atomic;
int ret;
if (mdbox_map_atomic_finish(&atomic) < 0)
ret = -1;
return ret;
}