mbox-sync.c revision 2cb565cd978aafd5714792b5161889986d49e431
/* Copyright (C) 2004 Timo Sirainen */
/*
Modifying mbox can be slow, so we try to do it all at once minimizing the
required disk I/O. We may need to:
- Update message flags in Status, X-Status and X-Keywords headers
- Write missing X-UID and X-IMAPbase headers
- Write missing or broken Content-Length header if there's space
- Expunge specified messages
Here's how we do it:
- Start reading the mails from the beginning
- X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
of them, remember how much each message has and offset to beginning of the
padding
- If header needs to be rewritten and there's enough space, do it
- If we didn't have enough space, remember how much was missing
- Continue reading and counting the padding in each message. If available
padding is enough to rewrite all the previous messages needing it, do it
- When we encounter expunged message, treat all of it as padding and
rewrite previous messages if needed (and there's enough space).
Afterwards keep moving messages backwards to fill the expunged space.
Moving is done by rewriting each message's headers, with possibly adding
missing Content-Length header and padding. Message bodies are moved
without modifications.
- If we encounter end of file, grow the file and rewrite needed messages
- Rewriting is done by moving message body forward, rewriting message's
header and doing the same for previous message, until all of them are
rewritten.
*/
#include "lib.h"
#include "ioloop.h"
#include "buffer.h"
#include "istream.h"
#include "file-set-size.h"
#include "str.h"
#include "write-full.h"
#include "istream-raw-mbox.h"
#include "mbox-storage.h"
#include "mbox-file.h"
#include "mbox-lock.h"
#include "mbox-sync-private.h"
#include <stddef.h>
#include <stdlib.h>
#define MBOX_SYNC_SECS 1
{
"Unexpectedly lost From-line at offset %"PRIuUOFF_T
" from mbox file %s", from_offset,
return -1;
}
return 0;
}
{
struct mail_index_sync_rec *sync;
/* keep it */
dest++;
}
}
}
static int
struct mbox_sync_mail_context *mail_ctx)
{
/* get EOF */
return 0;
/* need to add 'O' flag to Status-header */
}
}
return 1;
}
{
const struct mail_index_sync_rec *sync;
for (i = 0; i < size; i++) {
return TRUE;
}
return FALSE;
}
{
int ret;
*sync_expunge_r = FALSE;
return 0;
if (uid == 0) {
/* nothing for this or the future ones */
}
sizeof(*sync_rec));
*sync_expunge_r = TRUE;
}
if (ret < 0) {
return -1;
}
if (ret == 0) {
break;
}
}
}
}
if (!*sync_expunge_r)
return 0;
}
{
const struct mail_index_sync_rec *sync;
for (i = 0; i < size; i++) {
continue;
}
}
static int
{
int ret = 0;
if (ret < 0) {
return -1;
}
break;
/* externally expunged message, remove from index */
}
/* this UID was already in index and it was expunged */
"mbox sync: Expunged message reappeared in mailbox %s "
/* new UID in the middle of the mailbox - shouldn't happen */
"mbox sync: UID inserted in the middle of mailbox %s "
} else {
ret = 1;
}
return ret;
}
unsigned char hdr_md5_sum[],
const struct mail_index_record **rec_r)
{
const void *data;
int ret;
if (ret < 0) {
return -1;
}
&data) < 0) {
return -1;
}
break;
/* externally expunged message, remove from index */
}
return 0;
}
static int
struct mbox_sync_mail *mail,
int nocheck)
{
const void *data;
if (!nocheck) {
/* see if from_offset needs updating */
&data) < 0) {
return -1;
}
return 0;
}
return 0;
}
struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
/* new message */
}
/*FIXME:mail_cache_add(sync_ctx->cache_trans,
MAIL_CACHE_UID_STRING,
str_data(mail_ctx->uidl),
str_len(mail_ctx->uidl));*/
}
} else {
/* see if flags changed */
if ((idx_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
/* flags are dirty, ignore whatever was in the file.
but remove recent flag if needed. */
mbox_flags &= ~MAIL_RECENT;
} else {
}
else if (!sync_ctx->delay_writes)
if ((idx_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) ==
(mbox_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) &&
INDEX_KEYWORDS_BYTE_COUNT) == 0) {
if (idx_flags != mbox_flags) {
/* dirty flag state changed */
int dirty = (mbox_flags &
MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
memset(idx_keywords, 0,
}
} else if ((idx_flags & ~MAIL_RECENT) !=
(mbox_flags & ~MAIL_RECENT) ||
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
/* drop recent flag */
}
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
return -1;
}
return 0;
}
{
const unsigned char *data;
for (;;) {
if (size >= from_line_size)
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int
{
return -1;
}
return 0;
}
{
const struct mbox_sync_mail *mails;
continue;
}
}
{
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
}
}
{
int ret;
/* move the header backwards to fill expunged space */
if (sync_ctx->dest_first_mail) {
/* we're moving this mail to beginning of file.
skip the initial \n (it's already counted in
expunged_space) */
}
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
if (ret < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
if (mbox_write_from_line(mail_ctx) < 0)
return -1;
} else {
if (sync_ctx->dest_first_mail) {
/* didn't have enough space, move the offset
back so seeking into it doesn't fail */
}
}
} else if (mail_ctx->need_rewrite ||
sync_ctx->update_base_uid_last != 0)) {
if (sync_ctx->delay_writes) {
/* mark it dirty and do it later */
return 0;
}
return -1;
} else {
/* nothing to do */
return 0;
}
/* first mail with no space to write it */
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
sync_ctx->expunged_space = 0;
}
}
return 0;
}
static int
{
if (sync_ctx->space_diff < 0) {
if (sync_ctx->expunged_space > 0) {
sync_ctx->expunged_space = 0;
}
return 0;
}
/* we have enough space now */
/* this message was expunged. fill more or less of the space.
space_diff now consists of a negative "bytes needed" sum,
plus the expunged space of this message. so it contains how
many bytes of _extra_ space we have. */
/* don't waste too much on padding */
} else {
sync_ctx->expunged_space = 0;
}
} else {
/* this message gave enough space from headers. rewriting stops
at the end of this message's headers. */
sync_ctx->expunged_space = 0;
move_diff = 0;
}
return -1;
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
sync_ctx->need_space_seq = 0;
sync_ctx->space_diff = 0;
return 0;
}
static int
{
if (seq == 0) {
"Mailbox isn't a valid mbox file");
return -1;
}
seq++;
} else {
if (ret < 0)
return -1;
if (ret == 0) {
old_offset) < 0) {
"Error seeking back to original "
"offset %s in mbox file %s",
return -1;
}
return 0;
}
}
if (seq <= 1)
uid = 0;
return -1;
}
/* set to -1, since it's always increased later */
/* this mbox has pseudo mail which contains the X-IMAP header */
}
return 1;
}
static int
{
return -1;
}
if (seq1 == 0) {
/* doesn't exist anymore, seek to end of file */
file_size) < 0) {
"Error seeking to end of mbox file %s",
return -1;
}
return 1;
}
}
struct mbox_sync_mail_context *mail_ctx,
{
const struct mail_index_record *rec;
(!partial && min_message_count != 0)) {
messages_count : 0);
} else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
&expunged) < 0)
return -1;
/* nothing to do */
return 1;
}
}
if (size == 0)
}
if (ret <= 0)
return ret;
"UIDVALIDITY changed (%u -> %u) "
"in mbox file %s",
return -1;
}
/* UID ordering problems, resync everything to make
sure we get everything right */
return 0;
}
uid = 0;
if (uid != 0) {
if (ret < 0)
return -1;
}
if (ret == 0) {
/* UID found but it's broken */
uid = 0;
(sync_ctx->delay_writes ||
Also check for existing MD5 sums when we're actually
able to write X-UIDs. */
"header-md5", 0, 16, 1);
}
&rec) < 0)
return -1;
}
/* get all sync records related to this message */
&expunged) < 0)
return -1;
} else {
}
need new UIDs. */
}
}
if (!expunged) {
if (mbox_sync_handle_header(mail_ctx) < 0)
return -1;
} else {
}
if (!expunged) {
rec) < 0)
return -1;
}
}
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
return -1;
return -1;
}
/* if there's no sync records left,
we can stop */
break;
/* we can skip forward to next record which
needs updating. if it failes because the
offset is dirty, just ignore and continue
from where we are now. */
return -1;
}
}
}
/* rest of the messages in index don't exist -> expunge them */
}
if (!partial)
return 1;
}
struct mbox_sync_mail_context *mail_ctx)
{
return 0;
}
if (sync_ctx->need_space_seq != 0) {
sync_ctx->expunged_space = 0;
"file_set_size()");
return -1;
}
return -1;
sync_ctx->need_space_seq = 0;
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
/* everything deleted, the trailer_size still contains
the \n trailer though */
trailer_size = 0;
}
trailer_size) < 0)
return -1;
return -1;
}
sync_ctx->expunged_space = 0;
}
return 0;
}
{
return -1;
}
if ((sync_ctx->base_uid_validity != 0 &&
if (sync_ctx->base_uid_validity == 0) {
/* we didn't rewrite X-IMAPbase header because
a) mbox is read-only, b) we're lazy-writing,
c) it's empty */
}
sizeof(sync_ctx->base_uid_validity));
}
}
&sync_stamp, sizeof(sync_stamp));
}
}
return 0;
}
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->prev_msg_uid = 0;
sync_ctx->need_space_seq = 0;
sync_ctx->expunged_space = 0;
sync_ctx->space_diff = 0;
}
enum mbox_sync_flags flags)
{
struct mbox_sync_mail_context mail_ctx;
if ((flags & MBOX_SYNC_HEADER) != 0)
min_msg_count = 1;
else {
return -1;
}
/* file is fully synced */
min_msg_count = 0;
} else if ((flags & MBOX_SYNC_UNDIRTY) != 0 ||
/* we want to do full syncing. always do this if
file size hasn't changed but timestamp has. it most
likely means that someone had modified some header
and we probably want to know about it */
} else {
/* see if we can delay syncing the whole file.
normally we only notice expunges and appends
in partial syncing. */
}
}
if (ret <= 0) {
if (ret < 0)
return -1;
/* partial syncing didn't work, do it again */
if (ret <= 0) {
return -1;
}
}
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
{
const struct mail_index_header *hdr;
return -1;
}
/* fully synced */
return 0;
}
return 1;
}
{
struct mbox_sync_mail_context mail_ctx;
return -1;
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
unsigned int lock_id = 0;
if (!ibox->mbox_do_dirty_syncs)
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
return -1;
}
if ((flags & MBOX_SYNC_HEADER) != 0)
changed = 1;
else {
if ((flags & MBOX_SYNC_LOCK_READING) != 0)
return -1;
}
}
if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
/* we just want to lock it for reading. if mbox hasn't been
modified don't do any syncing. */
if (!changed)
return 0;
/* have to sync to make sure offsets have stayed the same */
lock_id = 0;
}
/* reopen input stream to make sure it has nothing buffered */
if (changed) {
/* we're most likely modifying the mbox while syncing, just
lock it for writing immediately. the mbox must be locked
before index syncing is started to avoid deadlocks, so we
don't have much choice either (well, easy ones anyway). */
return -1;
}
if ((flags & MBOX_SYNC_LAST_COMMIT) != 0) {
} else {
}
(flags & MBOX_SYNC_REWRITE) != 0);
if (ret <= 0) {
if (ret < 0)
if (lock_id != 0)
return ret;
}
/* nothing to do */
if (lock_id != 0)
/* index may need to do internal syncing though, so commit
instead of rollbacking. */
if (mail_index_sync_commit(index_sync_ctx) < 0) {
return -1;
}
return 0;
}
if (lock_id == 0) {
/* ok, we have something to do but no locks. we'll have to
restart syncing to avoid deadlocking. */
changed = 1;
goto __again;
}
if (mbox_file_open_stream(ibox) < 0) {
return -1;
}
((flags & MBOX_SYNC_REWRITE) == 0 &&
if (ret < 0)
ret = -1;
} else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
if (ret < 0)
else if (mail_index_sync_commit(index_sync_ctx) < 0) {
ret = -1;
}
if (sync_ctx.seen_first_mail &&
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
if (ret < 0)
else {
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0) {
ret = -1;
}
index_sync_ctx) < 0) {
ret = -1;
}
}
}
ret = -1;
}
}
/* drop to read lock */
unsigned int read_lock_id = 0;
ret = -1;
else {
ret = -1;
}
}
/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
notice changes made by others .. and this has to be done
even if lock_reading is set.. except if
mbox_sync_dirty = TRUE */
ret = -1;
}
return ret;
}
struct mailbox_sync_context *
{
enum mbox_sync_flags mbox_sync_flags = 0;
int ret = 0;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0)
if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0)
}
}