mbox-sync.c revision b9ac6179d3aee0d1641a4ee1d78da28628929c61
/* Copyright (C) 2004 Timo Sirainen */
/*
Modifying mbox can be slow, so we try to do it all at once minimizing the
required disk I/O. We may need to:
- Update message flags in Status, X-Status and X-Keywords headers
- Write missing X-UID and X-IMAPbase headers
- Write missing or broken Content-Length header if there's space
- Expunge specified messages
Here's how we do it:
- Start reading the mails from the beginning
- X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end
of them, remember how much each message has and offset to beginning of the
padding
- If header needs to be rewritten and there's enough space, do it
- If we didn't have enough space, remember how much was missing
- Continue reading and counting the padding in each message. If available
padding is enough to rewrite all the previous messages needing it, do it
- When we encounter expunged message, treat all of it as padding and
rewrite previous messages if needed (and there's enough space).
Afterwards keep moving messages backwards to fill the expunged space.
Moving is done by rewriting each message's headers, with possibly adding
missing Content-Length header and padding. Message bodies are moved
without modifications.
- If we encounter end of file, grow the file and rewrite needed messages
- Rewriting is done by moving message body forward, rewriting message's
header and doing the same for previous message, until all of them are
rewritten.
*/
#include "lib.h"
#include "ioloop.h"
#include "buffer.h"
#include "istream.h"
#include "file-set-size.h"
#include "str.h"
#include "write-full.h"
#include "istream-raw-mbox.h"
#include "mbox-storage.h"
#include "mbox-file.h"
#include "mbox-lock.h"
#include "mbox-sync-private.h"
#include <stddef.h>
#define MBOX_SYNC_SECS 1
/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
{
}
} else {
return -1;
}
}
return -1;
if (mbox_file_open_stream(ibox) < 0)
return -1;
/* we didn't have the file open before -> it changed */
return 0;
}
return -1;
}
return 0;
/* same as before. we'll have to fix mbox stream to contain
correct from_offset, hdr_offset and body_offset. so, seek
to from_offset and read through the header. */
"Message offset %s changed unexpectedly for mbox file "
return 0;
}
return 1;
}
{
"Unexpectedly lost From-line at offset %"PRIuUOFF_T
" from mbox file %s", from_offset,
return -1;
}
return 0;
}
struct mbox_sync_mail_context *mail_ctx,
{
/* put the padding between last message's header and body */
return -1;
}
return -1;
return 0;
}
{
struct mail_index_sync_rec *sync;
dest++;
}
}
}
static int
struct mbox_sync_mail_context *mail_ctx)
{
/* get EOF */
return 0;
/* First message was expunged and this is the next one.
Skip \n header */
mail_ctx->from_offset++;
}
/* save the offset permanently with recent flag state */
/* need to add 'O' flag to Status-header */
}
}
return 1;
}
{
const struct mail_index_sync_rec *sync;
for (i = 0; i < size; i++) {
return TRUE;
}
return FALSE;
}
{
int ret;
*sync_expunge_r = FALSE;
return 0;
if (uid == 0) {
/* nothing for this or the future ones */
}
sizeof(*sync_rec));
*sync_expunge_r = TRUE;
}
if (ret < 0) {
return -1;
}
if (ret == 0) {
break;
}
}
}
}
if (!*sync_expunge_r)
return 0;
}
{
const struct mail_index_sync_rec *sync;
for (i = 0; i < size; i++) {
continue;
}
}
static int
{
int ret = 0;
if (ret < 0) {
return -1;
}
break;
/* externally expunged message, remove from index */
}
/* this UID was already in index and it was expunged */
"mbox sync: Expunged message reappeared in mailbox %s "
ret = 0;
/* new UID in the middle of the mailbox - shouldn't happen */
"mbox sync: UID inserted in the middle of mailbox %s "
ret = 0;
} else {
ret = 1;
}
return ret;
}
unsigned char hdr_md5_sum[],
const struct mail_index_record **rec_r)
{
const void *data;
int ret;
if (ret < 0) {
return -1;
}
&data) < 0) {
return -1;
}
break;
/* externally expunged message, remove from index */
}
return 0;
}
{
const void *data;
/* see if from_offset needs updating */
&data) < 0) {
return -1;
}
return 0;
}
static int
struct mbox_sync_mail *mail,
int nocheck)
{
if (!nocheck) {
&offset) < 0)
return -1;
return 0;
}
return 0;
}
struct mbox_sync_mail_context *mail_ctx,
const struct mail_index_record *rec)
{
/* new message */
}
/*FIXME:mail_cache_add(sync_ctx->cache_trans,
MAIL_CACHE_UID_STRING,
str_data(mail_ctx->uidl),
str_len(mail_ctx->uidl));*/
}
} else {
/* see if flags changed */
mbox_flags &= ~MAIL_RECENT;
if (idx_flags != mbox_flags ||
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
}
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
return -1;
}
return 0;
}
{
const unsigned char *data;
for (;;) {
if (size >= from_line_size)
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int
{
return -1;
}
return 0;
}
{
const struct mbox_sync_mail *mails;
idx = 0;
if (idx_seq == 0) {
}
continue;
&offset);
}
}
{
int ret;
return -1;
if (ret == 0)
return -2;
}
return 0;
}
{
int ret;
return ret;
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
}
return 0;
}
{
int ret;
return 0;
/* move the header backwards to fill expunged space */
return ret;
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
if (ret < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
return -1;
}
} else if (mail_ctx->need_rewrite ||
sync_ctx->update_base_uid_last != 0)) {
return ret;
return -1;
} else {
/* nothing to do */
return 0;
}
/* first mail with no space to write it */
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
}
}
return 0;
}
static int
{
if (sync_ctx->space_diff < 0)
return 0;
/* we have enough space now */
/* don't waste too much on padding */
} else {
sync_ctx->expunged_space = 0;
}
return -1;
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
sync_ctx->need_space_seq = 0;
return 0;
}
static int
{
return -1;
}
if (seq1 == 0)
return 0;
return -1;
/* set to -1, since it's always increased later */
/* this mbox has pseudo mail which contains the X-IMAP header */
}
"Cached message offset %s is invalid for mbox file %s",
return -1;
}
return 1;
}
struct mbox_sync_mail_context *mail_ctx,
{
const struct mail_index_record *rec;
if (min_message_count != 0)
ret = 0;
else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
&expunged) < 0)
return -1;
/* nothing to do */
return 0;
}
}
if (size == 0)
if (ret < 0)
return -1;
}
if (ret == 0) {
/* doesn't begin with a From-line */
"Mailbox isn't a valid mbox file");
return -1;
}
}
uid = 0;
/* get all sync records related to this message */
return -1;
if (ret < 0)
return -1;
if (ret == 0)
uid = 0;
}
/* Use MD5 sums */
"header-md5", 16);
}
&rec) < 0)
return -1;
}
need new UIDs. */
}
}
if (!expunged) {
} else {
}
if (ret < 0) {
/* -1 = error, -2 = need to restart */
return ret;
}
if (!expunged) {
rec) < 0)
return -1;
}
}
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
return -1;
return -1;
}
/* if there's no sync records left,
we can stop */
break;
/* we can skip forward to next record which
needs updating. */
return -1;
}
}
}
/* rest of the messages in index don't exist -> expunge them */
}
return 0;
}
struct mbox_sync_mail_context *mail_ctx)
{
int need_rewrite;
return 0;
}
if (sync_ctx->need_space_seq != 0) {
sync_ctx->expunged_space = 0;
else
if (sync_ctx->space_diff < 0 &&
-sync_ctx->space_diff) < 0)
return -1;
return -1;
if (need_rewrite) {
return -1;
}
sync_ctx->need_space_seq = 0;
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
trailer_size) < 0)
return -1;
return -1;
}
sync_ctx->expunged_space = 0;
}
return 0;
}
{
return -1;
}
if ((sync_ctx->base_uid_validity != 0 &&
if (sync_ctx->base_uid_validity == 0) {
/* we couldn't rewrite X-IMAPbase because it's
a read-only mbox */
}
sizeof(sync_ctx->base_uid_validity));
}
}
&sync_stamp, sizeof(sync_stamp));
}
}
return 0;
}
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->prev_msg_uid = 0;
}
{
struct mbox_sync_mail_context mail_ctx;
int ret;
if (sync_header)
min_msg_count = 1;
else {
return -1;
}
0 : (uint32_t)-1;
}
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
FALSE);
return -1;
}
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
{
const struct mail_index_header *hdr;
return -1;
}
return -1;
}
}
{
struct mbox_sync_mail_context mail_ctx;
if (mbox_sync_seek(sync_ctx, 0) < 0)
return -1;
return -1;
return -1;
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int sync_header, int lock)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
unsigned int lock_id = 0;
if (lock) {
return -1;
}
if (sync_header)
ret = 0;
else {
if (lock)
return -1;
}
}
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
} else {
}
if (ret <= 0) {
if (ret < 0)
return ret;
}
lock_id = 0;
}
ret = -1;
ret = -1;
if (ret < 0)
ret = -1;
} else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
if (mail_index_sync_end(index_sync_ctx) < 0) {
ret = -1;
}
if (sync_ctx.seen_first_mail &&
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
if (ret < 0)
else {
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0) {
ret = -1;
}
ret = -1;
}
}
}
ret = -1;
}
}
/* drop to read lock */
unsigned int lock_id = 0;
ret = -1;
else {
ret = -1;
}
}
/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
notice changes made by others */
ret = -1;
}
return ret;
}
{
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
return -1;
}
}