mbox-sync.c revision abe8230dd1dd37d7ccf0163100e934bb5e658c20
0N/A/* Copyright (C) 2004 Timo Sirainen */
1472N/A
0N/A/*
0N/A Modifying mbox can be slow, so we try to do it all at once minimizing the
0N/A required disk I/O. We may need to:
0N/A
0N/A - Update message flags in Status, X-Status and X-Keywords headers
0N/A - Write missing X-UID and X-IMAPbase headers
0N/A - Write missing or broken Content-Length header if there's space
0N/A - Expunge specified messages
0N/A
0N/A Here's how we do it:
0N/A
0N/A - Start reading the mails mail headers from the beginning
0N/A - X-Keywords and X-UID headers may contain extra spaces at the end of them,
0N/A remember how much extra each message has and offset to beginning of the
0N/A spaces
0N/A - If message flags are dirty and there's enough space to write them, do it
1472N/A - If we didn't have enough space, remember how much was missing and keep
1472N/A the total amount of them
1472N/A - When we encounter expunged message, check if the amount of empty space in
0N/A previous messages plus size of expunged message is enough to cover the
0N/A missing space. If yes,
0N/A - execute the rewrite plan
0N/A - forget all the messages before the expunged message. only remember
0N/A how much data we still have to move to cover the expunged message
0N/A - If we encounter end of file, grow the file and execute the rewrite plan
0N/A
0N/A Rewrite plan goes:
0N/A
0N/A - Start from the first message that needs more space
0N/A - If there's expunged messages before us, we have to write over them.
0N/A - Move all messages after it backwards to fill it
0N/A - Each moved message's X-Keywords header should have n bytes extra
0N/A space, unless there's not enough space to do it.
0N/A - If there's no expunged messages, we can move data either forward or
0N/A backward to get it. Calculate which requires less moving. Forward
0N/A counting may encounter more messages which require extra space, count
0N/A that too.
0N/A - If we decide to move forwards and we had to go through dirty
0N/A messages, do the moving from last to first dirty message
0N/A - If we encounter end of file, grow the file enough to get the required
0N/A amount of space plus enough space to fill X-Keywords headers full of
0N/A spaces.
0N/A*/
0N/A
0N/A#include "lib.h"
0N/A#include "ioloop.h"
0N/A#include "buffer.h"
0N/A#include "istream.h"
0N/A#include "file-set-size.h"
0N/A#include "str.h"
0N/A#include "write-full.h"
0N/A#include "istream-raw-mbox.h"
0N/A#include "mbox-storage.h"
0N/A#include "mbox-file.h"
0N/A#include "mbox-lock.h"
0N/A#include "mbox-sync-private.h"
0N/A
0N/A#include <stddef.h>
0N/A#include <sys/stat.h>
0N/A
0N/A#define MBOX_SYNC_SECS 1
0N/A
0N/A/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
0N/Astatic int mbox_sync_lock(struct mbox_sync_context *sync_ctx, int lock_type)
0N/A{
0N/A struct index_mailbox *ibox = sync_ctx->ibox;
0N/A struct stat old_st, st;
0N/A uoff_t old_from_offset, old_offset = 0;
0N/A
0N/A if (sync_ctx->lock_id != 0) {
0N/A if (fstat(sync_ctx->fd, &old_st) < 0) {
0N/A mbox_set_syscall_error(ibox, "stat()");
0N/A return -1;
0N/A }
0N/A old_from_offset =
0N/A istream_raw_mbox_get_start_offset(sync_ctx->input);
0N/A old_offset = sync_ctx->input->v_offset;
0N/A
0N/A (void)mbox_unlock(ibox, sync_ctx->lock_id);
0N/A sync_ctx->lock_id = 0;
0N/A } else {
0N/A memset(&old_st, 0, sizeof(old_st));
0N/A }
0N/A
0N/A if (mbox_lock(ibox, lock_type, &sync_ctx->lock_id) <= 0)
0N/A return -1;
0N/A if (mbox_file_open_stream(ibox) < 0)
0N/A return -1;
0N/A
0N/A sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
0N/A sync_ctx->input = sync_ctx->ibox->mbox_stream;
0N/A sync_ctx->fd = sync_ctx->ibox->mbox_fd;
0N/A
0N/A if (old_st.st_mtime == 0) {
0N/A /* we didn't have the file open before -> it changed */
0N/A return 0;
0N/A }
0N/A
0N/A if (fstat(sync_ctx->fd, &st) < 0) {
0N/A mbox_set_syscall_error(ibox, "fstat()");
0N/A return -1;
0N/A }
0N/A
0N/A if (st.st_mtime != old_st.st_mtime || st.st_size != old_st.st_size ||
0N/A st.st_ino != old_st.st_ino ||
0N/A !CMP_DEV_T(st.st_dev, old_st.st_dev) ||
0N/A time(NULL) - st.st_mtime <= MBOX_SYNC_SECS)
0N/A return 0;
0N/A
0N/A /* same as before. we'll have to fix mbox stream to contain
0N/A correct from_offset, hdr_offset and body_offset. so, seek
0N/A to from_offset and read through the header. */
0N/A istream_raw_mbox_seek(sync_ctx->input, old_from_offset);
0N/A (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
0N/A i_stream_seek(sync_ctx->input, old_offset);
0N/A return 1;
0N/A}
0N/A
0N/Astatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
0N/A struct mbox_sync_mail_context *mail_ctx,
0N/A uoff_t grow_size)
0N/A{
0N/A uoff_t src_offset, file_size;
0N/A
0N/A i_assert(grow_size > 0);
0N/A
0N/A /* put the extra space between last message's header and body */
0N/A file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
0N/A if (file_set_size(sync_ctx->fd, file_size) < 0) {
0N/A mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
0N/A return -1;
0N/A }
0N/A
0N/A src_offset = mail_ctx->body_offset;
0N/A mail_ctx->body_offset += grow_size;
0N/A if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
0N/A file_size - mail_ctx->body_offset) < 0)
0N/A return -1;
0N/A
0N/A istream_raw_mbox_flush(sync_ctx->input);
0N/A return 0;
0N/A}
0N/A
0N/Astatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
0N/A{
0N/A struct mail_index_sync_rec *sync;
0N/A size_t size, src, dest;
sync = buffer_get_modifyable_data(syncs_buf, &size);
size /= sizeof(*sync);
for (src = dest = 0; src < size; src++) {
if (sync[src].uid2 >= uid) {
if (src != dest)
sync[dest] = sync[src];
dest++;
}
}
buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
}
static int
mbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
/* set input->eof */
(void)istream_raw_mbox_get_header_offset(sync_ctx->input);
if (sync_ctx->input->eof)
return 0;
memset(mail_ctx, 0, sizeof(*mail_ctx));
mail_ctx->sync_ctx = sync_ctx;
mail_ctx->seq = ++sync_ctx->seq;
mail_ctx->header = sync_ctx->header;
mail_ctx->from_offset =
istream_raw_mbox_get_start_offset(sync_ctx->input);
mail_ctx->mail.offset =
istream_raw_mbox_get_header_offset(sync_ctx->input);
if (mail_ctx->seq > 1 && sync_ctx->first_uid == mail_ctx->mail.uid) {
/* First message was expunged and this is the next one.
Skip \n header */
mail_ctx->from_offset++;
}
mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
mail_ctx->mail.body_size =
istream_raw_mbox_get_body_size(sync_ctx->input,
mail_ctx->content_length);
/* save the offset permanently with recent flag state */
mail_ctx->mail.from_offset = mail_ctx->from_offset;
if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
/* need to add 'O' flag to Status-header */
mail_ctx->need_rewrite = TRUE;
// FIXME: save it somewhere
}
return 1;
}
static int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
uint32_t uid, int *sync_expunge_r)
{
struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
int ret;
*sync_expunge_r = FALSE;
if (sync_ctx->ibox->readonly)
return 0;
mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
while (uid >= sync_rec->uid1) {
if (sync_rec->uid1 != 0) {
i_assert(uid <= sync_rec->uid2);
buffer_append(sync_ctx->syncs, sync_rec,
sizeof(*sync_rec));
if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
*sync_expunge_r = TRUE;
}
ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
if (ret < 0)
return -1;
if (ret == 0) {
memset(sync_rec, 0, sizeof(*sync_rec));
break;
}
}
return 0;
}
static void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
keywords_mask_t keywords)
{
const struct mail_index_sync_rec *sync;
size_t size, i;
sync = buffer_get_data(syncs_buf, &size);
size /= sizeof(*sync);
for (i = 0; i < size; i++)
mail_index_sync_flags_apply(&sync[i], flags, keywords);
}
static int
mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
uint32_t uid, const struct mail_index_record **rec_r)
{
const struct mail_index_record *rec = NULL;
uint32_t messages_count;
messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq < messages_count) {
if (mail_index_lookup(sync_ctx->sync_view,
++sync_ctx->idx_seq, &rec) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
if (uid <= rec->uid)
break;
/* externally expunged message, remove from index */
mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
rec = NULL;
}
if (rec != NULL && rec->uid != uid) {
/* new UID in the middle of the mailbox - shouldn't happen */
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"mbox sync: UID inserted in the middle of mailbox "
"(%u > %u)", rec->uid, uid);
mail_index_mark_corrupted(sync_ctx->ibox->index);
return -1;
}
*rec_r = rec;
return 0;
}
static int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
uint32_t seq, uint64_t *offset_r)
{
const void *data;
/* see if from_offset needs updating */
if (mail_index_lookup_extra(sync_ctx->sync_view, seq,
sync_ctx->ibox->mbox_extra_idx,
&data) < 0) {
mail_storage_set_index_error(sync_ctx->ibox);
return -1;
}
*offset_r = *((const uint64_t *)data);
return 0;
}
static int
mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
int nocheck)
{
uint64_t offset;
if (!nocheck) {
if (mbox_sync_get_from_offset(sync_ctx, sync_ctx->idx_seq,
&offset) < 0)
return -1;
if (offset == mail->from_offset)
return 0;
} else {
offset = mail->from_offset;
}
mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
sync_ctx->ibox->mbox_extra_idx, &offset);
return 0;
}
static int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mail,
const struct mail_index_record *rec)
{
keywords_mask_t idx_keywords;
uint8_t idx_flags, mbox_flags;
if (rec == NULL) {
/* new message */
mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags,
mail->keywords);
} else {
/* see if flags changed */
idx_flags = rec->flags;
memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
mbox_sync_apply_index_syncs(sync_ctx->syncs,
&idx_flags, idx_keywords);
mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
(mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
if (idx_flags != mbox_flags ||
memcmp(idx_keywords, mail->keywords,
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
MODIFY_REPLACE, mbox_flags,
mail->keywords);
}
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
return -1;
}
return 0;
}
static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
{
struct istream *input = ctx->sync_ctx->file_input;
const unsigned char *data;
size_t size, from_line_size;
buffer_set_used_size(ctx->sync_ctx->from_line, 0);
from_line_size = ctx->hdr_offset - ctx->from_offset;
i_stream_seek(input, ctx->from_offset);
for (;;) {
data = i_stream_get_data(input, &size);
if (size >= from_line_size)
size = from_line_size;
buffer_append(ctx->sync_ctx->from_line, data, size);
i_stream_skip(input, size);
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int
mbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
{
string_t *str = ctx->sync_ctx->from_line;
if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
ctx->from_offset + move_diff) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
return -1;
}
istream_raw_mbox_flush(ctx->sync_ctx->input);
return 0;
}
static void
update_from_offsets(struct index_mailbox *ibox,
struct mail_index_transaction *t, buffer_t *mails_buf,
uint32_t seq1, uint32_t seq2)
{
const struct mbox_sync_mail *mails;
uint32_t extra_idx = ibox->mbox_extra_idx;
uint64_t offset;
mails = buffer_get_modifyable_data(mails_buf, NULL);
for (; seq1 <= seq2; seq1++, mails++) {
if (mails->uid != 0) {
offset = mails->from_offset;
mail_index_update_extra_rec(t, seq1, extra_idx,
&offset);
}
}
}
static int mbox_sync_check_excl_lock(struct mbox_sync_context *sync_ctx)
{
int ret;
if (sync_ctx->ibox->mbox_lock_type == F_RDLCK) {
if ((ret = mbox_sync_lock(sync_ctx, F_WRLCK)) < 0)
return -1;
if (ret == 0)
return -2;
}
return 0;
}
static int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
{
int ret;
if ((ret = mbox_sync_check_excl_lock(mail_ctx->sync_ctx)) < 0)
return ret;
mail_ctx->mail.offset = mail_ctx->from_offset;
mail_ctx->mail.space =
mail_ctx->body_offset - mail_ctx->from_offset +
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
if (mail_ctx->sync_ctx->seq == 1) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
}
mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
return 0;
}
static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
off_t move_diff;
int ret;
if (sync_ctx->first_uid == 0)
sync_ctx->first_uid = mail_ctx->mail.uid;
if (sync_ctx->ibox->readonly)
return 0;
if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
/* move the header backwards to fill expunged space */
if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
return ret;
move_diff = -sync_ctx->expunged_space;
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
mail_ctx->mail.from_offset += move_diff;
mail_ctx->mail.offset += move_diff;
if (mbox_write_from_line(mail_ctx, move_diff) < 0)
return -1;
}
} else if (mail_ctx->need_rewrite ||
buffer_get_used_size(sync_ctx->syncs) != 0) {
if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
return ret;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
return -1;
} else {
/* nothing to do */
return 0;
}
if (ret == 0 && sync_ctx->need_space_seq == 0) {
/* first mail with no space to write it */
sync_ctx->need_space_seq = sync_ctx->seq;
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
memset(&mail, 0, sizeof(mail));
mail.offset = mail_ctx->from_offset -
sync_ctx->expunged_space;
mail.space = sync_ctx->expunged_space;
sync_ctx->need_space_seq--;
buffer_append(sync_ctx->mails, &mail, sizeof(mail));
}
}
return 0;
}
static int
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
uoff_t extra_space;
buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
sync_ctx->space_diff += mail_ctx->mail.space;
if (sync_ctx->space_diff < 0)
return 0;
/* we have enough space now */
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
if (mail_ctx->mail.uid == 0 &&
(uoff_t)sync_ctx->space_diff > extra_space) {
/* don't waste too much on extra spacing */
sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
sync_ctx->space_diff = extra_space;
} else {
sync_ctx->expunged_space = 0;
}
if (mbox_sync_rewrite(sync_ctx, sync_ctx->need_space_seq, sync_ctx->seq,
sync_ctx->space_diff) < 0)
return -1;
update_from_offsets(sync_ctx->ibox, sync_ctx->t, sync_ctx->mails,
sync_ctx->need_space_seq, sync_ctx->seq);
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
return 0;
}
static int
mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
{
uint32_t seq;
uint64_t offset;
if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, uid,
&seq, &seq) < 0)
return -1;
if (seq == 0)
return 0;
if (mbox_sync_get_from_offset(sync_ctx, seq, &offset) < 0)
return -1;
/* set to -1, since they're always increased later */
sync_ctx->seq = sync_ctx->idx_seq = seq-1;
istream_raw_mbox_seek(sync_ctx->input, offset);
return 0;
}
static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx,
uint32_t min_message_count)
{
const struct mail_index_record *rec;
uint32_t uid, messages_count;
uoff_t offset;
int ret, expunged;
if (min_message_count != 0)
istream_raw_mbox_seek(sync_ctx->input, 0);
else {
/* we sync only what we need to. jump to first record that
needs updating */
if (sync_ctx->sync_rec.uid1 == 0) {
if (mbox_sync_read_index_syncs(sync_ctx, 1,
&expunged) < 0)
return -1;
}
if (sync_ctx->sync_rec.uid1 == 0) {
/* nothing to do */
return 0;
}
uid = sync_ctx->sync_rec.uid1;
if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
return -1;
if (sync_ctx->seq > 0) {
if (mail_index_lookup_uid(sync_ctx->sync_view, 1,
&sync_ctx->first_uid) < 0)
return -1;
}
}
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
uid = mail_ctx->mail.uid;
/* get all sync records related to this message */
if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
return -1;
if (!expunged)
ret = mbox_sync_handle_header(mail_ctx);
else {
mail_ctx->mail.uid = 0;
ret = mbox_sync_handle_expunge(mail_ctx);
}
if (ret < 0) {
/* -1 = error, -2 = need to restart */
return ret;
}
if (mbox_sync_read_index_rec(sync_ctx, uid, &rec) < 0)
return -1;
if (!expunged) {
if (mbox_sync_update_index(sync_ctx, &mail_ctx->mail,
rec) < 0)
return -1;
}
istream_raw_mbox_next(sync_ctx->input,
mail_ctx->mail.body_size);
offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
i_stream_seek(sync_ctx->input, offset);
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
if (mbox_move(sync_ctx,
mail_ctx->body_offset -
sync_ctx->expunged_space,
mail_ctx->body_offset,
mail_ctx->mail.body_size) < 0)
return -1;
i_stream_seek(sync_ctx->input, offset);
}
} else if (sync_ctx->seq >= min_message_count) {
mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
if (buffer_get_used_size(sync_ctx->syncs) == 0) {
/* if there's no sync records left,
we can stop */
if (sync_ctx->sync_rec.uid1 == 0)
break;
/* we can skip forward to next record which
needs updating. */
uid = sync_ctx->sync_rec.uid1;
if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
return -1;
}
}
}
if (sync_ctx->input->eof) {
/* rest of the messages in index don't exist -> expunge them */
messages_count =
mail_index_view_get_message_count(sync_ctx->sync_view);
while (sync_ctx->idx_seq < messages_count)
mail_index_expunge(sync_ctx->t, ++sync_ctx->idx_seq);
} else {
/* we didn't go through everything. fake the headers and all */
i_assert(sync_ctx->next_uid <= sync_ctx->hdr->next_uid);
sync_ctx->next_uid = sync_ctx->hdr->next_uid;
sync_ctx->base_uid_last = sync_ctx->hdr->next_uid-1;
sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity;
}
return 0;
}
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t offset, extra_space, trailer_size;
if (!sync_ctx->input->eof) {
i_assert(sync_ctx->need_space_seq == 0);
i_assert(sync_ctx->expunged_space == 0);
return 0;
}
trailer_size = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->file_input->v_offset;
if (sync_ctx->need_space_seq != 0) {
i_assert(sync_ctx->space_diff < 0);
extra_space = MBOX_HEADER_EXTRA_SPACE *
(sync_ctx->seq - sync_ctx->need_space_seq + 1);
sync_ctx->space_diff -= extra_space;
sync_ctx->space_diff += sync_ctx->expunged_space;
sync_ctx->expunged_space -= -sync_ctx->space_diff;
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
if (sync_ctx->space_diff < 0 &&
mbox_sync_grow_file(sync_ctx, mail_ctx,
-sync_ctx->space_diff) < 0)
return -1;
if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
return -1;
if (sync_ctx->seq != sync_ctx->need_space_seq) {
buffer_set_used_size(sync_ctx->mails,
(sync_ctx->seq -
sync_ctx->need_space_seq) *
sizeof(mail_ctx->mail));
buffer_append(sync_ctx->mails, &mail_ctx->mail,
sizeof(mail_ctx->mail));
if (mbox_sync_rewrite(sync_ctx,
sync_ctx->need_space_seq,
sync_ctx->seq, extra_space) < 0)
return -1;
update_from_offsets(sync_ctx->ibox, sync_ctx->t,
sync_ctx->mails,
sync_ctx->need_space_seq,
sync_ctx->seq);
}
sync_ctx->need_space_seq = 0;
buffer_set_used_size(sync_ctx->mails, 0);
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
offset = i_stream_get_size(sync_ctx->file_input) -
sync_ctx->expunged_space - trailer_size;
if (mbox_move(sync_ctx, offset,
offset + sync_ctx->expunged_space,
trailer_size) < 0)
return -1;
if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
return -1;
}
istream_raw_mbox_flush(sync_ctx->input);
}
return 0;
}
static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
{
struct stat st;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
return -1;
}
if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, uid_validity),
&sync_ctx->base_uid_validity,
sizeof(sync_ctx->base_uid_validity));
}
if (sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, next_uid),
&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
}
if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
uint32_t sync_stamp = st.st_mtime;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_stamp),
&sync_stamp, sizeof(sync_stamp));
}
if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
uint64_t sync_size = st.st_size;
mail_index_update_header(sync_ctx->t,
offsetof(struct mail_index_header, sync_size),
&sync_size, sizeof(sync_size));
}
return 0;
}
static void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
sync_ctx->next_uid = 1;
sync_ctx->prev_msg_uid = sync_ctx->first_uid = 0;
sync_ctx->seq = sync_ctx->idx_seq = 0;
mail_index_transaction_rollback(sync_ctx->t);
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
struct stat st;
uint32_t min_msg_count;
int ret, lock_type;
lock_type = mail_index_sync_have_more(sync_ctx->index_sync_ctx) ?
F_WRLCK : F_RDLCK;
if (mbox_sync_lock(sync_ctx, lock_type) < 0)
return -1;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "stat()");
return -1;
}
min_msg_count =
(uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
0 : (uint32_t)-1;
mbox_sync_restart(sync_ctx);
if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
return -1;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
if (sync_ctx->base_uid_last != sync_ctx->next_uid-1) {
/* rewrite X-IMAPbase header */
if (mbox_sync_check_excl_lock(sync_ctx) == -1)
return -1;
sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
return -1;
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
}
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
while (mail_index_sync_next(sync_ctx->index_sync_ctx,
&sync_ctx->sync_rec) > 0)
;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
static int mbox_sync_has_changed(struct index_mailbox *ibox)
{
const struct mail_index_header *hdr;
struct stat st;
if (mail_index_get_header(ibox->view, &hdr) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (stat(ibox->path, &st) < 0) {
mbox_set_syscall_error(ibox, "stat()");
return -1;
}
return (uint32_t)st.st_mtime != hdr->sync_stamp ||
(uint64_t)st.st_size != hdr->sync_size;
}
int mbox_sync(struct index_mailbox *ibox, int last_commit)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
int ret;
if ((ret = mbox_sync_has_changed(ibox)) < 0)
return -1;
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
seq = ibox->commit_log_file_seq;
offset = ibox->commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
seq, offset);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(ibox);
return ret;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
i_assert(ret == 0);
if (mbox_sync_do(&sync_ctx) < 0)
ret = -1;
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0)
ret = -1;
else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
if (mail_index_sync_end(index_sync_ctx) < 0)
ret = -1;
if (sync_ctx.lock_id != 0) {
/* FIXME: drop to read locking and keep it MBOX_SYNC_SECS+1
to make sure we notice changes made by others */
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
}
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(sync_ctx.mails);
buffer_free(sync_ctx.syncs);
return ret;
}
int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
if (mbox_sync(ibox, FALSE) < 0)
return -1;
}
return index_storage_sync(box, flags);
}