mbox-sync.c revision 7f773564b94e6054a40d3785cb63c29f1e4d4dee
/* Copyright (C) 2004 Timo Sirainen */
/*
Modifying mbox can be slow, so we try to do it all at once minimizing the
required disk I/O. We may need to:
- Update message flags in Status, X-Status and X-Keywords headers
- Write missing X-UID and X-IMAPbase headers
- Write missing or broken Content-Length header if there's space
- Expunge specified messages
Here's how we do it:
- Start reading the mails mail headers from the beginning
- X-Keywords and X-UID headers may contain extra spaces at the end of them,
remember how much extra each message has and offset to beginning of the
spaces
- If message flags are dirty and there's enough space to write them, do it
- If we didn't have enough space, remember how much was missing and keep
the total amount of them
- When we encounter expunged message, check if the amount of empty space in
previous messages plus size of expunged message is enough to cover the
missing space. If yes,
- execute the rewrite plan
- forget all the messages before the expunged message. only remember
how much data we still have to move to cover the expunged message
- If we encounter end of file, grow the file and execute the rewrite plan
Rewrite plan goes:
- Start from the first message that needs more space
- If there's expunged messages before us, we have to write over them.
- Move all messages after it backwards to fill it
- Each moved message's X-Keywords header should have n bytes extra
space, unless there's not enough space to do it.
- If there's no expunged messages, we can move data either forward or
backward to get it. Calculate which requires less moving. Forward
counting may encounter more messages which require extra space, count
that too.
- If we decide to move forwards and we had to go through dirty
messages, do the moving from last to first dirty message
- If we encounter end of file, grow the file enough to get the required
amount of space plus enough space to fill X-Keywords headers full of
spaces.
*/
#include "lib.h"
#include "ioloop.h"
#include "buffer.h"
#include "istream.h"
#include "file-set-size.h"
#include "str.h"
#include "write-full.h"
#include "istream-raw-mbox.h"
#include "mbox-storage.h"
#include "mbox-file.h"
#include "mbox-lock.h"
#include "mbox-sync-private.h"
#include <stddef.h>
#define MBOX_SYNC_SECS 1
/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
{
return -1;
}
} else {
}
return -1;
if (mbox_file_open_stream(ibox) < 0)
return -1;
/* we didn't have the file open before -> it changed */
return 0;
}
return -1;
}
return 0;
/* same as before. we'll have to fix mbox stream to contain
correct from_offset, hdr_offset and body_offset. so, seek
to from_offset and read through the header. */
"Message offset %s changed unexpectedly for mbox file "
return 0;
}
return 1;
}
struct mbox_sync_mail_context *mail_ctx,
{
/* put the extra space between last message's header and body */
return -1;
}
return -1;
return 0;
}
{
struct mail_index_sync_rec *sync;
dest++;
}
}
}
static int
struct mbox_sync_mail_context *mail_ctx)
{
/* set input->eof */
return 0;
/* First message was expunged and this is the next one.
Skip \n header */
mail_ctx->from_offset++;
}
/* save the offset permanently with recent flag state */
/* need to add 'O' flag to Status-header */
// FIXME: save it somewhere
}
return 1;
}
{
const struct mail_index_sync_rec *sync;
for (i = 0; i < size; i++) {
return TRUE;
}
return FALSE;
}
{
int ret;
*sync_expunge_r = FALSE;
return 0;
sizeof(*sync_rec));
*sync_expunge_r = TRUE;
}
if (ret < 0)
return -1;
if (ret == 0) {
break;
}
}
if (!*sync_expunge_r)
return 0;
}
{
const struct mail_index_sync_rec *sync;
for (i = 0; i < size; i++) {
continue;
}
}
static int
{
return -1;
}
break;
/* externally expunged message, remove from index */
}
/* new UID in the middle of the mailbox - shouldn't happen */
"mbox sync: UID inserted in the middle of mailbox %s "
return -1;
}
return 0;
}
{
const void *data;
/* see if from_offset needs updating */
&data) < 0) {
return -1;
}
return 0;
}
static int
struct mbox_sync_mail *mail,
int nocheck)
{
if (!nocheck) {
&offset) < 0)
return -1;
return 0;
} else {
}
return 0;
}
struct mbox_sync_mail *mail,
const struct mail_index_record *rec)
{
/* new message */
} else {
/* see if flags changed */
if (idx_flags != mbox_flags ||
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
}
}
/* update from_offsets, but not if we're going to rewrite this message.
rewriting would just move it anyway. */
if (sync_ctx->need_space_seq == 0) {
return -1;
}
return 0;
}
{
const unsigned char *data;
for (;;) {
if (size >= from_line_size)
from_line_size -= size;
if (from_line_size == 0)
break;
if (i_stream_read(input) < 0)
return -1;
}
return 0;
}
static int
{
return -1;
}
return 0;
}
static void
{
const struct mbox_sync_mail *mails;
&offset);
}
}
}
{
int ret;
return -1;
if (ret == 0)
return -2;
}
return 0;
}
{
int ret;
return ret;
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
}
return 0;
}
{
int ret;
return 0;
/* move the header backwards to fill expunged space */
return ret;
/* read the From-line before rewriting overwrites it */
if (mbox_read_from_line(mail_ctx) < 0)
return -1;
return -1;
if (ret > 0) {
/* rewrite successful, write From-line to
new location */
return -1;
}
} else if (mail_ctx->need_rewrite ||
return ret;
return -1;
} else {
/* nothing to do */
return 0;
}
/* first mail with no space to write it */
sync_ctx->space_diff = 0;
if (sync_ctx->expunged_space > 0) {
/* create dummy message to describe the expunged data */
struct mbox_sync_mail mail;
}
}
return 0;
}
static int
{
if (sync_ctx->space_diff < 0)
return 0;
/* we have enough space now */
/* don't waste too much on extra spacing */
} else {
sync_ctx->expunged_space = 0;
}
sync_ctx->space_diff) < 0)
return -1;
/* mail_ctx may contain wrong data after rewrite, so make sure we
don't try to access it */
sync_ctx->need_space_seq = 0;
return 0;
}
static int
{
return -1;
if (seq == 0)
return 0;
return -1;
/* set to -1, since they're always increased later */
"Cached message offset %s is invalid for mbox file %s",
return -1;
}
return 0;
}
struct mbox_sync_mail_context *mail_ctx,
{
const struct mail_index_record *rec;
if (min_message_count != 0) {
/* doesn't begin with a From-line */
"Mailbox isn't a valid mbox file");
return -1;
}
} else {
/* we sync only what we need to. jump to first record that
needs updating */
&expunged) < 0)
return -1;
}
/* nothing to do */
return 0;
}
return -1;
return -1;
}
}
/* get all sync records related to this message */
return -1;
if (!expunged)
else {
}
if (ret < 0) {
/* -1 = error, -2 = need to restart */
return ret;
}
return -1;
if (!expunged) {
rec) < 0)
return -1;
}
if (sync_ctx->need_space_seq != 0) {
if (mbox_sync_handle_missing_space(mail_ctx) < 0)
return -1;
} else if (sync_ctx->expunged_space > 0) {
if (!expunged) {
/* move the body */
return -1;
}
/* if there's no sync records left,
we can stop */
break;
/* we can skip forward to next record which
needs updating. */
return -1;
}
}
}
/* rest of the messages in index don't exist -> expunge them */
}
return 0;
}
struct mbox_sync_mail_context *mail_ctx)
{
return 0;
}
if (sync_ctx->need_space_seq != 0) {
if (sync_ctx->space_diff < 0 &&
-sync_ctx->space_diff) < 0)
return -1;
if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
return -1;
return -1;
}
sync_ctx->need_space_seq = 0;
}
if (sync_ctx->expunged_space > 0) {
/* copy trailer, then truncate the file */
trailer_size) < 0)
return -1;
return -1;
}
sync_ctx->expunged_space = 0;
}
return 0;
}
{
return -1;
}
if (sync_ctx->base_uid_validity != 0 &&
sizeof(sync_ctx->base_uid_validity));
}
if (sync_ctx->base_uid_last != 0 &&
}
&sync_stamp, sizeof(sync_stamp));
}
}
return 0;
}
{
sync_ctx->base_uid_validity = 0;
sync_ctx->base_uid_last = 0;
}
{
struct mbox_sync_mail_context mail_ctx;
int ret;
return -1;
}
0 : (uint32_t)-1;
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
FALSE);
return -1;
}
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
{
const struct mail_index_header *hdr;
return -1;
}
return -1;
}
}
{
struct mbox_sync_mail_context mail_ctx;
return -1;
return -1;
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
unsigned int lock_id = 0;
if (lock) {
return -1;
}
if (lock)
return -1;
}
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
} else {
}
if (ret <= 0) {
if (ret < 0)
return ret;
}
lock_id = 0;
}
return -1;
if (mbox_sync_do(&sync_ctx) < 0)
ret = -1;
if (ret < 0)
ret = -1;
else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
if (mail_index_sync_end(index_sync_ctx) < 0)
ret = -1;
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
if (ret < 0)
else {
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0)
ret = -1;
ret = -1;
}
}
/* FIXME: drop to read locking and keep it MBOX_SYNC_SECS+1
to make sure we notice changes made by others */
ret = -1;
}
return ret;
}
{
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
return -1;
}
}