mbox-sync.c revision dda2c506c8fc8ac2f88272de4523ded42baa0aa0
2875N/A/* Copyright (C) 2004 Timo Sirainen */
2875N/A
2875N/A/*
2875N/A Modifying mbox can be slow, so we try to do it all at once minimizing the
2875N/A required disk I/O. We may need to:
2875N/A
2875N/A - Update message flags in Status, X-Status and X-Keywords headers
2875N/A - Write missing X-UID and X-IMAPbase headers
2875N/A - Write missing or broken Content-Length header if there's space
2875N/A - Expunge specified messages
2875N/A
2875N/A Here's how we do it:
2875N/A
2875N/A - Start reading the mails mail headers from the beginning
2875N/A - X-Keywords and X-UID headers may contain extra spaces at the end of them,
2875N/A remember how much extra each message has and offset to beginning of the
2875N/A spaces
2875N/A - If message flags are dirty and there's enough space to write them, do it
2875N/A - If we didn't have enough space, remember how much was missing and keep
2875N/A the total amount of them
2875N/A - When we encounter expunged message, check if the amount of empty space in
2875N/A previous messages plus size of expunged message is enough to cover the
2875N/A missing space. If yes,
2875N/A - execute the rewrite plan
2875N/A - forget all the messages before the expunged message. only remember
2875N/A how much data we still have to move to cover the expunged message
2875N/A - If we encounter end of file, grow the file and execute the rewrite plan
2875N/A
2875N/A Rewrite plan goes:
2875N/A
2875N/A - Start from the first message that needs more space
2875N/A - If there's expunged messages before us, we have to write over them.
2875N/A - Move all messages after it backwards to fill it
2875N/A - Each moved message's X-Keywords header should have n bytes extra
2875N/A space, unless there's not enough space to do it.
2875N/A - If there's no expunged messages, we can move data either forward or
2875N/A backward to get it. Calculate which requires less moving. Forward
2875N/A counting may encounter more messages which require extra space, count
2875N/A that too.
2875N/A - If we decide to move forwards and we had to go through dirty
2875N/A messages, do the moving from last to first dirty message
2875N/A - If we encounter end of file, grow the file enough to get the required
2875N/A amount of space plus enough space to fill X-Keywords headers full of
2875N/A spaces.
2875N/A*/
2875N/A
2875N/A#include "lib.h"
2875N/A#include "ioloop.h"
2875N/A#include "buffer.h"
2875N/A#include "istream.h"
2875N/A#include "file-set-size.h"
2875N/A#include "str.h"
2875N/A#include "write-full.h"
2875N/A#include "istream-raw-mbox.h"
2875N/A#include "mbox-storage.h"
2875N/A#include "mbox-file.h"
2875N/A#include "mbox-lock.h"
2875N/A#include "mbox-sync-private.h"
2875N/A
2875N/A#include <stddef.h>
2875N/A#include <sys/stat.h>
2875N/A
2875N/Astatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
2875N/A struct mbox_sync_mail_context *mail_ctx,
2875N/A uoff_t grow_size)
2875N/A{
2875N/A uoff_t src_offset, file_size;
2875N/A
2875N/A i_assert(grow_size > 0);
2875N/A
2875N/A /* put the extra space between last message's header and body */
2875N/A file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
2875N/A if (file_set_size(sync_ctx->fd, file_size) < 0) {
2875N/A mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
2875N/A return -1;
2875N/A }
2875N/A
2875N/A src_offset = mail_ctx->body_offset;
2875N/A mail_ctx->body_offset += grow_size;
2875N/A if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
2875N/A file_size - mail_ctx->body_offset) < 0)
2875N/A return -1;
2875N/A
2875N/A istream_raw_mbox_flush(sync_ctx->input);
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
2875N/A{
2875N/A struct mail_index_sync_rec *sync;
2875N/A size_t size, src, dest;
2875N/A
2875N/A sync = buffer_get_modifyable_data(syncs_buf, &size);
2875N/A size /= sizeof(*sync);
2875N/A
2875N/A for (src = dest = 0; src < size; src++) {
2875N/A if (sync[src].uid2 >= uid) {
2875N/A if (src != dest)
2875N/A sync[dest] = sync[src];
2875N/A dest++;
2875N/A }
2875N/A }
2875N/A
2875N/A buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
2875N/A}
2875N/A
2875N/Astatic int
2875N/Ambox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
2875N/A struct mbox_sync_mail_context *mail_ctx)
2875N/A{
2875N/A /* set input->eof */
2875N/A (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
2875N/A if (sync_ctx->input->eof)
2875N/A return 0;
2875N/A
2875N/A memset(mail_ctx, 0, sizeof(*mail_ctx));
2875N/A mail_ctx->sync_ctx = sync_ctx;
2875N/A mail_ctx->seq = ++sync_ctx->seq;
2875N/A mail_ctx->header = sync_ctx->header;
2875N/A
2875N/A mail_ctx->from_offset =
2875N/A istream_raw_mbox_get_start_offset(sync_ctx->input);
2875N/A mail_ctx->mail.offset =
2875N/A istream_raw_mbox_get_header_offset(sync_ctx->input);
2875N/A
2875N/A if (mail_ctx->seq > 1 && sync_ctx->first_uid == mail_ctx->mail.uid) {
2875N/A /* First message was expunged and this is the next one.
2875N/A Skip \n header */
2875N/A mail_ctx->from_offset++;
2875N/A }
2875N/A
2875N/A mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
2875N/A i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
2875N/A
2875N/A mail_ctx->mail.body_size =
2875N/A istream_raw_mbox_get_body_size(sync_ctx->input,
2875N/A mail_ctx->content_length);
2875N/A
2875N/A /* save the offset permanently with recent flag state */
2875N/A mail_ctx->mail.from_offset = mail_ctx->from_offset;
2875N/A if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
2875N/A /* need to add 'O' flag to Status-header */
2875N/A mail_ctx->need_rewrite = TRUE;
2875N/A // FIXME: save it somewhere
2875N/A }
2875N/A return 1;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
2875N/A uint32_t uid, int *sync_expunge_r)
2875N/A{
2875N/A struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
2875N/A int ret;
2875N/A
2875N/A *sync_expunge_r = FALSE;
2875N/A
2875N/A if (sync_ctx->ibox->readonly)
2875N/A return 0;
2875N/A
2875N/A mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
2875N/A while (uid >= sync_rec->uid1) {
2875N/A if (sync_rec->uid1 != 0) {
2875N/A i_assert(uid <= sync_rec->uid2);
2875N/A buffer_append(sync_ctx->syncs, sync_rec,
2875N/A sizeof(*sync_rec));
2875N/A
2875N/A if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
2875N/A *sync_expunge_r = TRUE;
2875N/A }
2875N/A
2875N/A ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
2875N/A if (ret < 0)
2875N/A return -1;
2875N/A
2875N/A if (ret == 0) {
2875N/A memset(sync_rec, 0, sizeof(*sync_rec));
2875N/A break;
2875N/A }
2875N/A }
2875N/A
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
2875N/A keywords_mask_t keywords)
2875N/A{
2875N/A const struct mail_index_sync_rec *sync;
2875N/A size_t size, i;
2875N/A
2875N/A sync = buffer_get_data(syncs_buf, &size);
2875N/A size /= sizeof(*sync);
2875N/A
2875N/A for (i = 0; i < size; i++)
2875N/A mail_index_sync_flags_apply(&sync[i], flags, keywords);
2875N/A}
2875N/A
2875N/Astatic int
2875N/Ambox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
2875N/A uint32_t uid, const struct mail_index_record **rec_r)
2875N/A{
2875N/A const struct mail_index_record *rec = NULL;
2875N/A uint32_t messages_count;
2875N/A
2875N/A messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
2875N/A while (sync_ctx->idx_seq < messages_count) {
2875N/A if (mail_index_lookup(sync_ctx->sync_view,
2875N/A ++sync_ctx->idx_seq, &rec) < 0) {
2875N/A mail_storage_set_index_error(sync_ctx->ibox);
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if (uid <= rec->uid)
2875N/A break;
2875N/A
2875N/A /* externally expunged message, remove from index */
2875N/A mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
2875N/A rec = NULL;
2875N/A }
2875N/A
2875N/A if (rec != NULL && rec->uid != uid) {
2875N/A /* new UID in the middle of the mailbox - shouldn't happen */
2875N/A mail_storage_set_critical(sync_ctx->ibox->box.storage,
2875N/A "mbox sync: UID inserted in the middle of mailbox "
2875N/A "(%u > %u)", rec->uid, uid);
2875N/A mail_index_mark_corrupted(sync_ctx->ibox->index);
2875N/A return -1;
2875N/A }
2875N/A
2875N/A *rec_r = rec;
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int
2875N/Ambox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
2875N/A struct mbox_sync_mail *mail,
2875N/A int nocheck)
2875N/A{
2875N/A const void *data;
2875N/A uint64_t offset;
2875N/A
2875N/A if (!nocheck) {
2875N/A /* see if from_offset needs updating */
2875N/A if (mail_index_lookup_extra(sync_ctx->sync_view,
2875N/A sync_ctx->idx_seq,
2875N/A sync_ctx->ibox->mbox_extra_idx,
2875N/A &data) < 0) {
2875N/A mail_storage_set_index_error(sync_ctx->ibox);
2875N/A return -1;
2875N/A }
2875N/A
2875N/A offset = *((const uint64_t *)data);
2875N/A if (offset == mail->from_offset)
2875N/A return 0;
2875N/A } else {
2875N/A offset = mail->from_offset;
2875N/A }
2875N/A
2875N/A mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
2875N/A sync_ctx->ibox->mbox_extra_idx, &offset);
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
2875N/A struct mbox_sync_mail *mail,
2875N/A const struct mail_index_record *rec)
2875N/A{
2875N/A keywords_mask_t idx_keywords;
2875N/A uint8_t idx_flags, mbox_flags;
2875N/A
2875N/A if (rec == NULL) {
2875N/A /* new message */
2875N/A mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
2875N/A mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
2875N/A mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
2875N/A MODIFY_REPLACE, mbox_flags,
2875N/A mail->keywords);
2875N/A } else {
2875N/A /* see if flags changed */
2875N/A idx_flags = rec->flags;
2875N/A memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
2875N/A mbox_sync_apply_index_syncs(sync_ctx->syncs,
2875N/A &idx_flags, idx_keywords);
2875N/A
2875N/A mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
2875N/A (mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
2875N/A
2875N/A if (idx_flags != mbox_flags ||
2875N/A memcmp(idx_keywords, mail->keywords,
2875N/A INDEX_KEYWORDS_BYTE_COUNT) != 0) {
2875N/A mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
2875N/A MODIFY_REPLACE, mbox_flags,
2875N/A mail->keywords);
2875N/A }
2875N/A }
2875N/A
2875N/A /* update from_offsets, but not if we're going to rewrite this message.
2875N/A rewriting would just move it anyway. */
2875N/A if (sync_ctx->need_space_seq == 0) {
2875N/A int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
2875N/A if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
2875N/A return -1;
2875N/A }
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
2875N/A{
2875N/A struct istream *input = ctx->sync_ctx->file_input;
2875N/A const unsigned char *data;
2875N/A size_t size, from_line_size;
2875N/A
2875N/A buffer_set_used_size(ctx->sync_ctx->from_line, 0);
2875N/A from_line_size = ctx->hdr_offset - ctx->from_offset;
2875N/A
2875N/A i_stream_seek(input, ctx->from_offset);
2875N/A for (;;) {
2875N/A data = i_stream_get_data(input, &size);
2875N/A if (size >= from_line_size)
2875N/A size = from_line_size;
2875N/A
2875N/A buffer_append(ctx->sync_ctx->from_line, data, size);
2875N/A i_stream_skip(input, size);
2875N/A from_line_size -= size;
2875N/A
2875N/A if (from_line_size == 0)
2875N/A break;
2875N/A
2875N/A if (i_stream_read(input) < 0)
2875N/A return -1;
2875N/A }
2875N/A
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int
2875N/Ambox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
2875N/A{
2875N/A string_t *str = ctx->sync_ctx->from_line;
2875N/A
2875N/A if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
2875N/A ctx->from_offset + move_diff) < 0) {
2875N/A mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
2875N/A return -1;
2875N/A }
2875N/A
2875N/A istream_raw_mbox_flush(ctx->sync_ctx->input);
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic void
2875N/Aupdate_from_offsets(struct index_mailbox *ibox,
2875N/A struct mail_index_transaction *t, buffer_t *mails_buf,
2875N/A uint32_t seq1, uint32_t seq2)
2875N/A{
2875N/A const struct mbox_sync_mail *mails;
2875N/A uint32_t extra_idx = ibox->mbox_extra_idx;
2875N/A uint64_t offset;
2875N/A
2875N/A mails = buffer_get_modifyable_data(mails_buf, NULL);
2875N/A
2875N/A for (; seq1 <= seq2; seq1++, mails++) {
2875N/A if (mails->uid != 0) {
2875N/A offset = mails->from_offset;
2875N/A mail_index_update_extra_rec(t, seq1, extra_idx,
2875N/A &offset);
2875N/A }
2875N/A }
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
2875N/A{
2875N/A if (mail_ctx->sync_ctx->ibox->mbox_lock_type == F_RDLCK)
2875N/A return -2;
2875N/A
2875N/A mail_ctx->mail.offset = mail_ctx->from_offset;
2875N/A mail_ctx->mail.space =
2875N/A mail_ctx->body_offset - mail_ctx->from_offset +
2875N/A mail_ctx->mail.body_size;
2875N/A mail_ctx->mail.body_size = 0;
2875N/A
2875N/A if (mail_ctx->sync_ctx->seq == 1) {
2875N/A /* expunging first message, fix space to contain next
2875N/A message's \n header too since it will be removed. */
2875N/A mail_ctx->mail.space++;
2875N/A }
2875N/A
2875N/A mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
2875N/A{
2875N/A struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
2875N/A off_t move_diff;
2875N/A int ret;
2875N/A
2875N/A if (sync_ctx->first_uid == 0)
2875N/A sync_ctx->first_uid = mail_ctx->mail.uid;
2875N/A
2875N/A if (sync_ctx->ibox->readonly)
2875N/A return 0;
2875N/A
2875N/A if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
2875N/A /* move the header backwards to fill expunged space */
2875N/A if (sync_ctx->ibox->mbox_lock_type == F_RDLCK)
2875N/A return -2;
2875N/A
2875N/A move_diff = -sync_ctx->expunged_space;
2875N/A
2875N/A /* read the From-line before rewriting overwrites it */
2875N/A if (mbox_read_from_line(mail_ctx) < 0)
2875N/A return -1;
2875N/A
2875N/A mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
2875N/A if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
2875N/A return -1;
2875N/A
2875N/A if (ret > 0) {
2875N/A /* rewrite successful, write From-line to
2875N/A new location */
2875N/A mail_ctx->mail.from_offset += move_diff;
2875N/A mail_ctx->mail.offset += move_diff;
2875N/A if (mbox_write_from_line(mail_ctx, move_diff) < 0)
2875N/A return -1;
2875N/A }
2875N/A } else if (mail_ctx->need_rewrite ||
2875N/A buffer_get_used_size(sync_ctx->syncs) != 0) {
2875N/A if (sync_ctx->ibox->mbox_lock_type == F_RDLCK)
2875N/A return -2;
2875N/A
2875N/A mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
2875N/A if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
2875N/A return -1;
2875N/A } else {
2893N/A /* nothing to do */
2893N/A return 0;
2893N/A }
2893N/A
2875N/A if (ret == 0 && sync_ctx->need_space_seq == 0) {
2875N/A /* first mail with no space to write it */
2875N/A sync_ctx->need_space_seq = sync_ctx->seq;
2875N/A sync_ctx->space_diff = 0;
2875N/A
2875N/A if (sync_ctx->expunged_space > 0) {
2875N/A /* create dummy message to describe the expunged data */
2875N/A struct mbox_sync_mail mail;
2875N/A
2875N/A memset(&mail, 0, sizeof(mail));
2875N/A mail.offset = mail_ctx->from_offset -
2875N/A sync_ctx->expunged_space;
2875N/A mail.space = sync_ctx->expunged_space;
2875N/A
2875N/A sync_ctx->need_space_seq--;
2875N/A buffer_append(sync_ctx->mails, &mail, sizeof(mail));
2875N/A }
2875N/A }
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int
2875N/Ambox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
2875N/A{
2875N/A struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
2875N/A uoff_t extra_space;
2875N/A
2875N/A buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
2875N/A
2875N/A sync_ctx->space_diff += mail_ctx->mail.space;
2875N/A if (sync_ctx->space_diff < 0)
2875N/A return 0;
2875N/A
2875N/A /* we have enough space now */
2875N/A extra_space = MBOX_HEADER_EXTRA_SPACE *
2875N/A (sync_ctx->seq - sync_ctx->need_space_seq + 1);
2875N/A
2875N/A if (mail_ctx->mail.uid == 0 &&
2875N/A (uoff_t)sync_ctx->space_diff > extra_space) {
2875N/A /* don't waste too much on extra spacing */
2875N/A sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
2875N/A sync_ctx->space_diff = extra_space;
2875N/A } else {
2875N/A sync_ctx->expunged_space = 0;
2875N/A }
2875N/A
2896N/A if (mbox_sync_rewrite(sync_ctx, sync_ctx->need_space_seq, sync_ctx->seq,
2875N/A sync_ctx->space_diff) < 0)
2875N/A return -1;
2875N/A
2875N/A update_from_offsets(sync_ctx->ibox, sync_ctx->t, sync_ctx->mails,
2875N/A sync_ctx->need_space_seq, sync_ctx->seq);
2875N/A
2875N/A /* mail_ctx may contain wrong data after rewrite, so make sure we
2875N/A don't try to access it */
2875N/A memset(mail_ctx, 0, sizeof(*mail_ctx));
2875N/A
2875N/A sync_ctx->need_space_seq = 0;
2875N/A buffer_set_used_size(sync_ctx->mails, 0);
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_parse_all(struct mbox_sync_context *sync_ctx,
2875N/A struct mbox_sync_mail_context *mail_ctx)
2875N/A{
2875N/A const struct mail_index_record *rec;
2875N/A uint32_t uid, messages_count;
2875N/A uoff_t offset;
2875N/A int ret, expunged;
2875N/A
2875N/A sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
2875N/A sync_ctx->input = sync_ctx->ibox->mbox_stream;
2875N/A sync_ctx->fd = sync_ctx->ibox->mbox_fd;
2875N/A
2875N/A istream_raw_mbox_seek(sync_ctx->input, 0);
2875N/A
2875N/A while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
2875N/A uid = mail_ctx->mail.uid;
2875N/A
2875N/A /* get all sync records related to this message */
2875N/A if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
2875N/A return -1;
2875N/A
2875N/A if (!expunged)
2875N/A ret = mbox_sync_handle_header(mail_ctx);
2875N/A else {
2875N/A mail_ctx->mail.uid = 0;
2875N/A ret = mbox_sync_handle_expunge(mail_ctx);
2875N/A }
2875N/A if (ret < 0) {
2875N/A /* -1 = error, -2 = need exclusive lock */
2875N/A return ret;
2875N/A }
2875N/A
2875N/A if (mbox_sync_read_index_rec(sync_ctx, uid, &rec) < 0)
2875N/A return -1;
2875N/A
2875N/A if (!expunged) {
2875N/A if (mbox_sync_update_index(sync_ctx, &mail_ctx->mail,
2875N/A rec) < 0)
2875N/A return -1;
2875N/A }
2875N/A
2875N/A istream_raw_mbox_next(sync_ctx->input,
2875N/A mail_ctx->mail.body_size);
2875N/A offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
2875N/A
2875N/A if (sync_ctx->need_space_seq != 0) {
2875N/A if (mbox_sync_handle_missing_space(mail_ctx) < 0)
2875N/A return -1;
2875N/A i_stream_seek(sync_ctx->input, offset);
2875N/A } else if (sync_ctx->expunged_space > 0 && !expunged) {
2875N/A /* move the body */
2875N/A if (mbox_move(sync_ctx,
2875N/A mail_ctx->body_offset -
2875N/A sync_ctx->expunged_space,
2875N/A mail_ctx->body_offset,
2875N/A mail_ctx->mail.body_size) < 0)
2875N/A return -1;
2875N/A i_stream_seek(sync_ctx->input, offset);
2875N/A }
2875N/A }
2875N/A
2875N/A /* rest of the messages in index don't exist -> expunge them */
2875N/A messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
2875N/A while (sync_ctx->idx_seq < messages_count)
2875N/A mail_index_expunge(sync_ctx->t, ++sync_ctx->idx_seq);
2875N/A
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
2875N/A struct mbox_sync_mail_context *mail_ctx)
2875N/A{
2875N/A uoff_t offset, extra_space, trailer_size;
2875N/A
2875N/A trailer_size = i_stream_get_size(sync_ctx->file_input) -
2875N/A sync_ctx->file_input->v_offset;
2875N/A
2875N/A if (sync_ctx->need_space_seq != 0) {
2875N/A i_assert(sync_ctx->space_diff < 0);
2875N/A extra_space = MBOX_HEADER_EXTRA_SPACE *
2875N/A (sync_ctx->seq - sync_ctx->need_space_seq + 1);
2875N/A sync_ctx->space_diff -= extra_space;
2875N/A
2875N/A sync_ctx->space_diff += sync_ctx->expunged_space;
2875N/A sync_ctx->expunged_space -= -sync_ctx->space_diff;
2875N/A
2875N/A if (mail_ctx->have_eoh && !mail_ctx->updated)
2875N/A str_append_c(mail_ctx->header, '\n');
2875N/A
2875N/A if (sync_ctx->space_diff < 0 &&
2875N/A mbox_sync_grow_file(sync_ctx, mail_ctx,
2875N/A -sync_ctx->space_diff) < 0)
2875N/A return -1;
2875N/A
2875N/A if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
2875N/A return -1;
2875N/A
2875N/A if (sync_ctx->seq != sync_ctx->need_space_seq) {
2875N/A buffer_set_used_size(sync_ctx->mails,
2875N/A (sync_ctx->seq -
2875N/A sync_ctx->need_space_seq) *
2875N/A sizeof(mail_ctx->mail));
2875N/A buffer_append(sync_ctx->mails, &mail_ctx->mail,
2875N/A sizeof(mail_ctx->mail));
2875N/A
2875N/A if (mbox_sync_rewrite(sync_ctx,
2875N/A sync_ctx->need_space_seq,
2875N/A sync_ctx->seq, extra_space) < 0)
2875N/A return -1;
2875N/A
2875N/A update_from_offsets(sync_ctx->ibox, sync_ctx->t,
2875N/A sync_ctx->mails,
2875N/A sync_ctx->need_space_seq,
2875N/A sync_ctx->seq);
2875N/A }
2875N/A }
2875N/A
2875N/A if (sync_ctx->expunged_space > 0) {
2875N/A /* copy trailer, then truncate the file */
2875N/A offset = i_stream_get_size(sync_ctx->file_input) -
2875N/A sync_ctx->expunged_space - trailer_size;
2875N/A
2875N/A if (mbox_move(sync_ctx, offset,
2875N/A offset + sync_ctx->expunged_space,
2875N/A trailer_size) < 0)
2875N/A return -1;
2875N/A if (ftruncate(sync_ctx->ibox->mbox_fd,
2875N/A offset + trailer_size) < 0) {
2875N/A mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
2875N/A return -1;
2875N/A }
2875N/A
2875N/A istream_raw_mbox_flush(sync_ctx->input);
2875N/A }
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
2875N/A{
2875N/A struct stat st;
2875N/A
2875N/A if (fstat(sync_ctx->ibox->mbox_fd, &st) < 0) {
2875N/A mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) {
2875N/A mail_index_update_header(sync_ctx->t,
2875N/A offsetof(struct mail_index_header, uid_validity),
2875N/A &sync_ctx->base_uid_validity,
2875N/A sizeof(sync_ctx->base_uid_validity));
2875N/A }
2875N/A if (sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
2875N/A mail_index_update_header(sync_ctx->t,
2875N/A offsetof(struct mail_index_header, next_uid),
2875N/A &sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
2875N/A }
2875N/A
2875N/A if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
2875N/A uint32_t sync_stamp = st.st_mtime;
2875N/A
2875N/A mail_index_update_header(sync_ctx->t,
2875N/A offsetof(struct mail_index_header, sync_stamp),
2875N/A &sync_stamp, sizeof(sync_stamp));
2875N/A }
2875N/A if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
2875N/A uint64_t sync_size = st.st_size;
2875N/A
2875N/A mail_index_update_header(sync_ctx->t,
2875N/A offsetof(struct mail_index_header, sync_size),
2875N/A &sync_size, sizeof(sync_size));
2875N/A }
2875N/A
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
2875N/A{
2875N/A sync_ctx->base_uid_validity = 0;
2875N/A sync_ctx->base_uid_last = 0;
2875N/A
2875N/A sync_ctx->next_uid = 1;
2875N/A sync_ctx->prev_msg_uid = sync_ctx->first_uid = 0;
2875N/A sync_ctx->seq = sync_ctx->idx_seq = 0;
2875N/A
2875N/A mail_index_transaction_rollback(sync_ctx->t);
2875N/A sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_do(struct mbox_sync_context *sync_ctx, int index_synced)
2925N/A{
2875N/A struct index_mailbox *ibox = sync_ctx->ibox;
2875N/A struct mbox_sync_mail_context mail_ctx;
2875N/A int ret, lock_type;
2875N/A
2875N/A lock_type = mail_index_sync_have_more(sync_ctx->index_sync_ctx) ?
2875N/A F_WRLCK : F_RDLCK;
2875N/A if ((ret = mbox_lock(ibox, lock_type, &sync_ctx->lock_id)) <= 0)
2875N/A return ret;
2875N/A
2875N/A if (mbox_file_open_stream(ibox) < 0)
2875N/A return -1;
2875N/A
2875N/A if ((ret = mbox_sync_parse_all(sync_ctx, &mail_ctx)) == -1)
2875N/A return -1;
2925N/A
2925N/A if (ret == -2) {
2875N/A /* we want to modify mbox, get exclusive lock. this requires
2875N/A dropping the read lock first, so we have to parse the whole
2875N/A mbox again */
2875N/A (void)mbox_unlock(ibox, sync_ctx->lock_id);
2875N/A sync_ctx->lock_id = 0;
2875N/A
2875N/A if ((ret = mbox_lock(ibox, F_WRLCK, &sync_ctx->lock_id)) <= 0)
2875N/A return ret;
2875N/A if (mbox_file_open_stream(ibox) < 0)
2875N/A return -1;
2875N/A
2875N/A /* FIXME: if mbox timestamp hasn't changed and it's older than
2875N/A 2 secs, we could continue from where we left */
2875N/A mbox_sync_restart(sync_ctx);
2875N/A
2875N/A if (mbox_sync_parse_all(sync_ctx, &mail_ctx) < 0)
2875N/A return -1;
2875N/A }
2875N/A
2893N/A if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
2875N/A return -1;
2875N/A
2875N/A if (sync_ctx->base_uid_last+1 != sync_ctx->next_uid) {
2875N/A // FIXME: rewrite X-IMAPbase header
2875N/A }
2875N/A
2875N/A /* only syncs left should be just appends (and their updates)
2875N/A which weren't synced yet for some reason (crash). we'll just
2875N/A ignore them, as we've overwritten them above. */
2875N/A while (mail_index_sync_next(sync_ctx->index_sync_ctx,
2875N/A &sync_ctx->sync_rec) > 0)
2875N/A ;
2875N/A
2875N/A if (mbox_sync_update_index_header(sync_ctx) < 0)
2875N/A return -1;
2875N/A
2875N/A return 0;
2875N/A}
2875N/A
2875N/Astatic int mbox_sync_has_changed(struct index_mailbox *ibox)
2875N/A{
2875N/A const struct mail_index_header *hdr;
2875N/A struct stat st;
2875N/A
2875N/A if (mail_index_get_header(ibox->view, &hdr) < 0) {
2875N/A mail_storage_set_index_error(ibox);
2875N/A return -1;
2875N/A }
2875N/A
2875N/A if (stat(ibox->path, &st) < 0) {
2875N/A mbox_set_syscall_error(ibox, "stat()");
2875N/A return -1;
2875N/A }
2875N/A
2875N/A return (uint32_t)st.st_mtime != hdr->sync_stamp ||
2875N/A (uint64_t)st.st_size != hdr->sync_size;
2875N/A}
2875N/A
2875N/Aint mbox_sync(struct index_mailbox *ibox, int last_commit)
2875N/A{
2875N/A struct mail_index_sync_ctx *index_sync_ctx;
2875N/A struct mail_index_view *sync_view;
2875N/A struct mbox_sync_context sync_ctx;
2875N/A uint32_t seq;
2875N/A uoff_t offset;
2875N/A int ret, index_synced;
2875N/A
2875N/A if ((ret = mbox_sync_has_changed(ibox)) < 0)
2875N/A return -1;
2875N/A if (ret == 0 && !last_commit)
2875N/A return 0;
2875N/A index_synced = ret > 0;
2875N/A
2875N/A if (last_commit) {
2875N/A seq = ibox->commit_log_file_seq;
2875N/A offset = ibox->commit_log_file_offset;
2875N/A } else {
2875N/A seq = (uint32_t)-1;
2875N/A offset = (uoff_t)-1;
2875N/A }
2875N/A
2875N/A ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
2875N/A seq, offset);
2875N/A if (ret <= 0) {
2875N/A if (ret < 0)
2875N/A mail_storage_set_index_error(ibox);
2875N/A return ret;
2875N/A }
2875N/A
2875N/A memset(&sync_ctx, 0, sizeof(sync_ctx));
2875N/A sync_ctx.ibox = ibox;
2875N/A sync_ctx.from_line = str_new(default_pool, 256);
2875N/A sync_ctx.header = str_new(default_pool, 4096);
2875N/A
2875N/A sync_ctx.index_sync_ctx = index_sync_ctx;
2875N/A sync_ctx.sync_view = sync_view;
2875N/A sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
2875N/A
2875N/A sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
2875N/A sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
2875N/A sync_ctx.next_uid = 1;
2875N/A
2875N/A ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
2875N/A i_assert(ret == 0);
2875N/A
2875N/A if (mbox_sync_do(&sync_ctx, index_synced) < 0)
2875N/A ret = -1;
2875N/A
2875N/A if (ret < 0)
2875N/A mail_index_transaction_rollback(sync_ctx.t);
2875N/A else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0)
2875N/A ret = -1;
2875N/A else {
2875N/A ibox->commit_log_file_seq = 0;
2875N/A ibox->commit_log_file_offset = 0;
2875N/A }
2875N/A
2875N/A if (mail_index_sync_end(index_sync_ctx) < 0)
2875N/A ret = -1;
2875N/A
2875N/A if (sync_ctx.lock_id != 0) {
2875N/A if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
2875N/A ret = -1;
2875N/A }
2875N/A
2875N/A str_free(sync_ctx.header);
2875N/A str_free(sync_ctx.from_line);
2875N/A buffer_free(sync_ctx.mails);
2875N/A buffer_free(sync_ctx.syncs);
2875N/A return ret;
2875N/A}
2875N/A
2875N/Aint mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
2875N/A{
2875N/A struct index_mailbox *ibox = (struct index_mailbox *)box;
2875N/A
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
if (mbox_sync(ibox, FALSE) < 0)
return -1;
}
return index_storage_sync(box, flags);
}