index/mbox/mbox-sync.c

	mbox-sync.c revision b2ecd50bb98c44816cb07c17aa17fae2b425f941
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen/*
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   Modifying mbox can be slow, so we try to do it all at once minimizing the
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   required disk I/O. We may need to:
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - Update message flags in Status, X-Status and X-Keywords headers
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - Write missing X-UID and X-IMAPbase headers
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - Write missing or broken Content-Length header if there's space
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - Expunge specified messages
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   Here's how we do it:
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - Start reading the mails mail headers from the beginning
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - X-Keywords and X-UID headers may contain extra spaces at the end of them,
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     remember how much extra each message has and offset to beginning of the
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     spaces
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - If message flags are dirty and there's enough space to write them, do it
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - If we didn't have enough space, remember how much was missing and keep
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     the total amount of them
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - When we encounter expunged message, check if the amount of empty space in
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     previous messages plus size of expunged message is enough to cover the
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     missing space. If yes,
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen       - execute the rewrite plan
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen       - forget all the messages before the expunged message. only remember
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen         how much data we still have to move to cover the expunged message
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - If we encounter end of file, grow the file and execute the rewrite plan
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   Rewrite plan goes:
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - Start from the first message that needs more space
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - If there's expunged messages before us, we have to write over them.
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen       - Move all messages after it backwards to fill it
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen       - Each moved message's X-Keywords header should have n bytes extra
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen         space, unless there's not enough space to do it.
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - If there's no expunged messages, we can move data either forward or
74ae32512357bdd4872bf160dc697ff7b54b54c5Timo Sirainen     backward to get it. Calculate which requires less moving. Forward
6868400686d0726ac2c08d67ed896f2c76e7b219Aki Tuomi     counting may encounter more messages which require extra space, count
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen     that too.
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen       - If we decide to move forwards and we had to go through dirty
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen         messages, do the moving from last to first dirty message
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen   - If we encounter end of file, grow the file enough to get the required
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     amount of space plus enough space to fill X-Keywords headers full of
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen     spaces.
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen*/
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "lib.h"
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "ioloop.h"
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "buffer.h"
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "istream.h"
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "file-set-size.h"
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "str.h"
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen#include "write-full.h"
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen#include "istream-raw-mbox.h"
4e1b3183abf33f3125247c8e28e30c788d2fa0abTimo Sirainen#include "mbox-storage.h"
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen#include "mbox-file.h"
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi#include "mbox-sync-private.h"
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen#include <sys/stat.h>
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen                   struct mbox_sync_mail *mail, uoff_t body_offset,
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen                   uoff_t grow_size)
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen{
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi    char spaces[1024];
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen    uoff_t offset, size;
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen
9625595c47c665f5aee57ebfcb1fcbe9ad1bf3a0Martti Rannanjärvi    i_assert(grow_size > 0);
58d16e83e0953ac98f4bdc905f2f5d24b8f0002dTimo Sirainen
74ae32512357bdd4872bf160dc697ff7b54b54c5Timo Sirainen    memset(spaces, ' ', sizeof(spaces));
74ae32512357bdd4872bf160dc697ff7b54b54c5Timo Sirainen
74ae32512357bdd4872bf160dc697ff7b54b54c5Timo Sirainen    size = sync_ctx->input->v_offset + grow_size;
7a7d97c082834dccdd89dbb95598977deec4d5a7Timo Sirainen    if (file_set_size(sync_ctx->fd, size) < 0)
        return -1;

    if (mail->space_offset == 0) {
        /* no X-Keywords header - place it at the end. */
        grow_size += 13;

        offset = body_offset-1;
        if (mbox_move(sync_ctx, body_offset-1 + size,
                  offset, (uoff_t)-1) < 0)
            return -1;
        if (pwrite_full(sync_ctx->fd, "X-Keywords: ", 12, offset) < 0)
            return -1;
        if (pwrite_full(sync_ctx->fd, "\n", 1,
                offset + grow_size-1) < 0)
            return -1;
        grow_size -= 13; offset += 12;

        /* FIXME: can this break anything? X-Keywords text might
           have been already included in space calculation. now we
           have more.. */
        mail->space_offset = offset;
        mail->space += grow_size;
    } else {
        offset = mail->space_offset;
        if (mbox_move(sync_ctx, mail->space_offset + grow_size,
                  offset, (uoff_t)-1) < 0)
            return -1;
    }

    while (grow_size >= sizeof(spaces)) {
        if (pwrite_full(sync_ctx->fd, spaces,
                sizeof(spaces), offset) < 0)
            return -1;
        grow_size -= sizeof(spaces);
        offset += sizeof(spaces);
    }

    if (grow_size > 0) {
        if (pwrite_full(sync_ctx->fd, spaces, grow_size, offset) < 0)
            return -1;
    }

    istream_raw_mbox_flush(sync_ctx->input);
    return 0;
}

int mbox_sync(struct index_mailbox *ibox, int last_commit)
{
    struct mbox_sync_context sync_ctx;
    struct mbox_sync_mail_context mail_ctx;
    struct mbox_sync_mail mail;
    struct mail_index_sync_ctx *index_sync_ctx;
    struct mail_index_sync_rec sync_rec;
    struct mail_index_view *sync_view;
    struct mail_index_transaction *t;
    const struct mail_index_header *hdr;
    const struct mail_index_record *rec;
    struct istream *input;
    uint32_t seq, need_space_seq, idx_seq, messages_count;
    off_t space_diff;
    uoff_t from_offset, offset;
    buffer_t *mails;
    string_t *header;
    size_t size;
    struct stat st;
    int readonly, ret = 0;

    if (last_commit) {
        seq = ibox->commit_log_file_seq;
        offset = ibox->commit_log_file_offset;
    } else {
        seq = (uint32_t)-1;
        offset = (uoff_t)-1;
    }

    ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
                    seq, offset);
    if (ret <= 0)
        return ret;

    if (mbox_file_open_stream(ibox) < 0)
        return -1;

    if (mail_index_get_header(sync_view, &hdr) < 0)
        return -1;

    t = mail_index_transaction_begin(sync_view, FALSE);

    if (ibox->mbox_data_buf == NULL) {
        ibox->mbox_data_buf =
            buffer_create_dynamic(default_pool, 512, (size_t)-1);
    } else {
        buffer_set_used_size(ibox->mbox_data_buf, 0);
    }

    readonly = TRUE; // FIXME

    // FIXME: lock the file

    memset(&sync_ctx, 0, sizeof(sync_ctx));
    sync_ctx.file_input = ibox->mbox_file_stream;
    sync_ctx.input = ibox->mbox_stream;
    sync_ctx.fd = ibox->mbox_fd;
    sync_ctx.hdr = hdr;

    input = sync_ctx.input;
    istream_raw_mbox_seek(input, 0);

    header = str_new(default_pool, 4096);

    mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
    memset(&sync_rec, 0, sizeof(sync_rec));
    messages_count = mail_index_view_get_message_count(sync_view);

    space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL;
    for (seq = 1; !input->eof; seq++) {
        if (sync_rec.seq2 < seq) {
            // FIXME: we may need more than one..
            ret = mail_index_sync_next(index_sync_ctx, &sync_rec);
            if (ret < 0)
                break;
        }

        from_offset = input->v_offset;

        memset(&mail, 0, sizeof(mail));
        memset(&mail_ctx, 0, sizeof(mail_ctx));
        mail_ctx.sync_ctx = &sync_ctx;
        mail_ctx.mail = &mail;
        mail_ctx.seq = seq;
        mail_ctx.header = header;

        mbox_sync_parse_next_mail(input, &mail_ctx);
        if (input->v_offset == from_offset) {
            /* this was the last mail */
            break;
        }

        mail.body_size =
            istream_raw_mbox_get_size(input,
                          mail_ctx.content_length);
        buffer_append(mails, &mail, sizeof(mail));

        /* save the offset permanently with recent flag state */
        from_offset <<= 1;
        if ((mail.flags & MBOX_NONRECENT) == 0)
            from_offset |= 1;
        buffer_append(ibox->mbox_data_buf,
                  &from_offset, sizeof(from_offset));

        /* update index */
        do {
            if (rec != NULL && rec->uid >= mail.uid)
                break;

            if (idx_seq >= messages_count) {
                rec = NULL;
                break;
            }

            if (rec != NULL)
                mail_index_expunge(t, idx_seq);

            ret = mail_index_lookup(sync_view, ++idx_seq, &rec);
        } while (ret == 0);

        if (ret < 0)
            break;
        if (rec != NULL && rec->uid != mail.uid) {
            /* new UID in the middle of the mailbox -
               shouldn't happen */
            mail_storage_set_critical(ibox->box.storage,
                "mbox sync: UID inserted in the middle "
                "of mailbox (%u > %u)", rec->uid, mail.uid);
            mail_index_mark_corrupted(ibox->index);
            ret = -1;
            break;
        }

        if (rec != NULL) {
            /* see if flags changed */
            if ((rec->flags & MAIL_FLAGS_MASK) !=
                (mail.flags & MAIL_FLAGS_MASK) ||
                memcmp(rec->keywords, mail.keywords,
                   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
                uint8_t new_flags =
                    (rec->flags & ~MAIL_FLAGS_MASK) |
                    (mail.flags & MAIL_FLAGS_MASK);
                mail_index_update_flags(t, idx_seq,
                            MODIFY_REPLACE,
                            new_flags,
                            mail.keywords);
            }
            rec = NULL;
        } else {
            /* new message */
            mail_index_append(t, mail.uid, &idx_seq);
            mail_index_update_flags(t, idx_seq, MODIFY_REPLACE,
                        mail.flags & MAIL_FLAGS_MASK,
                        mail.keywords);
        }

        if (mail_ctx.need_rewrite && !readonly) {
            mbox_sync_update_header(&mail_ctx, NULL);
            if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
                break;
        } else {
            ret = 1;
        }

        if (ret == 0 && need_space_seq == 0) {
            /* didn't have space to write it */
            need_space_seq = seq;
            space_diff = mail.space;
        } else if (need_space_seq != 0) {
            space_diff += mail.space;
            if (space_diff >= 0) {
                /* we have enough space now */
                if (mbox_sync_rewrite(&sync_ctx, mails,
                              need_space_seq, seq,
                              space_diff) < 0) {
                    ret = -1;
                    break;
                }
                need_space_seq = 0;
            }
        }

        istream_raw_mbox_next(input, mail.body_size);
    }

    if (need_space_seq != 0) {
        i_assert(space_diff < 0);
        if (mbox_sync_grow_file(&sync_ctx, &mail, mail_ctx.body_offset,
                    -space_diff) < 0)
            ret = -1;
        else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
                       seq-1, space_diff) < 0)
            ret = -1;
    }

    while ((ret = mail_index_sync_next(index_sync_ctx, &sync_rec)) > 0) {
        // FIXME: should be just appends
    }

    if (fstat(ibox->mbox_fd, &st) < 0) {
        mbox_set_syscall_error(ibox, "fstat()");
        ret = -1;
    }

    if (ret < 0)
        mail_index_transaction_rollback(t);
    else {
        if (mail_index_transaction_commit(t, &seq, &offset) < 0)
            ret = -1;
        else {
            ibox->commit_log_file_seq = seq;
            ibox->commit_log_file_offset = offset;
        }
    }

    if (ret < 0) {
        st.st_mtime = 0;
        st.st_size = 0;
    }

    if (mail_index_sync_end(index_sync_ctx, st.st_mtime, st.st_size) < 0)
        ret = -1;

    if (ret == 0) {
        ibox->commit_log_file_seq = 0;
        ibox->commit_log_file_offset = 0;
    } else {
        mail_storage_set_index_error(ibox);
    }

    ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size);
    ibox->mbox_data_count = size / sizeof(*ibox->mbox_data);

    str_free(header);
    return ret < 0 ? -1 : 0;
}

int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
    struct index_mailbox *ibox = (struct index_mailbox *)box;

    if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
        ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
        ibox->sync_last_check = ioloop_time;

        if (mbox_sync(ibox, FALSE) < 0)
            return -1;
    }

    return index_storage_sync(box, flags);
}