mbox-sync.c revision 9e59a1f3f095b3099478562cf3f3970a24736970
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen/* Copyright (C) 2004 Timo Sirainen */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen/*
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen Modifying mbox can be slow, so we try to do it all at once minimizing the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen required disk I/O. We may need to:
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Update message flags in Status, X-Status and X-Keywords headers
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Write missing X-UID and X-IMAPbase headers
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Write missing or broken Content-Length header if there's space
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen - Expunge specified messages
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen Here's how we do it:
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Start reading the mails mail headers from the beginning
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - X-Keywords and X-UID headers may contain extra spaces at the end of them,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen remember how much extra each message has and offset to beginning of the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen spaces
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If message flags are dirty and there's enough space to write them, do it
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If we didn't have enough space, remember how much was missing and keep
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen the total amount of them
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - When we encounter expunged message, check if the amount of empty space in
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen previous messages plus size of expunged message is enough to cover the
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen missing space. If yes,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - execute the rewrite plan
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - forget all the messages before the expunged message. only remember
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen how much data we still have to move to cover the expunged message
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If we encounter end of file, grow the file and execute the rewrite plan
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen Rewrite plan goes:
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Start from the first message that needs more space
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If there's expunged messages before us, we have to write over them.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Move all messages after it backwards to fill it
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - Each moved message's X-Keywords header should have n bytes extra
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen space, unless there's not enough space to do it.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If there's no expunged messages, we can move data either forward or
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen backward to get it. Calculate which requires less moving. Forward
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen counting may encounter more messages which require extra space, count
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen that too.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If we decide to move forwards and we had to go through dirty
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen messages, do the moving from last to first dirty message
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen - If we encounter end of file, grow the file enough to get the required
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen amount of space plus enough space to fill X-Keywords headers full of
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen spaces.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen*/
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "lib.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "ioloop.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "buffer.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "istream.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "file-set-size.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "str.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "write-full.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "istream-raw-mbox.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mbox-storage.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mbox-file.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mbox-lock.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include "mbox-sync-private.h"
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include <stddef.h>
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#include <sys/stat.h>
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen#define MBOX_SYNC_SECS 1
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen/* returns -1 = error, 0 = mbox changed since previous lock, 1 = didn't */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_sync_lock(struct mbox_sync_context *sync_ctx, int lock_type)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct index_mailbox *ibox = sync_ctx->ibox;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct stat old_st, st;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uoff_t old_from_offset = 0, old_offset = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(lock_type != F_WRLCK || !ibox->mbox_readonly);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync_ctx->lock_id != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (fstat(sync_ctx->fd, &old_st) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_set_syscall_error(ibox, "stat()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen old_from_offset =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen old_offset = sync_ctx->input->v_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (void)mbox_unlock(ibox, sync_ctx->lock_id);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->lock_id = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memset(&old_st, 0, sizeof(old_st));
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_lock(ibox, lock_type, &sync_ctx->lock_id) <= 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_file_open_stream(ibox) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->file_input = sync_ctx->ibox->mbox_file_stream;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->input = sync_ctx->ibox->mbox_stream;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->fd = sync_ctx->ibox->mbox_fd;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (old_st.st_mtime == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we didn't have the file open before -> it changed */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (fstat(sync_ctx->fd, &st) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_set_syscall_error(ibox, "fstat()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (st.st_mtime != old_st.st_mtime || st.st_size != old_st.st_size ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen st.st_ino != old_st.st_ino ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen !CMP_DEV_T(st.st_dev, old_st.st_dev) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen time(NULL) - st.st_mtime <= MBOX_SYNC_SECS)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen /* same as before. we'll have to fix mbox stream to contain
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen correct from_offset, hdr_offset and body_offset. so, seek
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen to from_offset and read through the header. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, old_from_offset) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_storage_set_critical(ibox->box.storage,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "Message offset %s changed unexpectedly for mbox file "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "%s", dec2str(old_from_offset), sync_ctx->ibox->path);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_stream_seek(sync_ctx->input, old_offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenint mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "Unexpectedly lost From-line at offset %"PRIuUOFF_T
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen " from mbox file %s", from_offset,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->ibox->path);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mbox_sync_mail_context *mail_ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uoff_t grow_size)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uoff_t src_offset, file_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(grow_size > 0);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* put the extra space between last message's header and body */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (file_set_size(sync_ctx->fd, file_size) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen src_offset = mail_ctx->body_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->body_offset += grow_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen file_size - mail_ctx->body_offset) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_flush(sync_ctx->input);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_sync_rec *sync;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size_t size, src, dest;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync = buffer_get_modifyable_data(syncs_buf, &size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size /= sizeof(*sync);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (src = dest = 0; src < size; src++) {
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen if (sync[src].uid2 >= uid) {
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen if (src != dest)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync[dest] = sync[src];
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dest++;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* get EOF */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (void)istream_raw_mbox_get_header_offset(sync_ctx->input);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input))
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen mail_ctx->sync_ctx = sync_ctx;
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen mail_ctx->seq = ++sync_ctx->seq;
2498b8003eb181001b0c4fd45763c462b45493d1Timo Sirainen mail_ctx->header = sync_ctx->header;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->from_offset =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_get_start_offset(sync_ctx->input);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.offset =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_get_header_offset(sync_ctx->input);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_ctx->seq == 1)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen sync_ctx->seen_first_mail = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* First message was expunged and this is the next one.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen Skip \n header */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->from_offset++;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.body_size =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_get_body_size(sync_ctx->input,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->content_length);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(mail_ctx->mail.body_size < OFF_T_MAX);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* save the offset permanently with recent flag state */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.from_offset = mail_ctx->from_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* need to add 'O' flag to Status-header */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->need_rewrite = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen // FIXME: save it somewhere
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_sync_buf_have_expunges(buffer_t *syncs_buf)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_sync_rec *sync;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size_t size, i;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size /= sizeof(*sync);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen for (i = 0; i < size; i++) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync[i].type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return TRUE;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t uid, int *sync_expunge_r)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mail_index_sync_rec *sync_rec = &sync_ctx->sync_rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen int ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *sync_expunge_r = FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen while (uid >= sync_rec->uid1) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (sync_rec->uid1 != 0) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen i_assert(uid <= sync_rec->uid2);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen buffer_append(sync_ctx->syncs, sync_rec,
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen sizeof(*sync_rec));
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE)
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen *sync_expunge_r = TRUE;
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen ret = mail_index_sync_next(sync_ctx->index_sync_ctx, sync_rec);
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (ret < 0)
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memset(sync_rec, 0, sizeof(*sync_rec));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen if (!*sync_expunge_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *sync_expunge_r = mbox_sync_buf_have_expunges(sync_ctx->syncs);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen keywords_mask_t keywords)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_sync_rec *sync;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size_t size, i;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync = buffer_get_data(syncs_buf, &size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size /= sizeof(*sync);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (i = 0; i < size; i++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync[i].type != MAIL_INDEX_SYNC_TYPE_FLAGS)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen continue;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_sync_flags_apply(&sync[i], flags, keywords);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t uid, const struct mail_index_record **rec_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mail_index_record *rec = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t messages_count;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen while (sync_ctx->idx_seq < messages_count) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_index_lookup(sync_ctx->sync_view,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen ++sync_ctx->idx_seq, &rec) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (uid <= rec->uid)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* externally expunged message, remove from index */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rec = NULL;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (rec != NULL && rec->uid != uid) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* new UID in the middle of the mailbox - shouldn't happen */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "mbox sync: UID inserted in the middle of mailbox %s "
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen "(%u > %u)", sync_ctx->ibox->path, rec->uid, uid);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen mail_index_mark_corrupted(sync_ctx->ibox->index);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *rec_r = rec;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen uint32_t seq, uint64_t *offset_r)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const void *data;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* see if from_offset needs updating */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_index_lookup_extra(sync_ctx->sync_view, seq,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->ibox->mbox_extra_idx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &data) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_storage_set_index_error(sync_ctx->ibox);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen *offset_r = *((const uint64_t *)data);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenmbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mbox_sync_mail *mail,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int nocheck)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint64_t offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!nocheck) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_sync_get_from_offset(sync_ctx, sync_ctx->idx_seq,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &offset) < 0)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen if (offset == mail->from_offset)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen offset = mail->from_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_update_extra_rec(sync_ctx->t, sync_ctx->idx_seq,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->ibox->mbox_extra_idx, &offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mbox_sync_mail *mail,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_record *rec)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen keywords_mask_t idx_keywords;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint8_t idx_flags, mbox_flags;
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (rec == NULL) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* new message */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen MODIFY_REPLACE, mbox_flags,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail->keywords);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* see if flags changed */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen idx_flags = rec->flags;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memcpy(idx_keywords, rec->keywords, INDEX_KEYWORDS_BYTE_COUNT);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_sync_apply_index_syncs(sync_ctx->syncs,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &idx_flags, idx_keywords);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (idx_flags != mbox_flags ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memcmp(idx_keywords, mail->keywords,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen INDEX_KEYWORDS_BYTE_COUNT) != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen MODIFY_REPLACE, mbox_flags,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail->keywords);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* update from_offsets, but not if we're going to rewrite this message.
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen rewriting would just move it anyway. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync_ctx->need_space_seq == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int nocheck = rec == NULL || sync_ctx->expunged_space > 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_sync_update_from_offset(sync_ctx, mail, nocheck) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct istream *input = ctx->sync_ctx->file_input;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const unsigned char *data;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen size_t size, from_line_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_set_used_size(ctx->sync_ctx->from_line, 0);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen from_line_size = ctx->hdr_offset - ctx->from_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_stream_seek(input, ctx->from_offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (;;) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen data = i_stream_get_data(input, &size);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (size >= from_line_size)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size = from_line_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_append(ctx->sync_ctx->from_line, data, size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_stream_skip(input, size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen from_line_size -= size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (from_line_size == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (i_stream_read(input) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen string_t *str = ctx->sync_ctx->from_line;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ctx->from_offset + move_diff) < 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_flush(ctx->sync_ctx->input);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic void
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenupdate_from_offsets(struct index_mailbox *ibox,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mail_index_transaction *t, buffer_t *mails_buf,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq1, uint32_t seq2)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen const struct mbox_sync_mail *mails;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t extra_idx = ibox->mbox_extra_idx;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint64_t offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mails = buffer_get_modifyable_data(mails_buf, NULL);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen for (; seq1 <= seq2; seq1++, mails++) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mails->uid != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen offset = mails->from_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_update_extra_rec(t, seq1, extra_idx,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &offset);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mbox_sync_check_excl_lock(struct mbox_sync_context *sync_ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync_ctx->ibox->mbox_lock_type == F_RDLCK) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((ret = mbox_sync_lock(sync_ctx, F_WRLCK)) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret == 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -2;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((ret = mbox_sync_check_excl_lock(mail_ctx->sync_ctx)) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.offset = mail_ctx->from_offset;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.space =
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->body_offset - mail_ctx->from_offset +
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.body_size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.body_size = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_ctx->sync_ctx->dest_first_mail) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* expunging first message, fix space to contain next
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen message's \n header too since it will be removed. */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.space++;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->sync_ctx->expunged_space += mail_ctx->mail.space;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen off_t move_diff;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen int ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync_ctx->ibox->mbox_readonly)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* move the header backwards to fill expunged space */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen move_diff = -sync_ctx->expunged_space;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* read the From-line before rewriting overwrites it */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_read_from_line(mail_ctx) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* rewrite successful, write From-line to
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen new location */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.from_offset += move_diff;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->mail.offset += move_diff;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_write_from_line(mail_ctx, move_diff) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else if (mail_ctx->need_rewrite ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_get_used_size(sync_ctx->syncs) != 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return ret;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* nothing to do */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (ret == 0 && sync_ctx->need_space_seq == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* first mail with no space to write it */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->need_space_seq = sync_ctx->seq;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->space_diff = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen if (sync_ctx->expunged_space > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* create dummy message to describe the expunged data */
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mbox_sync_mail mail;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen memset(&mail, 0, sizeof(mail));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail.offset = mail_ctx->from_offset -
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->expunged_space;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail.space = sync_ctx->expunged_space;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->need_space_seq--;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_append(sync_ctx->mails, &mail, sizeof(mail));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t extra_space;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->space_diff += mail_ctx->mail.space;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_ctx->space_diff < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we have enough space now */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen extra_space = MBOX_HEADER_EXTRA_SPACE *
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mail_ctx->mail.uid == 0 &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (uoff_t)sync_ctx->space_diff > extra_space) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* don't waste too much on extra spacing */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->expunged_space = sync_ctx->space_diff - extra_space;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->space_diff = extra_space;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->expunged_space = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_rewrite(sync_ctx, sync_ctx->need_space_seq, sync_ctx->seq,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->space_diff) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen update_from_offsets(sync_ctx->ibox, sync_ctx->t, sync_ctx->mails,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->need_space_seq, sync_ctx->seq);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* mail_ctx may contain wrong data after rewrite, so make sure we
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen don't try to access it */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen memset(mail_ctx, 0, sizeof(*mail_ctx));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->need_space_seq = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen buffer_set_used_size(sync_ctx->mails, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenmbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t seq;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint64_t offset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, uid,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &seq, &seq) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (seq == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_get_from_offset(sync_ctx, seq, &offset) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* set to -1, since they're always increased later */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->seq = sync_ctx->idx_seq = seq-1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->dest_first_mail = sync_ctx->seq == 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_storage_set_critical(sync_ctx->ibox->box.storage,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "Cached message offset %s is invalid for mbox file %s",
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen dec2str(offset), sync_ctx->ibox->path);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_mark_corrupted(sync_ctx->ibox->index);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct mbox_sync_mail_context *mail_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t min_message_count)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen const struct mail_index_record *rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uint32_t uid, messages_count;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t offset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen int ret, expunged;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (min_message_count != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (istream_raw_mbox_seek(sync_ctx->input, 0) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* doesn't begin with a From-line */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_storage_set_error(sync_ctx->ibox->box.storage,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen "Mailbox isn't a valid mbox file");
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->dest_first_mail = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* we sync only what we need to. jump to first record that
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen needs updating */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen const struct mail_index_sync_rec *sync_rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen size_t size;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_ctx->sync_rec.uid1 == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_sync_read_index_syncs(sync_ctx, 1,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &expunged) < 0)
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen return -1;
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen
51b979b6414b940f04677a7e2d064be119345954Timo Sirainen if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->sync_rec.uid1 == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* nothing to do */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_rec = buffer_get_data(sync_ctx->syncs, &size);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (size == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_rec = &sync_ctx->sync_rec;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uid = mail_ctx->mail.uid;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* get all sync records related to this message */
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen return -1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (!expunged) {
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen ret = mbox_sync_handle_header(mail_ctx);
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_ctx->dest_first_mail = FALSE;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen } else {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_ctx->mail.uid = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen ret = mbox_sync_handle_expunge(mail_ctx);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ret < 0) {
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen /* -1 = error, -2 = need to restart */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return ret;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_read_index_rec(sync_ctx, uid, &rec) < 0)
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!expunged) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_update_index(sync_ctx, &mail_ctx->mail,
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen rec) < 0)
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen return -1;
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen }
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen
51795bfe9d05d92fe942cb451aec2b9d16d32a11Timo Sirainen istream_raw_mbox_next(sync_ctx->input,
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen mail_ctx->mail.body_size);
8e57335924f5ff57cbd1929ec99764dc267c3312Timo Sirainen offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_ctx->need_space_seq != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_handle_missing_space(mail_ctx) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_sync_seek(sync_ctx, offset) < 0)
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen return -1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen } else if (sync_ctx->expunged_space > 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (!expunged) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* move the body */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (mbox_move(sync_ctx,
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen mail_ctx->body_offset -
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->expunged_space,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_ctx->body_offset,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_ctx->mail.body_size) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen if (mbox_sync_seek(sync_ctx, offset) < 0)
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen return -1;
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen } else if (sync_ctx->seq >= min_message_count) {
5c7aa03f959b8b9cab3eba8a585a90f4b50a4cdfTimo Sirainen mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (buffer_get_used_size(sync_ctx->syncs) == 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen /* if there's no sync records left,
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen we can stop */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_ctx->sync_rec.uid1 == 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen break;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we can skip forward to next record which
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen needs updating. */
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uid = sync_ctx->sync_rec.uid1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input)) {
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen /* rest of the messages in index don't exist -> expunge them */
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen messages_count =
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen mail_index_view_get_message_count(sync_ctx->sync_view);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen while (sync_ctx->idx_seq < messages_count)
2d49f150b4bce6f2f59a84e268e4777901c3e42cTimo Sirainen mail_index_expunge(sync_ctx->t, ++sync_ctx->idx_seq);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen struct mbox_sync_mail_context *mail_ctx)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen uoff_t offset, extra_space, trailer_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(sync_ctx->need_space_seq == 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(sync_ctx->expunged_space == 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen trailer_size = i_stream_get_size(sync_ctx->file_input) -
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_ctx->file_input->v_offset;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_ctx->need_space_seq != 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen i_assert(sync_ctx->space_diff < 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen extra_space = MBOX_HEADER_EXTRA_SPACE *
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (sync_ctx->seq - sync_ctx->need_space_seq + 1);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->space_diff -= extra_space;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->space_diff += sync_ctx->expunged_space;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (sync_ctx->expunged_space <= -sync_ctx->space_diff)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_ctx->expunged_space = 0;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen else
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen sync_ctx->expunged_space -= -sync_ctx->space_diff;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (mail_ctx->have_eoh && !mail_ctx->updated)
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen str_append_c(mail_ctx->header, '\n');
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen if (sync_ctx->space_diff < 0 &&
bf91bed88d4e294b4577ba2a3b14d87cf35ae135Timo Sirainen mbox_sync_grow_file(sync_ctx, mail_ctx,
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen -sync_ctx->space_diff) < 0)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen return -1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen return -1;
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if (sync_ctx->seq != sync_ctx->need_space_seq) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_set_used_size(sync_ctx->mails,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen (sync_ctx->seq -
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->need_space_seq) *
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sizeof(mail_ctx->mail));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_append(sync_ctx->mails, &mail_ctx->mail,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sizeof(mail_ctx->mail));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (mbox_sync_rewrite(sync_ctx,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->need_space_seq,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->seq, extra_space) < 0)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen update_from_offsets(sync_ctx->ibox, sync_ctx->t,
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen sync_ctx->mails,
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen sync_ctx->need_space_seq,
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen sync_ctx->seq);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->need_space_seq = 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen buffer_set_used_size(sync_ctx->mails, 0);
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (sync_ctx->expunged_space > 0) {
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen /* copy trailer, then truncate the file */
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen offset = i_stream_get_size(sync_ctx->file_input) -
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen sync_ctx->expunged_space - trailer_size;
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen
19779377be72c9fe8365bb9ba7a2e0d06dc99c3bTimo Sirainen if (mbox_move(sync_ctx, offset,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen offset + sync_ctx->expunged_space,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen trailer_size) < 0)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (ftruncate(sync_ctx->fd, offset + trailer_size) < 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "ftruncate()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->expunged_space = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen istream_raw_mbox_flush(sync_ctx->input);
b0be0bead3d6963149f7f2a9504b8ab5aced9af5Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainenstatic int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx)
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen{
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen struct stat st;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (fstat(sync_ctx->fd, &st) < 0) {
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mbox_set_syscall_error(sync_ctx->ibox, "fstat()");
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen return -1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if ((sync_ctx->base_uid_validity != 0 &&
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen (sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if (sync_ctx->base_uid_validity == 0) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen /* we couldn't rewrite X-IMAPbase because it's
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen a read-only mbox */
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(sync_ctx->ibox->mbox_readonly);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->base_uid_validity = time(NULL);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_update_header(sync_ctx->t,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen offsetof(struct mail_index_header, uid_validity),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &sync_ctx->base_uid_validity,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sizeof(sync_ctx->base_uid_validity));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen if (istream_raw_mbox_is_eof(sync_ctx->input) &&
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen sync_ctx->next_uid != sync_ctx->hdr->next_uid) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen i_assert(sync_ctx->next_uid != 0);
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen mail_index_update_header(sync_ctx->t,
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen offsetof(struct mail_index_header, next_uid),
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen &sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen }
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen uint32_t sync_stamp = st.st_mtime;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen mail_index_update_header(sync_ctx->t,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen offsetof(struct mail_index_header, sync_stamp),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &sync_stamp, sizeof(sync_stamp));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen uint64_t sync_size = st.st_size;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
14c474d9f4591c397ed0b5206af6537c7b52c924Timo Sirainen mail_index_update_header(sync_ctx->t,
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen offsetof(struct mail_index_header, sync_size),
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen &sync_size, sizeof(sync_size));
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen }
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen return 0;
2a6af811ea3de3cf9e2f15e446674dd21b0705f3Timo Sirainen}
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainenstatic void mbox_sync_restart(struct mbox_sync_context *sync_ctx)
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen{
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->base_uid_validity = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->base_uid_last = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->next_uid = 1;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->prev_msg_uid = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->seq = sync_ctx->idx_seq = 0;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->dest_first_mail = TRUE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen sync_ctx->seen_first_mail = FALSE;
2a34e2be33f8a17d21384a5527ed9f75f4d270e0Timo Sirainen}
static int mbox_sync_do(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
struct stat st;
uint32_t min_msg_count;
int ret;
if (fstat(sync_ctx->fd, &st) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "stat()");
return -1;
}
min_msg_count =
(uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
(uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
0 : (uint32_t)-1;
mbox_sync_restart(sync_ctx);
if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
return -1;
if (ret == -2) {
/* initially we had mbox read-locked, but later we needed a
write-lock. doing it required dropping the read lock.
we're here because mbox was modified before we got the
write-lock. so, restart the whole syncing. */
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mail_index_transaction_rollback(sync_ctx->t);
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
FALSE);
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
return -1;
}
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
/* only syncs left should be just appends (and their updates)
which weren't synced yet for some reason (crash). we'll just
ignore them, as we've overwritten them above. */
buffer_set_used_size(sync_ctx->syncs, 0);
memset(&sync_ctx->sync_rec, 0, sizeof(sync_ctx->sync_rec));
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
static int mbox_sync_has_changed(struct index_mailbox *ibox)
{
const struct mail_index_header *hdr;
struct stat st;
if (mail_index_get_header(ibox->view, &hdr) < 0) {
mail_storage_set_index_error(ibox);
return -1;
}
if (stat(ibox->path, &st) < 0) {
mbox_set_syscall_error(ibox, "stat()");
return -1;
}
return (uint32_t)st.st_mtime != hdr->sync_stamp ||
(uint64_t)st.st_size != hdr->sync_size;
}
static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
{
struct mbox_sync_mail_context mail_ctx;
if (mbox_sync_seek(sync_ctx, 0) < 0)
return -1;
sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view, FALSE);
sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
if (mbox_sync_check_excl_lock(sync_ctx) == -1)
return -1;
mbox_sync_restart(sync_ctx);
if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
return -1;
if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
return -1;
if (mbox_sync_update_index_header(sync_ctx) < 0)
return -1;
return 0;
}
int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock)
{
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_view *sync_view;
struct mbox_sync_context sync_ctx;
uint32_t seq;
uoff_t offset;
unsigned int lock_id = 0;
int ret, lock_type;
if (lock) {
if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
return -1;
}
if ((ret = mbox_sync_has_changed(ibox)) < 0) {
if (lock)
(void)mbox_unlock(ibox, lock_id);
return -1;
}
if (ret == 0 && !last_commit)
return 0;
if (last_commit) {
seq = ibox->commit_log_file_seq;
offset = ibox->commit_log_file_offset;
} else {
seq = (uint32_t)-1;
offset = (uoff_t)-1;
}
ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
seq, offset);
if (ret <= 0) {
if (ret < 0)
mail_storage_set_index_error(ibox);
return ret;
}
memset(&sync_ctx, 0, sizeof(sync_ctx));
sync_ctx.ibox = ibox;
sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.index_sync_ctx = index_sync_ctx;
sync_ctx.sync_view = sync_view;
sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE);
sync_ctx.mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
sync_ctx.syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
ret = mail_index_get_header(sync_view, &sync_ctx.hdr);
i_assert(ret == 0);
lock_type = mail_index_sync_have_more(index_sync_ctx) ?
F_WRLCK : F_RDLCK;
if (lock_type == F_WRLCK && lock) {
(void)mbox_unlock(ibox, lock_id);
lock_id = 0;
}
if (mbox_sync_lock(&sync_ctx, lock_type) < 0)
return -1;
if (mbox_sync_do(&sync_ctx) < 0)
ret = -1;
if (ret < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t, &seq, &offset) < 0)
ret = -1;
else {
ibox->commit_log_file_seq = 0;
ibox->commit_log_file_offset = 0;
}
sync_ctx.t = NULL;
if (mail_index_sync_end(index_sync_ctx) < 0)
ret = -1;
if (sync_ctx.base_uid_last != sync_ctx.next_uid-1 && ret == 0 &&
!ibox->mbox_readonly) {
/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
so previous transactions have been committed. */
/* FIXME: ugly .. */
ret = mail_index_sync_begin(ibox->index,
&sync_ctx.index_sync_ctx,
&sync_ctx.sync_view,
(uint32_t)-1, (uoff_t)-1);
if (ret < 0)
mail_storage_set_index_error(ibox);
else {
ret = mail_index_get_header(sync_ctx.sync_view,
&sync_ctx.hdr);
if ((ret = mbox_sync_update_imap_base(&sync_ctx)) < 0)
mail_index_transaction_rollback(sync_ctx.t);
else if (mail_index_transaction_commit(sync_ctx.t,
&seq,
&offset) < 0)
ret = -1;
if (mail_index_sync_end(sync_ctx.index_sync_ctx) < 0)
ret = -1;
}
}
if (sync_ctx.lock_id != 0 && (ret < 0 || !lock)) {
/* FIXME: drop to read locking and keep it MBOX_SYNC_SECS+1
to make sure we notice changes made by others */
if (mbox_unlock(ibox, sync_ctx.lock_id) < 0)
ret = -1;
}
str_free(sync_ctx.header);
str_free(sync_ctx.from_line);
buffer_free(sync_ctx.mails);
buffer_free(sync_ctx.syncs);
return ret;
}
int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
{
struct index_mailbox *ibox = (struct index_mailbox *)box;
if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
ibox->sync_last_check = ioloop_time;
if (mbox_sync(ibox, FALSE, FALSE) < 0)
return -1;
}
return index_storage_sync(box, flags);
}